[clang] [llvm] [mlir] [clang][NVPTX][MLIR][NVVM] Add overloaded fadd intrinsics (PR #221681)
Srinivasa Ravi via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 01:55:49 PDT 2026
https://github.com/Wolfram70 created https://github.com/llvm/llvm-project/pull/221681
This change adds the following overloaded `fadd` intrinsics with
NVPTX codegen:
- `llvm.nvvm.fadd`
- `llvm.nvvm.fadd.ftz`
- `llvm.nvvm.fadd.sat`
- `llvm.nvvm.fadd.ftz.sat`
The rounding mode is passed in as an `i32` immediate operand.
Auto-upgrades the older non-overloaded intrinsics to the new
ones, and updates clang builtins, CIR codegen, and MLIR NVVM ops
to lower to the new intrinsics.
In the interest of completion, this also:
- Adds intrinsics support for lowering some half-precision additions
that were omitted earlier (`f16/f16x2` without saturation, and
`bf16/bf16x2` additions), and support for the `f32x2` type.
- Extends the DAG combine pattern for lowering `fneg + add-intrinsic`
to `sub` instructions to all supported types.
- Adds tests for constant folding of these intrinsics with the newly
supported scalar types.
PTX Spec Reference: https://docs.nvidia.com/cuda/developer-preview/13.4/parallel-thread-execution/index.html#floating-point-instructions-add
Assisted-by: Claude Opus 5
>From 8be38eaa0a7108760e2aa066eb87ae0b407daa73 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 11 Aug 2026 14:18:11 +0000
Subject: [PATCH 01/16] [clang][NVPTX][MLIR][NVVM] Add overloaded fadd
intrinsics
This change adds the following overloaded fadd intrinsics with
NVPTX codegen:
- `llvm.nvvm.fadd.<rn/rz/rm/rp>`
- `llvm.nvvm.fadd.<rn/rz/rm/rp>.ftz`
- `llvm.nvvm.fadd.<rn/rz/rm/rp>.sat`
- `llvm.nvvm.fadd.<rn/rz/rm/rp>.ftz.sat`
Auto-upgrades the older non-overloaded intrinsics to the new
ones, and updates clang builtins, CIR codegen, and MLIR NVVM ops
to lower to the new intrinsics.
In the interest of completion, this also adds intrinsics support
for lowering some half-precision additions that were omitted
earlier (`f16/f16x2` without saturation, and `bf16/bf16x2`
additions), and also adds support for the `f32x2` type.
PTX Spec Reference: https://docs.nvidia.com/cuda/developer-preview/13.4/parallel-thread-execution/index.html#floating-point-instructions-add
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 54 ++++
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 61 ++++
.../CIR/CodeGenCUDA/builtins-nvvm-math.cu | 24 ++
clang/test/CodeGen/builtins-nvptx.c | 26 +-
llvm/include/llvm/IR/IntrinsicsNVVM.td | 30 +-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 81 +++--
llvm/lib/Analysis/ConstantFolding.cpp | 40 +--
llvm/lib/IR/AutoUpgrade.cpp | 16 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 113 +++++--
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 122 +++++---
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 24 ++
llvm/test/CodeGen/NVPTX/bf16-add.ll | 33 ++
llvm/test/CodeGen/NVPTX/bf16-sub.ll | 35 +++
llvm/test/CodeGen/NVPTX/f16-add-sat.ll | 63 ----
llvm/test/CodeGen/NVPTX/f16-add.ll | 123 ++++++++
llvm/test/CodeGen/NVPTX/f16-sub-sat.ll | 69 -----
llvm/test/CodeGen/NVPTX/f16-sub.ll | 133 ++++++++
llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll | 60 ++++
llvm/test/CodeGen/NVPTX/fp-add-invalid.ll | 47 +++
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 32 +-
llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll | 64 ++++
llvm/test/CodeGen/NVPTX/fp-fold-sub.ll | 20 +-
llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll | 64 ++--
.../InstCombine/NVPTX/nvvm-intrins.ll | 18 +-
.../InstSimplify/const-fold-nvvm-add.ll | 216 ++++++-------
llvm/unittests/IR/IntrinsicsTest.cpp | 2 +-
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 8 -
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 104 ++-----
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 68 +++--
.../Target/LLVMIR/nvvm/addf/addf_invalid.mlir | 10 -
.../Target/LLVMIR/nvvm/addf/addf_vector.mlir | 257 ++++++----------
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 69 +++--
.../Target/LLVMIR/nvvm/subf/subf_invalid.mlir | 10 -
.../Target/LLVMIR/nvvm/subf/subf_vector.mlir | 284 +++++++-----------
34 files changed, 1429 insertions(+), 951 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/bf16-add.ll
create mode 100644 llvm/test/CodeGen/NVPTX/bf16-sub.ll
delete mode 100644 llvm/test/CodeGen/NVPTX/f16-add-sat.ll
create mode 100644 llvm/test/CodeGen/NVPTX/f16-add.ll
delete mode 100644 llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
create mode 100644 llvm/test/CodeGen/NVPTX/f16-sub.ll
create mode 100644 llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
create mode 100644 llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
create mode 100644 llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 5f284c11c3272..9937b206aca60 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -69,6 +69,20 @@ static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
.getResult();
}
+/// Emit a CIR LLVMIntrinsicCallOp for a binary NVVM intrinsic.
+/// The result type is inferred from the first argument.
+static mlir::Value emitBinaryNVVMIntrinsic(CIRGenFunction &cgf,
+ const CallExpr *expr,
+ llvm::StringRef intrinsicName) {
+ auto &builder = cgf.getBuilder();
+ mlir::Value lhs = cgf.emitScalarExpr(expr->getArg(0));
+ mlir::Value rhs = cgf.emitScalarExpr(expr->getArg(1));
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, cgf.getLoc(expr->getExprLoc()),
+ builder.getStringAttr(intrinsicName), lhs.getType(), {lhs, rhs})
+ .getResult();
+}
+
static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr,
llvm::StringRef intrinsicName,
bool returnsPred) {
@@ -822,6 +836,46 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx");
case NVPTX::BI__nvvm_ex2_approx_ftz_f:
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
+ case NVPTX::BI__nvvm_add_rn_f:
+ case NVPTX::BI__nvvm_add_rn_d:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn");
+ case NVPTX::BI__nvvm_add_rz_f:
+ case NVPTX::BI__nvvm_add_rz_d:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz");
+ case NVPTX::BI__nvvm_add_rm_f:
+ case NVPTX::BI__nvvm_add_rm_d:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm");
+ case NVPTX::BI__nvvm_add_rp_f:
+ case NVPTX::BI__nvvm_add_rp_d:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp");
+ case NVPTX::BI__nvvm_add_rn_ftz_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.ftz");
+ case NVPTX::BI__nvvm_add_rz_ftz_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.ftz");
+ case NVPTX::BI__nvvm_add_rm_ftz_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.ftz");
+ case NVPTX::BI__nvvm_add_rp_ftz_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.ftz");
+ case NVPTX::BI__nvvm_add_rn_sat_f:
+ case NVPTX::BI__nvvm_add_rn_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_sat_v2f16:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.sat");
+ case NVPTX::BI__nvvm_add_rz_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.sat");
+ case NVPTX::BI__nvvm_add_rm_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.sat");
+ case NVPTX::BI__nvvm_add_rp_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.sat");
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.ftz.sat");
+ case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.ftz.sat");
+ case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.ftz.sat");
+ case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.ftz.sat");
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
cgm.errorNYI(expr->getSourceRange(),
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index e3ef3ae488a0c..896afea982385 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -433,6 +433,13 @@ static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
CGF.EmitScalarExpr(E->getArg(2))});
}
+static Value *MakeBinaryIntrinsic(unsigned IntrinsicID, const CallExpr *E,
+ CodeGenFunction &CGF) {
+ return CGF.Builder.CreateBinaryIntrinsic(IntrinsicID,
+ CGF.EmitScalarExpr(E->getArg(0)),
+ CGF.EmitScalarExpr(E->getArg(1)));
+}
+
} // namespace
Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
@@ -1181,6 +1188,60 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
case NVPTX::BI__nvvm_ex2_approx_ftz_f:
return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_ex2_approx_ftz,
EmitScalarExpr(E->getArg(0)));
+ case NVPTX::BI__nvvm_add_rn_f:
+ case NVPTX::BI__nvvm_add_rn_d:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn, E, *this);
+ case NVPTX::BI__nvvm_add_rz_f:
+ case NVPTX::BI__nvvm_add_rz_d:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz, E, *this);
+ case NVPTX::BI__nvvm_add_rm_f:
+ case NVPTX::BI__nvvm_add_rm_d:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm, E, *this);
+ case NVPTX::BI__nvvm_add_rp_f:
+ case NVPTX::BI__nvvm_add_rp_d:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_ftz, E, *this);
+ case NVPTX::BI__nvvm_add_rz_ftz_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_ftz, E, *this);
+ case NVPTX::BI__nvvm_add_rm_ftz_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_ftz, E, *this);
+ case NVPTX::BI__nvvm_add_rp_ftz_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_ftz, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rm_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rp_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_f16:
+ return MakeHalfType(
+ CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_sat, Builder.getHalfTy()),
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_v2f16:
+ return MakeHalfType(
+ CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_sat,
+ FixedVectorType::get(Builder.getHalfTy(), 2)),
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
+ return MakeHalfType(
+ CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat, Builder.getHalfTy()),
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
+ return MakeHalfType(
+ CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat,
+ FixedVectorType::get(Builder.getHalfTy(), 2)),
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
return MakeLdg(*this, E);
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index a2f0edb6d073b..ede4c0605c98b 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -63,3 +63,27 @@ __device__ double test_ex2_approx_d(double x) {
__device__ float test_ex2_approx_ftz_f(float x) {
return __nvvm_ex2_approx_ftz_f(x);
}
+
+// CIR-LABEL: @_Z13test_add_rn_fff
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rn" {{.*}} : (!cir.float, !cir.float) -> !cir.float
+// LLVM-LABEL: @_Z13test_add_rn_fff
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.rn.f32(float
+__device__ float test_add_rn_f(float x, float y) {
+ return __nvvm_add_rn_f(x, y);
+}
+
+// CIR-LABEL: @_Z13test_add_rz_ddd
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rz" {{.*}} : (!cir.double, !cir.double) -> !cir.double
+// LLVM-LABEL: @_Z13test_add_rz_ddd
+// LLVM: call {{.*}}double @llvm.nvvm.fadd.rz.f64(double
+__device__ double test_add_rz_d(double x, double y) {
+ return __nvvm_add_rz_d(x, y);
+}
+
+// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rm.ftz.sat" {{.*}} : (!cir.float, !cir.float) -> !cir.float
+// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.rm.ftz.sat.f32(float
+__device__ float test_add_rm_ftz_sat_f(float x, float y) {
+ return __nvvm_add_rm_ftz_sat_f(x, y);
+}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index c72634025dbc7..e529eceffc84f 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -251,7 +251,7 @@ __device__ void nvvm_math(float f1, float f2, double d1, double d2) {
float t3 = __nvvm_sqrt_rn_f(f1);
// CHECK: call float @llvm.nvvm.rcp.rn.f
float t4 = __nvvm_rcp_rn_f(f2);
-// CHECK: call float @llvm.nvvm.add.rn.f
+// CHECK: call float @llvm.nvvm.fadd.rn.f32
float t5 = __nvvm_add_rn_f(f1, f2);
// CHECK: call double @llvm.nvvm.fmax.d
@@ -1628,21 +1628,21 @@ __device__ void nvvm_min_max_sm86() {
// CHECK-LABEL: nvvm_add_fma_f32_sat
__device__ void nvvm_add_fma_f32_sat() {
- // CHECK: call float @llvm.nvvm.add.rn.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rn.sat.f32
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rn.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rn.ftz.sat.f32
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rz.sat.f32
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rz.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rz.ftz.sat.f32
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rm.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rm.sat.f32
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rm.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rm.ftz.sat.f32
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rp.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rp.sat.f32
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rp.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rp.ftz.sat.f32
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1672,13 +1672,13 @@ __device__ void nvvm_add_fma_f32_sat() {
// CHECK-LABEL: nvvm_add_mul_f16_sat
__device__ void nvvm_add_mul_f16_sat() {
- // CHECK: call half @llvm.nvvm.add.rn.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.rn.sat.f16
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.add.rn.ftz.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.rn.ftz.sat.f16
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
- // CHECK: call <2 x half> @llvm.nvvm.add.rn.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index a85b7de7693d6..efd45523d5cd2 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1810,31 +1810,17 @@ let TargetPrefix = "nvvm" in {
}
//
- // Add
+ // FAdd
//
let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative,
- IntrNoCreateUndefOrPoison] in {
- foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in {
- def int_nvvm_add # rnd # ftz # sat # _f : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty]>;
- } // sat
- } // ftz
- def int_nvvm_add # rnd # _d : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty]>;
- }
-
- foreach ftz = ["", "_ftz"] in {
- def int_nvvm_add_rn # ftz # _sat_f16 : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_half_ty], [llvm_half_ty, llvm_half_ty]>;
-
- def int_nvvm_add_rn # ftz # _sat_v2f16 : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_v2f16_ty, llvm_v2f16_ty]>;
-
- } // ftz
- }
+ IntrNoCreateUndefOrPoison] in
+ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def int_nvvm_fadd # rnd # ftz # sat :
+ DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>]>;
//
// Dot Product
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index a6f7555be838a..0421400678c71 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -612,42 +612,75 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd_rm_ftz_sat:
+ case Intrinsic::nvvm_fadd_rn_ftz_sat:
+ case Intrinsic::nvvm_fadd_rp_ftz_sat:
+ case Intrinsic::nvvm_fadd_rz_ftz_sat:
return true;
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_sat:
+ case Intrinsic::nvvm_fadd_rn_sat:
+ case Intrinsic::nvvm_fadd_rp_sat:
+ case Intrinsic::nvvm_fadd_rz_sat:
return false;
}
llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
}
+inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
+ switch (IntrinsicID) {
+ case Intrinsic::nvvm_fadd_rm_sat:
+ case Intrinsic::nvvm_fadd_rn_sat:
+ case Intrinsic::nvvm_fadd_rp_sat:
+ case Intrinsic::nvvm_fadd_rz_sat:
+ case Intrinsic::nvvm_fadd_rm_ftz_sat:
+ case Intrinsic::nvvm_fadd_rn_ftz_sat:
+ case Intrinsic::nvvm_fadd_rp_ftz_sat:
+ case Intrinsic::nvvm_fadd_rz_ftz_sat:
+ return true;
+
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
+ return false;
+ }
+ llvm_unreachable("Checking sat flag for invalid NVVM add intrinsic");
+}
+
inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rm_ftz_f:
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rm_sat:
+ case Intrinsic::nvvm_fadd_rm_ftz_sat:
return APFloat::rmTowardNegative;
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rn_ftz_f:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rn_sat:
+ case Intrinsic::nvvm_fadd_rn_ftz_sat:
return APFloat::rmNearestTiesToEven;
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rp_ftz_f:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rp_sat:
+ case Intrinsic::nvvm_fadd_rp_ftz_sat:
return APFloat::rmTowardPositive;
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rz_ftz_f:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd_rz_sat:
+ case Intrinsic::nvvm_fadd_rz_ftz_sat:
return APFloat::rmTowardZero;
}
llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index c819e188cac95..2b6708a8a6a38 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1995,18 +1995,14 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
return !IsStrictFP;
// NVVM add intrinsics with explicit rounding modes
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f:
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
// NVVM div intrinsics with explicit rounding modes
case Intrinsic::nvvm_div_rm_d:
@@ -3618,18 +3614,14 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
return ConstantFP::get(Ty, Res);
}
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f: {
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz: {
bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 378a0b5ef53f9..e42013219963f 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1993,6 +1993,22 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
return NewFn != F;
}
+ // Upgrade the FP add intrinsics, which are overloaded on the operand type
+ // llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
+ // llvm.nvvm.fadd.<rnd>{.ftz}{.sat}.<mangled type>
+ if (Name.starts_with("add.")) {
+ auto [Base, TypeSuffix] = Name.rsplit('.');
+ if (TypeSuffix == "f" || TypeSuffix == "d" || TypeSuffix == "f16" ||
+ TypeSuffix == "v2f16") {
+ IID = Intrinsic::lookupIntrinsicID(("llvm.nvvm.f" + Base).str());
+ if (IID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
+ {F->getReturnType()});
+ return true;
+ }
+ }
+ }
+
// The following nvvm intrinsics correspond exactly to an LLVM idiom, but
// not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
//
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 70fb1aa11b5d6..a8b26cba631cc 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -53,6 +53,7 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCContext.h"
@@ -7179,22 +7180,42 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
}
}
-static unsigned getF16SubOpc(Intrinsic::ID AddIntrinsicID) {
- switch (AddIntrinsicID) {
+static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID) {
+ const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
+ switch (VT.getScalarType().getSimpleVT().SimpleTy) {
+ case MVT::f16:
+ if (nvvm::FAddShouldSaturate(IID))
+ return IsFTZ ? NVPTXISD::SUB_RN_FTZ_SAT : NVPTXISD::SUB_RN_SAT;
+ return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
+ case MVT::bf16:
+ return NVPTXISD::SUB_RN;
+ case MVT::f32:
+ if (!VT.isVector() || nvvm::FAddShouldSaturate(IID))
+ return 0;
+ switch (nvvm::GetFAddRoundingMode(IID)) {
+ case APFloat::rmNearestTiesToEven:
+ return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
+ case APFloat::rmTowardZero:
+ return IsFTZ ? NVPTXISD::SUB_RZ_FTZ : NVPTXISD::SUB_RZ;
+ case APFloat::rmTowardNegative:
+ return IsFTZ ? NVPTXISD::SUB_RM_FTZ : NVPTXISD::SUB_RM;
+ case APFloat::rmTowardPositive:
+ return IsFTZ ? NVPTXISD::SUB_RP_FTZ : NVPTXISD::SUB_RP;
+ default:
+ llvm_unreachable("Unexpected fadd rounding mode");
+ }
default:
- break;
- case Intrinsic::nvvm_add_rn_sat_f16:
- case Intrinsic::nvvm_add_rn_sat_v2f16:
- return NVPTXISD::SUB_RN_SAT;
- case Intrinsic::nvvm_add_rn_ftz_sat_f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_v2f16:
- return NVPTXISD::SUB_RN_FTZ_SAT;
+ return 0;
}
- llvm_unreachable("Invalid F16 add intrinsic");
}
-static SDValue combineF16AddWithNeg(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID AddIntrinsicID) {
+static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
+ Intrinsic::ID AddIntrinsicID) {
+ const EVT VT = N->getValueType(0);
+ const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID);
+ if (!Opc)
+ return SDValue();
+
SDValue Op1 = N->getOperand(1);
SDValue Op2 = N->getOperand(2);
@@ -7210,24 +7231,74 @@ static SDValue combineF16AddWithNeg(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
- SDLoc DL(N);
- return DAG.getNode(getF16SubOpc(AddIntrinsicID), DL, N->getValueType(0),
- SubOp1, SubOp2);
+ return DAG.getNode(Opc, SDLoc(N), VT, SubOp1, SubOp2);
+}
+
+static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID,
+ const NVPTXSubtarget &STI) {
+ if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(2))
+ return false;
+
+ const bool IsSat = nvvm::FAddShouldSaturate(IID);
+ switch (VT.getScalarType().getSimpleVT().SimpleTy) {
+ case MVT::f16:
+ return nvvm::GetFAddRoundingMode(IID) == APFloat::rmNearestTiesToEven;
+ case MVT::bf16:
+ return nvvm::GetFAddRoundingMode(IID) == APFloat::rmNearestTiesToEven &&
+ !IsSat && !nvvm::FAddShouldFTZ(IID) &&
+ STI.hasNativeBF16Support(ISD::FADD);
+ case MVT::f32:
+ return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
+ case MVT::f64:
+ return !VT.isVector() && !IsSat && !nvvm::FAddShouldFTZ(IID);
+ default:
+ return false;
+ }
+}
+
+static SDValue diagnoseInvalidFAdd(SDNode *N, SelectionDAG &DAG,
+ Intrinsic::ID IID,
+ const NVPTXSubtarget &STI) {
+ const EVT VT = N->getValueType(0);
+ if (isSupportedFAdd(VT, IID, STI))
+ return SDValue();
+
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ Twine(Intrinsic::getBaseName(IID)) + " with operand type " +
+ VT.getEVTString() + " is not supported on this target",
+ SDLoc(N).getDebugLoc()));
+ return DAG.getPOISON(VT);
}
static SDValue combineIntrinsicWOChain(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const NVPTXSubtarget &STI) {
- unsigned IID = N->getConstantOperandVal(0);
+ const Intrinsic::ID IID =
+ static_cast<Intrinsic::ID>(N->getConstantOperandVal(0));
switch (IID) {
default:
break;
- case Intrinsic::nvvm_add_rn_sat_f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_f16:
- case Intrinsic::nvvm_add_rn_sat_v2f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_v2f16:
- return combineF16AddWithNeg(N, DCI.DAG, IID);
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd_rm_sat:
+ case Intrinsic::nvvm_fadd_rn_sat:
+ case Intrinsic::nvvm_fadd_rp_sat:
+ case Intrinsic::nvvm_fadd_rz_sat:
+ case Intrinsic::nvvm_fadd_rm_ftz_sat:
+ case Intrinsic::nvvm_fadd_rn_ftz_sat:
+ case Intrinsic::nvvm_fadd_rp_ftz_sat:
+ case Intrinsic::nvvm_fadd_rz_ftz_sat:
+ if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, STI))
+ return V;
+ return combineFAddWithNeg(N, DCI.DAG, IID);
}
return SDValue();
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index b7801b3496bcb..3d418ea5c81f0 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1687,6 +1687,14 @@ class F_MATH_2<string OpcStr, NVPTXRegClass t_regclass,
[(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1))]>,
Requires<Preds>;
+class F_MATH_2_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP,
+ list<Predicate> Preds = []>
+ : BasicNVPTXInst<(outs t.RC:$dst),
+ (ins t.RC:$src0, t.RC:$src1),
+ OpcStr,
+ [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1))]>,
+ Requires<Preds>;
+
class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,
NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass,
NVPTXRegClass s2_regclass, Intrinsic IntOP, list<Predicate> Preds = []>
@@ -2256,32 +2264,50 @@ let Predicates = [doRsqrtOpt] in {
// Add
//
-def INT_NVVM_ADD_RN_SAT_F16 : F_MATH_2<"add.rn.sat.f16", B16, B16, B16, int_nvvm_add_rn_sat_f16>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16 : F_MATH_2<"add.rn.ftz.sat.f16", B16, B16, B16, int_nvvm_add_rn_ftz_sat_f16>;
-def INT_NVVM_ADD_RN_SAT_F16X2 : F_MATH_2<"add.rn.sat.f16x2", B32, B32, B32, int_nvvm_add_rn_sat_v2f16>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16X2 : F_MATH_2<"add.rn.ftz.sat.f16x2", B32, B32, B32, int_nvvm_add_rn_ftz_sat_v2f16>;
-
-def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2<"add.rn.ftz.f32", B32, B32, B32, int_nvvm_add_rn_ftz_f>;
-def INT_NVVM_ADD_RN_SAT_FTZ_F : F_MATH_2<"add.rn.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rn_ftz_sat_f>;
-def INT_NVVM_ADD_RN_F : F_MATH_2<"add.rn.f32", B32, B32, B32, int_nvvm_add_rn_f>;
-def INT_NVVM_ADD_RN_SAT_F : F_MATH_2<"add.rn.sat.f32", B32, B32, B32, int_nvvm_add_rn_sat_f>;
-def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2<"add.rz.ftz.f32", B32, B32, B32, int_nvvm_add_rz_ftz_f>;
-def INT_NVVM_ADD_RZ_SAT_FTZ_F : F_MATH_2<"add.rz.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rz_ftz_sat_f>;
-def INT_NVVM_ADD_RZ_F : F_MATH_2<"add.rz.f32", B32, B32, B32, int_nvvm_add_rz_f>;
-def INT_NVVM_ADD_RZ_SAT_F : F_MATH_2<"add.rz.sat.f32", B32, B32, B32, int_nvvm_add_rz_sat_f>;
-def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2<"add.rm.ftz.f32", B32, B32, B32, int_nvvm_add_rm_ftz_f>;
-def INT_NVVM_ADD_RM_SAT_FTZ_F : F_MATH_2<"add.rm.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rm_ftz_sat_f>;
-def INT_NVVM_ADD_RM_F : F_MATH_2<"add.rm.f32", B32, B32, B32, int_nvvm_add_rm_f>;
-def INT_NVVM_ADD_RM_SAT_F : F_MATH_2<"add.rm.sat.f32", B32, B32, B32, int_nvvm_add_rm_sat_f>;
-def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2<"add.rp.ftz.f32", B32, B32, B32, int_nvvm_add_rp_ftz_f>;
-def INT_NVVM_ADD_RP_SAT_FTZ_F : F_MATH_2<"add.rp.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rp_ftz_sat_f>;
-def INT_NVVM_ADD_RP_F : F_MATH_2<"add.rp.f32", B32, B32, B32, int_nvvm_add_rp_f>;
-def INT_NVVM_ADD_RP_SAT_F : F_MATH_2<"add.rp.sat.f32", B32, B32, B32, int_nvvm_add_rp_sat_f>;
-
-def INT_NVVM_ADD_RN_D : F_MATH_2<"add.rn.f64", B64, B64, B64, int_nvvm_add_rn_d>;
-def INT_NVVM_ADD_RZ_D : F_MATH_2<"add.rz.f64", B64, B64, B64, int_nvvm_add_rz_d>;
-def INT_NVVM_ADD_RM_D : F_MATH_2<"add.rm.f64", B64, B64, B64, int_nvvm_add_rm_d>;
-def INT_NVVM_ADD_RP_D : F_MATH_2<"add.rp.f64", B64, B64, B64, int_nvvm_add_rp_d>;
+defvar BF16ArithPreds = [hasBF16Math, hasPTX<78>, hasSM<90>];
+
+def INT_NVVM_ADD_RN_F16 : F_MATH_2_TY<"add.rn.f16", F16RT, int_nvvm_fadd_rn>;
+def INT_NVVM_ADD_RN_FTZ_F16 : F_MATH_2_TY<"add.rn.ftz.f16", F16RT, int_nvvm_fadd_rn_ftz>;
+def INT_NVVM_ADD_RN_SAT_F16 : F_MATH_2_TY<"add.rn.sat.f16", F16RT, int_nvvm_fadd_rn_sat>;
+def INT_NVVM_ADD_RN_FTZ_SAT_F16 : F_MATH_2_TY<"add.rn.ftz.sat.f16", F16RT, int_nvvm_fadd_rn_ftz_sat>;
+def INT_NVVM_ADD_RN_F16X2 : F_MATH_2_TY<"add.rn.f16x2", F16X2RT, int_nvvm_fadd_rn>;
+def INT_NVVM_ADD_RN_FTZ_F16X2 : F_MATH_2_TY<"add.rn.ftz.f16x2", F16X2RT, int_nvvm_fadd_rn_ftz>;
+def INT_NVVM_ADD_RN_SAT_F16X2 : F_MATH_2_TY<"add.rn.sat.f16x2", F16X2RT, int_nvvm_fadd_rn_sat>;
+def INT_NVVM_ADD_RN_FTZ_SAT_F16X2 : F_MATH_2_TY<"add.rn.ftz.sat.f16x2", F16X2RT, int_nvvm_fadd_rn_ftz_sat>;
+
+def INT_NVVM_ADD_RN_BF16 :
+ F_MATH_2_TY<"add.rn.bf16", BF16RT, int_nvvm_fadd_rn, BF16ArithPreds>;
+def INT_NVVM_ADD_RN_BF16X2 :
+ F_MATH_2_TY<"add.rn.bf16x2", BF16X2RT, int_nvvm_fadd_rn, BF16ArithPreds>;
+
+def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2_TY<"add.rn.ftz.f32", F32RT, int_nvvm_fadd_rn_ftz>;
+def INT_NVVM_ADD_RN_SAT_FTZ_F : F_MATH_2_TY<"add.rn.sat.ftz.f32", F32RT, int_nvvm_fadd_rn_ftz_sat>;
+def INT_NVVM_ADD_RN_F : F_MATH_2_TY<"add.rn.f32", F32RT, int_nvvm_fadd_rn>;
+def INT_NVVM_ADD_RN_SAT_F : F_MATH_2_TY<"add.rn.sat.f32", F32RT, int_nvvm_fadd_rn_sat>;
+def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2_TY<"add.rz.ftz.f32", F32RT, int_nvvm_fadd_rz_ftz>;
+def INT_NVVM_ADD_RZ_SAT_FTZ_F : F_MATH_2_TY<"add.rz.sat.ftz.f32", F32RT, int_nvvm_fadd_rz_ftz_sat>;
+def INT_NVVM_ADD_RZ_F : F_MATH_2_TY<"add.rz.f32", F32RT, int_nvvm_fadd_rz>;
+def INT_NVVM_ADD_RZ_SAT_F : F_MATH_2_TY<"add.rz.sat.f32", F32RT, int_nvvm_fadd_rz_sat>;
+def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2_TY<"add.rm.ftz.f32", F32RT, int_nvvm_fadd_rm_ftz>;
+def INT_NVVM_ADD_RM_SAT_FTZ_F : F_MATH_2_TY<"add.rm.sat.ftz.f32", F32RT, int_nvvm_fadd_rm_ftz_sat>;
+def INT_NVVM_ADD_RM_F : F_MATH_2_TY<"add.rm.f32", F32RT, int_nvvm_fadd_rm>;
+def INT_NVVM_ADD_RM_SAT_F : F_MATH_2_TY<"add.rm.sat.f32", F32RT, int_nvvm_fadd_rm_sat>;
+def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2_TY<"add.rp.ftz.f32", F32RT, int_nvvm_fadd_rp_ftz>;
+def INT_NVVM_ADD_RP_SAT_FTZ_F : F_MATH_2_TY<"add.rp.sat.ftz.f32", F32RT, int_nvvm_fadd_rp_ftz_sat>;
+def INT_NVVM_ADD_RP_F : F_MATH_2_TY<"add.rp.f32", F32RT, int_nvvm_fadd_rp>;
+def INT_NVVM_ADD_RP_SAT_F : F_MATH_2_TY<"add.rp.sat.f32", F32RT, int_nvvm_fadd_rp_sat>;
+
+def INT_NVVM_ADD_RN_D : F_MATH_2_TY<"add.rn.f64", F64RT, int_nvvm_fadd_rn>;
+def INT_NVVM_ADD_RZ_D : F_MATH_2_TY<"add.rz.f64", F64RT, int_nvvm_fadd_rz>;
+def INT_NVVM_ADD_RM_D : F_MATH_2_TY<"add.rm.f64", F64RT, int_nvvm_fadd_rm>;
+def INT_NVVM_ADD_RP_D : F_MATH_2_TY<"add.rp.f64", F64RT, int_nvvm_fadd_rp>;
+
+foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach ftz = ["", "_ftz"] in
+ def INT_NVVM_ADD # rnd # ftz # _F32X2 :
+ F_MATH_2_TY<!subst("_", ".", "add" # rnd # ftz # "_f32x2"), F32X2RT,
+ !cast<Intrinsic>("int_nvvm_fadd" # rnd # ftz),
+ [hasF32x2Instructions]>;
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach sat = ["", "_sat"] in {
@@ -2290,7 +2316,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
!subst("_", ".", "add" # rnd # sat # "_f32_" # type),
[(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_add" # rnd # sat # "_f")
+ (!cast<Intrinsic>("int_nvvm_fadd" # rnd # sat)
(f32 (fpextend type:$a)),
f32:$b))]>,
Requires<[SM100]>;
@@ -2310,25 +2336,45 @@ let Predicates = [SM100, doNoF32FTZ] in {
// Sub
//
+// These nodes are created by combineFAddWithNeg.
def sub_rn_sat : SDNode<"NVPTXISD::SUB_RN_SAT", SDTFPBinOp>;
def sub_rn_ftz_sat :
SDNode<"NVPTXISD::SUB_RN_FTZ_SAT", SDTFPBinOp>;
-
-class INT_NVVM_SUB_RN<RegTyInfo TyInfo, string variant> :
+
+foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach ftz = ["", "_ftz"] in
+ def sub # rnd # ftz :
+ SDNode<"NVPTXISD::SUB" # !toupper(rnd # ftz), SDTFPBinOp>;
+
+class INT_NVVM_SUB<RegTyInfo TyInfo, string variant> :
BasicNVPTXInst<(outs TyInfo.RC:$dst), (ins TyInfo.RC:$a, TyInfo.RC:$b),
- !subst("_", ".", "sub.rn" # variant # "." # TyInfo.PtxType),
+ !subst("_", ".", "sub" # variant # "." # TyInfo.PtxType),
[(set TyInfo.Ty:$dst,
- (!cast<SDNode>("sub_rn" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
+ (!cast<SDNode>("sub" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
+
+def INT_NVVM_SUB_RN_F16 : INT_NVVM_SUB<F16RT, "_rn">;
+def INT_NVVM_SUB_RN_FTZ_F16 : INT_NVVM_SUB<F16RT, "_rn_ftz">;
+def INT_NVVM_SUB_RN_SAT_F16 : INT_NVVM_SUB<F16RT, "_rn_sat">;
+def INT_NVVM_SUB_RN_FTZ_SAT_F16 : INT_NVVM_SUB<F16RT, "_rn_ftz_sat">;
+def INT_NVVM_SUB_RN_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn">;
+def INT_NVVM_SUB_RN_FTZ_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_ftz">;
+def INT_NVVM_SUB_RN_SAT_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_sat">;
+def INT_NVVM_SUB_RN_FTZ_SAT_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_ftz_sat">;
+
+let Predicates = BF16ArithPreds in {
+ def INT_NVVM_SUB_RN_BF16 : INT_NVVM_SUB<BF16RT, "_rn">;
+ def INT_NVVM_SUB_RN_BF16X2 : INT_NVVM_SUB<BF16X2RT, "_rn">;
+}
-def INT_NVVM_SUB_RN_SAT_F16 : INT_NVVM_SUB_RN<F16RT, "_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16 : INT_NVVM_SUB_RN<F16RT, "_ftz_sat">;
-def INT_NVVM_SUB_RN_SAT_F16X2 : INT_NVVM_SUB_RN<F16X2RT, "_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16X2 : INT_NVVM_SUB_RN<F16X2RT, "_ftz_sat">;
+let Predicates = [hasF32x2Instructions] in
+ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach ftz = ["", "_ftz"] in
+ def INT_NVVM_SUB # rnd # ftz # _F32X2 : INT_NVVM_SUB<F32X2RT, rnd # ftz>;
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_add" # rnd # ftz # sat # "_f");
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # rnd # ftz # sat);
def INT_NVVM_SUB # rnd # ftz # sat # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
@@ -2336,7 +2382,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
}
}
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_add" # rnd # "_d");
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # rnd);
def INT_NVVM_SUB # rnd # _D :
BasicNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
!subst("_", ".", "sub" # rnd # "_f64"),
@@ -2350,7 +2396,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # "_f32_" # type),
[(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_add" # rnd # sat # "_f")
+ (!cast<Intrinsic>("int_nvvm_fadd" # rnd # sat)
(f32 (fpextend type:$a)),
(f32 (fneg f32:$b))))]>,
Requires<[SM100]>;
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index bcbeef260f2bb..8a0c23e0e4f09 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -711,3 +711,27 @@ define void @nvvm_ex2_approx(float %a, double %b, half %c, <2 x half> %d) {
%r4 = call float @llvm.nvvm.ex2.approx.ftz.f(float %a)
ret void
}
+
+define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
+; CHECK: call float @llvm.nvvm.fadd.rn.f32(float %a, float %a)
+; CHECK: call float @llvm.nvvm.fadd.rz.ftz.f32(float %a, float %a)
+; CHECK: call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %a)
+; CHECK: call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %a)
+; CHECK: call double @llvm.nvvm.fadd.rn.f64(double %b, double %b)
+; CHECK: call double @llvm.nvvm.fadd.rz.f64(double %b, double %b)
+; CHECK: call half @llvm.nvvm.fadd.rn.sat.f16(half %c, half %c)
+; CHECK: call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %c, half %c)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %d, <2 x half> %d)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d)
+ %r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
+ %r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
+ %r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
+ %r4 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %a)
+ %r5 = call double @llvm.nvvm.add.rn.d(double %b, double %b)
+ %r6 = call double @llvm.nvvm.add.rz.d(double %b, double %b)
+ %r7 = call half @llvm.nvvm.add.rn.sat.f16(half %c, half %c)
+ %r8 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %c, half %c)
+ %r9 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %d, <2 x half> %d)
+ %r10 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/bf16-add.ll b/llvm/test/CodeGen/NVPTX/bf16-add.ll
new file mode 100644
index 0000000000000..71d90a3ae675c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/bf16-add.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-7.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
+
+define bfloat @add_rn_bf16(bfloat %a, bfloat %b) {
+; CHECK-LABEL: add_rn_bf16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_bf16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_bf16_param_1];
+; CHECK-NEXT: add.rn.bf16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %b)
+ ret bfloat %1
+}
+
+define <2 x bfloat> @add_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; CHECK-LABEL: add_rn_bf16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_bf16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_bf16x2_param_1];
+; CHECK-NEXT: add.rn.bf16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ ret <2 x bfloat> %1
+}
diff --git a/llvm/test/CodeGen/NVPTX/bf16-sub.ll b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
new file mode 100644
index 0000000000000..3d685ebf65cf2
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-7.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
+
+define bfloat @sub_rn_bf16(bfloat %a, bfloat %b) {
+; CHECK-LABEL: sub_rn_bf16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_bf16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_bf16_param_1];
+; CHECK-NEXT: sub.rn.bf16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg bfloat %b
+ %res = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %1)
+ ret bfloat %res
+}
+
+define <2 x bfloat> @sub_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; CHECK-LABEL: sub_rn_bf16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_bf16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_bf16x2_param_1];
+; CHECK-NEXT: sub.rn.bf16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x bfloat> %b
+ %res = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %a, <2 x bfloat> %1)
+ ret <2 x bfloat> %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/f16-add-sat.ll b/llvm/test/CodeGen/NVPTX/f16-add-sat.ll
deleted file mode 100644
index c2ffc126694c4..0000000000000
--- a/llvm/test/CodeGen/NVPTX/f16-add-sat.ll
+++ /dev/null
@@ -1,63 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
-; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
-
-define half @add_rn_sat_f16(half %a, half %b) {
-; CHECK-LABEL: add_rn_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_sat_f16_param_1];
-; CHECK-NEXT: add.rn.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.add.rn.sat.f16(half %a, half %b)
- ret half %1
-}
-
-define <2 x half> @add_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: add_rn_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [add_rn_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [add_rn_sat_f16x2_param_1];
-; CHECK-NEXT: add.rn.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
- ret <2 x half> %1
-}
-
-define half @add_rn_ftz_sat_f16(half %a, half %b) {
-; CHECK-LABEL: add_rn_ftz_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_sat_f16_param_1];
-; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %b)
- ret half %1
-}
-
-define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: add_rn_ftz_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_sat_f16x2_param_1];
-; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
- ret <2 x half> %1
-}
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
new file mode 100644
index 0000000000000..63d7f1e2705d6
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
+; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
+
+define half @add_rn_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_f16_param_1];
+; CHECK-NEXT: add.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.rn.f16(half %a, half %b)
+ ret half %1
+}
+
+define <2 x half> @add_rn_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_f16x2_param_1];
+; CHECK-NEXT: add.rn.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %a, <2 x half> %b)
+ ret <2 x half> %1
+}
+
+define half @add_rn_ftz_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_ftz_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_f16_param_1];
+; CHECK-NEXT: add.rn.ftz.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %a, half %b)
+ ret half %1
+}
+
+define <2 x half> @add_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_ftz_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_f16x2_param_1];
+; CHECK-NEXT: add.rn.ftz.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %a, <2 x half> %b)
+ ret <2 x half> %1
+}
+
+define half @add_rn_sat_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_sat_f16_param_1];
+; CHECK-NEXT: add.rn.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.rn.sat.f16(half %a, half %b)
+ ret half %1
+}
+
+define <2 x half> @add_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_sat_f16x2_param_1];
+; CHECK-NEXT: add.rn.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
+ ret <2 x half> %1
+}
+
+define half @add_rn_ftz_sat_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_ftz_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_sat_f16_param_1];
+; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %a, half %b)
+ ret half %1
+}
+
+define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_ftz_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_sat_f16x2_param_1];
+; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
+ ret <2 x half> %1
+}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll b/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
deleted file mode 100644
index 774ce7ccb2f95..0000000000000
--- a/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
+++ /dev/null
@@ -1,69 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | FileCheck %s
-; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
-; RUN: %if ptxas-isa-6.0 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | %ptxas-verify%}
-
-define half @sub_rn_sat_f16(half %a, half %b) {
-; CHECK-LABEL: sub_rn_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_sat_f16_param_1];
-; CHECK-NEXT: sub.rn.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = fneg half %b
- %res = call half @llvm.nvvm.add.rn.sat.f16(half %a, half %1)
- ret half %res
-}
-
-define <2 x half> @sub_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: sub_rn_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_sat_f16x2_param_1];
-; CHECK-NEXT: sub.rn.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %1)
- ret <2 x half> %res
-}
-
-define half @sub_rn_ftz_sat_f16(half %a, half %b) {
-; CHECK-LABEL: sub_rn_ftz_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_sat_f16_param_1];
-; CHECK-NEXT: sub.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = fneg half %b
- %res = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %1)
- ret half %res
-}
-
-define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: sub_rn_ftz_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_sat_f16x2_param_1];
-; CHECK-NEXT: sub.rn.ftz.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1)
- ret <2 x half> %res
-}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
new file mode 100644
index 0000000000000..e76c033c6705f
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | FileCheck %s
+; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
+; RUN: %if ptxas-isa-6.0 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | %ptxas-verify%}
+
+define half @sub_rn_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_f16_param_1];
+; CHECK-NEXT: sub.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.rn.f16(half %a, half %1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_f16x2_param_1];
+; CHECK-NEXT: sub.rn.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %a, <2 x half> %1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_ftz_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_ftz_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_f16_param_1];
+; CHECK-NEXT: sub.rn.ftz.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.rn.ftz.f16(half %a, half %1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_ftz_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_f16x2_param_1];
+; CHECK-NEXT: sub.rn.ftz.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %a, <2 x half> %1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_sat_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_sat_f16_param_1];
+; CHECK-NEXT: sub.rn.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.rn.sat.f16(half %a, half %1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_sat_f16x2_param_1];
+; CHECK-NEXT: sub.rn.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %a, <2 x half> %1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_ftz_sat_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_ftz_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_sat_f16_param_1];
+; CHECK-NEXT: sub.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %a, half %1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_ftz_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_sat_f16x2_param_1];
+; CHECK-NEXT: sub.rn.ftz.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1)
+ ret <2 x half> %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
new file mode 100644
index 0000000000000..f837c6d5d9a54
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
@@ -0,0 +1,60 @@
+; RUN: llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 | FileCheck %s
+; RUN: %if ptxas-sm_100 && ptxas-isa-8.8 %{ llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 | %ptxas-verify -arch=sm_100 %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define <2 x float> @add_rn(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn(
+; CHECK: add.rn.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz(
+; CHECK: add.rz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rm(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rm(
+; CHECK: add.rm.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rp(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rp(
+; CHECK: add.rp.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rn_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_ftz(
+; CHECK: add.rn.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rz_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_ftz(
+; CHECK: add.rz.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rm_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rm_ftz(
+; CHECK: add.rm.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rp_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rp_ftz(
+; CHECK: add.rp.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
new file mode 100644
index 0000000000000..3114592aaccdc
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
@@ -0,0 +1,47 @@
+; RUN: not llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 2>&1 | FileCheck %s
+; RUN: not llc < %s -mcpu=sm_90 -mattr=+ptx78 -march=nvptx64 2>&1 | FileCheck %s --check-prefix=NOF32X2
+; RUN: not llc < %s -mcpu=sm_80 -mattr=+ptx78 -march=nvptx64 2>&1 | FileCheck %s --check-prefix=NOBF16
+
+target triple = "nvptx64-nvidia-cuda"
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.sat with operand type v2f32 is not supported
+define <2 x float> @sat_f32x2(<2 x float> %a, <2 x float> %b) {
+ %r = call <2 x float> @llvm.nvvm.fadd.rn.sat.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+; NOF32X2: error: {{.*}}llvm.nvvm.fadd.rn with operand type v2f32 is not supported
+define <2 x float> @unsupported_f32x2(<2 x float> %a, <2 x float> %b) {
+ %r = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.ftz with operand type f64 is not supported
+define double @ftz_f64(double %a, double %b) {
+ %r = call double @llvm.nvvm.fadd.rn.ftz.f64(double %a, double %b)
+ ret double %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rz.sat with operand type f16 is not supported
+define half @rz_f16(half %a, half %b) {
+ %r = call half @llvm.nvvm.fadd.rz.sat.f16(half %a, half %b)
+ ret half %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.ftz with operand type bf16 is not supported
+define bfloat @ftz_bf16(bfloat %a, bfloat %b) {
+ %r = call bfloat @llvm.nvvm.fadd.rn.ftz.bf16(bfloat %a, bfloat %b)
+ ret bfloat %r
+}
+
+; NOBF16: error: {{.*}}llvm.nvvm.fadd.rn with operand type bf16 is not supported
+define bfloat @unsupported_bf16(bfloat %a, bfloat %b) {
+ %r = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %b)
+ ret bfloat %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rn with operand type v4f32 is not supported
+define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) {
+ %r = call <4 x float> @llvm.nvvm.fadd.rn.v4f32(<4 x float> %a, <4 x float> %b)
+ ret <4 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index c6b3b649aae06..44c09cc177595 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -20,17 +20,17 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: add.rp.sat.ftz.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.add.rn.sat.f(float %a, float %b)
- %r2 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %a, float %r1)
+ %r1 = call float @llvm.nvvm.fadd.rn.sat.f32(float %a, float %b)
+ %r2 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %a, float %r1)
- %r3 = call float @llvm.nvvm.add.rz.sat.f(float %a, float %r2)
- %r4 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %a, float %r3)
+ %r3 = call float @llvm.nvvm.fadd.rz.sat.f32(float %a, float %r2)
+ %r4 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %a, float %r3)
- %r5 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %r4)
- %r6 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %a, float %r5)
+ %r5 = call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %r4)
+ %r6 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %a, float %r5)
- %r7 = call float @llvm.nvvm.add.rp.sat.f(float %a, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %r7)
+ %r7 = call float @llvm.nvvm.fadd.rp.sat.f32(float %a, float %r6)
+ %r8 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %r7)
ret float %r8
}
@@ -54,28 +54,28 @@ define float @sub_sat_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.sat.f(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.rn.sat.f32(float %a, float %f0)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %a, float %f1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rz.sat.f(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.rz.sat.f32(float %a, float %f2)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %a, float %f3)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %f4)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %a, float %f5)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rp.sat.f(float %a, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.rp.sat.f32(float %a, float %f6)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %f7)
ret float %r8
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
new file mode 100644
index 0000000000000..513926de4451d
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s
+; RUN: %if ptxas-sm_100 && ptxas-isa-8.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | %ptxas-verify -arch=sm_100 %}
+
+define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<11>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [sub_f32x2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [sub_f32x2_param_1];
+; CHECK-NEXT: sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NEXT: sub.rz.f32x2 %rd5, %rd1, %rd4;
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd6, %rd1, %rd5;
+; CHECK-NEXT: sub.rm.f32x2 %rd7, %rd1, %rd6;
+; CHECK-NEXT: sub.rm.ftz.f32x2 %rd8, %rd1, %rd7;
+; CHECK-NEXT: sub.rp.f32x2 %rd9, %rd1, %rd8;
+; CHECK-NEXT: sub.rp.ftz.f32x2 %rd10, %rd1, %rd9;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd10;
+; CHECK-NEXT: ret;
+ %f0 = fneg <2 x float> %b
+ %r1 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %f0)
+
+ %f1 = fneg <2 x float> %r1
+ %r2 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %a, <2 x float> %f1)
+
+ %f2 = fneg <2 x float> %r2
+ %r3 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %a, <2 x float> %f2)
+
+ %f3 = fneg <2 x float> %r3
+ %r4 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %a, <2 x float> %f3)
+
+ %f4 = fneg <2 x float> %r4
+ %r5 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %a, <2 x float> %f4)
+
+ %f5 = fneg <2 x float> %r5
+ %r6 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %a, <2 x float> %f5)
+
+ %f6 = fneg <2 x float> %r6
+ %r7 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %a, <2 x float> %f6)
+
+ %f7 = fneg <2 x float> %r7
+ %r8 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %a, <2 x float> %f7)
+
+ ret <2 x float> %r8
+}
+
+define <2 x float> @sub_f32x2_negated_lhs(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_negated_lhs(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [sub_f32x2_negated_lhs_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [sub_f32x2_negated_lhs_param_1];
+; CHECK-NEXT: sub.rz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
+; CHECK-NEXT: ret;
+ %f = fneg <2 x float> %a
+ %r = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %f, <2 x float> %b)
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
index 351f45ccbcc6b..a3084392c7853 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
@@ -20,22 +20,22 @@ define float @sub_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r8;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %a, float %f0)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rn.ftz.f(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %f1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rz.f(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.rz.f32(float %a, float %f2)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %a, float %f3)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rm.f(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.rm.f32(float %a, float %f4)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rm.ftz.f(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %a, float %f5)
ret float %r6
}
@@ -55,16 +55,16 @@ define double @sub_f64(double %a, double %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd6;
; CHECK-NEXT: ret;
%f0 = fneg double %b
- %r1 = call double @llvm.nvvm.add.rn.d(double %a, double %f0)
+ %r1 = call double @llvm.nvvm.fadd.rn.f64(double %a, double %f0)
%f1 = fneg double %r1
- %r2 = call double @llvm.nvvm.add.rz.d(double %a, double %f1)
+ %r2 = call double @llvm.nvvm.fadd.rz.f64(double %a, double %f1)
%f2 = fneg double %r2
- %r3 = call double @llvm.nvvm.add.rm.d(double %a, double %f2)
+ %r3 = call double @llvm.nvvm.fadd.rm.f64(double %a, double %f2)
%f3 = fneg double %r3
- %r4 = call double @llvm.nvvm.add.rp.d(double %a, double %f3)
+ %r4 = call double @llvm.nvvm.fadd.rp.f64(double %a, double %f3)
ret double %r4
}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
index 5f82d2e77a468..aa80dc2185bd5 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
@@ -27,16 +27,16 @@ define float @test_add_f32_f16_1(half %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext half %a to float
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %b)
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %b)
+ %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %r1)
+ %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %r2)
+ %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %r3)
; SAT
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %r4)
+ %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %r5)
+ %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %r6)
+ %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %r7)
ret float %r8
}
@@ -93,16 +93,16 @@ define float @test_add_f32_bf16_1(bfloat %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext bfloat %a to float
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %b)
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %b)
+ %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %r1)
+ %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %r2)
+ %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %r3)
; SAT
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %r4)
+ %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %r5)
+ %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %r6)
+ %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %r7)
ret float %r8
}
@@ -160,29 +160,29 @@ define float @test_sub_f32_f16_1(half %a, float %b) {
%r0 = fpext half %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %f0)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %f1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f2)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rm.f(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f3)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %f4)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %f5)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %f6)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %f7)
ret float %r7
}
@@ -240,29 +240,29 @@ define float @test_sub_f32_bf16_1(bfloat %a, float %b) {
%r0 = fpext bfloat %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %f0)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %f1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f2)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %f3)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %f4)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %f5)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %f6)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %f7)
ret float %r8
}
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index db172c5a34cdc..d0acf650d02c5 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -299,20 +299,20 @@ define float @test_ull2f(i64 %a) #0 {
; CHECK-LABEL: @test_add_rn_d
define double @test_add_rn_d(double %a, double %b) #0 {
-; CHECK: call double @llvm.nvvm.add.rn.d
- %ret = call double @llvm.nvvm.add.rn.d(double %a, double %b)
+; CHECK: call double @llvm.nvvm.fadd.rn.f64
+ %ret = call double @llvm.nvvm.fadd.rn.f64(double %a, double %b)
ret double %ret
}
; CHECK-LABEL: @test_add_rn_f
define float @test_add_rn_f(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.add.rn.f
- %ret = call float @llvm.nvvm.add.rn.f(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.rn.f32
+ %ret = call float @llvm.nvvm.fadd.rn.f32(float %a, float %b)
ret float %ret
}
; CHECK-LABEL: @test_add_rn_f_ftz
define float @test_add_rn_f_ftz(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)
- %ret = call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %b)
+ %ret = call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %b)
ret float %ret
}
@@ -437,9 +437,9 @@ define i32 @test_fshr_clamp_3(i32 %a, i32 %b, i32 %c) {
ret i32 %call
}
-declare double @llvm.nvvm.add.rn.d(double, double)
-declare float @llvm.nvvm.add.rn.f(float, float)
-declare float @llvm.nvvm.add.rn.ftz.f(float, float)
+declare double @llvm.nvvm.fadd.rn.f64(double, double)
+declare float @llvm.nvvm.fadd.rn.f32(float, float)
+declare float @llvm.nvvm.fadd.rn.ftz.f32(float, float)
declare double @llvm.nvvm.ceil.d(double)
declare float @llvm.nvvm.ceil.f(float)
declare float @llvm.nvvm.ceil.ftz.f(float)
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
index b23ae275b71b7..f3129ae0c5f48 100644
--- a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
@@ -13,7 +13,7 @@ define double @test_1_25_minus_2_rm_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rm_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 1.25, double -2.0)
ret double %res
}
@@ -21,7 +21,7 @@ define double @test_1_25_minus_2_rn_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rn_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 1.25, double -2.0)
ret double %res
}
@@ -29,7 +29,7 @@ define double @test_1_25_minus_2_rp_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rp_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 1.25, double -2.0)
ret double %res
}
@@ -37,7 +37,7 @@ define double @test_1_25_minus_2_rz_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rz_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 1.25, double -2.0)
ret double %res
}
@@ -45,7 +45,7 @@ define float @test_1_25_minus_2_rm_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 1.25, float -2.0)
ret float %res
}
@@ -53,7 +53,7 @@ define float @test_1_25_minus_2_rn_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 1.25, float -2.0)
ret float %res
}
@@ -61,7 +61,7 @@ define float @test_1_25_minus_2_rp_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 1.25, float -2.0)
ret float %res
}
@@ -69,7 +69,7 @@ define float @test_1_25_minus_2_rz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -77,7 +77,7 @@ define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -85,7 +85,7 @@ define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -93,7 +93,7 @@ define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -101,7 +101,7 @@ define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -113,109 +113,109 @@ define float @test_1_25_minus_2_rz_ftz_f() {
define double @test_zero_plus_nan_rm_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rm_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rm.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rm.f64(double 0.000000e+00, double +snan(0x4444400000000))
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rm.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 0.0, double 0x7ff4444400000000)
ret double %res
}
define double @test_zero_plus_nan_rn_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rn_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rn.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rn.f64(double 0.000000e+00, double +snan(0x4444400000000))
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rn.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 0.0, double 0x7ff4444400000000)
ret double %res
}
define double @test_zero_plus_nan_rp_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rp_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rp.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rp.f64(double 0.000000e+00, double +snan(0x4444400000000))
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rp.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 0.0, double 0x7ff4444400000000)
ret double %res
}
define double @test_zero_plus_nan_rz_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rz_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rz.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rz.f64(double 0.000000e+00, double +snan(0x4444400000000))
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rz.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 0.0, double 0x7ff4444400000000)
ret double %res
}
define float @test_zero_plus_nan_rm_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rm.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rm.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rm.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rn_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rn.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rn.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rn.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rp_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rp.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rp.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rp.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rm_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rm.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rn_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rn.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rp_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rp.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rz_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rz.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
@@ -230,7 +230,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rm.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 0x3800000000000000, double 0x3800000000000000)
ret double %res
}
@@ -238,7 +238,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rn.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 0x3800000000000000, double 0x3800000000000000)
ret double %res
}
@@ -246,7 +246,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rp.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 0x3800000000000000, double 0x3800000000000000)
ret double %res
}
@@ -254,7 +254,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rz.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 0x3800000000000000, double 0x3800000000000000)
ret double %res
}
@@ -262,7 +262,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rm.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -270,7 +270,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rn.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -278,7 +278,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rp.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -286,7 +286,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -294,7 +294,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -302,7 +302,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -310,7 +310,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -318,7 +318,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -335,7 +335,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rm.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 0x3810000000000000, double 0xB800000000000000)
ret double %res
}
@@ -343,7 +343,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rn.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 0x3810000000000000, double 0xB800000000000000)
ret double %res
}
@@ -351,7 +351,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rp.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 0x3810000000000000, double 0xB800000000000000)
ret double %res
}
@@ -359,7 +359,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rz.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 0x3810000000000000, double 0xB800000000000000)
ret double %res
}
@@ -367,7 +367,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rm.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -375,7 +375,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rn.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -383,7 +383,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rp.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -391,7 +391,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -399,7 +399,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -407,7 +407,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -415,7 +415,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -423,7 +423,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -439,7 +439,7 @@ define float @test_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -447,7 +447,7 @@ define float @test_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -455,7 +455,7 @@ define float @test_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -463,7 +463,7 @@ define float @test_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -471,7 +471,7 @@ define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -479,7 +479,7 @@ define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -487,7 +487,7 @@ define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -495,7 +495,7 @@ define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -511,7 +511,7 @@ define double @test_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 1.0, double 0x3C90000000000000)
ret double %res
}
@@ -519,7 +519,7 @@ define double @test_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 1.0, double 0x3C90000000000000)
ret double %res
}
@@ -527,7 +527,7 @@ define double @test_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0x3FF0000000000001
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 1.0, double 0x3C90000000000000)
ret double %res
}
@@ -535,7 +535,7 @@ define double @test_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 1.0, double 0x3C90000000000000)
ret double %res
}
@@ -551,7 +551,7 @@ define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -559,7 +559,7 @@ define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -567,7 +567,7 @@ define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rp.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -575,7 +575,7 @@ define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -583,7 +583,7 @@ define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -591,7 +591,7 @@ define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -599,7 +599,7 @@ define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -607,7 +607,7 @@ define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -623,7 +623,7 @@ define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rm.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double -1.0, double 0x3C90000000000000)
ret double %res
}
@@ -631,7 +631,7 @@ define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double -1.0, double 0x3C90000000000000)
ret double %res
}
@@ -639,7 +639,7 @@ define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rp.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double -1.0, double 0x3C90000000000000)
ret double %res
}
@@ -647,7 +647,7 @@ define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rz.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double -1.0, double 0x3C90000000000000)
ret double %res
}
@@ -663,7 +663,7 @@ define float @test_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -671,7 +671,7 @@ define float @test_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -679,7 +679,7 @@ define float @test_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -687,7 +687,7 @@ define float @test_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -695,7 +695,7 @@ define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -703,7 +703,7 @@ define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -711,7 +711,7 @@ define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -719,7 +719,7 @@ define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -735,7 +735,7 @@ define double @test_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 1.0, double 0xBC90000000000000)
ret double %res
}
@@ -743,7 +743,7 @@ define double @test_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 1.0, double 0xBC90000000000000)
ret double %res
}
@@ -751,7 +751,7 @@ define double @test_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 1.0, double 0xBC90000000000000)
ret double %res
}
@@ -759,7 +759,7 @@ define double @test_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 1.0, double 0xBC90000000000000)
ret double %res
}
@@ -775,7 +775,7 @@ define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.add.rm.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -783,7 +783,7 @@ define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -791,7 +791,7 @@ define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -799,7 +799,7 @@ define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -807,7 +807,7 @@ define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -815,7 +815,7 @@ define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -823,7 +823,7 @@ define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -831,7 +831,7 @@ define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -847,7 +847,7 @@ define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0xBFF0000000000001
;
- %res = call double @llvm.nvvm.add.rm.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double -1.0, double 0xBC90000000000000)
ret double %res
}
@@ -855,7 +855,7 @@ define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double -1.0, double 0xBC90000000000000)
ret double %res
}
@@ -863,7 +863,7 @@ define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rp.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double -1.0, double 0xBC90000000000000)
ret double %res
}
@@ -871,6 +871,6 @@ define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rz.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double -1.0, double 0xBC90000000000000)
ret double %res
}
diff --git a/llvm/unittests/IR/IntrinsicsTest.cpp b/llvm/unittests/IR/IntrinsicsTest.cpp
index 47a1ea7d73906..f982715154bc0 100644
--- a/llvm/unittests/IR/IntrinsicsTest.cpp
+++ b/llvm/unittests/IR/IntrinsicsTest.cpp
@@ -118,7 +118,7 @@ TEST(IntrinsicNameLookup, ClangBuiltinLookup) {
{"__builtin_HEXAGON_A2_tfr", "hexagon", hexagon_A2_tfr},
{"__builtin_lasx_xbz_w", "loongarch", loongarch_lasx_xbz_w},
{"__builtin_mips_bitrev", "mips", mips_bitrev},
- {"__nvvm_add_rn_d", "nvvm", nvvm_add_rn_d},
+ {"__nvvm_mul_rn_d", "nvvm", nvvm_mul_rn_d},
{"__builtin_altivec_dss", "ppc", ppc_altivec_dss},
{"__builtin_riscv_sha512sum1r", "riscv", riscv_sha512sum1r},
{"__builtin_tend", "s390", s390_tend},
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 843344703d0a5..43280c4a6f00a 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -3609,14 +3609,6 @@ static LogicalResult verifyAddSubFOp(OpType op) {
"vector<2xbf16> additions/subtractions");
}
- // FIXME: This is a temporary check disallowing lowering to add.rn.ftz.f16(x2)
- // PTX instructions since the corresponding LLVM intrinsic is missing. This
- // should be removed once the intrinsics for f16 addition (with FTZ only) are
- // available.
- if (opBaseType.isF16() && isFTZ && satMode == NVVM::SaturationMode::NONE)
- return op.emitOpError("FTZ with no saturation is not supported for f16 and "
- "vector<2xf16> additions/subtractions");
-
return success();
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 4201aabc02c3d..8b823fa425206 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -481,87 +481,39 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
llvm::Type *opTypeLLVM = argLHS->getType();
- bool isVectorOp = opTypeLLVM->isVectorTy();
bool isSat = satMode != NVVM::SaturationMode::NONE;
- // FIXME: Add intrinsics for add.rn.ftz.f16x2 and add.rn.ftz.f16 here when
- // they are available.
- static constexpr llvm::Intrinsic::ID f16IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_sat_f16,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f16,
- llvm::Intrinsic::nvvm_add_rn_sat_v2f16,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_v2f16,
- };
-
- static constexpr llvm::Intrinsic::ID f32IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_f,
- llvm::Intrinsic::nvvm_add_rm_f,
- llvm::Intrinsic::nvvm_add_rp_f,
- llvm::Intrinsic::nvvm_add_rz_f,
- llvm::Intrinsic::nvvm_add_rn_sat_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_sat_f,
- llvm::Intrinsic::nvvm_add_rm_sat_f,
- llvm::Intrinsic::nvvm_add_rp_sat_f,
- llvm::Intrinsic::nvvm_add_rz_sat_f,
- llvm::Intrinsic::nvvm_add_rn_ftz_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_ftz_f,
- llvm::Intrinsic::nvvm_add_rm_ftz_f,
- llvm::Intrinsic::nvvm_add_rp_ftz_f,
- llvm::Intrinsic::nvvm_add_rz_ftz_f,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rm_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rp_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rz_ftz_sat_f,
- };
-
- static constexpr llvm::Intrinsic::ID f64IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_d, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_d, llvm::Intrinsic::nvvm_add_rm_d,
- llvm::Intrinsic::nvvm_add_rp_d, llvm::Intrinsic::nvvm_add_rz_d};
-
- auto addIntrinsic = [&](llvm::Intrinsic::ID IID) -> llvm::Value * {
- return createScalarizedIntrinsicCall(builder, IID, opTypeLLVM,
- {argLHS, argRHS}, opTypeLLVM);
- };
-
- // f16 + f16 -> f16 / vector<2xf16> + vector<2xf16> -> vector<2xf16>
- // FIXME: Allow lowering to add.rn.ftz.f16x2 and add.rn.ftz.f16 here when the
- // intrinsics are available.
- if (opTypeLLVM->getScalarType()->isHalfTy()) {
- llvm::Value *result;
- if (isSat) {
- unsigned index = (isVectorOp << 1) | isFTZ;
- result = addIntrinsic(f16IDs[index]);
- } else {
- result = builder.CreateFAdd(argLHS, argRHS);
- }
- mt.mapValue(res, result);
+ static constexpr llvm::Intrinsic::ID addIDs[2][2][5] = {
+ {{llvm::Intrinsic::nvvm_fadd_rn, llvm::Intrinsic::nvvm_fadd_rn,
+ llvm::Intrinsic::nvvm_fadd_rm, llvm::Intrinsic::nvvm_fadd_rp,
+ llvm::Intrinsic::nvvm_fadd_rz},
+ {llvm::Intrinsic::nvvm_fadd_rn_sat, llvm::Intrinsic::nvvm_fadd_rn_sat,
+ llvm::Intrinsic::nvvm_fadd_rm_sat, llvm::Intrinsic::nvvm_fadd_rp_sat,
+ llvm::Intrinsic::nvvm_fadd_rz_sat}},
+ {{llvm::Intrinsic::nvvm_fadd_rn_ftz, llvm::Intrinsic::nvvm_fadd_rn_ftz,
+ llvm::Intrinsic::nvvm_fadd_rm_ftz, llvm::Intrinsic::nvvm_fadd_rp_ftz,
+ llvm::Intrinsic::nvvm_fadd_rz_ftz},
+ {llvm::Intrinsic::nvvm_fadd_rn_ftz_sat,
+ llvm::Intrinsic::nvvm_fadd_rn_ftz_sat,
+ llvm::Intrinsic::nvvm_fadd_rm_ftz_sat,
+ llvm::Intrinsic::nvvm_fadd_rp_ftz_sat,
+ llvm::Intrinsic::nvvm_fadd_rz_ftz_sat}}};
+
+ llvm::Intrinsic::ID id = addIDs[isFTZ][isSat][static_cast<unsigned>(rndMode)];
+
+ // For f64 vector addition, and f32 vector addition with saturation,
+ // we need to scalarize the intrinsic call.
+ llvm::Type *scalarTypeLLVM = opTypeLLVM->getScalarType();
+ if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() ||
+ (isSat && scalarTypeLLVM->isFloatTy()))) {
+ mt.mapValue(res, createScalarizedIntrinsicCall(builder, id, opTypeLLVM,
+ {argLHS, argRHS},
+ scalarTypeLLVM));
return;
}
- // bf16 + bf16 -> bf16 / vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
- if (opTypeLLVM->getScalarType()->isBFloatTy()) {
- mt.mapValue(res, builder.CreateFAdd(argLHS, argRHS));
- return;
- }
-
- // f64 + f64 -> f64 / vector<2xf64> + vector<2xf64> -> vector<2xf64>
- if (opTypeLLVM->getScalarType()->isDoubleTy()) {
- unsigned index = static_cast<unsigned>(rndMode);
- mt.mapValue(res, addIntrinsic(f64IDs[index]));
- return;
- }
-
- // f32 + f32 -> f32 / vector<2xf32> + vector<2xf32> -> vector<2xf32>
- const unsigned numRndModes = 5; // NONE, RM, RN, RP, RZ
- if (opTypeLLVM->getScalarType()->isFloatTy()) {
- unsigned index =
- ((isFTZ << 1) | isSat) * numRndModes + static_cast<unsigned>(rndMode);
- mt.mapValue(res, addIntrinsic(f32IDs[index]));
- return;
- }
+ mt.mapValue(res,
+ createIntrinsicCall(builder, id, opTypeLLVM, {argLHS, argRHS}));
}
void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index 201cfb7e098fc..a5e5a204a36e3 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -3,24 +3,26 @@
// f16 + f16 -> f16
llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fadd_f16_f16(half %0, half %1) {
- // CHECK-NEXT: %3 = fadd half %0, %1
- // CHECK-NEXT: %4 = fadd half %3, %3
- // CHECK-NEXT: %5 = call half @llvm.nvvm.add.rn.sat.f16(half %4, half %4)
- // CHECK-NEXT: %6 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %5, half %5)
- // CHECK-NEXT: ret half %6
+ // CHECK-NEXT: %3 = call half @llvm.nvvm.fadd.rn.f16(half %0, half %1)
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.rn.f16(half %3, half %3)
+ // CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %4, half %4)
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.rn.sat.f16(half %5, half %5)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %6, half %6)
+ // CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
- %f2 = nvvm.addf %f1, %f1 rnd = <rn> : f16
- %f3 = nvvm.addf %f2, %f2 rnd = <rn> sat = <sat> : f16
- %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : f16
- llvm.return %f4 : f16
+ %f2 = nvvm.addf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : f16
+ %f3 = nvvm.addf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : f16
+ %f4 = nvvm.addf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : f16
+ %f5 = nvvm.addf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : f16
+ llvm.return %f5 : f16
}
// bf16 + bf16 -> bf16
llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fadd_bf16_bf16(bfloat %0, bfloat %1) {
- // CHECK-NEXT: %3 = fadd bfloat %0, %1
- // CHECK-NEXT: %4 = fadd bfloat %3, %3
+ // CHECK-NEXT: %3 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %0, bfloat %1)
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %3, bfloat %3)
// CHECK-NEXT: ret bfloat %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : bf16
@@ -31,23 +33,23 @@ llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// f32 + f32 -> f32
llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fadd_f32_f32(float %0, float %1) {
- // CHECK-NEXT: %3 = call float @llvm.nvvm.add.rn.f(float %0, float %1)
- // CHECK-NEXT: %4 = call float @llvm.nvvm.add.rn.f(float %3, float %3)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rn.sat.f(float %4, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.ftz.f(float %5, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %6, float %6)
- // CHECK-NEXT: %8 = call float @llvm.nvvm.add.rm.f(float %7, float %7)
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rm.sat.f(float %8, float %8)
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rm.ftz.f(float %9, float %9)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %10, float %10)
- // CHECK-NEXT: %12 = call float @llvm.nvvm.add.rp.f(float %11, float %11)
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rp.sat.f(float %12, float %12)
- // CHECK-NEXT: %14 = call float @llvm.nvvm.add.rp.ftz.f(float %13, float %13)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %14, float %14)
- // CHECK-NEXT: %16 = call float @llvm.nvvm.add.rz.f(float %15, float %15)
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rz.sat.f(float %16, float %16)
- // CHECK-NEXT: %18 = call float @llvm.nvvm.add.rz.ftz.f(float %17, float %17)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %18, float %18)
+ // CHECK-NEXT: %3 = call float @llvm.nvvm.fadd.rn.f32(float %0, float %1)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.rn.f32(float %3, float %3)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %4, float %4)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %5, float %5)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %6, float %6)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rm.f32(float %7, float %7)
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.rm.sat.f32(float %8, float %8)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %9, float %9)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %10, float %10)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rp.f32(float %11, float %11)
+ // CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.rp.sat.f32(float %12, float %12)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rp.ftz.f32(float %13, float %13)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %14, float %14)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rz.f32(float %15, float %15)
+ // CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.rz.sat.f32(float %16, float %16)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %17, float %17)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %18, float %18)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
@@ -73,11 +75,11 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// f64 + f64 -> f64
llvm.func @fadd_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fadd_f64_f64(double %0, double %1) {
- // CHECK-NEXT: %3 = call double @llvm.nvvm.add.rn.d(double %0, double %1)
- // CHECK-NEXT: %4 = call double @llvm.nvvm.add.rn.d(double %3, double %3)
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rm.d(double %4, double %4)
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rp.d(double %5, double %5)
- // CHECK-NEXT: %7 = call double @llvm.nvvm.add.rz.d(double %6, double %6)
+ // CHECK-NEXT: %3 = call double @llvm.nvvm.fadd.rn.f64(double %0, double %1)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.rn.f64(double %3, double %3)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rm.f64(double %4, double %4)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rp.f64(double %5, double %5)
+ // CHECK-NEXT: %7 = call double @llvm.nvvm.fadd.rz.f64(double %6, double %6)
// CHECK-NEXT: ret double %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
index de9a53c8e95d6..58fb8814903ad 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
@@ -55,13 +55,3 @@ llvm.func @addf_invalid_bf16_sat_ftz(%a : bf16, %b : bf16) -> bf16 {
%f1 = nvvm.addf %a, %b sat = <sat> ftz = true : bf16
llvm.return %f1 : bf16
}
-
-// -----
-
-// FIXME: Remove this test once intrinsics for f16 addition (with FTZ only) are
-// available.
-llvm.func @addf_invalid_f16_ftz_no_sat(%a : f16, %b : f16) -> f16 {
- // expected-error at +1 {{FTZ with no saturation is not supported for f16 and vector<2xf16> additions/subtractions}}
- %f1 = nvvm.addf %a, %b ftz = true : f16
- llvm.return %f1 : f16
-}
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index d8bf3dfcc8fb6..1f7daa6c3c66a 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -3,24 +3,26 @@
// vector<2xf16> + vector<2xf16> -> vector<2xf16>
llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @addf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
- // CHECK-NEXT: %3 = fadd <2 x half> %0, %1
- // CHECK-NEXT: %4 = fadd <2 x half> %3, %3
- // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %4, <2 x half> %4)
- // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %5, <2 x half> %5)
+ // CHECK-NEXT: %3 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %0, <2 x half> %1)
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %3, <2 x half> %3)
+ // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %4, <2 x half> %4)
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %5, <2 x half> %5)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
- %f2 = nvvm.addf %f1, %f1 rnd = <rn> : vector<2xf16>
- %f3 = nvvm.addf %f2, %f2 rnd = <rn> sat = <sat> : vector<2xf16>
- %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
+ %f2 = nvvm.addf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xf16>
+ %f3 = nvvm.addf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : vector<2xf16>
+ %f4 = nvvm.addf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : vector<2xf16>
+ %f5 = nvvm.addf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
// vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @addf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
- // CHECK-NEXT: %3 = fadd <2 x bfloat> %0, %1
- // CHECK-NEXT: %4 = fadd <2 x bfloat> %3, %3
+ // CHECK-NEXT: %3 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1)
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %3, <2 x bfloat> %3)
// CHECK-NEXT: ret <2 x bfloat> %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xbf16>
@@ -31,47 +33,26 @@ llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
// vector<2xf32> + vector<2xf32> -> vector<2xf32>
llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rn.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rn.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rn.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rn.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rn.sat.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rn.sat.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rn.ftz.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rn.ftz.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: %35 = extractelement <2 x float> %34, i32 0
- // CHECK-NEXT: %36 = extractelement <2 x float> %34, i32 0
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> poison, float %37, i32 0
- // CHECK-NEXT: %39 = extractelement <2 x float> %34, i32 1
- // CHECK-NEXT: %40 = extractelement <2 x float> %34, i32 1
- // CHECK-NEXT: %41 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %39, float %40)
- // CHECK-NEXT: %42 = insertelement <2 x float> %38, float %41, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %3, <2 x float> %3)
+ // CHECK-NEXT: %5 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.rn.sat.f32(float %5, float %6)
+ // CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
+ // CHECK-NEXT: %9 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.rn.sat.f32(float %9, float %10)
+ // CHECK-NEXT: %12 = insertelement <2 x float> %8, float %11, i32 1
+ // CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %12, <2 x float> %12)
+ // CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
+ // CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %14, float %15)
+ // CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
+ // CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %18, float %19)
+ // CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
+ // CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rn> : vector<2xf32>
@@ -83,39 +64,25 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rm.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rm.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rm.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rm.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rm.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rm.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rm.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rm.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rm> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rm> sat = <sat> : vector<2xf32>
@@ -126,39 +93,25 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rp.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rp.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rp.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rp.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rp.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rp.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rp.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rp.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rp> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rp> sat = <sat> : vector<2xf32>
@@ -169,39 +122,25 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rz.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rz.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rz.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rz.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rz.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rz.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rz.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rz> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rz> sat = <sat> : vector<2xf32>
@@ -215,19 +154,19 @@ llvm.func @addf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rn(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rn.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rn.f64(double %3, double %4)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rn.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rn.f64(double %7, double %8)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: %11 = extractelement <2 x double> %10, i32 0
// CHECK-NEXT: %12 = extractelement <2 x double> %10, i32 0
- // CHECK-NEXT: %13 = call double @llvm.nvvm.add.rn.d(double %11, double %12)
+ // CHECK-NEXT: %13 = call double @llvm.nvvm.fadd.rn.f64(double %11, double %12)
// CHECK-NEXT: %14 = insertelement <2 x double> poison, double %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x double> %10, i32 1
// CHECK-NEXT: %16 = extractelement <2 x double> %10, i32 1
- // CHECK-NEXT: %17 = call double @llvm.nvvm.add.rn.d(double %15, double %16)
+ // CHECK-NEXT: %17 = call double @llvm.nvvm.fadd.rn.f64(double %15, double %16)
// CHECK-NEXT: %18 = insertelement <2 x double> %14, double %17, i32 1
// CHECK-NEXT: ret <2 x double> %18
// CHECK-NEXT: }
@@ -240,11 +179,11 @@ llvm.func @addf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rm(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rm.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rm.f64(double %3, double %4)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rm.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rm.f64(double %7, double %8)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -256,11 +195,11 @@ llvm.func @addf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rp(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rp.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rp.f64(double %3, double %4)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rp.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rp.f64(double %7, double %8)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -272,11 +211,11 @@ llvm.func @addf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rz(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rz.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rz.f64(double %3, double %4)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rz.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rz.f64(double %7, double %8)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index b2c6163045cb7..a540c0338a59b 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -4,29 +4,32 @@
llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fsub_f16_f16(half %0, half %1) {
// CHECK-NEXT: %3 = fneg half %1
- // CHECK-NEXT: %4 = fadd half %0, %3
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.rn.f16(half %0, half %3)
// CHECK-NEXT: %5 = fneg half %4
- // CHECK-NEXT: %6 = fadd half %4, %5
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.rn.f16(half %4, half %5)
// CHECK-NEXT: %7 = fneg half %6
- // CHECK-NEXT: %8 = call half @llvm.nvvm.add.rn.sat.f16(half %6, half %7)
+ // CHECK-NEXT: %8 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %6, half %7)
// CHECK-NEXT: %9 = fneg half %8
- // CHECK-NEXT: %10 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %8, half %9)
- // CHECK-NEXT: ret half %10
+ // CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.rn.sat.f16(half %8, half %9)
+ // CHECK-NEXT: %11 = fneg half %10
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %10, half %11)
+ // CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
- %f2 = nvvm.subf %f1, %f1 rnd = <rn> : f16
- %f3 = nvvm.subf %f2, %f2 rnd = <rn> sat = <sat> : f16
- %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : f16
- llvm.return %f4 : f16
+ %f2 = nvvm.subf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : f16
+ %f3 = nvvm.subf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : f16
+ %f4 = nvvm.subf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : f16
+ %f5 = nvvm.subf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : f16
+ llvm.return %f5 : f16
}
// bf16 - bf16 -> bf16
llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fsub_bf16_bf16(bfloat %0, bfloat %1) {
// CHECK-NEXT: %3 = fneg bfloat %1
- // CHECK-NEXT: %4 = fadd bfloat %0, %3
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %0, bfloat %3)
// CHECK-NEXT: %5 = fneg bfloat %4
- // CHECK-NEXT: %6 = fadd bfloat %4, %5
+ // CHECK-NEXT: %6 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %4, bfloat %5)
// CHECK-NEXT: ret bfloat %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : bf16
@@ -38,39 +41,39 @@ llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fsub_f32_f32(float %0, float %1) {
// CHECK-NEXT: %3 = fneg float %1
- // CHECK-NEXT: %4 = call float @llvm.nvvm.add.rn.f(float %0, float %3)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.rn.f32(float %0, float %3)
// CHECK-NEXT: %5 = fneg float %4
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.f(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rn.f32(float %4, float %5)
// CHECK-NEXT: %7 = fneg float %6
- // CHECK-NEXT: %8 = call float @llvm.nvvm.add.rn.sat.f(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rn.sat.f32(float %6, float %7)
// CHECK-NEXT: %9 = fneg float %8
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rn.ftz.f(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %8, float %9)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %10, float %11)
// CHECK-NEXT: %13 = fneg float %12
- // CHECK-NEXT: %14 = call float @llvm.nvvm.add.rm.f(float %12, float %13)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rm.f32(float %12, float %13)
// CHECK-NEXT: %15 = fneg float %14
- // CHECK-NEXT: %16 = call float @llvm.nvvm.add.rm.sat.f(float %14, float %15)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rm.sat.f32(float %14, float %15)
// CHECK-NEXT: %17 = fneg float %16
- // CHECK-NEXT: %18 = call float @llvm.nvvm.add.rm.ftz.f(float %16, float %17)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %16, float %17)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %18, float %19)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %18, float %19)
// CHECK-NEXT: %21 = fneg float %20
- // CHECK-NEXT: %22 = call float @llvm.nvvm.add.rp.f(float %20, float %21)
+ // CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.rp.f32(float %20, float %21)
// CHECK-NEXT: %23 = fneg float %22
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rp.sat.f(float %22, float %23)
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fadd.rp.sat.f32(float %22, float %23)
// CHECK-NEXT: %25 = fneg float %24
- // CHECK-NEXT: %26 = call float @llvm.nvvm.add.rp.ftz.f(float %24, float %25)
+ // CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.rp.ftz.f32(float %24, float %25)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %26, float %27)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %26, float %27)
// CHECK-NEXT: %29 = fneg float %28
- // CHECK-NEXT: %30 = call float @llvm.nvvm.add.rz.f(float %28, float %29)
+ // CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.rz.f32(float %28, float %29)
// CHECK-NEXT: %31 = fneg float %30
- // CHECK-NEXT: %32 = call float @llvm.nvvm.add.rz.sat.f(float %30, float %31)
+ // CHECK-NEXT: %32 = call float @llvm.nvvm.fadd.rz.sat.f32(float %30, float %31)
// CHECK-NEXT: %33 = fneg float %32
- // CHECK-NEXT: %34 = call float @llvm.nvvm.add.rz.ftz.f(float %32, float %33)
+ // CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %32, float %33)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %34, float %35)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %34, float %35)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
@@ -97,15 +100,15 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
llvm.func @fsub_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fsub_f64_f64(double %0, double %1) {
// CHECK-NEXT: %3 = fneg double %1
- // CHECK-NEXT: %4 = call double @llvm.nvvm.add.rn.d(double %0, double %3)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.rn.f64(double %0, double %3)
// CHECK-NEXT: %5 = fneg double %4
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rn.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rn.f64(double %4, double %5)
// CHECK-NEXT: %7 = fneg double %6
- // CHECK-NEXT: %8 = call double @llvm.nvvm.add.rm.d(double %6, double %7)
+ // CHECK-NEXT: %8 = call double @llvm.nvvm.fadd.rm.f64(double %6, double %7)
// CHECK-NEXT: %9 = fneg double %8
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rp.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rp.f64(double %8, double %9)
// CHECK-NEXT: %11 = fneg double %10
- // CHECK-NEXT: %12 = call double @llvm.nvvm.add.rz.d(double %10, double %11)
+ // CHECK-NEXT: %12 = call double @llvm.nvvm.fadd.rz.f64(double %10, double %11)
// CHECK-NEXT: ret double %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
index d5cff844ed1d3..38a2ca14bfa90 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
@@ -55,13 +55,3 @@ llvm.func @subf_invalid_bf16_sat_ftz(%a : bf16, %b : bf16) -> bf16 {
%f1 = nvvm.subf %a, %b sat = <sat> ftz = true : bf16
llvm.return %f1 : bf16
}
-
-// -----
-
-// FIXME: Remove this test once intrinsics for f16 addition (with FTZ only) are
-// available.
-llvm.func @subf_invalid_f16_ftz_no_sat(%a : f16, %b : f16) -> f16 {
- // expected-error at +1 {{FTZ with no saturation is not supported for f16 and vector<2xf16> additions/subtractions}}
- %f1 = nvvm.subf %a, %b ftz = true : f16
- llvm.return %f1 : f16
-}
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index 4c0f143806329..b72d3b0ebecde 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -4,19 +4,22 @@
llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @subf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
// CHECK-NEXT: %3 = fneg <2 x half> %1
- // CHECK-NEXT: %4 = fadd <2 x half> %0, %3
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %0, <2 x half> %3)
// CHECK-NEXT: %5 = fneg <2 x half> %4
- // CHECK-NEXT: %6 = fadd <2 x half> %4, %5
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %4, <2 x half> %5)
// CHECK-NEXT: %7 = fneg <2 x half> %6
- // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %6, <2 x half> %7)
+ // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %6, <2 x half> %7)
// CHECK-NEXT: %9 = fneg <2 x half> %8
- // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %8, <2 x half> %9)
+ // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %8, <2 x half> %9)
+ // CHECK-NEXT: %11 = fneg <2 x half> %10
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
- %f2 = nvvm.subf %f1, %f1 rnd = <rn> : vector<2xf16>
- %f3 = nvvm.subf %f2, %f2 rnd = <rn> sat = <sat> : vector<2xf16>
- %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
+ %f2 = nvvm.subf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xf16>
+ %f3 = nvvm.subf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : vector<2xf16>
+ %f4 = nvvm.subf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : vector<2xf16>
+ %f5 = nvvm.subf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
@@ -24,9 +27,9 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @subf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
// CHECK-NEXT: %3 = fneg <2 x bfloat> %1
- // CHECK-NEXT: %4 = fadd <2 x bfloat> %0, %3
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %0, <2 x bfloat> %3)
// CHECK-NEXT: %5 = fneg <2 x bfloat> %4
- // CHECK-NEXT: %6 = fadd <2 x bfloat> %4, %5
+ // CHECK-NEXT: %6 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %4, <2 x bfloat> %5)
// CHECK-NEXT: ret <2 x bfloat> %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xbf16>
@@ -38,51 +41,30 @@ llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rn.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rn.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rn.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rn.sat.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rn.sat.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rn.ftz.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rn.ftz.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: %39 = fneg <2 x float> %38
- // CHECK-NEXT: %40 = extractelement <2 x float> %38, i32 0
- // CHECK-NEXT: %41 = extractelement <2 x float> %39, i32 0
- // CHECK-NEXT: %42 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %40, float %41)
- // CHECK-NEXT: %43 = insertelement <2 x float> poison, float %42, i32 0
- // CHECK-NEXT: %44 = extractelement <2 x float> %38, i32 1
- // CHECK-NEXT: %45 = extractelement <2 x float> %39, i32 1
- // CHECK-NEXT: %46 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %44, float %45)
- // CHECK-NEXT: %47 = insertelement <2 x float> %43, float %46, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %4, <2 x float> %5)
+ // CHECK-NEXT: %7 = fneg <2 x float> %6
+ // CHECK-NEXT: %8 = extractelement <2 x float> %6, i32 0
+ // CHECK-NEXT: %9 = extractelement <2 x float> %7, i32 0
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rn.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %11 = insertelement <2 x float> poison, float %10, i32 0
+ // CHECK-NEXT: %12 = extractelement <2 x float> %6, i32 1
+ // CHECK-NEXT: %13 = extractelement <2 x float> %7, i32 1
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rn.sat.f32(float %12, float %13)
+ // CHECK-NEXT: %15 = insertelement <2 x float> %11, float %14, i32 1
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %15, <2 x float> %16)
+ // CHECK-NEXT: %18 = fneg <2 x float> %17
+ // CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
+ // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %19, float %20)
+ // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
+ // CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
+ // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %23, float %24)
+ // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
+ // CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rn> : vector<2xf32>
@@ -95,42 +77,28 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rm.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rm.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rm.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rm.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rm.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rm.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rm.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rm.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rm> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rm> sat = <sat> : vector<2xf32>
@@ -142,42 +110,28 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rp.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rp.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rp.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rp.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rp.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rp.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rp.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rp> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rp> sat = <sat> : vector<2xf32>
@@ -189,42 +143,28 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rz.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rz.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rz.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rz.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rz.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rz.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rz.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rz.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rz> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rz> sat = <sat> : vector<2xf32>
@@ -239,20 +179,20 @@ llvm.func @subf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rn.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rn.f64(double %4, double %5)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rn.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rn.f64(double %8, double %9)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: %12 = fneg <2 x double> %11
// CHECK-NEXT: %13 = extractelement <2 x double> %11, i32 0
// CHECK-NEXT: %14 = extractelement <2 x double> %12, i32 0
- // CHECK-NEXT: %15 = call double @llvm.nvvm.add.rn.d(double %13, double %14)
+ // CHECK-NEXT: %15 = call double @llvm.nvvm.fadd.rn.f64(double %13, double %14)
// CHECK-NEXT: %16 = insertelement <2 x double> poison, double %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x double> %11, i32 1
// CHECK-NEXT: %18 = extractelement <2 x double> %12, i32 1
- // CHECK-NEXT: %19 = call double @llvm.nvvm.add.rn.d(double %17, double %18)
+ // CHECK-NEXT: %19 = call double @llvm.nvvm.fadd.rn.f64(double %17, double %18)
// CHECK-NEXT: %20 = insertelement <2 x double> %16, double %19, i32 1
// CHECK-NEXT: ret <2 x double> %20
// CHECK-NEXT: }
@@ -266,11 +206,11 @@ llvm.func @subf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rm.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rm.f64(double %4, double %5)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rm.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rm.f64(double %8, double %9)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -283,11 +223,11 @@ llvm.func @subf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rp.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rp.f64(double %4, double %5)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rp.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rp.f64(double %8, double %9)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -300,11 +240,11 @@ llvm.func @subf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rz.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rz.f64(double %4, double %5)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rz.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rz.f64(double %8, double %9)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
>From 004776bbdbe01faf77f926a87e9230d4b21f7955 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 18 Aug 2026 11:12:52 +0000
Subject: [PATCH 02/16] use immediate argument for rounding mode
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 66 +-
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 82 +--
.../CIR/CodeGenCUDA/builtins-nvvm-math.cu | 12 +-
clang/test/CodeGen/builtins-nvptx.c | 26 +-
llvm/include/llvm/IR/IntrinsicsNVVM.td | 18 +-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 81 +--
llvm/lib/Analysis/ConstantFolding.cpp | 68 +-
llvm/lib/IR/AutoUpgrade.cpp | 47 +-
llvm/lib/IR/NVVMIntrinsicUtils.cpp | 5 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 53 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 130 ++--
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 20 +-
llvm/test/CodeGen/NVPTX/bf16-add.ll | 4 +-
llvm/test/CodeGen/NVPTX/bf16-sub.ll | 4 +-
llvm/test/CodeGen/NVPTX/f16-add.ll | 16 +-
llvm/test/CodeGen/NVPTX/f16-sub.ll | 16 +-
llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll | 16 +-
llvm/test/CodeGen/NVPTX/fp-add-invalid.ll | 28 +-
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 32 +-
llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll | 18 +-
llvm/test/CodeGen/NVPTX/fp-fold-sub.ll | 20 +-
llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll | 64 +-
.../InstCombine/NVPTX/nvvm-intrins.ll | 18 +-
.../InstSimplify/const-fold-nvvm-add.ll | 592 ++++++++++++++----
llvm/test/Verifier/NVPTX/fadd.ll | 16 +
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 39 +-
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 58 +-
.../Target/LLVMIR/nvvm/addf/addf_vector.mlir | 84 +--
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 58 +-
.../Target/LLVMIR/nvvm/subf/subf_vector.mlir | 84 +--
30 files changed, 1094 insertions(+), 681 deletions(-)
create mode 100644 llvm/test/Verifier/NVPTX/fadd.ll
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 9937b206aca60..0b19fb18cd430 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -69,17 +69,20 @@ static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
.getResult();
}
-/// Emit a CIR LLVMIntrinsicCallOp for a binary NVVM intrinsic.
-/// The result type is inferred from the first argument.
-static mlir::Value emitBinaryNVVMIntrinsic(CIRGenFunction &cgf,
- const CallExpr *expr,
- llvm::StringRef intrinsicName) {
+/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd intrinsic, which takes the
+/// rounding mode as a trailing operand.
+static mlir::Value emitNVVMFAdd(CIRGenFunction &cgf, const CallExpr *expr,
+ llvm::StringRef intrinsicName,
+ llvm::APFloat::roundingMode rm) {
auto &builder = cgf.getBuilder();
+ mlir::Location loc = cgf.getLoc(expr->getExprLoc());
mlir::Value lhs = cgf.emitScalarExpr(expr->getArg(0));
mlir::Value rhs = cgf.emitScalarExpr(expr->getArg(1));
- return cir::LLVMIntrinsicCallOp::create(
- builder, cgf.getLoc(expr->getExprLoc()),
- builder.getStringAttr(intrinsicName), lhs.getType(), {lhs, rhs})
+ mlir::Value rnd =
+ builder.getConstInt(loc, builder.getSInt32Ty(), static_cast<int>(rm));
+ return cir::LLVMIntrinsicCallOp::create(builder, loc,
+ builder.getStringAttr(intrinsicName),
+ lhs.getType(), {lhs, rhs, rnd})
.getResult();
}
@@ -838,44 +841,59 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
case NVPTX::BI__nvvm_add_rn_f:
case NVPTX::BI__nvvm_add_rn_d:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_f:
case NVPTX::BI__nvvm_add_rz_d:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd", llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_f:
case NVPTX::BI__nvvm_add_rm_d:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_f:
case NVPTX::BI__nvvm_add_rp_d:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_add_rn_ftz_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.ftz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_ftz_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.ftz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_ftz_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.ftz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_ftz_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.ftz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_add_rn_sat_f:
case NVPTX::BI__nvvm_add_rn_sat_f16:
case NVPTX::BI__nvvm_add_rn_sat_v2f16:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.ftz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.ftz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.ftz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.ftz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
cgm.errorNYI(expr->getSourceRange(),
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index 896afea982385..09ad23985b092 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -411,6 +411,7 @@ static Value *MakeHalfType(Function *Intrinsic, unsigned BuiltinID,
ArgValue = CGF.Builder.CreateBitCast(ArgValue, PTy);
Args.push_back(ArgValue);
}
+ Args.append(TrailingArgs.begin(), TrailingArgs.end());
llvm::append_range(Args, TrailingArgs);
appendDefaultIntrinsicArgs(Args, Intrinsic);
@@ -433,11 +434,12 @@ static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
CGF.EmitScalarExpr(E->getArg(2))});
}
-static Value *MakeBinaryIntrinsic(unsigned IntrinsicID, const CallExpr *E,
- CodeGenFunction &CGF) {
- return CGF.Builder.CreateBinaryIntrinsic(IntrinsicID,
- CGF.EmitScalarExpr(E->getArg(0)),
- CGF.EmitScalarExpr(E->getArg(1)));
+static Value *MakeFAdd(unsigned IntrinsicID, APFloat::roundingMode RM,
+ unsigned BuiltinID, const CallExpr *E,
+ CodeGenFunction &CGF) {
+ llvm::Type *Ty = CGF.ConvertType(E->getType());
+ return MakeHalfType(CGF.CGM.getIntrinsic(IntrinsicID, Ty), BuiltinID, E, CGF,
+ {CGF.Builder.getInt32(static_cast<int>(RM))});
}
} // namespace
@@ -1190,58 +1192,60 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
EmitScalarExpr(E->getArg(0)));
case NVPTX::BI__nvvm_add_rn_f:
case NVPTX::BI__nvvm_add_rn_d:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_f:
case NVPTX::BI__nvvm_add_rz_d:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardZero, BuiltinID, E,
+ *this);
case NVPTX::BI__nvvm_add_rm_f:
case NVPTX::BI__nvvm_add_rm_d:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardNegative, BuiltinID,
+ E, *this);
case NVPTX::BI__nvvm_add_rp_f:
case NVPTX::BI__nvvm_add_rp_d:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardPositive, BuiltinID,
+ E, *this);
case NVPTX::BI__nvvm_add_rn_ftz_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_ftz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_ftz_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_ftz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardZero, BuiltinID,
+ E, *this);
case NVPTX::BI__nvvm_add_rm_ftz_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_ftz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_ftz_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_ftz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rn_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_sat_v2f16:
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardZero, BuiltinID,
+ E, *this);
case NVPTX::BI__nvvm_add_rm_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_ftz_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardZero,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_ftz_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_ftz_sat, E, *this);
- case NVPTX::BI__nvvm_add_rn_sat_f16:
- return MakeHalfType(
- CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_sat, Builder.getHalfTy()),
- BuiltinID, E, *this);
- case NVPTX::BI__nvvm_add_rn_sat_v2f16:
- return MakeHalfType(
- CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_sat,
- FixedVectorType::get(Builder.getHalfTy(), 2)),
- BuiltinID, E, *this);
- case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
- return MakeHalfType(
- CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat, Builder.getHalfTy()),
- BuiltinID, E, *this);
- case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return MakeHalfType(
- CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat,
- FixedVectorType::get(Builder.getHalfTy(), 2)),
- BuiltinID, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
return MakeLdg(*this, E);
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index ede4c0605c98b..69df1d376f7f7 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -65,25 +65,25 @@ __device__ float test_ex2_approx_ftz_f(float x) {
}
// CIR-LABEL: @_Z13test_add_rn_fff
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rn" {{.*}} : (!cir.float, !cir.float) -> !cir.float
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
// LLVM-LABEL: @_Z13test_add_rn_fff
-// LLVM: call {{.*}}float @llvm.nvvm.fadd.rn.f32(float
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.f32(float {{.*}}, float {{.*}}, /* rnd=rn */ i32 1)
__device__ float test_add_rn_f(float x, float y) {
return __nvvm_add_rn_f(x, y);
}
// CIR-LABEL: @_Z13test_add_rz_ddd
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rz" {{.*}} : (!cir.double, !cir.double) -> !cir.double
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd" {{.*}} : (!cir.double, !cir.double, !s32i) -> !cir.double
// LLVM-LABEL: @_Z13test_add_rz_ddd
-// LLVM: call {{.*}}double @llvm.nvvm.fadd.rz.f64(double
+// LLVM: call {{.*}}double @llvm.nvvm.fadd.f64(double {{.*}}, double {{.*}}, /* rnd=rz */ i32 0)
__device__ double test_add_rz_d(double x, double y) {
return __nvvm_add_rz_d(x, y);
}
// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rm.ftz.sat" {{.*}} : (!cir.float, !cir.float) -> !cir.float
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.ftz.sat" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// LLVM: call {{.*}}float @llvm.nvvm.fadd.rm.ftz.sat.f32(float
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.ftz.sat.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
__device__ float test_add_rm_ftz_sat_f(float x, float y) {
return __nvvm_add_rm_ftz_sat_f(x, y);
}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index e529eceffc84f..bed1498236b06 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -251,7 +251,7 @@ __device__ void nvvm_math(float f1, float f2, double d1, double d2) {
float t3 = __nvvm_sqrt_rn_f(f1);
// CHECK: call float @llvm.nvvm.rcp.rn.f
float t4 = __nvvm_rcp_rn_f(f2);
-// CHECK: call float @llvm.nvvm.fadd.rn.f32
+// CHECK: call float @llvm.nvvm.fadd.f32({{.*}}i32 1)
float t5 = __nvvm_add_rn_f(f1, f2);
// CHECK: call double @llvm.nvvm.fmax.d
@@ -1628,21 +1628,21 @@ __device__ void nvvm_min_max_sm86() {
// CHECK-LABEL: nvvm_add_fma_f32_sat
__device__ void nvvm_add_fma_f32_sat() {
- // CHECK: call float @llvm.nvvm.fadd.rn.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 1)
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rn.ftz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 0)
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rz.ftz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 0)
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rm.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 3)
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rm.ftz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 3)
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rp.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 2)
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rp.ftz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 2)
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1672,13 +1672,13 @@ __device__ void nvvm_add_fma_f32_sat() {
// CHECK-LABEL: nvvm_add_mul_f16_sat
__device__ void nvvm_add_mul_f16_sat() {
- // CHECK: call half @llvm.nvvm.fadd.rn.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.sat.f16({{.*}}i32 1)
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.fadd.rn.ftz.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
- // CHECK: call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index efd45523d5cd2..489f45e19cab0 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1814,13 +1814,17 @@ let TargetPrefix = "nvvm" in {
//
let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative,
- IntrNoCreateUndefOrPoison] in
- foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
- foreach ftz = ["", "_ftz"] in
- foreach sat = ["", "_sat"] in
- def int_nvvm_fadd # rnd # ftz # sat :
- DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
- [LLVMMatchType<0>, LLVMMatchType<0>]>;
+ IntrNoCreateUndefOrPoison, ImmArg<ArgIndex<2>>,
+ Range<ArgIndex<2>, 0, 4>,
+ ArgInfo<ArgIndex<2>,
+ [ArgName<"rnd">,
+ ImmArgPrinter<"printFAddRoundingMode">]>] in
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def int_nvvm_fadd # ftz # sat :
+ DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
+ llvm_i32_ty]>;
//
// Dot Product
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index 0421400678c71..c0800f217e50e 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -178,6 +178,7 @@ LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
const Constant *ImmArgVal);
LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
const Constant *ImmArgVal);
+LLVM_ABI void printFAddRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
@@ -612,24 +613,12 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz:
- case Intrinsic::nvvm_fadd_rm_ftz_sat:
- case Intrinsic::nvvm_fadd_rn_ftz_sat:
- case Intrinsic::nvvm_fadd_rp_ftz_sat:
- case Intrinsic::nvvm_fadd_rz_ftz_sat:
+ case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fadd_ftz_sat:
return true;
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_sat:
- case Intrinsic::nvvm_fadd_rn_sat:
- case Intrinsic::nvvm_fadd_rp_sat:
- case Intrinsic::nvvm_fadd_rz_sat:
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_sat:
return false;
}
llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
@@ -637,53 +626,35 @@ inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd_rm_sat:
- case Intrinsic::nvvm_fadd_rn_sat:
- case Intrinsic::nvvm_fadd_rp_sat:
- case Intrinsic::nvvm_fadd_rz_sat:
- case Intrinsic::nvvm_fadd_rm_ftz_sat:
- case Intrinsic::nvvm_fadd_rn_ftz_sat:
- case Intrinsic::nvvm_fadd_rp_ftz_sat:
- case Intrinsic::nvvm_fadd_rz_ftz_sat:
+ case Intrinsic::nvvm_fadd_sat:
+ case Intrinsic::nvvm_fadd_ftz_sat:
return true;
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
return false;
}
llvm_unreachable("Checking sat flag for invalid NVVM add intrinsic");
}
-inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rm_sat:
- case Intrinsic::nvvm_fadd_rm_ftz_sat:
- return APFloat::rmTowardNegative;
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rn_sat:
- case Intrinsic::nvvm_fadd_rn_ftz_sat:
- return APFloat::rmNearestTiesToEven;
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rp_sat:
- case Intrinsic::nvvm_fadd_rp_ftz_sat:
- return APFloat::rmTowardPositive;
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rz_ftz:
- case Intrinsic::nvvm_fadd_rz_sat:
- case Intrinsic::nvvm_fadd_rz_ftz_sat:
- return APFloat::rmTowardZero;
+inline APFloat::roundingMode GetFAddRoundingMode(const Value *ImmArgVal) {
+ return static_cast<APFloat::roundingMode>(
+ cast<ConstantInt>(ImmArgVal)->getSExtValue());
+}
+
+inline StringRef GetRoundingModeName(APFloat::roundingMode RM) {
+ switch (RM) {
+ case APFloat::rmNearestTiesToEven:
+ return "rn";
+ case APFloat::rmTowardZero:
+ return "rz";
+ case APFloat::rmTowardNegative:
+ return "rm";
+ case APFloat::rmTowardPositive:
+ return "rp";
+ default:
+ return "";
}
- llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
}
inline bool FMulShouldFTZ(Intrinsic::ID IntrinsicID) {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 2b6708a8a6a38..8092250ec8f5d 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1995,14 +1995,8 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
return !IsStrictFP;
// NVVM add intrinsics with explicit rounding modes
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
// NVVM div intrinsics with explicit rounding modes
case Intrinsic::nvvm_div_rm_d:
@@ -3614,33 +3608,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
return ConstantFP::get(Ty, Res);
}
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz: {
-
- bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
- APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
- APFloat B = IsFTZ ? FTZPreserveSign(Op2V) : Op2V;
-
- APFloat::roundingMode RoundMode =
- nvvm::GetFAddRoundingMode(IntrinsicID);
-
- APFloat Res = A;
- APFloat::opStatus Status = Res.add(B, RoundMode);
-
- if (!Res.isNaN() &&
- (Status == APFloat::opOK || Status == APFloat::opInexact)) {
- Res = IsFTZ ? FTZPreserveSign(Res) : Res;
- return ConstantFP::get(Ty, Res);
- }
- return nullptr;
- }
-
case Intrinsic::nvvm_mul_rm_f:
case Intrinsic::nvvm_mul_rn_f:
case Intrinsic::nvvm_mul_rp_f:
@@ -4191,6 +4158,31 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
}
}
+ if (IntrinsicID == Intrinsic::nvvm_fadd ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz) {
+ const auto *Op1 = dyn_cast<ConstantFP>(Operands[0]);
+ const auto *Op2 = dyn_cast<ConstantFP>(Operands[1]);
+ if (!Op1 || !Op2)
+ return nullptr;
+
+ bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
+ APFloat A =
+ IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
+ APFloat B =
+ IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
+
+ APFloat Res = A;
+ APFloat::opStatus Status =
+ Res.add(B, nvvm::GetFAddRoundingMode(Operands[2]));
+
+ if (!Res.isNaN() &&
+ (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+ Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+ return ConstantFP::get(Ty, Res);
+ }
+ return nullptr;
+ }
+
if (IntrinsicID == Intrinsic::smul_fix ||
IntrinsicID == Intrinsic::smul_fix_sat) {
const APInt *C0, *C1;
@@ -4474,6 +4466,12 @@ static Constant *ConstantFoldFixedVectorCall(
return ConstantVector::get(Result);
}
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
+ // The rounding mode operand is a scalar, so the lane-wise folding below
+ // does not apply.
+ // TODO: Fold these by passing the rounding mode through to every lane.
+ return nullptr;
default:
break;
}
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index e42013219963f..7039a06af3ecf 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1421,6 +1421,29 @@ static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
return F->getIntrinsicID();
}
+static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
+getNVVMFAddUpgrade(StringRef Modifiers) {
+ std::optional<RoundingMode> RM =
+ StringSwitch<std::optional<RoundingMode>>(Modifiers.take_front(2))
+ .Case("rn", RoundingMode::NearestTiesToEven)
+ .Case("rz", RoundingMode::TowardZero)
+ .Case("rm", RoundingMode::TowardNegative)
+ .Case("rp", RoundingMode::TowardPositive)
+ .Default(std::nullopt);
+ if (!RM)
+ return std::nullopt;
+
+ Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
+ .Case("", Intrinsic::nvvm_fadd)
+ .Case(".ftz", Intrinsic::nvvm_fadd_ftz)
+ .Case(".sat", Intrinsic::nvvm_fadd_sat)
+ .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
+ .Default(Intrinsic::not_intrinsic);
+ if (IID == Intrinsic::not_intrinsic)
+ return std::nullopt;
+ return std::make_pair(IID, *RM);
+}
+
static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
return Name.consume_front("local") || Name.consume_front("shared") ||
Name.consume_front("global") || Name.consume_front("constant") ||
@@ -1994,18 +2017,17 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
}
// Upgrade the FP add intrinsics, which are overloaded on the operand type
+ // and take the rounding mode as an operand:
// llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
- // llvm.nvvm.fadd.<rnd>{.ftz}{.sat}.<mangled type>
+ // llvm.nvvm.fadd{.ftz}{.sat}.<mangled type>
+ // The extra operand means these are expanded in UpgradeIntrinsicCall.
if (Name.starts_with("add.")) {
auto [Base, TypeSuffix] = Name.rsplit('.');
- if (TypeSuffix == "f" || TypeSuffix == "d" || TypeSuffix == "f16" ||
- TypeSuffix == "v2f16") {
- IID = Intrinsic::lookupIntrinsicID(("llvm.nvvm.f" + Base).str());
- if (IID != Intrinsic::not_intrinsic) {
- NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
- {F->getReturnType()});
- return true;
- }
+ if ((TypeSuffix == "f" || TypeSuffix == "d" || TypeSuffix == "f16" ||
+ TypeSuffix == "v2f16") &&
+ getNVVMFAddUpgrade(Base.drop_front(strlen("add.")))) {
+ NewFn = nullptr;
+ return true;
}
}
@@ -3100,6 +3122,13 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
: Intrinsic::nvvm_fabs;
Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
+ } else if (Name.consume_front("add.")) {
+ // nvvm.add.<rnd>[.ftz][.sat].{f,d,f16,v2f16}
+ auto [IID, RM] = *getNVVMFAddUpgrade(Name.rsplit('.').first);
+ Value *A = CI->getArgOperand(0);
+ Rep = Builder.CreateIntrinsic(
+ A->getType(), IID,
+ {A, CI->getArgOperand(1), Builder.getInt32(static_cast<int>(RM))});
} else if (Name.consume_front("ex2.approx.")) {
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index b67f70d2568dc..35fffb0be73ff 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -161,6 +161,11 @@ void nvvm::printTensormapSwizzleAtomicity(raw_ostream &OS,
}
}
+void nvvm::printFAddRoundingMode(raw_ostream &OS, const Constant *ImmArgVal) {
+ if (isa<ConstantInt>(ImmArgVal))
+ OS << nvvm::GetRoundingModeName(nvvm::GetFAddRoundingMode(ImmArgVal));
+}
+
void nvvm::printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal) {
if (const auto *CI = dyn_cast<ConstantInt>(ImmArgVal)) {
uint64_t Val = CI->getZExtValue();
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index a8b26cba631cc..5ad3786063c44 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7180,7 +7180,8 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
}
}
-static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID) {
+static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
+ APFloat::roundingMode RM) {
const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
@@ -7192,7 +7193,7 @@ static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID) {
case MVT::f32:
if (!VT.isVector() || nvvm::FAddShouldSaturate(IID))
return 0;
- switch (nvvm::GetFAddRoundingMode(IID)) {
+ switch (RM) {
case APFloat::rmNearestTiesToEven:
return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
case APFloat::rmTowardZero:
@@ -7210,9 +7211,10 @@ static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID) {
}
static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID AddIntrinsicID) {
+ Intrinsic::ID AddIntrinsicID,
+ APFloat::roundingMode RM) {
const EVT VT = N->getValueType(0);
- const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID);
+ const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID, RM);
if (!Opc)
return SDValue();
@@ -7234,7 +7236,7 @@ static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(Opc, SDLoc(N), VT, SubOp1, SubOp2);
}
-static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID,
+static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID, APFloat::roundingMode RM,
const NVPTXSubtarget &STI) {
if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(2))
return false;
@@ -7242,11 +7244,10 @@ static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID,
const bool IsSat = nvvm::FAddShouldSaturate(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
- return nvvm::GetFAddRoundingMode(IID) == APFloat::rmNearestTiesToEven;
+ return RM == APFloat::rmNearestTiesToEven;
case MVT::bf16:
- return nvvm::GetFAddRoundingMode(IID) == APFloat::rmNearestTiesToEven &&
- !IsSat && !nvvm::FAddShouldFTZ(IID) &&
- STI.hasNativeBF16Support(ISD::FADD);
+ return RM == APFloat::rmNearestTiesToEven && !IsSat &&
+ !nvvm::FAddShouldFTZ(IID) && STI.hasNativeBF16Support(ISD::FADD);
case MVT::f32:
return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
case MVT::f64:
@@ -7257,15 +7258,16 @@ static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID,
}
static SDValue diagnoseInvalidFAdd(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID IID,
+ Intrinsic::ID IID, APFloat::roundingMode RM,
const NVPTXSubtarget &STI) {
const EVT VT = N->getValueType(0);
- if (isSupportedFAdd(VT, IID, STI))
+ if (isSupportedFAdd(VT, IID, RM, STI))
return SDValue();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
DAG.getMachineFunction().getFunction(),
- Twine(Intrinsic::getBaseName(IID)) + " with operand type " +
+ Twine(Intrinsic::getBaseName(IID)) + " with rounding mode " +
+ nvvm::GetRoundingModeName(RM) + " and operand type " +
VT.getEVTString() + " is not supported on this target",
SDLoc(N).getDebugLoc()));
return DAG.getPOISON(VT);
@@ -7280,25 +7282,16 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
switch (IID) {
default:
break;
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz:
- case Intrinsic::nvvm_fadd_rm_sat:
- case Intrinsic::nvvm_fadd_rn_sat:
- case Intrinsic::nvvm_fadd_rp_sat:
- case Intrinsic::nvvm_fadd_rz_sat:
- case Intrinsic::nvvm_fadd_rm_ftz_sat:
- case Intrinsic::nvvm_fadd_rn_ftz_sat:
- case Intrinsic::nvvm_fadd_rp_ftz_sat:
- case Intrinsic::nvvm_fadd_rz_ftz_sat:
- if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, STI))
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fadd_sat:
+ case Intrinsic::nvvm_fadd_ftz_sat: {
+ const auto RM = static_cast<APFloat::roundingMode>(
+ N->getConstantOperandAPInt(3).getSExtValue());
+ if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, RM, STI))
return V;
- return combineFAddWithNeg(N, DCI.DAG, IID);
+ return combineFAddWithNeg(N, DCI.DAG, IID, RM);
+ }
}
return SDValue();
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 3d418ea5c81f0..b74222037314d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1687,12 +1687,12 @@ class F_MATH_2<string OpcStr, NVPTXRegClass t_regclass,
[(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1))]>,
Requires<Preds>;
-class F_MATH_2_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP,
- list<Predicate> Preds = []>
+class F_MATH_2_RND_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP, TImmLeaf rnd,
+ list<Predicate> Preds = []>
: BasicNVPTXInst<(outs t.RC:$dst),
(ins t.RC:$src0, t.RC:$src1),
OpcStr,
- [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1))]>,
+ [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1, rnd))]>,
Requires<Preds>;
class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,
@@ -2266,59 +2266,60 @@ let Predicates = [doRsqrtOpt] in {
defvar BF16ArithPreds = [hasBF16Math, hasPTX<78>, hasSM<90>];
-def INT_NVVM_ADD_RN_F16 : F_MATH_2_TY<"add.rn.f16", F16RT, int_nvvm_fadd_rn>;
-def INT_NVVM_ADD_RN_FTZ_F16 : F_MATH_2_TY<"add.rn.ftz.f16", F16RT, int_nvvm_fadd_rn_ftz>;
-def INT_NVVM_ADD_RN_SAT_F16 : F_MATH_2_TY<"add.rn.sat.f16", F16RT, int_nvvm_fadd_rn_sat>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16 : F_MATH_2_TY<"add.rn.ftz.sat.f16", F16RT, int_nvvm_fadd_rn_ftz_sat>;
-def INT_NVVM_ADD_RN_F16X2 : F_MATH_2_TY<"add.rn.f16x2", F16X2RT, int_nvvm_fadd_rn>;
-def INT_NVVM_ADD_RN_FTZ_F16X2 : F_MATH_2_TY<"add.rn.ftz.f16x2", F16X2RT, int_nvvm_fadd_rn_ftz>;
-def INT_NVVM_ADD_RN_SAT_F16X2 : F_MATH_2_TY<"add.rn.sat.f16x2", F16X2RT, int_nvvm_fadd_rn_sat>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16X2 : F_MATH_2_TY<"add.rn.ftz.sat.f16x2", F16X2RT, int_nvvm_fadd_rn_ftz_sat>;
-
-def INT_NVVM_ADD_RN_BF16 :
- F_MATH_2_TY<"add.rn.bf16", BF16RT, int_nvvm_fadd_rn, BF16ArithPreds>;
-def INT_NVVM_ADD_RN_BF16X2 :
- F_MATH_2_TY<"add.rn.bf16x2", BF16X2RT, int_nvvm_fadd_rn, BF16ArithPreds>;
-
-def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2_TY<"add.rn.ftz.f32", F32RT, int_nvvm_fadd_rn_ftz>;
-def INT_NVVM_ADD_RN_SAT_FTZ_F : F_MATH_2_TY<"add.rn.sat.ftz.f32", F32RT, int_nvvm_fadd_rn_ftz_sat>;
-def INT_NVVM_ADD_RN_F : F_MATH_2_TY<"add.rn.f32", F32RT, int_nvvm_fadd_rn>;
-def INT_NVVM_ADD_RN_SAT_F : F_MATH_2_TY<"add.rn.sat.f32", F32RT, int_nvvm_fadd_rn_sat>;
-def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2_TY<"add.rz.ftz.f32", F32RT, int_nvvm_fadd_rz_ftz>;
-def INT_NVVM_ADD_RZ_SAT_FTZ_F : F_MATH_2_TY<"add.rz.sat.ftz.f32", F32RT, int_nvvm_fadd_rz_ftz_sat>;
-def INT_NVVM_ADD_RZ_F : F_MATH_2_TY<"add.rz.f32", F32RT, int_nvvm_fadd_rz>;
-def INT_NVVM_ADD_RZ_SAT_F : F_MATH_2_TY<"add.rz.sat.f32", F32RT, int_nvvm_fadd_rz_sat>;
-def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2_TY<"add.rm.ftz.f32", F32RT, int_nvvm_fadd_rm_ftz>;
-def INT_NVVM_ADD_RM_SAT_FTZ_F : F_MATH_2_TY<"add.rm.sat.ftz.f32", F32RT, int_nvvm_fadd_rm_ftz_sat>;
-def INT_NVVM_ADD_RM_F : F_MATH_2_TY<"add.rm.f32", F32RT, int_nvvm_fadd_rm>;
-def INT_NVVM_ADD_RM_SAT_F : F_MATH_2_TY<"add.rm.sat.f32", F32RT, int_nvvm_fadd_rm_sat>;
-def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2_TY<"add.rp.ftz.f32", F32RT, int_nvvm_fadd_rp_ftz>;
-def INT_NVVM_ADD_RP_SAT_FTZ_F : F_MATH_2_TY<"add.rp.sat.ftz.f32", F32RT, int_nvvm_fadd_rp_ftz_sat>;
-def INT_NVVM_ADD_RP_F : F_MATH_2_TY<"add.rp.f32", F32RT, int_nvvm_fadd_rp>;
-def INT_NVVM_ADD_RP_SAT_F : F_MATH_2_TY<"add.rp.sat.f32", F32RT, int_nvvm_fadd_rp_sat>;
-
-def INT_NVVM_ADD_RN_D : F_MATH_2_TY<"add.rn.f64", F64RT, int_nvvm_fadd_rn>;
-def INT_NVVM_ADD_RZ_D : F_MATH_2_TY<"add.rz.f64", F64RT, int_nvvm_fadd_rz>;
-def INT_NVVM_ADD_RM_D : F_MATH_2_TY<"add.rm.f64", F64RT, int_nvvm_fadd_rm>;
-def INT_NVVM_ADD_RP_D : F_MATH_2_TY<"add.rp.f64", F64RT, int_nvvm_fadd_rp>;
+class RndModeImm<string mode> : TImmLeaf<i32,
+ "return Imm == static_cast<int>(RoundingMode::" # mode # ");">;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+def rnd_rn_imm : RndModeImm<"NearestTiesToEven">;
+def rnd_rz_imm : RndModeImm<"TowardZero">;
+def rnd_rm_imm : RndModeImm<"TowardNegative">;
+def rnd_rp_imm : RndModeImm<"TowardPositive">;
+
+foreach t = [F16RT, F16X2RT] in
foreach ftz = ["", "_ftz"] in
- def INT_NVVM_ADD # rnd # ftz # _F32X2 :
- F_MATH_2_TY<!subst("_", ".", "add" # rnd # ftz # "_f32x2"), F32X2RT,
- !cast<Intrinsic>("int_nvvm_fadd" # rnd # ftz),
- [hasF32x2Instructions]>;
+ foreach sat = ["", "_sat"] in
+ def INT_NVVM_ADD_RN # !toupper(ftz # sat # "_" # t.PtxType) :
+ F_MATH_2_RND_TY<!subst("_", ".", "add_rn" # ftz # sat # "_" # t.PtxType),
+ t, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ rnd_rn_imm>;
+
+foreach t = [BF16RT, BF16X2RT] in
+ def INT_NVVM_ADD_RN_ # !toupper(t.PtxType) :
+ F_MATH_2_RND_TY<"add.rn." # t.PtxType, t, int_nvvm_fadd, rnd_rn_imm,
+ BF16ArithPreds>;
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+ defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+
+ foreach ftz = ["", "_ftz"] in {
+ foreach sat = ["", "_sat"] in
+ def INT_NVVM_ADD # !toupper(rnd # sat # ftz) # _F :
+ F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # sat # ftz # "_f32"),
+ F32RT, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ rnd_imm>;
+
+ def INT_NVVM_ADD # !toupper(rnd # ftz) # _F32X2 :
+ F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # ftz # "_f32x2"), F32X2RT,
+ !cast<Intrinsic>("int_nvvm_fadd" # ftz), rnd_imm,
+ [hasF32x2Instructions]>;
+ }
+
+ def INT_NVVM_ADD # !toupper(rnd) # _D :
+ F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # "_f64"), F64RT,
+ int_nvvm_fadd, rnd_imm>;
+}
+
+foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+ defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
def INT_NVVM_MIXED_ADD # rnd # sat # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
!subst("_", ".", "add" # rnd # sat # "_f32_" # type),
[(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fadd" # rnd # sat)
+ (!cast<Intrinsic>("int_nvvm_fadd" # sat)
(f32 (fpextend type:$a)),
- f32:$b))]>,
+ f32:$b, rnd_imm))]>,
Requires<[SM100]>;
}
}
@@ -2352,19 +2353,15 @@ class INT_NVVM_SUB<RegTyInfo TyInfo, string variant> :
[(set TyInfo.Ty:$dst,
(!cast<SDNode>("sub" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
-def INT_NVVM_SUB_RN_F16 : INT_NVVM_SUB<F16RT, "_rn">;
-def INT_NVVM_SUB_RN_FTZ_F16 : INT_NVVM_SUB<F16RT, "_rn_ftz">;
-def INT_NVVM_SUB_RN_SAT_F16 : INT_NVVM_SUB<F16RT, "_rn_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16 : INT_NVVM_SUB<F16RT, "_rn_ftz_sat">;
-def INT_NVVM_SUB_RN_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn">;
-def INT_NVVM_SUB_RN_FTZ_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_ftz">;
-def INT_NVVM_SUB_RN_SAT_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_ftz_sat">;
+foreach t = [F16RT, F16X2RT] in
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def INT_NVVM_SUB_RN # !toupper(ftz # sat # "_" # t.PtxType) :
+ INT_NVVM_SUB<t, "_rn" # ftz # sat>;
-let Predicates = BF16ArithPreds in {
- def INT_NVVM_SUB_RN_BF16 : INT_NVVM_SUB<BF16RT, "_rn">;
- def INT_NVVM_SUB_RN_BF16X2 : INT_NVVM_SUB<BF16X2RT, "_rn">;
-}
+let Predicates = BF16ArithPreds in
+ foreach t = [BF16RT, BF16X2RT] in
+ def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, "_rn">;
let Predicates = [hasF32x2Instructions] in
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
@@ -2372,33 +2369,38 @@ let Predicates = [hasF32x2Instructions] in
def INT_NVVM_SUB # rnd # ftz # _F32X2 : INT_NVVM_SUB<F32X2RT, rnd # ftz>;
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+ defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # rnd # ftz # sat);
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat);
def INT_NVVM_SUB # rnd # ftz # sat # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
- [(set f32:$dst, (add_intrin f32:$a, (f32 (fneg f32:$b))))]>;
+ [(set f32:$dst,
+ (add_intrin f32:$a, (f32 (fneg f32:$b)), rnd_imm))]>;
}
}
-
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # rnd);
+
def INT_NVVM_SUB # rnd # _D :
BasicNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
!subst("_", ".", "sub" # rnd # "_f64"),
- [(set f64:$dst, (add_intrin f64:$a, (f64 (fneg f64:$b))))]>;
+ [(set f64:$dst,
+ (int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), rnd_imm))]>;
}
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+ defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
def INT_NVVM_MIXED_SUB # rnd # sat # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # "_f32_" # type),
[(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fadd" # rnd # sat)
+ (!cast<Intrinsic>("int_nvvm_fadd" # sat)
(f32 (fpextend type:$a)),
- (f32 (fneg f32:$b))))]>,
+ (f32 (fneg f32:$b)), rnd_imm))]>,
Requires<[SM100]>;
}
}
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 8a0c23e0e4f09..2871ddab068bc 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -713,16 +713,16 @@ define void @nvvm_ex2_approx(float %a, double %b, half %c, <2 x half> %d) {
}
define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
-; CHECK: call float @llvm.nvvm.fadd.rn.f32(float %a, float %a)
-; CHECK: call float @llvm.nvvm.fadd.rz.ftz.f32(float %a, float %a)
-; CHECK: call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %a)
-; CHECK: call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %a)
-; CHECK: call double @llvm.nvvm.fadd.rn.f64(double %b, double %b)
-; CHECK: call double @llvm.nvvm.fadd.rz.f64(double %b, double %b)
-; CHECK: call half @llvm.nvvm.fadd.rn.sat.f16(half %c, half %c)
-; CHECK: call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %c, half %c)
-; CHECK: call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %d, <2 x half> %d)
-; CHECK: call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d)
+; CHECK: call float @llvm.nvvm.fadd.f32(float %a, float %a, /* rnd=rn */ i32 1)
+; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %a, /* rnd=rz */ i32 0)
+; CHECK: call float @llvm.nvvm.fadd.sat.f32(float %a, float %a, /* rnd=rm */ i32 3)
+; CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %a, /* rnd=rp */ i32 2)
+; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rn */ i32 1)
+; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rz */ i32 0)
+; CHECK: call half @llvm.nvvm.fadd.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
%r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
%r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
%r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
diff --git a/llvm/test/CodeGen/NVPTX/bf16-add.ll b/llvm/test/CodeGen/NVPTX/bf16-add.ll
index 71d90a3ae675c..374c37b564cf1 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-add.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-add.ll
@@ -13,7 +13,7 @@ define bfloat @add_rn_bf16(bfloat %a, bfloat %b) {
; CHECK-NEXT: add.rn.bf16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %b)
+ %1 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 1)
ret bfloat %1
}
@@ -28,6 +28,6 @@ define <2 x bfloat> @add_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
; CHECK-NEXT: add.rn.bf16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ %1 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, i32 1)
ret <2 x bfloat> %1
}
diff --git a/llvm/test/CodeGen/NVPTX/bf16-sub.ll b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
index 3d685ebf65cf2..b095a534d3c35 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
@@ -14,7 +14,7 @@ define bfloat @sub_rn_bf16(bfloat %a, bfloat %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg bfloat %b
- %res = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %1)
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %1, i32 1)
ret bfloat %res
}
@@ -30,6 +30,6 @@ define <2 x bfloat> @sub_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x bfloat> %b
- %res = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %a, <2 x bfloat> %1)
+ %res = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %1, i32 1)
ret <2 x bfloat> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
index 63d7f1e2705d6..0b470daf7a535 100644
--- a/llvm/test/CodeGen/NVPTX/f16-add.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -13,7 +13,7 @@ define half @add_rn_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.rn.f16(half %a, half %b)
+ %1 = call half @llvm.nvvm.fadd.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -28,7 +28,7 @@ define <2 x half> @add_rn_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %a, <2 x half> %b)
+ %1 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
@@ -43,7 +43,7 @@ define half @add_rn_ftz_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.ftz.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %a, half %b)
+ %1 = call half @llvm.nvvm.fadd.ftz.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -58,7 +58,7 @@ define <2 x half> @add_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.ftz.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %a, <2 x half> %b)
+ %1 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
@@ -73,7 +73,7 @@ define half @add_rn_sat_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.sat.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.rn.sat.f16(half %a, half %b)
+ %1 = call half @llvm.nvvm.fadd.sat.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -88,7 +88,7 @@ define <2 x half> @add_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.sat.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
+ %1 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
@@ -103,7 +103,7 @@ define half @add_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %a, half %b)
+ %1 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -118,6 +118,6 @@ define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
+ %1 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
index e76c033c6705f..c59875af1d4f9 100644
--- a/llvm/test/CodeGen/NVPTX/f16-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -16,7 +16,7 @@ define half @sub_rn_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.rn.f16(half %a, half %1)
+ %res = call half @llvm.nvvm.fadd.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -32,7 +32,7 @@ define <2 x half> @sub_rn_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %a, <2 x half> %1)
+ %res = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
@@ -48,7 +48,7 @@ define half @sub_rn_ftz_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.rn.ftz.f16(half %a, half %1)
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -64,7 +64,7 @@ define <2 x half> @sub_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %a, <2 x half> %1)
+ %res = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
@@ -80,7 +80,7 @@ define half @sub_rn_sat_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.rn.sat.f16(half %a, half %1)
+ %res = call half @llvm.nvvm.fadd.sat.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -96,7 +96,7 @@ define <2 x half> @sub_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %a, <2 x half> %1)
+ %res = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
@@ -112,7 +112,7 @@ define half @sub_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %a, half %1)
+ %res = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -128,6 +128,6 @@ define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1)
+ %res = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
index f837c6d5d9a54..01f0c6cac37b6 100644
--- a/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
@@ -6,55 +6,55 @@ target triple = "nvptx64-nvidia-cuda"
define <2 x float> @add_rn(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rn(
; CHECK: add.rn.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
ret <2 x float> %r
}
define <2 x float> @add_rz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rz(
; CHECK: add.rz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
ret <2 x float> %r
}
define <2 x float> @add_rm(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rm(
; CHECK: add.rm.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 3)
ret <2 x float> %r
}
define <2 x float> @add_rp(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rp(
; CHECK: add.rp.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 2)
ret <2 x float> %r
}
define <2 x float> @add_rn_ftz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rn_ftz(
; CHECK: add.rn.ftz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
ret <2 x float> %r
}
define <2 x float> @add_rz_ftz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rz_ftz(
; CHECK: add.rz.ftz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
ret <2 x float> %r
}
define <2 x float> @add_rm_ftz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rm_ftz(
; CHECK: add.rm.ftz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 3)
ret <2 x float> %r
}
define <2 x float> @add_rp_ftz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rp_ftz(
; CHECK: add.rp.ftz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 2)
ret <2 x float> %r
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
index 3114592aaccdc..776935ddd0f91 100644
--- a/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
@@ -4,44 +4,44 @@
target triple = "nvptx64-nvidia-cuda"
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.sat with operand type v2f32 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd.sat with rounding mode rn and operand type v2f32 is not supported
define <2 x float> @sat_f32x2(<2 x float> %a, <2 x float> %b) {
- %r = call <2 x float> @llvm.nvvm.fadd.rn.sat.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.sat.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
ret <2 x float> %r
}
-; NOF32X2: error: {{.*}}llvm.nvvm.fadd.rn with operand type v2f32 is not supported
+; NOF32X2: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type v2f32 is not supported
define <2 x float> @unsupported_f32x2(<2 x float> %a, <2 x float> %b) {
- %r = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
ret <2 x float> %r
}
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.ftz with operand type f64 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd.ftz with rounding mode rn and operand type f64 is not supported
define double @ftz_f64(double %a, double %b) {
- %r = call double @llvm.nvvm.fadd.rn.ftz.f64(double %a, double %b)
+ %r = call double @llvm.nvvm.fadd.ftz.f64(double %a, double %b, i32 1)
ret double %r
}
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rz.sat with operand type f16 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd.sat with rounding mode rz and operand type f16 is not supported
define half @rz_f16(half %a, half %b) {
- %r = call half @llvm.nvvm.fadd.rz.sat.f16(half %a, half %b)
+ %r = call half @llvm.nvvm.fadd.sat.f16(half %a, half %b, i32 0)
ret half %r
}
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.ftz with operand type bf16 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd.ftz with rounding mode rn and operand type bf16 is not supported
define bfloat @ftz_bf16(bfloat %a, bfloat %b) {
- %r = call bfloat @llvm.nvvm.fadd.rn.ftz.bf16(bfloat %a, bfloat %b)
+ %r = call bfloat @llvm.nvvm.fadd.ftz.bf16(bfloat %a, bfloat %b, i32 1)
ret bfloat %r
}
-; NOBF16: error: {{.*}}llvm.nvvm.fadd.rn with operand type bf16 is not supported
+; NOBF16: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type bf16 is not supported
define bfloat @unsupported_bf16(bfloat %a, bfloat %b) {
- %r = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %b)
+ %r = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 1)
ret bfloat %r
}
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rn with operand type v4f32 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type v4f32 is not supported
define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) {
- %r = call <4 x float> @llvm.nvvm.fadd.rn.v4f32(<4 x float> %a, <4 x float> %b)
+ %r = call <4 x float> @llvm.nvvm.fadd.v4f32(<4 x float> %a, <4 x float> %b, i32 1)
ret <4 x float> %r
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index 44c09cc177595..ad0e77d119e65 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -20,17 +20,17 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: add.rp.sat.ftz.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.fadd.rn.sat.f32(float %a, float %b)
- %r2 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %a, float %r1)
+ %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, i32 1)
- %r3 = call float @llvm.nvvm.fadd.rz.sat.f32(float %a, float %r2)
- %r4 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %a, float %r3)
+ %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, i32 0)
- %r5 = call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %r4)
- %r6 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %a, float %r5)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, i32 3)
- %r7 = call float @llvm.nvvm.fadd.rp.sat.f32(float %a, float %r6)
- %r8 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %r7)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, i32 2)
ret float %r8
}
@@ -54,28 +54,28 @@ define float @sub_sat_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.rn.sat.f32(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.rz.sat.f32(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, i32 3)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.rp.sat.f32(float %a, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, i32 2)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, i32 2)
ret float %r8
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
index 513926de4451d..ca854106fa31b 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
@@ -21,28 +21,28 @@ define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd10;
; CHECK-NEXT: ret;
%f0 = fneg <2 x float> %b
- %r1 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %f0)
+ %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f0, i32 1)
%f1 = fneg <2 x float> %r1
- %r2 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %a, <2 x float> %f1)
+ %r2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f1, i32 1)
%f2 = fneg <2 x float> %r2
- %r3 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %a, <2 x float> %f2)
+ %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f2, i32 0)
%f3 = fneg <2 x float> %r3
- %r4 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %a, <2 x float> %f3)
+ %r4 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f3, i32 0)
%f4 = fneg <2 x float> %r4
- %r5 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %a, <2 x float> %f4)
+ %r5 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f4, i32 3)
%f5 = fneg <2 x float> %r5
- %r6 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %a, <2 x float> %f5)
+ %r6 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f5, i32 3)
%f6 = fneg <2 x float> %r6
- %r7 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %a, <2 x float> %f6)
+ %r7 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f6, i32 2)
%f7 = fneg <2 x float> %r7
- %r8 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %a, <2 x float> %f7)
+ %r8 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f7, i32 2)
ret <2 x float> %r8
}
@@ -59,6 +59,6 @@ define <2 x float> @sub_f32x2_negated_lhs(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
; CHECK-NEXT: ret;
%f = fneg <2 x float> %a
- %r = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %f, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %f, <2 x float> %b, i32 0)
ret <2 x float> %r
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
index a3084392c7853..31edae75a48e7 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
@@ -20,22 +20,22 @@ define float @sub_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r8;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.rz.f32(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.rm.f32(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f5, i32 3)
ret float %r6
}
@@ -55,16 +55,16 @@ define double @sub_f64(double %a, double %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd6;
; CHECK-NEXT: ret;
%f0 = fneg double %b
- %r1 = call double @llvm.nvvm.fadd.rn.f64(double %a, double %f0)
+ %r1 = call double @llvm.nvvm.fadd.f64(double %a, double %f0, i32 1)
%f1 = fneg double %r1
- %r2 = call double @llvm.nvvm.fadd.rz.f64(double %a, double %f1)
+ %r2 = call double @llvm.nvvm.fadd.f64(double %a, double %f1, i32 0)
%f2 = fneg double %r2
- %r3 = call double @llvm.nvvm.fadd.rm.f64(double %a, double %f2)
+ %r3 = call double @llvm.nvvm.fadd.f64(double %a, double %f2, i32 3)
%f3 = fneg double %r3
- %r4 = call double @llvm.nvvm.fadd.rp.f64(double %a, double %f3)
+ %r4 = call double @llvm.nvvm.fadd.f64(double %a, double %f3, i32 2)
ret double %r4
}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
index aa80dc2185bd5..9da6c9a39a34e 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
@@ -27,16 +27,16 @@ define float @test_add_f32_f16_1(half %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext half %a to float
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %b)
- %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, i32 0)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, i32 2)
; SAT
- %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, i32 1)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, i32 0)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, i32 3)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, i32 2)
ret float %r8
}
@@ -93,16 +93,16 @@ define float @test_add_f32_bf16_1(bfloat %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext bfloat %a to float
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %b)
- %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, i32 0)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, i32 2)
; SAT
- %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, i32 1)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, i32 0)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, i32 3)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, i32 2)
ret float %r8
}
@@ -160,29 +160,29 @@ define float @test_sub_f32_f16_1(half %a, float %b) {
%r0 = fpext half %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, i32 0)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, i32 3)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, i32 3)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, i32 1)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, i32 0)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, i32 3)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, i32 2)
ret float %r7
}
@@ -240,29 +240,29 @@ define float @test_sub_f32_bf16_1(bfloat %a, float %b) {
%r0 = fpext bfloat %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, i32 0)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, i32 3)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, i32 2)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, i32 1)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, i32 0)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, i32 3)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, i32 2)
ret float %r8
}
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index d0acf650d02c5..b4b04ef371e94 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -299,20 +299,20 @@ define float @test_ull2f(i64 %a) #0 {
; CHECK-LABEL: @test_add_rn_d
define double @test_add_rn_d(double %a, double %b) #0 {
-; CHECK: call double @llvm.nvvm.fadd.rn.f64
- %ret = call double @llvm.nvvm.fadd.rn.f64(double %a, double %b)
+; CHECK: call double @llvm.nvvm.fadd.f64
+ %ret = call double @llvm.nvvm.fadd.f64(double %a, double %b, /* rnd=rn */ i32 1)
ret double %ret
}
; CHECK-LABEL: @test_add_rn_f
define float @test_add_rn_f(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.fadd.rn.f32
- %ret = call float @llvm.nvvm.fadd.rn.f32(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.f32
+ %ret = call float @llvm.nvvm.fadd.f32(float %a, float %b, /* rnd=rn */ i32 1)
ret float %ret
}
; CHECK-LABEL: @test_add_rn_f_ftz
define float @test_add_rn_f_ftz(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %b)
- %ret = call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %b, /* rnd=rn */ i32 1)
+ %ret = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %b, /* rnd=rn */ i32 1)
ret float %ret
}
@@ -437,9 +437,9 @@ define i32 @test_fshr_clamp_3(i32 %a, i32 %b, i32 %c) {
ret i32 %call
}
-declare double @llvm.nvvm.fadd.rn.f64(double, double)
-declare float @llvm.nvvm.fadd.rn.f32(float, float)
-declare float @llvm.nvvm.fadd.rn.ftz.f32(float, float)
+declare double @llvm.nvvm.fadd.f64(double, double, i32 immarg)
+declare float @llvm.nvvm.fadd.f32(float, float, i32 immarg)
+declare float @llvm.nvvm.fadd.ftz.f32(float, float, i32 immarg)
declare double @llvm.nvvm.ceil.d(double)
declare float @llvm.nvvm.ceil.f(float)
declare float @llvm.nvvm.ceil.ftz.f(float)
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
index f3129ae0c5f48..02942634327af 100644
--- a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
@@ -13,7 +13,7 @@ define double @test_1_25_minus_2_rm_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rm_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rm */ i32 3)
ret double %res
}
@@ -21,7 +21,7 @@ define double @test_1_25_minus_2_rn_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rn_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rn */ i32 1)
ret double %res
}
@@ -29,7 +29,7 @@ define double @test_1_25_minus_2_rp_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rp_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rp */ i32 2)
ret double %res
}
@@ -37,7 +37,7 @@ define double @test_1_25_minus_2_rz_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rz_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rz */ i32 0)
ret double %res
}
@@ -45,7 +45,7 @@ define float @test_1_25_minus_2_rm_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -53,7 +53,7 @@ define float @test_1_25_minus_2_rn_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -61,7 +61,7 @@ define float @test_1_25_minus_2_rp_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -69,7 +69,7 @@ define float @test_1_25_minus_2_rz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -77,7 +77,7 @@ define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -85,7 +85,7 @@ define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -93,7 +93,7 @@ define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -101,10 +101,74 @@ define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_1_25_minus_2_rm_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rm_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rn_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rn_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rp_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rp_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rz_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rz_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_1_25_minus_2_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rm_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rn_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rp_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rz_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(0.0, NaN) #
;###############################################################
@@ -113,112 +177,184 @@ define float @test_1_25_minus_2_rz_ftz_f() {
define double @test_zero_plus_nan_rm_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rm_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rm.f64(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rm */ i32 3)
ret double %res
}
define double @test_zero_plus_nan_rn_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rn_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rn.f64(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rn */ i32 1)
ret double %res
}
define double @test_zero_plus_nan_rp_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rp_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rp.f64(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rp */ i32 2)
ret double %res
}
define double @test_zero_plus_nan_rz_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rz_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rz.f64(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rz */ i32 0)
ret double %res
}
define float @test_zero_plus_nan_rm_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rm.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rm */ i32 3)
ret float %res
}
define float @test_zero_plus_nan_rn_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rn.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_zero_plus_nan_rp_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rp.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_zero_plus_nan_rz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rz */ i32 0)
ret float %res
}
define float @test_zero_plus_nan_rm_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rm */ i32 3)
ret float %res
}
define float @test_zero_plus_nan_rn_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_zero_plus_nan_rp_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_zero_plus_nan_rz_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_zero_plus_nan_rm_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rm_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rm */ i32 3)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rn_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rn_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rn */ i32 1)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rp_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rp_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rp */ i32 2)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rz_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rz_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rz */ i32 0)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_zero_plus_nan_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rm_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rm */ i32 3)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rn_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rn */ i32 1)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rp_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rp */ i32 2)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rz_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rz */ i32 0)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(Subnormal, Subnormal) -> Normal #
;###############################################################
@@ -230,7 +366,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -238,7 +374,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -246,7 +382,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -254,7 +390,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -262,7 +398,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -270,7 +406,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -278,7 +414,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -286,7 +422,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -294,7 +430,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -302,7 +438,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -310,7 +446,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -318,10 +454,106 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_subnorm_plus_subnorm_to_normal_rm_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rm_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rn_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rn_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rp_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rp_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rm_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rn_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rp_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rz_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rm_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rn_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rp_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rz_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(Normal, -Subnormal) -> Subnormal #
;###############################################################
@@ -335,7 +567,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -343,7 +575,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -351,7 +583,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -359,7 +591,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -367,7 +599,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -375,7 +607,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -383,7 +615,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -391,7 +623,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -399,7 +631,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -407,7 +639,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -415,7 +647,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -423,10 +655,74 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_normal_minus_subnorm_to_subnorm_rm_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rm_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rn_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rn_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rp_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rp_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rz_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rn_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rp_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rz_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(1.0, 2^(-25)) #
;###############################################################
@@ -439,7 +735,7 @@ define float @test_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -447,7 +743,7 @@ define float @test_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -455,7 +751,7 @@ define float @test_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -463,7 +759,7 @@ define float @test_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -471,7 +767,7 @@ define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -479,7 +775,7 @@ define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -487,7 +783,7 @@ define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -495,7 +791,7 @@ define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -511,7 +807,7 @@ define double @test_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -519,7 +815,7 @@ define double @test_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -527,7 +823,7 @@ define double @test_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0x3FF0000000000001
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -535,10 +831,90 @@ define double @test_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
+;###############################################################
+;# Add(1.0, 2^(-12)) #
+;###############################################################
+; Tests addition of 1.0 and 2^(-12) where the exact result falls between
+; 1.0 and 1.0 + 2^(-10):
+; - RN, RZ, RM: Return 1.0 (rounding to nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-10) (rounding up)
+
+define half @test_1_plus_ulp_rm_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rm_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rn_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rn_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rp_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rp_f16() {
+; CHECK-NEXT: ret half 1.000980e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rz_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rz_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+;###############################################################
+;# Add(1.0, 2^(-9)) #
+;###############################################################
+; Tests addition of 1.0 and 2^(-9) where the exact result falls between
+; 1.0 and 1.0 + 2^(-7):
+; - RN, RZ, RM: Return 1.0 (rounding to nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-7) (rounding up)
+
+define bfloat @test_1_plus_ulp_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rm_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rn_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rp_bf16() {
+; CHECK-NEXT: ret bfloat 1.007810e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rz_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(-1.0, 2^(-25)) #
;###############################################################
@@ -551,7 +927,7 @@ define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -559,7 +935,7 @@ define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -567,7 +943,7 @@ define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -575,7 +951,7 @@ define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -583,7 +959,7 @@ define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -591,7 +967,7 @@ define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -599,7 +975,7 @@ define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -607,7 +983,7 @@ define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -623,7 +999,7 @@ define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -631,7 +1007,7 @@ define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -639,7 +1015,7 @@ define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -647,7 +1023,7 @@ define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -663,7 +1039,7 @@ define float @test_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -671,7 +1047,7 @@ define float @test_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -679,7 +1055,7 @@ define float @test_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -687,7 +1063,7 @@ define float @test_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -695,7 +1071,7 @@ define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -703,7 +1079,7 @@ define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -711,7 +1087,7 @@ define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -719,7 +1095,7 @@ define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -735,7 +1111,7 @@ define double @test_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -743,7 +1119,7 @@ define double @test_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -751,7 +1127,7 @@ define double @test_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -759,7 +1135,7 @@ define double @test_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -775,7 +1151,7 @@ define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -783,7 +1159,7 @@ define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -791,7 +1167,7 @@ define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -799,7 +1175,7 @@ define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -807,7 +1183,7 @@ define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -815,7 +1191,7 @@ define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -823,7 +1199,7 @@ define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -831,7 +1207,7 @@ define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -847,7 +1223,7 @@ define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0xBFF0000000000001
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -855,7 +1231,7 @@ define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -863,7 +1239,7 @@ define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -871,6 +1247,6 @@ define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
diff --git a/llvm/test/Verifier/NVPTX/fadd.ll b/llvm/test/Verifier/NVPTX/fadd.ll
new file mode 100644
index 0000000000000..d3a03abf6f1f2
--- /dev/null
+++ b/llvm/test/Verifier/NVPTX/fadd.ll
@@ -0,0 +1,16 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+declare float @llvm.nvvm.fadd.f32(float, float, i32 immarg)
+
+define void @test_fadd_rounding_mode(float %a) {
+ ; CHECK: immarg value 4 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 4)
+
+ ; CHECK: immarg value 7 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 7)
+
+ ; CHECK: immarg value -1 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 -1)
+
+ ret void
+}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 8b823fa425206..b3c953de574b9 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -465,7 +465,9 @@ createScalarizedIntrinsicCall(llvm::IRBuilderBase &builder,
llvm::SmallVector<llvm::Value *> scalarArgs;
for (llvm::Value *op : operands)
scalarArgs.push_back(
- builder.CreateExtractElement(op, builder.getInt32(i)));
+ op->getType()->isVectorTy()
+ ? builder.CreateExtractElement(op, builder.getInt32(i))
+ : op);
llvm::Value *res = createIntrinsicCall(builder, IID, retType, scalarArgs);
result = builder.CreateInsertElement(result, res, builder.getInt32(i));
}
@@ -483,23 +485,18 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
llvm::Type *opTypeLLVM = argLHS->getType();
bool isSat = satMode != NVVM::SaturationMode::NONE;
- static constexpr llvm::Intrinsic::ID addIDs[2][2][5] = {
- {{llvm::Intrinsic::nvvm_fadd_rn, llvm::Intrinsic::nvvm_fadd_rn,
- llvm::Intrinsic::nvvm_fadd_rm, llvm::Intrinsic::nvvm_fadd_rp,
- llvm::Intrinsic::nvvm_fadd_rz},
- {llvm::Intrinsic::nvvm_fadd_rn_sat, llvm::Intrinsic::nvvm_fadd_rn_sat,
- llvm::Intrinsic::nvvm_fadd_rm_sat, llvm::Intrinsic::nvvm_fadd_rp_sat,
- llvm::Intrinsic::nvvm_fadd_rz_sat}},
- {{llvm::Intrinsic::nvvm_fadd_rn_ftz, llvm::Intrinsic::nvvm_fadd_rn_ftz,
- llvm::Intrinsic::nvvm_fadd_rm_ftz, llvm::Intrinsic::nvvm_fadd_rp_ftz,
- llvm::Intrinsic::nvvm_fadd_rz_ftz},
- {llvm::Intrinsic::nvvm_fadd_rn_ftz_sat,
- llvm::Intrinsic::nvvm_fadd_rn_ftz_sat,
- llvm::Intrinsic::nvvm_fadd_rm_ftz_sat,
- llvm::Intrinsic::nvvm_fadd_rp_ftz_sat,
- llvm::Intrinsic::nvvm_fadd_rz_ftz_sat}}};
-
- llvm::Intrinsic::ID id = addIDs[isFTZ][isSat][static_cast<unsigned>(rndMode)];
+ static constexpr llvm::Intrinsic::ID addIDs[2][2] = {
+ {llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
+ {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
+
+ static constexpr llvm::RoundingMode roundingModes[5] = {
+ llvm::RoundingMode::NearestTiesToEven,
+ llvm::RoundingMode::NearestTiesToEven, llvm::RoundingMode::TowardNegative,
+ llvm::RoundingMode::TowardPositive, llvm::RoundingMode::TowardZero};
+
+ llvm::Intrinsic::ID id = addIDs[isFTZ][isSat];
+ llvm::Value *rnd = builder.getInt32(
+ static_cast<int>(roundingModes[static_cast<unsigned>(rndMode)]));
// For f64 vector addition, and f32 vector addition with saturation,
// we need to scalarize the intrinsic call.
@@ -507,13 +504,13 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() ||
(isSat && scalarTypeLLVM->isFloatTy()))) {
mt.mapValue(res, createScalarizedIntrinsicCall(builder, id, opTypeLLVM,
- {argLHS, argRHS},
+ {argLHS, argRHS, rnd},
scalarTypeLLVM));
return;
}
- mt.mapValue(res,
- createIntrinsicCall(builder, id, opTypeLLVM, {argLHS, argRHS}));
+ mt.mapValue(
+ res, createIntrinsicCall(builder, id, opTypeLLVM, {argLHS, argRHS, rnd}));
}
void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index a5e5a204a36e3..a376351ac20ec 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -3,11 +3,11 @@
// f16 + f16 -> f16
llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fadd_f16_f16(half %0, half %1) {
- // CHECK-NEXT: %3 = call half @llvm.nvvm.fadd.rn.f16(half %0, half %1)
- // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.rn.f16(half %3, half %3)
- // CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %4, half %4)
- // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.rn.sat.f16(half %5, half %5)
- // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %6, half %6)
+ // CHECK-NEXT: %3 = call half @llvm.nvvm.fadd.f16(half %0, half %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %3, half %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.ftz.f16(half %4, half %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.sat.f16(half %5, half %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %6, half %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
@@ -21,8 +21,8 @@ llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// bf16 + bf16 -> bf16
llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fadd_bf16_bf16(bfloat %0, bfloat %1) {
- // CHECK-NEXT: %3 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %0, bfloat %1)
- // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %3, bfloat %3)
+ // CHECK-NEXT: %3 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %0, bfloat %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %3, bfloat %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret bfloat %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : bf16
@@ -33,23 +33,23 @@ llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// f32 + f32 -> f32
llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fadd_f32_f32(float %0, float %1) {
- // CHECK-NEXT: %3 = call float @llvm.nvvm.fadd.rn.f32(float %0, float %1)
- // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.rn.f32(float %3, float %3)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %4, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %5, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %6, float %6)
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rm.f32(float %7, float %7)
- // CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.rm.sat.f32(float %8, float %8)
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %9, float %9)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %10, float %10)
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rp.f32(float %11, float %11)
- // CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.rp.sat.f32(float %12, float %12)
- // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rp.ftz.f32(float %13, float %13)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %14, float %14)
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rz.f32(float %15, float %15)
- // CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.rz.sat.f32(float %16, float %16)
- // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %17, float %17)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %18, float %18)
+ // CHECK-NEXT: %3 = call float @llvm.nvvm.fadd.f32(float %0, float %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %3, float %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.ftz.f32(float %5, float %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %6, float %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.f32(float %7, float %7, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %8, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %9, float %9, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %10, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.f32(float %11, float %11, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %12, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.ftz.f32(float %13, float %13, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.f32(float %15, float %15, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.sat.f32(float %16, float %16, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %17, float %17, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
@@ -75,11 +75,11 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// f64 + f64 -> f64
llvm.func @fadd_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fadd_f64_f64(double %0, double %1) {
- // CHECK-NEXT: %3 = call double @llvm.nvvm.fadd.rn.f64(double %0, double %1)
- // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.rn.f64(double %3, double %3)
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rm.f64(double %4, double %4)
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rp.f64(double %5, double %5)
- // CHECK-NEXT: %7 = call double @llvm.nvvm.fadd.rz.f64(double %6, double %6)
+ // CHECK-NEXT: %3 = call double @llvm.nvvm.fadd.f64(double %0, double %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.f64(double %3, double %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %4, double %4, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %5, double %5, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %7 = call double @llvm.nvvm.fadd.f64(double %6, double %6, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret double %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index 1f7daa6c3c66a..d3421ac025f2b 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -3,11 +3,11 @@
// vector<2xf16> + vector<2xf16> -> vector<2xf16>
llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @addf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
- // CHECK-NEXT: %3 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %0, <2 x half> %1)
- // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %3, <2 x half> %3)
- // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %4, <2 x half> %4)
- // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %5, <2 x half> %5)
- // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6)
+ // CHECK-NEXT: %3 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %0, <2 x half> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %3, <2 x half> %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %4, <2 x half> %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %5, <2 x half> %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
@@ -21,8 +21,8 @@ llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @addf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
- // CHECK-NEXT: %3 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1)
- // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %3, <2 x bfloat> %3)
+ // CHECK-NEXT: %3 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %3, <2 x bfloat> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x bfloat> %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xbf16>
@@ -33,24 +33,24 @@ llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
// vector<2xf32> + vector<2xf32> -> vector<2xf32>
llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %0, <2 x float> %1)
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %3, <2 x float> %3)
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %3, <2 x float> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = extractelement <2 x float> %4, i32 0
// CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.rn.sat.f32(float %5, float %6)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.sat.f32(float %5, float %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
// CHECK-NEXT: %9 = extractelement <2 x float> %4, i32 1
// CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
- // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.rn.sat.f32(float %9, float %10)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.sat.f32(float %9, float %10, /* rnd=rn */ i32 1)
// CHECK-NEXT: %12 = insertelement <2 x float> %8, float %11, i32 1
- // CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %12, <2 x float> %12)
+ // CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %12, <2 x float> %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %14, float %15)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %15, /* rnd=rn */ i32 1)
// CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
// CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %18, float %19)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rn */ i32 1)
// CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
// CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
@@ -64,23 +64,23 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rm */ i32 3)
// CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
// CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rm.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rm */ i32 3)
// CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
// CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rm.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rm */ i32 3)
// CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rm */ i32 3)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rm */ i32 3)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -93,23 +93,23 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rp */ i32 2)
// CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
// CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rp.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rp */ i32 2)
// CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
// CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rp.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rp */ i32 2)
// CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -122,23 +122,23 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rz */ i32 0)
// CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
// CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rz */ i32 0)
// CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
// CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rz.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rz */ i32 0)
// CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rz */ i32 0)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -154,19 +154,19 @@ llvm.func @addf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rn(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rn.f64(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rn.f64(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rn */ i32 1)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: %11 = extractelement <2 x double> %10, i32 0
// CHECK-NEXT: %12 = extractelement <2 x double> %10, i32 0
- // CHECK-NEXT: %13 = call double @llvm.nvvm.fadd.rn.f64(double %11, double %12)
+ // CHECK-NEXT: %13 = call double @llvm.nvvm.fadd.f64(double %11, double %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = insertelement <2 x double> poison, double %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x double> %10, i32 1
// CHECK-NEXT: %16 = extractelement <2 x double> %10, i32 1
- // CHECK-NEXT: %17 = call double @llvm.nvvm.fadd.rn.f64(double %15, double %16)
+ // CHECK-NEXT: %17 = call double @llvm.nvvm.fadd.f64(double %15, double %16, /* rnd=rn */ i32 1)
// CHECK-NEXT: %18 = insertelement <2 x double> %14, double %17, i32 1
// CHECK-NEXT: ret <2 x double> %18
// CHECK-NEXT: }
@@ -179,11 +179,11 @@ llvm.func @addf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rm(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rm.f64(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rm */ i32 3)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rm.f64(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rm */ i32 3)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -195,11 +195,11 @@ llvm.func @addf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rp(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rp.f64(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rp */ i32 2)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rp.f64(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rp */ i32 2)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -211,11 +211,11 @@ llvm.func @addf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rz(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rz.f64(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rz */ i32 0)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rz.f64(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rz */ i32 0)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index a540c0338a59b..eaf33ae64ee76 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -4,15 +4,15 @@
llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fsub_f16_f16(half %0, half %1) {
// CHECK-NEXT: %3 = fneg half %1
- // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.rn.f16(half %0, half %3)
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %0, half %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg half %4
- // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.rn.f16(half %4, half %5)
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.f16(half %4, half %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg half %6
- // CHECK-NEXT: %8 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %6, half %7)
+ // CHECK-NEXT: %8 = call half @llvm.nvvm.fadd.ftz.f16(half %6, half %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg half %8
- // CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.rn.sat.f16(half %8, half %9)
+ // CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.sat.f16(half %8, half %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg half %10
- // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %10, half %11)
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %10, half %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
@@ -27,9 +27,9 @@ llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fsub_bf16_bf16(bfloat %0, bfloat %1) {
// CHECK-NEXT: %3 = fneg bfloat %1
- // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %0, bfloat %3)
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %0, bfloat %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg bfloat %4
- // CHECK-NEXT: %6 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %4, bfloat %5)
+ // CHECK-NEXT: %6 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %4, bfloat %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret bfloat %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : bf16
@@ -41,39 +41,39 @@ llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fsub_f32_f32(float %0, float %1) {
// CHECK-NEXT: %3 = fneg float %1
- // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.rn.f32(float %0, float %3)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %0, float %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg float %4
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rn.f32(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.f32(float %4, float %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg float %6
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rn.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg float %8
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: %13 = fneg float %12
- // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rm.f32(float %12, float %13)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.f32(float %12, float %13, /* rnd=rm */ i32 3)
// CHECK-NEXT: %15 = fneg float %14
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rm.sat.f32(float %14, float %15)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.sat.f32(float %14, float %15, /* rnd=rm */ i32 3)
// CHECK-NEXT: %17 = fneg float %16
- // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %16, float %17)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %16, float %17, /* rnd=rm */ i32 3)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %18, float %19)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rm */ i32 3)
// CHECK-NEXT: %21 = fneg float %20
- // CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.rp.f32(float %20, float %21)
+ // CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.f32(float %20, float %21, /* rnd=rp */ i32 2)
// CHECK-NEXT: %23 = fneg float %22
- // CHECK-NEXT: %24 = call float @llvm.nvvm.fadd.rp.sat.f32(float %22, float %23)
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fadd.sat.f32(float %22, float %23, /* rnd=rp */ i32 2)
// CHECK-NEXT: %25 = fneg float %24
- // CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.rp.ftz.f32(float %24, float %25)
+ // CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.ftz.f32(float %24, float %25, /* rnd=rp */ i32 2)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %26, float %27)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %26, float %27, /* rnd=rp */ i32 2)
// CHECK-NEXT: %29 = fneg float %28
- // CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.rz.f32(float %28, float %29)
+ // CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.f32(float %28, float %29, /* rnd=rz */ i32 0)
// CHECK-NEXT: %31 = fneg float %30
- // CHECK-NEXT: %32 = call float @llvm.nvvm.fadd.rz.sat.f32(float %30, float %31)
+ // CHECK-NEXT: %32 = call float @llvm.nvvm.fadd.sat.f32(float %30, float %31, /* rnd=rz */ i32 0)
// CHECK-NEXT: %33 = fneg float %32
- // CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %32, float %33)
+ // CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.ftz.f32(float %32, float %33, /* rnd=rz */ i32 0)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %34, float %35)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %34, float %35, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
@@ -100,15 +100,15 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
llvm.func @fsub_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fsub_f64_f64(double %0, double %1) {
// CHECK-NEXT: %3 = fneg double %1
- // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.rn.f64(double %0, double %3)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.f64(double %0, double %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg double %4
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rn.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg double %6
- // CHECK-NEXT: %8 = call double @llvm.nvvm.fadd.rm.f64(double %6, double %7)
+ // CHECK-NEXT: %8 = call double @llvm.nvvm.fadd.f64(double %6, double %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = fneg double %8
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rp.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rp */ i32 2)
// CHECK-NEXT: %11 = fneg double %10
- // CHECK-NEXT: %12 = call double @llvm.nvvm.fadd.rz.f64(double %10, double %11)
+ // CHECK-NEXT: %12 = call double @llvm.nvvm.fadd.f64(double %10, double %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret double %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index b72d3b0ebecde..78109d5ef9e7d 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -4,15 +4,15 @@
llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @subf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
// CHECK-NEXT: %3 = fneg <2 x half> %1
- // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %0, <2 x half> %3)
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %0, <2 x half> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg <2 x half> %4
- // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %4, <2 x half> %5)
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %4, <2 x half> %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg <2 x half> %6
- // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %6, <2 x half> %7)
+ // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %6, <2 x half> %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg <2 x half> %8
- // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %8, <2 x half> %9)
+ // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %8, <2 x half> %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg <2 x half> %10
- // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11)
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
@@ -27,9 +27,9 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @subf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
// CHECK-NEXT: %3 = fneg <2 x bfloat> %1
- // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %0, <2 x bfloat> %3)
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %0, <2 x bfloat> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg <2 x bfloat> %4
- // CHECK-NEXT: %6 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %4, <2 x bfloat> %5)
+ // CHECK-NEXT: %6 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %4, <2 x bfloat> %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x bfloat> %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xbf16>
@@ -41,28 +41,28 @@ llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg <2 x float> %4
- // CHECK-NEXT: %6 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %4, <2 x float> %5)
+ // CHECK-NEXT: %6 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %4, <2 x float> %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg <2 x float> %6
// CHECK-NEXT: %8 = extractelement <2 x float> %6, i32 0
// CHECK-NEXT: %9 = extractelement <2 x float> %7, i32 0
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rn.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = insertelement <2 x float> poison, float %10, i32 0
// CHECK-NEXT: %12 = extractelement <2 x float> %6, i32 1
// CHECK-NEXT: %13 = extractelement <2 x float> %7, i32 1
- // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rn.sat.f32(float %12, float %13)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %13, /* rnd=rn */ i32 1)
// CHECK-NEXT: %15 = insertelement <2 x float> %11, float %14, i32 1
// CHECK-NEXT: %16 = fneg <2 x float> %15
- // CHECK-NEXT: %17 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %15, <2 x float> %16)
+ // CHECK-NEXT: %17 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %15, <2 x float> %16, /* rnd=rn */ i32 1)
// CHECK-NEXT: %18 = fneg <2 x float> %17
// CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
// CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %19, float %20)
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %19, float %20, /* rnd=rn */ i32 1)
// CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
// CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
// CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %23, float %24)
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %23, float %24, /* rnd=rn */ i32 1)
// CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
// CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
@@ -77,26 +77,26 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rm */ i32 3)
// CHECK-NEXT: %5 = fneg <2 x float> %4
// CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
// CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rm.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
// CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
// CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rm.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rm */ i32 3)
// CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
// CHECK-NEXT: %14 = fneg <2 x float> %13
- // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rm */ i32 3)
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rm */ i32 3)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -110,26 +110,26 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rp */ i32 2)
// CHECK-NEXT: %5 = fneg <2 x float> %4
// CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
// CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rp */ i32 2)
// CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
// CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
// CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rp.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
// CHECK-NEXT: %14 = fneg <2 x float> %13
- // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rp */ i32 2)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -143,26 +143,26 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rz */ i32 0)
// CHECK-NEXT: %5 = fneg <2 x float> %4
// CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
// CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rz.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rz */ i32 0)
// CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
// CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
// CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rz.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
// CHECK-NEXT: %14 = fneg <2 x float> %13
- // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rz */ i32 0)
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rz */ i32 0)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -179,20 +179,20 @@ llvm.func @subf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rn.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rn.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: %12 = fneg <2 x double> %11
// CHECK-NEXT: %13 = extractelement <2 x double> %11, i32 0
// CHECK-NEXT: %14 = extractelement <2 x double> %12, i32 0
- // CHECK-NEXT: %15 = call double @llvm.nvvm.fadd.rn.f64(double %13, double %14)
+ // CHECK-NEXT: %15 = call double @llvm.nvvm.fadd.f64(double %13, double %14, /* rnd=rn */ i32 1)
// CHECK-NEXT: %16 = insertelement <2 x double> poison, double %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x double> %11, i32 1
// CHECK-NEXT: %18 = extractelement <2 x double> %12, i32 1
- // CHECK-NEXT: %19 = call double @llvm.nvvm.fadd.rn.f64(double %17, double %18)
+ // CHECK-NEXT: %19 = call double @llvm.nvvm.fadd.f64(double %17, double %18, /* rnd=rn */ i32 1)
// CHECK-NEXT: %20 = insertelement <2 x double> %16, double %19, i32 1
// CHECK-NEXT: ret <2 x double> %20
// CHECK-NEXT: }
@@ -206,11 +206,11 @@ llvm.func @subf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rm.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rm */ i32 3)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rm.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rm */ i32 3)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -223,11 +223,11 @@ llvm.func @subf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rp.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rp */ i32 2)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rp.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rp */ i32 2)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -240,11 +240,11 @@ llvm.func @subf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rz.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rz */ i32 0)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rz.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rz */ i32 0)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
>From 0de0b78b196242fc651638371a7bccc7668939e0 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 18 Aug 2026 11:23:15 +0000
Subject: [PATCH 03/16] fix some constant fold tests and add some missing tests
---
.../InstSimplify/const-fold-nvvm-add.ll | 186 +++++++++++++++++-
1 file changed, 177 insertions(+), 9 deletions(-)
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
index 02942634327af..edc4b2233f235 100644
--- a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
@@ -557,11 +557,10 @@ define bfloat @test_subnorm_plus_subnorm_to_normal_rz_bf16() {
;###############################################################
;# Add(Normal, -Subnormal) -> Subnormal #
;###############################################################
-; Tests addition of 2^-126 (the smallest normal number) and -(2^127).
-; - Without FTZ: The result is correctly computed as a subnormal (2^127)
-; - With FTZ: The result is flushed to zero.
-; This verifies that the output is also flushed to zero, as we'd end up
-; with 2^-126 if we only flushed the inputs.
+; Tests addition of 2^-126 (the smallest normal number) and -(2^-127).
+; - Without FTZ: The result is correctly computed as a subnormal (2^-127)
+; - With FTZ: The subnormal input is flushed to zero, so the result is the
+; normal input (2^-126)
define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
@@ -645,17 +644,17 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
-; CHECK-NEXT: ret float 0.000000e+00
+; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
-; CHECK-NEXT: ret float 0.000000e+00
+; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -691,6 +690,38 @@ define half @test_normal_minus_subnorm_to_subnorm_rz_f16() {
ret half %res
}
+define half @test_normal_minus_subnorm_to_subnorm_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rm_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rn_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rp_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rz_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
; CHECK-NEXT: ret bfloat 5.877470e-39
@@ -723,6 +754,143 @@ define bfloat @test_normal_minus_subnorm_to_subnorm_rz_bf16() {
ret bfloat %res
}
+;###############################################################
+;# Add(Normal, -Normal) -> Subnormal #
+;###############################################################
+; Tests addition of 1.5*(2^-126) and -(2^-126), where both inputs are normal
+; but the exact result is subnormal.
+; - Without FTZ: The result is the exact difference (2^-127)
+; - With FTZ: The result is flushed to zero. Flushing the inputs alone would
+; leave it untouched, as neither input is subnormal.
+
+define float @test_normal_minus_normal_to_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rm_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rm */ i32 3)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rn_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rn */ i32 1)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rp_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rp */ i32 2)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rz_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rz */ i32 0)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rm_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rm */ i32 3)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rn_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rn */ i32 1)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rp_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rp */ i32 2)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rz_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rz */ i32 0)
+ ret float %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rm_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rm_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rn_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rn_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rp_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rp_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rz_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rm_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rn_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rp_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rz_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
;###############################################################
;# Add(1.0, 2^(-25)) #
;###############################################################
>From 1113223427c441dbb567883fdfd3ae390994ac96 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 19 Aug 2026 11:58:22 +0000
Subject: [PATCH 04/16] move ftz modifier to the end of the intrinsic name
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 8 ++++----
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 8 ++++----
.../test/CIR/CodeGenCUDA/builtins-nvvm-math.cu | 4 ++--
clang/test/CodeGen/builtins-nvptx.c | 12 ++++++------
llvm/include/llvm/IR/IntrinsicsNVVM.td | 6 +++---
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 4 ++--
llvm/lib/IR/AutoUpgrade.cpp | 4 ++--
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 8 ++++----
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 6 +++---
llvm/test/CodeGen/NVPTX/f16-add.ll | 4 ++--
llvm/test/CodeGen/NVPTX/f16-sub.ll | 4 ++--
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 16 ++++++++--------
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 2 +-
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 10 +++++-----
.../Target/LLVMIR/nvvm/addf/addf_vector.mlir | 18 +++++++++---------
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 10 +++++-----
.../Target/LLVMIR/nvvm/subf/subf_vector.mlir | 18 +++++++++---------
18 files changed, 72 insertions(+), 72 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 0b19fb18cd430..221d093d4b26d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -883,16 +883,16 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index 09ad23985b092..42cf0114be4a4 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -1235,16 +1235,16 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmNearestTiesToEven,
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmNearestTiesToEven,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardZero,
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardZero,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardNegative,
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardNegative,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardPositive,
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardPositive,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index 69df1d376f7f7..c2f4d19322cfe 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -81,9 +81,9 @@ __device__ double test_add_rz_d(double x, double y) {
}
// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.ftz.sat" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.sat.ftz" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// LLVM: call {{.*}}float @llvm.nvvm.fadd.ftz.sat.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.sat.ftz.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
__device__ float test_add_rm_ftz_sat_f(float x, float y) {
return __nvvm_add_rm_ftz_sat_f(x, y);
}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index bed1498236b06..34bb697a3f2a0 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -1630,19 +1630,19 @@ __device__ void nvvm_min_max_sm86() {
__device__ void nvvm_add_fma_f32_sat() {
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 1)
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 1)
+ // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 0)
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 0)
+ // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 0)
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 3)
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 3)
+ // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 3)
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 2)
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 2)
+ // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 2)
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1674,11 +1674,11 @@ __device__ void nvvm_add_fma_f32_sat() {
__device__ void nvvm_add_mul_f16_sat() {
// CHECK: call half @llvm.nvvm.fadd.sat.f16({{.*}}i32 1)
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16({{.*}}i32 1)
+ // CHECK: call half @llvm.nvvm.fadd.sat.ftz.f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
// CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16({{.*}}i32 1)
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 489f45e19cab0..e240df285fdda 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1819,9 +1819,9 @@ let TargetPrefix = "nvvm" in {
ArgInfo<ArgIndex<2>,
[ArgName<"rnd">,
ImmArgPrinter<"printFAddRoundingMode">]>] in
- foreach ftz = ["", "_ftz"] in
- foreach sat = ["", "_sat"] in
- def int_nvvm_fadd # ftz # sat :
+ foreach sat = ["", "_sat"] in
+ foreach ftz = ["", "_ftz"] in
+ def int_nvvm_fadd # sat # ftz :
DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
[LLVMMatchType<0>, LLVMMatchType<0>,
llvm_i32_ty]>;
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index c0800f217e50e..15b05f3ff9467 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -614,7 +614,7 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_ftz:
- case Intrinsic::nvvm_fadd_ftz_sat:
+ case Intrinsic::nvvm_fadd_sat_ftz:
return true;
case Intrinsic::nvvm_fadd:
@@ -627,7 +627,7 @@ inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_ftz_sat:
+ case Intrinsic::nvvm_fadd_sat_ftz:
return true;
case Intrinsic::nvvm_fadd:
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 7039a06af3ecf..4d804514b410c 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1437,7 +1437,7 @@ getNVVMFAddUpgrade(StringRef Modifiers) {
.Case("", Intrinsic::nvvm_fadd)
.Case(".ftz", Intrinsic::nvvm_fadd_ftz)
.Case(".sat", Intrinsic::nvvm_fadd_sat)
- .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
+ .Case(".ftz.sat", Intrinsic::nvvm_fadd_sat_ftz)
.Default(Intrinsic::not_intrinsic);
if (IID == Intrinsic::not_intrinsic)
return std::nullopt;
@@ -2019,7 +2019,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
// Upgrade the FP add intrinsics, which are overloaded on the operand type
// and take the rounding mode as an operand:
// llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
- // llvm.nvvm.fadd{.ftz}{.sat}.<mangled type>
+ // llvm.nvvm.fadd{.sat}{.ftz}.<mangled type>
// The extra operand means these are expanded in UpgradeIntrinsicCall.
if (Name.starts_with("add.")) {
auto [Base, TypeSuffix] = Name.rsplit('.');
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 5ad3786063c44..29668a2024480 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7285,7 +7285,7 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_ftz_sat: {
+ case Intrinsic::nvvm_fadd_sat_ftz: {
const auto RM = static_cast<APFloat::roundingMode>(
N->getConstantOperandAPInt(3).getSExtValue());
if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, RM, STI))
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index b74222037314d..8a9443c458992 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2264,7 +2264,7 @@ let Predicates = [doRsqrtOpt] in {
// Add
//
-defvar BF16ArithPreds = [hasBF16Math, hasPTX<78>, hasSM<90>];
+defvar BF16ArithPreds = [hasBF16Math, PTX78, SM90];
class RndModeImm<string mode> : TImmLeaf<i32,
"return Imm == static_cast<int>(RoundingMode::" # mode # ");">;
@@ -2279,7 +2279,7 @@ foreach t = [F16RT, F16X2RT] in
foreach sat = ["", "_sat"] in
def INT_NVVM_ADD_RN # !toupper(ftz # sat # "_" # t.PtxType) :
F_MATH_2_RND_TY<!subst("_", ".", "add_rn" # ftz # sat # "_" # t.PtxType),
- t, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ t, !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz),
rnd_rn_imm>;
foreach t = [BF16RT, BF16X2RT] in
@@ -2294,7 +2294,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach sat = ["", "_sat"] in
def INT_NVVM_ADD # !toupper(rnd # sat # ftz) # _F :
F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # sat # ftz # "_f32"),
- F32RT, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ F32RT, !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz),
rnd_imm>;
def INT_NVVM_ADD # !toupper(rnd # ftz) # _F32X2 :
@@ -2373,7 +2373,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat);
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz);
def INT_NVVM_SUB # rnd # ftz # sat # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 2871ddab068bc..96fc033a03f30 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -716,13 +716,13 @@ define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
; CHECK: call float @llvm.nvvm.fadd.f32(float %a, float %a, /* rnd=rn */ i32 1)
; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %a, /* rnd=rz */ i32 0)
; CHECK: call float @llvm.nvvm.fadd.sat.f32(float %a, float %a, /* rnd=rm */ i32 3)
-; CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %a, /* rnd=rp */ i32 2)
+; CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %a, /* rnd=rp */ i32 2)
; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rn */ i32 1)
; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rz */ i32 0)
; CHECK: call half @llvm.nvvm.fadd.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
-; CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fadd.sat.ftz.f16(half %c, half %c, /* rnd=rn */ i32 1)
; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
-; CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
%r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
%r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
%r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
index 0b470daf7a535..3a3c2c66cf180 100644
--- a/llvm/test/CodeGen/NVPTX/f16-add.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -103,7 +103,7 @@ define half @add_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %b, i32 1)
+ %1 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -118,6 +118,6 @@ define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
+ %1 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
index c59875af1d4f9..e3d32ac1a52ec 100644
--- a/llvm/test/CodeGen/NVPTX/f16-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -112,7 +112,7 @@ define half @sub_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %1, i32 1)
+ %res = call half @llvm.nvvm.fadd.sat.ftz.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -128,6 +128,6 @@ define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index ad0e77d119e65..061155f995472 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -21,16 +21,16 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, i32 1)
- %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r1, i32 1)
%r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r3, i32 0)
%r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, i32 3)
- %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r5, i32 3)
%r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, i32 2)
- %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r7, i32 2)
ret float %r8
}
@@ -57,25 +57,25 @@ define float @sub_sat_f32(float %a, float %b) {
%r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
%r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
%r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f5, i32 3)
%f6 = fneg float %r6
%r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, i32 2)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f7, i32 2)
ret float %r8
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index b3c953de574b9..1cfa2eb1712db 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -487,7 +487,7 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
static constexpr llvm::Intrinsic::ID addIDs[2][2] = {
{llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
- {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
+ {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_sat_ftz}};
static constexpr llvm::RoundingMode roundingModes[5] = {
llvm::RoundingMode::NearestTiesToEven,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index a376351ac20ec..d3eea29fe372d 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -7,7 +7,7 @@ llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %3, half %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.ftz.f16(half %4, half %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.sat.f16(half %5, half %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %6, half %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %6, half %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
@@ -37,19 +37,19 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %3, float %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.ftz.f32(float %5, float %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %6, float %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %6, float %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.f32(float %7, float %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %8, /* rnd=rm */ i32 3)
// CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %9, float %9, /* rnd=rm */ i32 3)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %10, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %10, float %10, /* rnd=rm */ i32 3)
// CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.f32(float %11, float %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %12, /* rnd=rp */ i32 2)
// CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.ftz.f32(float %13, float %13, /* rnd=rp */ i32 2)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %14, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.f32(float %15, float %15, /* rnd=rz */ i32 0)
// CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.sat.f32(float %16, float %16, /* rnd=rz */ i32 0)
// CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %17, float %17, /* rnd=rz */ i32 0)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index d3421ac025f2b..7b892cf1c691f 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -7,7 +7,7 @@ llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %3, <2 x half> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %4, <2 x half> %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %5, <2 x half> %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
@@ -46,11 +46,11 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %12, <2 x float> %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %15, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %14, float %15, /* rnd=rn */ i32 1)
// CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
// CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %19, /* rnd=rn */ i32 1)
// CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
// CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
@@ -76,11 +76,11 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rm */ i32 3)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rm */ i32 3)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -105,11 +105,11 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -134,11 +134,11 @@ llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rz */ i32 0)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index eaf33ae64ee76..cb0d21945ddbc 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -12,7 +12,7 @@ llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: %9 = fneg half %8
// CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.sat.f16(half %8, half %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg half %10
- // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %10, half %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %10, half %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
@@ -49,7 +49,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %9 = fneg float %8
// CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %10, float %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: %13 = fneg float %12
// CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.f32(float %12, float %13, /* rnd=rm */ i32 3)
// CHECK-NEXT: %15 = fneg float %14
@@ -57,7 +57,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %17 = fneg float %16
// CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %16, float %17, /* rnd=rm */ i32 3)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %19, /* rnd=rm */ i32 3)
// CHECK-NEXT: %21 = fneg float %20
// CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.f32(float %20, float %21, /* rnd=rp */ i32 2)
// CHECK-NEXT: %23 = fneg float %22
@@ -65,7 +65,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %25 = fneg float %24
// CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.ftz.f32(float %24, float %25, /* rnd=rp */ i32 2)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %26, float %27, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %26, float %27, /* rnd=rp */ i32 2)
// CHECK-NEXT: %29 = fneg float %28
// CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.f32(float %28, float %29, /* rnd=rz */ i32 0)
// CHECK-NEXT: %31 = fneg float %30
@@ -73,7 +73,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %33 = fneg float %32
// CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.ftz.f32(float %32, float %33, /* rnd=rz */ i32 0)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %34, float %35, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %34, float %35, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index 78109d5ef9e7d..c184da42b2b97 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -12,7 +12,7 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: %9 = fneg <2 x half> %8
// CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %8, <2 x half> %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg <2 x half> %10
- // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
@@ -58,11 +58,11 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %18 = fneg <2 x float> %17
// CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
// CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %19, float %20, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %19, float %20, /* rnd=rn */ i32 1)
// CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
// CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
// CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %23, float %24, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %23, float %24, /* rnd=rn */ i32 1)
// CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
// CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
@@ -92,11 +92,11 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rm */ i32 3)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -125,11 +125,11 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rp */ i32 2)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -158,11 +158,11 @@ llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rz */ i32 0)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
>From edf8ce682fd0cc8fb32b7c60f8ad62fddb478b2f Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 19 Aug 2026 12:49:56 +0000
Subject: [PATCH 05/16] fix tests
---
llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll | 12 ++++++------
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 8 ++++----
mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir | 8 ++++----
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 8 ++++----
mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir | 8 ++++----
5 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
index ca854106fa31b..7aebe89a1c1fd 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
@@ -8,8 +8,8 @@ define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: .reg .b64 %rd<11>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [sub_f32x2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [sub_f32x2_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_param_1];
; CHECK-NEXT: sub.rn.f32x2 %rd3, %rd1, %rd2;
; CHECK-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
; CHECK-NEXT: sub.rz.f32x2 %rd5, %rd1, %rd4;
@@ -18,7 +18,7 @@ define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: sub.rm.ftz.f32x2 %rd8, %rd1, %rd7;
; CHECK-NEXT: sub.rp.f32x2 %rd9, %rd1, %rd8;
; CHECK-NEXT: sub.rp.ftz.f32x2 %rd10, %rd1, %rd9;
-; CHECK-NEXT: st.param.b64 [func_retval0], %rd10;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd10;
; CHECK-NEXT: ret;
%f0 = fneg <2 x float> %b
%r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f0, i32 1)
@@ -53,10 +53,10 @@ define <2 x float> @sub_f32x2_negated_lhs(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [sub_f32x2_negated_lhs_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [sub_f32x2_negated_lhs_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_negated_lhs_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_negated_lhs_param_1];
; CHECK-NEXT: sub.rz.f32x2 %rd3, %rd2, %rd1;
-; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd3;
; CHECK-NEXT: ret;
%f = fneg <2 x float> %a
%r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %f, <2 x float> %b, i32 0)
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index d3eea29fe372d..c9bfde9ebc88a 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -11,10 +11,10 @@ llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
- %f2 = nvvm.addf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : f16
- %f3 = nvvm.addf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : f16
- %f4 = nvvm.addf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : f16
- %f5 = nvvm.addf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : f16
+ %f2 = nvvm.addf %f1, %f1 rnd = <rn> : f16
+ %f3 = nvvm.addf %f2, %f2 rnd = <rn> ftz = true : f16
+ %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> : f16
+ %f5 = nvvm.addf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : f16
llvm.return %f5 : f16
}
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index 7b892cf1c691f..bb83fd11b467e 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -11,10 +11,10 @@ llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
- %f2 = nvvm.addf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xf16>
- %f3 = nvvm.addf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : vector<2xf16>
- %f4 = nvvm.addf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : vector<2xf16>
- %f5 = nvvm.addf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : vector<2xf16>
+ %f2 = nvvm.addf %f1, %f1 rnd = <rn> : vector<2xf16>
+ %f3 = nvvm.addf %f2, %f2 rnd = <rn> ftz = true : vector<2xf16>
+ %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> : vector<2xf16>
+ %f5 = nvvm.addf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index cb0d21945ddbc..d269ba88fe9fd 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -16,10 +16,10 @@ llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
- %f2 = nvvm.subf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : f16
- %f3 = nvvm.subf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : f16
- %f4 = nvvm.subf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : f16
- %f5 = nvvm.subf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : f16
+ %f2 = nvvm.subf %f1, %f1 rnd = <rn> : f16
+ %f3 = nvvm.subf %f2, %f2 rnd = <rn> ftz = true : f16
+ %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> : f16
+ %f5 = nvvm.subf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : f16
llvm.return %f5 : f16
}
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index c184da42b2b97..890d63a8921e9 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -16,10 +16,10 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
- %f2 = nvvm.subf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xf16>
- %f3 = nvvm.subf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : vector<2xf16>
- %f4 = nvvm.subf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : vector<2xf16>
- %f5 = nvvm.subf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : vector<2xf16>
+ %f2 = nvvm.subf %f1, %f1 rnd = <rn> : vector<2xf16>
+ %f3 = nvvm.subf %f2, %f2 rnd = <rn> ftz = true : vector<2xf16>
+ %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> : vector<2xf16>
+ %f5 = nvvm.subf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
>From 0cfed960329ec39432f1a425c27cf8b953c6b48d Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 19 Aug 2026 13:08:34 +0000
Subject: [PATCH 06/16] add docs
---
llvm/docs/NVPTXUsage.md | 83 ++++++++++++++++++++++++++++++++++++-----
1 file changed, 73 insertions(+), 10 deletions(-)
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index e05af78012691..27e96725d9887 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1185,6 +1185,33 @@ For more information, see [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread
### Arithmetic Intrinsics
+Some of these intrinsics take the rounding mode as an `i32` immediate operand
+instead of encoding it in the intrinsic name. The accepted values match the
+`llvm::RoundingMode` enumeration and are described in the following table:
+
+(fp-rounding-modes)=
+
+```{list-table} Floating-Point Rounding Modes
+:widths: 15 15 70
+:header-rows: 1
+
+ * - Value
+ - Rounding Mode
+ - Description
+ * - 0
+ - `rz`
+ - Round towards zero
+ * - 1
+ - `rn`
+ - Round to nearest, with ties to even
+ * - 2
+ - `rp`
+ - Round towards positive infinity
+ * - 3
+ - `rm`
+ - Round towards negative infinity
+```
+
#### '`llvm.nvvm.fabs.*`' Intrinsic
##### Syntax:
@@ -1284,29 +1311,65 @@ used in the '`llvm.nvvm.idp4a.[us].u`' variants, while sign-extension is used
with '`llvm.nvvm.idp4a.[us].s`' variants. The dot product of these 4-element
vectors is added to `%c` to produce the return.
-#### '`llvm.nvvm.add.*`' Half-precision Intrinsics
+#### '`llvm.nvvm.fadd.*`' Intrinsics
##### Syntax:
-```llvm
-declare half @llvm.nvvm.add.rn.sat.f16(half %a, half %b)
-declare <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
+This is an overloaded intrinsic. The '`.sat`' and '`.ftz`' modifiers are
+optional.
-declare half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %b)
-declare <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
+```llvm
+declare half @llvm.nvvm.fadd{.sat}{.ftz}.f16(half %a, half %b, i32 immarg %rnd)
+declare <2 x half> @llvm.nvvm.fadd{.sat}{.ftz}.v2f16(<2 x half> %a, <2 x half> %b, i32 immarg %rnd)
+declare bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 immarg %rnd)
+declare <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, i32 immarg %rnd)
+declare float @llvm.nvvm.fadd{.sat}{.ftz}.f32(float %a, float %b, i32 immarg %rnd)
+declare <2 x float> @llvm.nvvm.fadd{.ftz}.v2f32(<2 x float> %a, <2 x float> %b, i32 immarg %rnd)
+declare double @llvm.nvvm.fadd.f64(double %a, double %b, i32 immarg %rnd)
```
##### Overview:
-The '`llvm.nvvm.add.*`' intrinsics perform an addition operation with the
-specified rounding mode and modifiers.
+The '`llvm.nvvm.fadd.*`' intrinsics add `%a` and `%b` using the rounding mode
+selected by `%rnd` and the modifiers present in the intrinsic name. They
+correspond directly to the `add` PTX instruction.
##### Semantics:
-The '`.sat`' modifier performs a saturating addition where the result is
-clamped to `[0.0, 1.0]` and `NaN` results are flushed to `+0.0f`.
+`%rnd` selects the rounding mode applied to the result, see
+{ref}`fp-rounding-modes`.
+
The '`.ftz`' modifier flushes subnormal inputs and results to sign-preserving
zero.
+The '`.sat`' modifier performs a saturating addition where the result is
+clamped to `[0.0, 1.0]` and `NaN` results are flushed to `+0.0f`.
+
+Not every combination of operand type, rounding mode and modifier maps to a
+PTX instruction. The supported combinations are:
+
+```{list-table}
+:widths: 25 25 25 25
+:header-rows: 1
+
+ * - Operand Type
+ - Rounding Modes
+ - Modifiers
+ * - `half`, `<2 x half>`
+ - `rn`
+ - `.sat`, `.ftz`
+ * - `bfloat`, `<2 x bfloat>`
+ - `rn`
+ - None
+ * - `float`
+ - `rn`, `rz`, `rp`, `rm`
+ - `.sat`, `.ftz`
+ * - `<2 x float>`
+ - `rn`, `rz`, `rp`, `rm`
+ - `.ftz`
+ * - `double`
+ - `rn`, `rz`, `rp`, `rm`
+ - None
+```
#### '`llvm.nvvm.mul.*`' Half-precision Intrinsics
>From 7b78e0ba41df7dfd270532b0f9bc74b93f38c3bc Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 19 Aug 2026 13:17:02 +0000
Subject: [PATCH 07/16] fix docs
---
llvm/docs/NVPTXUsage.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index 27e96725d9887..5c9797691efbe 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1348,7 +1348,7 @@ Not every combination of operand type, rounding mode and modifier maps to a
PTX instruction. The supported combinations are:
```{list-table}
-:widths: 25 25 25 25
+:widths: 25 25 25
:header-rows: 1
* - Operand Type
>From 889cb45927b549c3ced46154fc8e8db1bac3655a Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 21 Aug 2026 16:38:51 +0000
Subject: [PATCH 08/16] revert to older order of modifiers
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 8 ++---
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 8 ++---
.../CIR/CodeGenCUDA/builtins-nvvm-math.cu | 4 +--
clang/test/CodeGen/builtins-nvptx.c | 12 +++----
llvm/docs/NVPTXUsage.md | 12 +++----
llvm/include/llvm/IR/IntrinsicsNVVM.td | 6 ++--
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 4 +--
llvm/lib/IR/AutoUpgrade.cpp | 4 +--
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 12 +++----
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 6 ++--
llvm/test/CodeGen/NVPTX/f16-add.ll | 4 +--
llvm/test/CodeGen/NVPTX/f16-sub.ll | 4 +--
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 32 +++++++++----------
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 2 +-
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 10 +++---
.../Target/LLVMIR/nvvm/addf/addf_vector.mlir | 18 +++++------
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 10 +++---
.../Target/LLVMIR/nvvm/subf/subf_vector.mlir | 18 +++++------
19 files changed, 88 insertions(+), 88 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 221d093d4b26d..0b19fb18cd430 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -883,16 +883,16 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index 42cf0114be4a4..09ad23985b092 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -1235,16 +1235,16 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmNearestTiesToEven,
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmNearestTiesToEven,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardZero,
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardZero,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardNegative,
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardNegative,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardPositive,
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardPositive,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index c2f4d19322cfe..69df1d376f7f7 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -81,9 +81,9 @@ __device__ double test_add_rz_d(double x, double y) {
}
// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.sat.ftz" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.ftz.sat" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// LLVM: call {{.*}}float @llvm.nvvm.fadd.sat.ftz.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.ftz.sat.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
__device__ float test_add_rm_ftz_sat_f(float x, float y) {
return __nvvm_add_rm_ftz_sat_f(x, y);
}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 34bb697a3f2a0..bed1498236b06 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -1630,19 +1630,19 @@ __device__ void nvvm_min_max_sm86() {
__device__ void nvvm_add_fma_f32_sat() {
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 1)
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 1)
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 0)
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 0)
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 0)
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 3)
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 3)
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 3)
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 2)
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 2)
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 2)
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1674,11 +1674,11 @@ __device__ void nvvm_add_fma_f32_sat() {
__device__ void nvvm_add_mul_f16_sat() {
// CHECK: call half @llvm.nvvm.fadd.sat.f16({{.*}}i32 1)
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.fadd.sat.ftz.f16({{.*}}i32 1)
+ // CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
// CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16({{.*}}i32 1)
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index 5c9797691efbe..d2ee19c42a602 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1315,15 +1315,15 @@ vectors is added to `%c` to produce the return.
##### Syntax:
-This is an overloaded intrinsic. The '`.sat`' and '`.ftz`' modifiers are
+This is an overloaded intrinsic. The '`.ftz`' and '`.sat`' modifiers are
optional.
```llvm
-declare half @llvm.nvvm.fadd{.sat}{.ftz}.f16(half %a, half %b, i32 immarg %rnd)
-declare <2 x half> @llvm.nvvm.fadd{.sat}{.ftz}.v2f16(<2 x half> %a, <2 x half> %b, i32 immarg %rnd)
+declare half @llvm.nvvm.fadd{.ftz}{.sat}.f16(half %a, half %b, i32 immarg %rnd)
+declare <2 x half> @llvm.nvvm.fadd{.ftz}{.sat}.v2f16(<2 x half> %a, <2 x half> %b, i32 immarg %rnd)
declare bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 immarg %rnd)
declare <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, i32 immarg %rnd)
-declare float @llvm.nvvm.fadd{.sat}{.ftz}.f32(float %a, float %b, i32 immarg %rnd)
+declare float @llvm.nvvm.fadd{.ftz}{.sat}.f32(float %a, float %b, i32 immarg %rnd)
declare <2 x float> @llvm.nvvm.fadd{.ftz}.v2f32(<2 x float> %a, <2 x float> %b, i32 immarg %rnd)
declare double @llvm.nvvm.fadd.f64(double %a, double %b, i32 immarg %rnd)
```
@@ -1356,13 +1356,13 @@ PTX instruction. The supported combinations are:
- Modifiers
* - `half`, `<2 x half>`
- `rn`
- - `.sat`, `.ftz`
+ - `.ftz`, `.sat`
* - `bfloat`, `<2 x bfloat>`
- `rn`
- None
* - `float`
- `rn`, `rz`, `rp`, `rm`
- - `.sat`, `.ftz`
+ - `.ftz`, `.sat`
* - `<2 x float>`
- `rn`, `rz`, `rp`, `rm`
- `.ftz`
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index e240df285fdda..489f45e19cab0 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1819,9 +1819,9 @@ let TargetPrefix = "nvvm" in {
ArgInfo<ArgIndex<2>,
[ArgName<"rnd">,
ImmArgPrinter<"printFAddRoundingMode">]>] in
- foreach sat = ["", "_sat"] in
- foreach ftz = ["", "_ftz"] in
- def int_nvvm_fadd # sat # ftz :
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def int_nvvm_fadd # ftz # sat :
DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
[LLVMMatchType<0>, LLVMMatchType<0>,
llvm_i32_ty]>;
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index 15b05f3ff9467..c0800f217e50e 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -614,7 +614,7 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_ftz:
- case Intrinsic::nvvm_fadd_sat_ftz:
+ case Intrinsic::nvvm_fadd_ftz_sat:
return true;
case Intrinsic::nvvm_fadd:
@@ -627,7 +627,7 @@ inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_sat_ftz:
+ case Intrinsic::nvvm_fadd_ftz_sat:
return true;
case Intrinsic::nvvm_fadd:
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 4d804514b410c..7039a06af3ecf 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1437,7 +1437,7 @@ getNVVMFAddUpgrade(StringRef Modifiers) {
.Case("", Intrinsic::nvvm_fadd)
.Case(".ftz", Intrinsic::nvvm_fadd_ftz)
.Case(".sat", Intrinsic::nvvm_fadd_sat)
- .Case(".ftz.sat", Intrinsic::nvvm_fadd_sat_ftz)
+ .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
.Default(Intrinsic::not_intrinsic);
if (IID == Intrinsic::not_intrinsic)
return std::nullopt;
@@ -2019,7 +2019,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
// Upgrade the FP add intrinsics, which are overloaded on the operand type
// and take the rounding mode as an operand:
// llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
- // llvm.nvvm.fadd{.sat}{.ftz}.<mangled type>
+ // llvm.nvvm.fadd{.ftz}{.sat}.<mangled type>
// The extra operand means these are expanded in UpgradeIntrinsicCall.
if (Name.starts_with("add.")) {
auto [Base, TypeSuffix] = Name.rsplit('.');
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 29668a2024480..5ad3786063c44 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7285,7 +7285,7 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_sat_ftz: {
+ case Intrinsic::nvvm_fadd_ftz_sat: {
const auto RM = static_cast<APFloat::roundingMode>(
N->getConstantOperandAPInt(3).getSExtValue());
if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, RM, STI))
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 8a9443c458992..aa49675bcc9dd 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2279,7 +2279,7 @@ foreach t = [F16RT, F16X2RT] in
foreach sat = ["", "_sat"] in
def INT_NVVM_ADD_RN # !toupper(ftz # sat # "_" # t.PtxType) :
F_MATH_2_RND_TY<!subst("_", ".", "add_rn" # ftz # sat # "_" # t.PtxType),
- t, !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz),
+ t, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
rnd_rn_imm>;
foreach t = [BF16RT, BF16X2RT] in
@@ -2292,9 +2292,9 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in
- def INT_NVVM_ADD # !toupper(rnd # sat # ftz) # _F :
- F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # sat # ftz # "_f32"),
- F32RT, !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz),
+ def INT_NVVM_ADD # !toupper(rnd # ftz # sat) # _F :
+ F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # ftz # sat # "_f32"),
+ F32RT, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
rnd_imm>;
def INT_NVVM_ADD # !toupper(rnd # ftz) # _F32X2 :
@@ -2373,10 +2373,10 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz);
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat);
def INT_NVVM_SUB # rnd # ftz # sat # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
- !subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
+ !subst("_", ".", "sub" # rnd # ftz # sat # "_f32"),
[(set f32:$dst,
(add_intrin f32:$a, (f32 (fneg f32:$b)), rnd_imm))]>;
}
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 96fc033a03f30..2871ddab068bc 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -716,13 +716,13 @@ define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
; CHECK: call float @llvm.nvvm.fadd.f32(float %a, float %a, /* rnd=rn */ i32 1)
; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %a, /* rnd=rz */ i32 0)
; CHECK: call float @llvm.nvvm.fadd.sat.f32(float %a, float %a, /* rnd=rm */ i32 3)
-; CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %a, /* rnd=rp */ i32 2)
+; CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %a, /* rnd=rp */ i32 2)
; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rn */ i32 1)
; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rz */ i32 0)
; CHECK: call half @llvm.nvvm.fadd.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
-; CHECK: call half @llvm.nvvm.fadd.sat.ftz.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
-; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
%r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
%r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
%r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
index 3a3c2c66cf180..0b470daf7a535 100644
--- a/llvm/test/CodeGen/NVPTX/f16-add.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -103,7 +103,7 @@ define half @add_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %a, half %b, i32 1)
+ %1 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -118,6 +118,6 @@ define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
+ %1 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
index e3d32ac1a52ec..c59875af1d4f9 100644
--- a/llvm/test/CodeGen/NVPTX/f16-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -112,7 +112,7 @@ define half @sub_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.sat.ftz.f16(half %a, half %1, i32 1)
+ %res = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -128,6 +128,6 @@ define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index 061155f995472..0b93da7af17d0 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -11,26 +11,26 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: ld.param.b32 %r1, [add_sat_f32_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [add_sat_f32_param_1];
; CHECK-NEXT: add.rn.sat.f32 %r3, %r1, %r2;
-; CHECK-NEXT: add.rn.sat.ftz.f32 %r4, %r1, %r3;
+; CHECK-NEXT: add.rn.ftz.sat.f32 %r4, %r1, %r3;
; CHECK-NEXT: add.rz.sat.f32 %r5, %r1, %r4;
-; CHECK-NEXT: add.rz.sat.ftz.f32 %r6, %r1, %r5;
+; CHECK-NEXT: add.rz.ftz.sat.f32 %r6, %r1, %r5;
; CHECK-NEXT: add.rm.sat.f32 %r7, %r1, %r6;
-; CHECK-NEXT: add.rm.sat.ftz.f32 %r8, %r1, %r7;
+; CHECK-NEXT: add.rm.ftz.sat.f32 %r8, %r1, %r7;
; CHECK-NEXT: add.rp.sat.f32 %r9, %r1, %r8;
-; CHECK-NEXT: add.rp.sat.ftz.f32 %r10, %r1, %r9;
+; CHECK-NEXT: add.rp.ftz.sat.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, i32 1)
- %r2 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, i32 1)
%r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, i32 0)
%r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, i32 3)
- %r6 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, i32 3)
%r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, i32 2)
- %r8 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, i32 2)
ret float %r8
}
@@ -44,38 +44,38 @@ define float @sub_sat_f32(float %a, float %b) {
; CHECK-NEXT: ld.param.b32 %r1, [sub_sat_f32_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [sub_sat_f32_param_1];
; CHECK-NEXT: sub.rn.sat.f32 %r3, %r1, %r2;
-; CHECK-NEXT: sub.rn.sat.ftz.f32 %r4, %r1, %r3;
+; CHECK-NEXT: sub.rn.ftz.sat.f32 %r4, %r1, %r3;
; CHECK-NEXT: sub.rz.sat.f32 %r5, %r1, %r4;
-; CHECK-NEXT: sub.rz.sat.ftz.f32 %r6, %r1, %r5;
+; CHECK-NEXT: sub.rz.ftz.sat.f32 %r6, %r1, %r5;
; CHECK-NEXT: sub.rm.sat.f32 %r7, %r1, %r6;
-; CHECK-NEXT: sub.rm.sat.ftz.f32 %r8, %r1, %r7;
+; CHECK-NEXT: sub.rm.ftz.sat.f32 %r8, %r1, %r7;
; CHECK-NEXT: sub.rp.sat.f32 %r9, %r1, %r8;
-; CHECK-NEXT: sub.rp.sat.ftz.f32 %r10, %r1, %r9;
+; CHECK-NEXT: sub.rp.ftz.sat.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%f0 = fneg float %b
%r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
%r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
%r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, i32 3)
%f6 = fneg float %r6
%r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, i32 2)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, i32 2)
ret float %r8
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 1cfa2eb1712db..b3c953de574b9 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -487,7 +487,7 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
static constexpr llvm::Intrinsic::ID addIDs[2][2] = {
{llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
- {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_sat_ftz}};
+ {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
static constexpr llvm::RoundingMode roundingModes[5] = {
llvm::RoundingMode::NearestTiesToEven,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index c9bfde9ebc88a..17c7e3f36406b 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -7,7 +7,7 @@ llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %3, half %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.ftz.f16(half %4, half %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.sat.f16(half %5, half %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %6, half %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %6, half %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
@@ -37,19 +37,19 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %3, float %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.ftz.f32(float %5, float %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %6, float %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %6, float %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.f32(float %7, float %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %8, /* rnd=rm */ i32 3)
// CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %9, float %9, /* rnd=rm */ i32 3)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %10, float %10, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %10, /* rnd=rm */ i32 3)
// CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.f32(float %11, float %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %12, /* rnd=rp */ i32 2)
// CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.ftz.f32(float %13, float %13, /* rnd=rp */ i32 2)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %14, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.f32(float %15, float %15, /* rnd=rz */ i32 0)
// CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.sat.f32(float %16, float %16, /* rnd=rz */ i32 0)
// CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %17, float %17, /* rnd=rz */ i32 0)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index bb83fd11b467e..504b0e5cae6bf 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -7,7 +7,7 @@ llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %3, <2 x half> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %4, <2 x half> %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %5, <2 x half> %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
@@ -46,11 +46,11 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %12, <2 x float> %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %14, float %15, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %15, /* rnd=rn */ i32 1)
// CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
// CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %19, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rn */ i32 1)
// CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
// CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
@@ -76,11 +76,11 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rm */ i32 3)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rm */ i32 3)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -105,11 +105,11 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -134,11 +134,11 @@ llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rz */ i32 0)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index d269ba88fe9fd..7226b0e172d75 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -12,7 +12,7 @@ llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: %9 = fneg half %8
// CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.sat.f16(half %8, half %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg half %10
- // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %10, half %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %10, half %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
@@ -49,7 +49,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %9 = fneg float %8
// CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %10, float %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: %13 = fneg float %12
// CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.f32(float %12, float %13, /* rnd=rm */ i32 3)
// CHECK-NEXT: %15 = fneg float %14
@@ -57,7 +57,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %17 = fneg float %16
// CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %16, float %17, /* rnd=rm */ i32 3)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %19, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rm */ i32 3)
// CHECK-NEXT: %21 = fneg float %20
// CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.f32(float %20, float %21, /* rnd=rp */ i32 2)
// CHECK-NEXT: %23 = fneg float %22
@@ -65,7 +65,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %25 = fneg float %24
// CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.ftz.f32(float %24, float %25, /* rnd=rp */ i32 2)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %26, float %27, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %26, float %27, /* rnd=rp */ i32 2)
// CHECK-NEXT: %29 = fneg float %28
// CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.f32(float %28, float %29, /* rnd=rz */ i32 0)
// CHECK-NEXT: %31 = fneg float %30
@@ -73,7 +73,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %33 = fneg float %32
// CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.ftz.f32(float %32, float %33, /* rnd=rz */ i32 0)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %34, float %35, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %34, float %35, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index 890d63a8921e9..ee4ab35a0132f 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -12,7 +12,7 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: %9 = fneg <2 x half> %8
// CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %8, <2 x half> %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg <2 x half> %10
- // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
@@ -58,11 +58,11 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %18 = fneg <2 x float> %17
// CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
// CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %19, float %20, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %19, float %20, /* rnd=rn */ i32 1)
// CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
// CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
// CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %23, float %24, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %23, float %24, /* rnd=rn */ i32 1)
// CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
// CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
@@ -92,11 +92,11 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rm */ i32 3)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -125,11 +125,11 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rp */ i32 2)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -158,11 +158,11 @@ llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rz */ i32 0)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
>From c5d4c99a9684813e180be1127a677ebaf4b07cbe Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Mon, 24 Aug 2026 15:40:10 +0000
Subject: [PATCH 09/16] address comments
---
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 31 +++++------
llvm/lib/IR/AutoUpgrade.cpp | 50 ++++++++---------
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 36 +++++++------
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 54 +++++++++++--------
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 25 ++++++---
5 files changed, 107 insertions(+), 89 deletions(-)
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index c0800f217e50e..c887608aaf350 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -15,6 +15,7 @@
#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
#define LLVM_IR_NVVMINTRINSICUTILS_H
+#include <cassert>
#include <stdint.h>
#include "llvm/ADT/APFloat.h"
@@ -611,30 +612,30 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
return DenormalMode::getIEEE();
}
-inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
+inline bool IsFAddIntrinsic(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
+ case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fadd_sat:
case Intrinsic::nvvm_fadd_ftz_sat:
return true;
-
- case Intrinsic::nvvm_fadd:
- case Intrinsic::nvvm_fadd_sat:
+ default:
return false;
}
- llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
}
-inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_ftz_sat:
- return true;
+inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
+ assert(IsFAddIntrinsic(IntrinsicID) &&
+ "Checking FTZ flag for invalid NVVM add intrinsic");
+ return IntrinsicID == Intrinsic::nvvm_fadd_ftz ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz_sat;
+}
- case Intrinsic::nvvm_fadd:
- case Intrinsic::nvvm_fadd_ftz:
- return false;
- }
- llvm_unreachable("Checking sat flag for invalid NVVM add intrinsic");
+inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
+ assert(IsFAddIntrinsic(IntrinsicID) &&
+ "Checking sat flag for invalid NVVM add intrinsic");
+ return IntrinsicID == Intrinsic::nvvm_fadd_sat ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz_sat;
}
inline APFloat::roundingMode GetFAddRoundingMode(const Value *ImmArgVal) {
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 7039a06af3ecf..e860a8f92f0f7 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1422,15 +1422,19 @@ static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
}
static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
-getNVVMFAddUpgrade(StringRef Modifiers) {
- std::optional<RoundingMode> RM =
- StringSwitch<std::optional<RoundingMode>>(Modifiers.take_front(2))
- .Case("rn", RoundingMode::NearestTiesToEven)
- .Case("rz", RoundingMode::TowardZero)
- .Case("rm", RoundingMode::TowardNegative)
- .Case("rp", RoundingMode::TowardPositive)
+getNVVMFAddUpgrade(StringRef Name) {
+ auto [Modifiers, Type] = Name.rsplit('.');
+ if (Type != "f" && Type != "d" && Type != "f16" && Type != "v2f16")
+ return std::nullopt;
+
+ std::optional<llvm::RoundingMode> RoundingMode =
+ StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
+ .Case("rn", llvm::RoundingMode::NearestTiesToEven)
+ .Case("rz", llvm::RoundingMode::TowardZero)
+ .Case("rm", llvm::RoundingMode::TowardNegative)
+ .Case("rp", llvm::RoundingMode::TowardPositive)
.Default(std::nullopt);
- if (!RM)
+ if (!RoundingMode)
return std::nullopt;
Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
@@ -1441,7 +1445,8 @@ getNVVMFAddUpgrade(StringRef Modifiers) {
.Default(Intrinsic::not_intrinsic);
if (IID == Intrinsic::not_intrinsic)
return std::nullopt;
- return std::make_pair(IID, *RM);
+
+ return std::make_pair(IID, *RoundingMode);
}
static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
@@ -2016,21 +2021,6 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
return NewFn != F;
}
- // Upgrade the FP add intrinsics, which are overloaded on the operand type
- // and take the rounding mode as an operand:
- // llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
- // llvm.nvvm.fadd{.ftz}{.sat}.<mangled type>
- // The extra operand means these are expanded in UpgradeIntrinsicCall.
- if (Name.starts_with("add.")) {
- auto [Base, TypeSuffix] = Name.rsplit('.');
- if ((TypeSuffix == "f" || TypeSuffix == "d" || TypeSuffix == "f16" ||
- TypeSuffix == "v2f16") &&
- getNVVMFAddUpgrade(Base.drop_front(strlen("add.")))) {
- NewFn = nullptr;
- return true;
- }
- }
-
// The following nvvm intrinsics correspond exactly to an LLVM idiom, but
// not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
//
@@ -2043,6 +2033,9 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
else if (Name.consume_front("fabs."))
// nvvm.fabs.{f,ftz.f,d}
Expand = Name == "f" || Name == "ftz.f" || Name == "d";
+ else if (Name.consume_front("add."))
+ // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
+ Expand = getNVVMFAddUpgrade(Name).has_value();
else if (Name.consume_front("ex2.approx."))
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Expand =
@@ -3123,12 +3116,15 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
: Intrinsic::nvvm_fabs;
Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
} else if (Name.consume_front("add.")) {
- // nvvm.add.<rnd>[.ftz][.sat].{f,d,f16,v2f16}
- auto [IID, RM] = *getNVVMFAddUpgrade(Name.rsplit('.').first);
+ // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
+ auto FAdd = getNVVMFAddUpgrade(Name);
+ assert(FAdd && "unsupported nvvm.add.* intrinsic");
+ auto [IID, RoundingMode] = *FAdd;
Value *A = CI->getArgOperand(0);
Rep = Builder.CreateIntrinsic(
A->getType(), IID,
- {A, CI->getArgOperand(1), Builder.getInt32(static_cast<int>(RM))});
+ {A, CI->getArgOperand(1),
+ Builder.getInt32(static_cast<int>(RoundingMode))});
} else if (Name.consume_front("ex2.approx.")) {
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 5ad3786063c44..b3a597a0a6708 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7181,7 +7181,7 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
}
static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
- APFloat::roundingMode RM) {
+ APFloat::roundingMode RoundingMode) {
const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
@@ -7193,7 +7193,7 @@ static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
case MVT::f32:
if (!VT.isVector() || nvvm::FAddShouldSaturate(IID))
return 0;
- switch (RM) {
+ switch (RoundingMode) {
case APFloat::rmNearestTiesToEven:
return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
case APFloat::rmTowardZero:
@@ -7212,9 +7212,9 @@ static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
Intrinsic::ID AddIntrinsicID,
- APFloat::roundingMode RM) {
+ APFloat::roundingMode RoundingMode) {
const EVT VT = N->getValueType(0);
- const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID, RM);
+ const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID, RoundingMode);
if (!Opc)
return SDValue();
@@ -7236,38 +7236,40 @@ static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(Opc, SDLoc(N), VT, SubOp1, SubOp2);
}
-static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID, APFloat::roundingMode RM,
- const NVPTXSubtarget &STI) {
+static bool isSupportedFAdd(EVT VT, const NVPTXSubtarget &STI,
+ Intrinsic::ID IID,
+ APFloat::roundingMode RoundingMode) {
if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(2))
return false;
+ const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
+ const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
const bool IsSat = nvvm::FAddShouldSaturate(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
- return RM == APFloat::rmNearestTiesToEven;
+ return IsRN;
case MVT::bf16:
- return RM == APFloat::rmNearestTiesToEven && !IsSat &&
- !nvvm::FAddShouldFTZ(IID) && STI.hasNativeBF16Support(ISD::FADD);
+ return IsRN && !IsSat && !IsFTZ && STI.hasNativeBF16Support(ISD::FADD);
case MVT::f32:
return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
case MVT::f64:
- return !VT.isVector() && !IsSat && !nvvm::FAddShouldFTZ(IID);
+ return !VT.isVector() && !IsSat && !IsFTZ;
default:
return false;
}
}
static SDValue diagnoseInvalidFAdd(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID IID, APFloat::roundingMode RM,
- const NVPTXSubtarget &STI) {
+ const NVPTXSubtarget &STI, Intrinsic::ID IID,
+ APFloat::roundingMode RoundingMode) {
const EVT VT = N->getValueType(0);
- if (isSupportedFAdd(VT, IID, RM, STI))
+ if (isSupportedFAdd(VT, STI, IID, RoundingMode))
return SDValue();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
DAG.getMachineFunction().getFunction(),
Twine(Intrinsic::getBaseName(IID)) + " with rounding mode " +
- nvvm::GetRoundingModeName(RM) + " and operand type " +
+ nvvm::GetRoundingModeName(RoundingMode) + " and operand type " +
VT.getEVTString() + " is not supported on this target",
SDLoc(N).getDebugLoc()));
return DAG.getPOISON(VT);
@@ -7286,11 +7288,11 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
case Intrinsic::nvvm_fadd_ftz:
case Intrinsic::nvvm_fadd_sat:
case Intrinsic::nvvm_fadd_ftz_sat: {
- const auto RM = static_cast<APFloat::roundingMode>(
+ const auto RoundingMode = static_cast<APFloat::roundingMode>(
N->getConstantOperandAPInt(3).getSExtValue());
- if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, RM, STI))
+ if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, STI, IID, RoundingMode))
return V;
- return combineFAddWithNeg(N, DCI.DAG, IID, RM);
+ return combineFAddWithNeg(N, DCI.DAG, IID, RoundingMode);
}
}
return SDValue();
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index aa49675bcc9dd..127991d012593 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2054,6 +2054,8 @@ def : Pat<(int_nvvm_cos_approx_f f32:$a), (COS_APPROX_f32 $a, NoFTZ)>;
// Fma
//
+defvar FPRoundingModes = ["_rn", "_rz", "_rm", "_rp"];
+
class FMA_TUPLE<string V, Intrinsic I, NVPTXRegClass RC,
list<Predicate> Preds = []> {
string Variant = V;
@@ -2128,7 +2130,7 @@ multiclass FMA_INST {
defm INT_NVVM_FMA : FMA_INST;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+foreach rnd = FPRoundingModes in {
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
def INT_NVVM_MIXED_FMA # rnd # sat # _f32_ # type :
@@ -2266,29 +2268,35 @@ let Predicates = [doRsqrtOpt] in {
defvar BF16ArithPreds = [hasBF16Math, PTX78, SM90];
-class RndModeImm<string mode> : TImmLeaf<i32,
+class FPRndModeImm<string mode> : TImmLeaf<i32,
"return Imm == static_cast<int>(RoundingMode::" # mode # ");">;
-def rnd_rn_imm : RndModeImm<"NearestTiesToEven">;
-def rnd_rz_imm : RndModeImm<"TowardZero">;
-def rnd_rm_imm : RndModeImm<"TowardNegative">;
-def rnd_rp_imm : RndModeImm<"TowardPositive">;
+def fp_rnd_rn_imm : FPRndModeImm<"NearestTiesToEven">;
+def fp_rnd_rz_imm : FPRndModeImm<"TowardZero">;
+def fp_rnd_rm_imm : FPRndModeImm<"TowardNegative">;
+def fp_rnd_rp_imm : FPRndModeImm<"TowardPositive">;
foreach t = [F16RT, F16X2RT] in
- foreach ftz = ["", "_ftz"] in
- foreach sat = ["", "_sat"] in
- def INT_NVVM_ADD_RN # !toupper(ftz # sat # "_" # t.PtxType) :
- F_MATH_2_RND_TY<!subst("_", ".", "add_rn" # ftz # sat # "_" # t.PtxType),
- t, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
- rnd_rn_imm>;
+ foreach ftz = ["", "ftz"] in
+ foreach sat = ["", "sat"] in {
+ // drop empty strings from the modifier list
+ defvar mods = !filter(m, [ftz, sat], !not(!empty(m)));
+ defvar suffix = !listconcat(mods, [t.PtxType]);
+ defvar add_intrin =
+ !cast<Intrinsic>(!interleave(!listconcat(["int_nvvm_fadd"], mods), "_"));
+
+ def INT_NVVM_ADD_RN_ # !toupper(!interleave(suffix, "_")) :
+ F_MATH_2_RND_TY<!interleave(!listconcat(["add", "rn"], suffix), "."), t,
+ add_intrin, fp_rnd_rn_imm>;
+ }
foreach t = [BF16RT, BF16X2RT] in
def INT_NVVM_ADD_RN_ # !toupper(t.PtxType) :
- F_MATH_2_RND_TY<"add.rn." # t.PtxType, t, int_nvvm_fadd, rnd_rn_imm,
+ F_MATH_2_RND_TY<"add.rn." # t.PtxType, t, int_nvvm_fadd, fp_rnd_rn_imm,
BF16ArithPreds>;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in
@@ -2308,8 +2316,8 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
int_nvvm_fadd, rnd_imm>;
}
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
@@ -2342,7 +2350,7 @@ def sub_rn_sat : SDNode<"NVPTXISD::SUB_RN_SAT", SDTFPBinOp>;
def sub_rn_ftz_sat :
SDNode<"NVPTXISD::SUB_RN_FTZ_SAT", SDTFPBinOp>;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+foreach rnd = FPRoundingModes in
foreach ftz = ["", "_ftz"] in
def sub # rnd # ftz :
SDNode<"NVPTXISD::SUB" # !toupper(rnd # ftz), SDTFPBinOp>;
@@ -2364,12 +2372,12 @@ let Predicates = BF16ArithPreds in
def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, "_rn">;
let Predicates = [hasF32x2Instructions] in
- foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach rnd = FPRoundingModes in
foreach ftz = ["", "_ftz"] in
def INT_NVVM_SUB # rnd # ftz # _F32X2 : INT_NVVM_SUB<F32X2RT, rnd # ftz>;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
@@ -2389,8 +2397,8 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
(int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), rnd_imm))]>;
}
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index b3c953de574b9..ba6eecb58144d 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -448,6 +448,22 @@ getFenceProxySyncRestrictID(NVVM::MemOrderKind order) {
nvvm_fence_proxy_async_generic_release_sync_restrict_space_cta_scope_cluster;
}
+static llvm::RoundingMode getLLVMRoundingMode(NVVM::FPRoundingMode rndMode) {
+ switch (rndMode) {
+ case NVVM::FPRoundingMode::NONE:
+ case NVVM::FPRoundingMode::RN:
+ return llvm::RoundingMode::NearestTiesToEven;
+ case NVVM::FPRoundingMode::RM:
+ return llvm::RoundingMode::TowardNegative;
+ case NVVM::FPRoundingMode::RP:
+ return llvm::RoundingMode::TowardPositive;
+ case NVVM::FPRoundingMode::RZ:
+ return llvm::RoundingMode::TowardZero;
+ default:
+ llvm_unreachable("unsupported rounding mode for nvvm fp arithmetic");
+ }
+}
+
// Calls an LLVM intrinsic on the given operands. For f32/f64 vector types,
// the intrinsic is called per-element and the results are packed back into a
// vector. If retType is non-null, it is forwarded as the return-type
@@ -489,14 +505,9 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
{llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
{llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
- static constexpr llvm::RoundingMode roundingModes[5] = {
- llvm::RoundingMode::NearestTiesToEven,
- llvm::RoundingMode::NearestTiesToEven, llvm::RoundingMode::TowardNegative,
- llvm::RoundingMode::TowardPositive, llvm::RoundingMode::TowardZero};
-
llvm::Intrinsic::ID id = addIDs[isFTZ][isSat];
- llvm::Value *rnd = builder.getInt32(
- static_cast<int>(roundingModes[static_cast<unsigned>(rndMode)]));
+ llvm::Value *rnd =
+ builder.getInt32(static_cast<int>(getLLVMRoundingMode(rndMode)));
// For f64 vector addition, and f32 vector addition with saturation,
// we need to scalarize the intrinsic call.
>From d0b4cdc8db9ad27625eadc1ca6d8104b76fd362a Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 25 Aug 2026 13:08:42 +0000
Subject: [PATCH 10/16] address comments
---
llvm/lib/IR/AutoUpgrade.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 35 +++++++++----------
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 9 ++---
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 13 ++++---
4 files changed, 29 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index e860a8f92f0f7..a19b4ef1a78ab 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1424,7 +1424,7 @@ static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
getNVVMFAddUpgrade(StringRef Name) {
auto [Modifiers, Type] = Name.rsplit('.');
- if (Type != "f" && Type != "d" && Type != "f16" && Type != "v2f16")
+ if (!is_contained({"f", "d", "f16", "v2f16"}, Type))
return std::nullopt;
std::optional<llvm::RoundingMode> RoundingMode =
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index b3a597a0a6708..b91da8f464d90 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7183,28 +7183,27 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
APFloat::roundingMode RoundingMode) {
const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
+ const bool IsSat = nvvm::FAddShouldSaturate(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
- case MVT::f16:
- if (nvvm::FAddShouldSaturate(IID))
- return IsFTZ ? NVPTXISD::SUB_RN_FTZ_SAT : NVPTXISD::SUB_RN_SAT;
- return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
+ case MVT::f16: {
+ static constexpr unsigned SubRNOpcodes[2][2] = {
+ {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_SAT},
+ {NVPTXISD::SUB_RN_FTZ, NVPTXISD::SUB_RN_FTZ_SAT}};
+ return SubRNOpcodes[IsFTZ][IsSat];
+ }
case MVT::bf16:
return NVPTXISD::SUB_RN;
- case MVT::f32:
- if (!VT.isVector() || nvvm::FAddShouldSaturate(IID))
+ case MVT::f32: {
+ // for f32x2 inputs
+ if (!VT.isVector() || IsSat)
return 0;
- switch (RoundingMode) {
- case APFloat::rmNearestTiesToEven:
- return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
- case APFloat::rmTowardZero:
- return IsFTZ ? NVPTXISD::SUB_RZ_FTZ : NVPTXISD::SUB_RZ;
- case APFloat::rmTowardNegative:
- return IsFTZ ? NVPTXISD::SUB_RM_FTZ : NVPTXISD::SUB_RM;
- case APFloat::rmTowardPositive:
- return IsFTZ ? NVPTXISD::SUB_RP_FTZ : NVPTXISD::SUB_RP;
- default:
- llvm_unreachable("Unexpected fadd rounding mode");
- }
+ static constexpr unsigned SubF32x2Opcodes[4][2] = {
+ {NVPTXISD::SUB_RZ, NVPTXISD::SUB_RZ_FTZ}, // RZ
+ {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_FTZ}, // RN
+ {NVPTXISD::SUB_RP, NVPTXISD::SUB_RP_FTZ}, // RP
+ {NVPTXISD::SUB_RM, NVPTXISD::SUB_RM_FTZ}}; // RM
+ return SubF32x2Opcodes[static_cast<unsigned>(RoundingMode)][IsFTZ];
+ }
default:
return 0;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 127991d012593..93d2490f7e500 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2279,14 +2279,11 @@ def fp_rnd_rp_imm : FPRndModeImm<"TowardPositive">;
foreach t = [F16RT, F16X2RT] in
foreach ftz = ["", "ftz"] in
foreach sat = ["", "sat"] in {
- // drop empty strings from the modifier list
- defvar mods = !filter(m, [ftz, sat], !not(!empty(m)));
- defvar suffix = !listconcat(mods, [t.PtxType]);
defvar add_intrin =
- !cast<Intrinsic>(!interleave(!listconcat(["int_nvvm_fadd"], mods), "_"));
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret);
- def INT_NVVM_ADD_RN_ # !toupper(!interleave(suffix, "_")) :
- F_MATH_2_RND_TY<!interleave(!listconcat(["add", "rn"], suffix), "."), t,
+ def INT_NVVM_ADD_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
+ F_MATH_2_RND_TY<StrJoin<".", ["add", "rn", ftz, sat, t.PtxType]>.ret, t,
add_intrin, fp_rnd_rn_imm>;
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index ba6eecb58144d..612425754e620 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -448,9 +448,9 @@ getFenceProxySyncRestrictID(NVVM::MemOrderKind order) {
nvvm_fence_proxy_async_generic_release_sync_restrict_space_cta_scope_cluster;
}
-static llvm::RoundingMode getLLVMRoundingMode(NVVM::FPRoundingMode rndMode) {
+static llvm::RoundingMode
+getLLVMRoundingModeForFPArith(NVVM::FPRoundingMode rndMode) {
switch (rndMode) {
- case NVVM::FPRoundingMode::NONE:
case NVVM::FPRoundingMode::RN:
return llvm::RoundingMode::NearestTiesToEven;
case NVVM::FPRoundingMode::RM:
@@ -460,7 +460,10 @@ static llvm::RoundingMode getLLVMRoundingMode(NVVM::FPRoundingMode rndMode) {
case NVVM::FPRoundingMode::RZ:
return llvm::RoundingMode::TowardZero;
default:
- llvm_unreachable("unsupported rounding mode for nvvm fp arithmetic");
+ // default rounding mode is RN
+ assert(rndMode == NVVM::FPRoundingMode::NONE &&
+ "unsupported rounding mode for nvvm fp arithmetic");
+ return llvm::RoundingMode::NearestTiesToEven;
}
}
@@ -506,8 +509,8 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
{llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
llvm::Intrinsic::ID id = addIDs[isFTZ][isSat];
- llvm::Value *rnd =
- builder.getInt32(static_cast<int>(getLLVMRoundingMode(rndMode)));
+ llvm::Value *rnd = builder.getInt32(
+ static_cast<int>(getLLVMRoundingModeForFPArith(rndMode)));
// For f64 vector addition, and f32 vector addition with saturation,
// we need to scalarize the intrinsic call.
>From 2129c7b4f66f4454ec2b6c285ff89ce305a2e59d Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 25 Aug 2026 14:26:58 +0000
Subject: [PATCH 11/16] address comments
---
llvm/include/llvm/IR/IntrinsicsNVVM.td | 2 +-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 31 ++--------------
llvm/lib/Analysis/ConstantFolding.cpp | 40 ++++++++++-----------
llvm/lib/IR/NVVMIntrinsicUtils.cpp | 4 +--
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 12 ++++---
5 files changed, 31 insertions(+), 58 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 489f45e19cab0..c50609f9fc5ee 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1818,7 +1818,7 @@ let TargetPrefix = "nvvm" in {
Range<ArgIndex<2>, 0, 4>,
ArgInfo<ArgIndex<2>,
[ArgName<"rnd">,
- ImmArgPrinter<"printFAddRoundingMode">]>] in
+ ImmArgPrinter<"printFPRoundingMode">]>] in
foreach ftz = ["", "_ftz"] in
foreach sat = ["", "_sat"] in
def int_nvvm_fadd # ftz # sat :
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index c887608aaf350..950f852ba820b 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -15,7 +15,6 @@
#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
#define LLVM_IR_NVVMINTRINSICUTILS_H
-#include <cassert>
#include <stdint.h>
#include "llvm/ADT/APFloat.h"
@@ -179,7 +178,7 @@ LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
const Constant *ImmArgVal);
LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
const Constant *ImmArgVal);
-LLVM_ABI void printFAddRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
+LLVM_ABI void printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
@@ -612,33 +611,7 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
return DenormalMode::getIEEE();
}
-inline bool IsFAddIntrinsic(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd:
- case Intrinsic::nvvm_fadd_ftz:
- case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_ftz_sat:
- return true;
- default:
- return false;
- }
-}
-
-inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
- assert(IsFAddIntrinsic(IntrinsicID) &&
- "Checking FTZ flag for invalid NVVM add intrinsic");
- return IntrinsicID == Intrinsic::nvvm_fadd_ftz ||
- IntrinsicID == Intrinsic::nvvm_fadd_ftz_sat;
-}
-
-inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
- assert(IsFAddIntrinsic(IntrinsicID) &&
- "Checking sat flag for invalid NVVM add intrinsic");
- return IntrinsicID == Intrinsic::nvvm_fadd_sat ||
- IntrinsicID == Intrinsic::nvvm_fadd_ftz_sat;
-}
-
-inline APFloat::roundingMode GetFAddRoundingMode(const Value *ImmArgVal) {
+inline APFloat::roundingMode GetRoundingModeFromImmArg(const Value *ImmArgVal) {
return static_cast<APFloat::roundingMode>(
cast<ConstantInt>(ImmArgVal)->getSExtValue());
}
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 8092250ec8f5d..7eb44a398ebca 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -4155,32 +4155,28 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
}
}
}
- }
- }
-
- if (IntrinsicID == Intrinsic::nvvm_fadd ||
- IntrinsicID == Intrinsic::nvvm_fadd_ftz) {
- const auto *Op1 = dyn_cast<ConstantFP>(Operands[0]);
- const auto *Op2 = dyn_cast<ConstantFP>(Operands[1]);
- if (!Op1 || !Op2)
- return nullptr;
- bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
- APFloat A =
- IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
- APFloat B =
- IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
+ // TODO: Add constant folding for the _sat variants.
+ if (IntrinsicID == Intrinsic::nvvm_fadd ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz) {
+ bool IsFTZ = IntrinsicID == Intrinsic::nvvm_fadd_ftz;
+ APFloat A =
+ IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
+ APFloat B =
+ IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
- APFloat Res = A;
- APFloat::opStatus Status =
- Res.add(B, nvvm::GetFAddRoundingMode(Operands[2]));
+ APFloat Res = A;
+ APFloat::opStatus Status =
+ Res.add(B, nvvm::GetRoundingModeFromImmArg(Operands[2]));
- if (!Res.isNaN() &&
- (Status == APFloat::opOK || Status == APFloat::opInexact)) {
- Res = IsFTZ ? FTZPreserveSign(Res) : Res;
- return ConstantFP::get(Ty, Res);
+ if (!Res.isNaN() &&
+ (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+ Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+ return ConstantFP::get(Ty, Res);
+ }
+ return nullptr;
+ }
}
- return nullptr;
}
if (IntrinsicID == Intrinsic::smul_fix ||
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index 35fffb0be73ff..8069cf7eb6f1e 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -161,9 +161,9 @@ void nvvm::printTensormapSwizzleAtomicity(raw_ostream &OS,
}
}
-void nvvm::printFAddRoundingMode(raw_ostream &OS, const Constant *ImmArgVal) {
+void nvvm::printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal) {
if (isa<ConstantInt>(ImmArgVal))
- OS << nvvm::GetRoundingModeName(nvvm::GetFAddRoundingMode(ImmArgVal));
+ OS << nvvm::GetRoundingModeName(nvvm::GetRoundingModeFromImmArg(ImmArgVal));
}
void nvvm::printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index b91da8f464d90..6557dac09a679 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7182,8 +7182,10 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
APFloat::roundingMode RoundingMode) {
- const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
- const bool IsSat = nvvm::FAddShouldSaturate(IID);
+ const bool IsFTZ =
+ IID == Intrinsic::nvvm_fadd_ftz || IID == Intrinsic::nvvm_fadd_ftz_sat;
+ const bool IsSat =
+ IID == Intrinsic::nvvm_fadd_sat || IID == Intrinsic::nvvm_fadd_ftz_sat;
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16: {
static constexpr unsigned SubRNOpcodes[2][2] = {
@@ -7242,8 +7244,10 @@ static bool isSupportedFAdd(EVT VT, const NVPTXSubtarget &STI,
return false;
const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
- const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
- const bool IsSat = nvvm::FAddShouldSaturate(IID);
+ const bool IsFTZ =
+ IID == Intrinsic::nvvm_fadd_ftz || IID == Intrinsic::nvvm_fadd_ftz_sat;
+ const bool IsSat =
+ IID == Intrinsic::nvvm_fadd_sat || IID == Intrinsic::nvvm_fadd_ftz_sat;
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
return IsRN;
>From 9be0517655f654c1d59c2a30aba90647e2284e08 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 25 Aug 2026 14:45:56 +0000
Subject: [PATCH 12/16] use StrJoin instead of subst
---
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 130 +++++++++++------------
1 file changed, 65 insertions(+), 65 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 93d2490f7e500..fa3df8abc9223 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2054,7 +2054,7 @@ def : Pat<(int_nvvm_cos_approx_f f32:$a), (COS_APPROX_f32 $a, NoFTZ)>;
// Fma
//
-defvar FPRoundingModes = ["_rn", "_rz", "_rm", "_rp"];
+defvar FPRoundingModes = ["rn", "rz", "rm", "rp"];
class FMA_TUPLE<string V, Intrinsic I, NVPTXRegClass RC,
list<Predicate> Preds = []> {
@@ -2130,21 +2130,18 @@ multiclass FMA_INST {
defm INT_NVVM_FMA : FMA_INST;
-foreach rnd = FPRoundingModes in {
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_FMA # rnd # sat # _f32_ # type :
+foreach rnd = FPRoundingModes in
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_FMA_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B16:$b, B32:$c),
- !subst("_", ".", "fma" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fma" # rnd # sat # "_f")
+ StrJoin<".", ["fma", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", rnd, sat, "f"]>.ret)
(f32 (fpextend type:$a)),
(f32 (fpextend type:$b)),
f32:$c))]>,
Requires<[SM100]>;
- }
- }
-}
// Pattern for llvm.fma.f32 intrinsic when there is no FTZ flag
let Predicates = [SM100, doNoF32FTZ] in {
@@ -2293,41 +2290,41 @@ foreach t = [BF16RT, BF16X2RT] in
BF16ArithPreds>;
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
-
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in
- def INT_NVVM_ADD # !toupper(rnd # ftz # sat) # _F :
- F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # ftz # sat # "_f32"),
- F32RT, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
+
+ foreach ftz = ["", "ftz"] in {
+ foreach sat = ["", "sat"] in
+ def INT_NVVM_ADD_ # !toupper(StrJoin<"_", [rnd, ftz, sat]>.ret) # _F :
+ F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, ftz, sat, "f32"]>.ret, F32RT,
+ !cast<Intrinsic>(
+ StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret),
rnd_imm>;
- def INT_NVVM_ADD # !toupper(rnd # ftz) # _F32X2 :
- F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # ftz # "_f32x2"), F32X2RT,
- !cast<Intrinsic>("int_nvvm_fadd" # ftz), rnd_imm,
- [hasF32x2Instructions]>;
+ def INT_NVVM_ADD_ # !toupper(StrJoin<"_", [rnd, ftz]>.ret) # _F32X2 :
+ F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, ftz, "f32x2"]>.ret, F32X2RT,
+ !cast<Intrinsic>(
+ StrJoin<"_", ["int_nvvm_fadd", ftz]>.ret),
+ rnd_imm, [hasF32x2Instructions]>;
}
- def INT_NVVM_ADD # !toupper(rnd) # _D :
- F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # "_f64"), F64RT,
- int_nvvm_fadd, rnd_imm>;
+ def INT_NVVM_ADD_ # !toupper(rnd) # _D :
+ F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, "f64"]>.ret, F64RT, int_nvvm_fadd,
+ rnd_imm>;
}
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_ADD # rnd # sat # _f32_ # type :
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_ADD_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
- !subst("_", ".", "add" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fadd" # sat)
+ StrJoin<".", ["add", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", sat]>.ret)
(f32 (fpextend type:$a)),
f32:$b, rnd_imm))]>,
Requires<[SM100]>;
- }
- }
}
// Pattern for fadd when there is no FTZ flag
@@ -2348,67 +2345,70 @@ def sub_rn_ftz_sat :
SDNode<"NVPTXISD::SUB_RN_FTZ_SAT", SDTFPBinOp>;
foreach rnd = FPRoundingModes in
- foreach ftz = ["", "_ftz"] in
- def sub # rnd # ftz :
- SDNode<"NVPTXISD::SUB" # !toupper(rnd # ftz), SDTFPBinOp>;
+ foreach ftz = ["", "ftz"] in {
+ defvar variant = StrJoin<"_", [rnd, ftz]>.ret;
+ def sub_ # variant :
+ SDNode<"NVPTXISD::SUB_" # !toupper(variant), SDTFPBinOp>;
+ }
-class INT_NVVM_SUB<RegTyInfo TyInfo, string variant> :
+class INT_NVVM_SUB<RegTyInfo TyInfo, list<string> variant> :
BasicNVPTXInst<(outs TyInfo.RC:$dst), (ins TyInfo.RC:$a, TyInfo.RC:$b),
- !subst("_", ".", "sub" # variant # "." # TyInfo.PtxType),
- [(set TyInfo.Ty:$dst,
- (!cast<SDNode>("sub" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
+ StrJoin<".", !listconcat(["sub"], variant, [TyInfo.PtxType])>.ret,
+ [(set TyInfo.Ty:$dst,
+ (!cast<SDNode>(StrJoin<"_", !listconcat(["sub"], variant)>.ret)
+ TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
foreach t = [F16RT, F16X2RT] in
- foreach ftz = ["", "_ftz"] in
- foreach sat = ["", "_sat"] in
- def INT_NVVM_SUB_RN # !toupper(ftz # sat # "_" # t.PtxType) :
- INT_NVVM_SUB<t, "_rn" # ftz # sat>;
+ foreach ftz = ["", "ftz"] in
+ foreach sat = ["", "sat"] in
+ def INT_NVVM_SUB_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
+ INT_NVVM_SUB<t, ["rn", ftz, sat]>;
let Predicates = BF16ArithPreds in
foreach t = [BF16RT, BF16X2RT] in
- def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, "_rn">;
+ def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, ["rn"]>;
let Predicates = [hasF32x2Instructions] in
foreach rnd = FPRoundingModes in
- foreach ftz = ["", "_ftz"] in
- def INT_NVVM_SUB # rnd # ftz # _F32X2 : INT_NVVM_SUB<F32X2RT, rnd # ftz>;
+ foreach ftz = ["", "ftz"] in
+ def INT_NVVM_SUB_ # StrJoin<"_", [rnd, ftz]>.ret # _F32X2 :
+ INT_NVVM_SUB<F32X2RT, [rnd, ftz]>;
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat);
- def INT_NVVM_SUB # rnd # ftz # sat # _F :
+ foreach ftz = ["", "ftz"] in {
+ foreach sat = ["", "sat"] in {
+ defvar add_intrin =
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret);
+ def INT_NVVM_SUB_ # StrJoin<"_", [rnd, ftz, sat]>.ret # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
- !subst("_", ".", "sub" # rnd # ftz # sat # "_f32"),
+ StrJoin<".", ["sub", rnd, ftz, sat, "f32"]>.ret,
[(set f32:$dst,
(add_intrin f32:$a, (f32 (fneg f32:$b)), rnd_imm))]>;
}
}
- def INT_NVVM_SUB # rnd # _D :
+ def INT_NVVM_SUB_ # rnd # _D :
BasicNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
- !subst("_", ".", "sub" # rnd # "_f64"),
+ StrJoin<".", ["sub", rnd, "f64"]>.ret,
[(set f64:$dst,
(int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), rnd_imm))]>;
}
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_SUB # rnd # sat # _f32_ # type :
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_SUB_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
- !subst("_", ".", "sub" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fadd" # sat)
+ StrJoin<".", ["sub", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", sat]>.ret)
(f32 (fpextend type:$a)),
(f32 (fneg f32:$b)), rnd_imm))]>,
Requires<[SM100]>;
- }
- }
}
// Pattern for fsub when there is no FTZ flag
>From 01f2defed32b32c619f7d0aa5471969919d1728d Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 26 Aug 2026 12:21:40 +0000
Subject: [PATCH 13/16] address comment
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 6557dac09a679..a946ceaa35d81 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7237,6 +7237,9 @@ static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(Opc, SDLoc(N), VT, SubOp1, SubOp2);
}
+// TODO: Remove the type-legality checks here once
+// https://github.com/llvm/llvm-project/pull/172442 lands, adding support for
+// explicit type constraints for overloaded intrinsics in tablegen.
static bool isSupportedFAdd(EVT VT, const NVPTXSubtarget &STI,
Intrinsic::ID IID,
APFloat::roundingMode RoundingMode) {
@@ -7262,13 +7265,10 @@ static bool isSupportedFAdd(EVT VT, const NVPTXSubtarget &STI,
}
}
-static SDValue diagnoseInvalidFAdd(SDNode *N, SelectionDAG &DAG,
- const NVPTXSubtarget &STI, Intrinsic::ID IID,
- APFloat::roundingMode RoundingMode) {
+static SDValue diagnoseUnsupportedFAdd(SDNode *N, SelectionDAG &DAG,
+ Intrinsic::ID IID,
+ APFloat::roundingMode RoundingMode) {
const EVT VT = N->getValueType(0);
- if (isSupportedFAdd(VT, STI, IID, RoundingMode))
- return SDValue();
-
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
DAG.getMachineFunction().getFunction(),
Twine(Intrinsic::getBaseName(IID)) + " with rounding mode " +
@@ -7293,8 +7293,8 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
case Intrinsic::nvvm_fadd_ftz_sat: {
const auto RoundingMode = static_cast<APFloat::roundingMode>(
N->getConstantOperandAPInt(3).getSExtValue());
- if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, STI, IID, RoundingMode))
- return V;
+ if (!isSupportedFAdd(N->getValueType(0), STI, IID, RoundingMode))
+ return diagnoseUnsupportedFAdd(N, DCI.DAG, IID, RoundingMode);
return combineFAddWithNeg(N, DCI.DAG, IID, RoundingMode);
}
}
>From 3342eb662dc72afa0efc62ca5b092427f9898439 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 2 Sep 2026 14:13:55 +0000
Subject: [PATCH 14/16] clean-up
---
.../NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp | 10 ++
.../NVPTX/MCTargetDesc/NVPTXInstPrinter.h | 2 +
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 4 +
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 91 ++++++++++---------
4 files changed, 64 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index 0875875611f2c..19855fd647eca 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -150,6 +150,16 @@ void NVPTXInstPrinter::printCvtMode(const MCInst *MI, int OpNum,
llvm_unreachable("Invalid conversion modifier");
}
+void NVPTXInstPrinter::printFPRoundingMode(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &,
+ raw_ostream &O) {
+ const auto RM =
+ static_cast<APFloat::roundingMode>(MI->getOperand(OpNum).getImm());
+ const StringRef Name = nvvm::GetRoundingModeName(RM);
+ assert(!Name.empty() && "Invalid FP rounding mode");
+ O << Name;
+}
+
void NVPTXInstPrinter::printFTZFlag(const MCInst *MI, int OpNum,
const MCSubtargetInfo &, raw_ostream &O) {
const MCOperand &MO = MI->getOperand(OpNum);
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index f6a086e6a3379..d98b9998a9179 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -42,6 +42,8 @@ class NVPTXInstPrinter : public MCInstPrinter {
raw_ostream &O, StringRef Modifier = {});
void printCmpMode(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
+ void printFPRoundingMode(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &STI, raw_ostream &O);
void printAtomicCode(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
void printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 53d682468f7f5..846bcc0557464 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -114,6 +114,10 @@ def PrmtMode : Operand<i32> {
let PrintMethod = "printPrmtMode";
}
+def FPRoundingMode : Operand<i32> {
+ let PrintMethod = "printFPRoundingMode";
+}
+
class NVPTXAddressSpace<string enum, string name, string suffix = "." # name> {
string Suffix = suffix;
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index fa3df8abc9223..f41a1fffc187d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1695,6 +1695,14 @@ class F_MATH_2_RND_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP, TImmLeaf rnd,
[(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1, rnd))]>,
Requires<Preds>;
+class F_MATH_2_RNDOP_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP,
+ list<Predicate> Preds = []>
+ : BasicFlagsNVPTXInst<(outs t.RC:$dst),
+ (ins t.RC:$src0, t.RC:$src1), (ins FPRoundingMode:$rnd),
+ OpcStr,
+ [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1, timm:$rnd))]>,
+ Requires<Preds>;
+
class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,
NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass,
NVPTXRegClass s2_regclass, Intrinsic IntOP, list<Predicate> Preds = []>
@@ -2273,45 +2281,42 @@ def fp_rnd_rz_imm : FPRndModeImm<"TowardZero">;
def fp_rnd_rm_imm : FPRndModeImm<"TowardNegative">;
def fp_rnd_rp_imm : FPRndModeImm<"TowardPositive">;
+// f16/f16x2
foreach t = [F16RT, F16X2RT] in
foreach ftz = ["", "ftz"] in
- foreach sat = ["", "sat"] in {
- defvar add_intrin =
- !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret);
-
+ foreach sat = ["", "sat"] in
def INT_NVVM_ADD_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
- F_MATH_2_RND_TY<StrJoin<".", ["add", "rn", ftz, sat, t.PtxType]>.ret, t,
- add_intrin, fp_rnd_rn_imm>;
- }
+ F_MATH_2_RND_TY<StrJoin<".", ["add.rn", ftz, sat, t.PtxType]>.ret, t,
+ !cast<Intrinsic>(
+ StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret),
+ fp_rnd_rn_imm>;
+// bf16/bf16x2
foreach t = [BF16RT, BF16X2RT] in
def INT_NVVM_ADD_RN_ # !toupper(t.PtxType) :
F_MATH_2_RND_TY<"add.rn." # t.PtxType, t, int_nvvm_fadd, fp_rnd_rn_imm,
BF16ArithPreds>;
-foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
-
- foreach ftz = ["", "ftz"] in {
- foreach sat = ["", "sat"] in
- def INT_NVVM_ADD_ # !toupper(StrJoin<"_", [rnd, ftz, sat]>.ret) # _F :
- F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, ftz, sat, "f32"]>.ret, F32RT,
+// f32/f32x2/f64
+foreach ftz = ["", "ftz"] in {
+ foreach sat = ["", "sat"] in
+ def StrJoin<"_", ["INT_NVVM_ADD", !toupper(ftz), !toupper(sat), "F"]>.ret :
+ F_MATH_2_RNDOP_TY<StrJoin<".", ["add.${rnd}", ftz, sat, "f32"]>.ret,
+ F32RT,
!cast<Intrinsic>(
- StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret),
- rnd_imm>;
+ StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret)>;
- def INT_NVVM_ADD_ # !toupper(StrJoin<"_", [rnd, ftz]>.ret) # _F32X2 :
- F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, ftz, "f32x2"]>.ret, F32X2RT,
+ def StrJoin<"_", ["INT_NVVM_ADD", !toupper(ftz), "F32X2"]>.ret :
+ F_MATH_2_RNDOP_TY<StrJoin<".", ["add.${rnd}", ftz, "f32x2"]>.ret, F32X2RT,
!cast<Intrinsic>(
StrJoin<"_", ["int_nvvm_fadd", ftz]>.ret),
- rnd_imm, [hasF32x2Instructions]>;
- }
-
- def INT_NVVM_ADD_ # !toupper(rnd) # _D :
- F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, "f64"]>.ret, F64RT, int_nvvm_fadd,
- rnd_imm>;
+ [hasF32x2Instructions]>;
}
+def INT_NVVM_ADD_D :
+ F_MATH_2_RNDOP_TY<"add.${rnd}.f64", F64RT, int_nvvm_fadd>;
+
+// mixed precision
foreach rnd = FPRoundingModes in {
defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
@@ -2358,44 +2363,44 @@ class INT_NVVM_SUB<RegTyInfo TyInfo, list<string> variant> :
(!cast<SDNode>(StrJoin<"_", !listconcat(["sub"], variant)>.ret)
TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
+// f16/f16x2
foreach t = [F16RT, F16X2RT] in
foreach ftz = ["", "ftz"] in
foreach sat = ["", "sat"] in
def INT_NVVM_SUB_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
INT_NVVM_SUB<t, ["rn", ftz, sat]>;
+// bf16/bf16x2
let Predicates = BF16ArithPreds in
foreach t = [BF16RT, BF16X2RT] in
def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, ["rn"]>;
+// f32x2
let Predicates = [hasF32x2Instructions] in
foreach rnd = FPRoundingModes in
foreach ftz = ["", "ftz"] in
def INT_NVVM_SUB_ # StrJoin<"_", [rnd, ftz]>.ret # _F32X2 :
INT_NVVM_SUB<F32X2RT, [rnd, ftz]>;
-foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
-
- foreach ftz = ["", "ftz"] in {
- foreach sat = ["", "sat"] in {
- defvar add_intrin =
- !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret);
- def INT_NVVM_SUB_ # StrJoin<"_", [rnd, ftz, sat]>.ret # _F :
- BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
- StrJoin<".", ["sub", rnd, ftz, sat, "f32"]>.ret,
- [(set f32:$dst,
- (add_intrin f32:$a, (f32 (fneg f32:$b)), rnd_imm))]>;
- }
+// f32/f64
+foreach ftz = ["", "ftz"] in
+ foreach sat = ["", "sat"] in {
+ defvar add_intrin =
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret);
+ def StrJoin<"_", ["INT_NVVM_SUB", !toupper(ftz), !toupper(sat), "F"]>.ret :
+ BasicFlagsNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
+ (ins FPRoundingMode:$rnd),
+ StrJoin<".", ["sub.${rnd}", ftz, sat, "f32"]>.ret,
+ [(set f32:$dst,
+ (add_intrin f32:$a, (f32 (fneg f32:$b)), timm:$rnd))]>;
}
- def INT_NVVM_SUB_ # rnd # _D :
- BasicNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
- StrJoin<".", ["sub", rnd, "f64"]>.ret,
- [(set f64:$dst,
- (int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), rnd_imm))]>;
-}
+def INT_NVVM_SUB_D :
+ BasicFlagsNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
+ (ins FPRoundingMode:$rnd), "sub.${rnd}.f64",
+ [(set f64:$dst, (int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), timm:$rnd))]>;
+// mixed precision
foreach rnd = FPRoundingModes in {
defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
>From 1fc66e4a60fb48db20589c19a0d10edaaecd71e3 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Thu, 3 Sep 2026 18:06:31 +0000
Subject: [PATCH 15/16] remove duplication after rebase
---
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index 09ad23985b092..06c5069d6f984 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -411,7 +411,6 @@ static Value *MakeHalfType(Function *Intrinsic, unsigned BuiltinID,
ArgValue = CGF.Builder.CreateBitCast(ArgValue, PTy);
Args.push_back(ArgValue);
}
- Args.append(TrailingArgs.begin(), TrailingArgs.end());
llvm::append_range(Args, TrailingArgs);
appendDefaultIntrinsicArgs(Args, Intrinsic);
>From ec665a1d109a182a293a61d67e9c2fa77bc78604 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Mon, 7 Sep 2026 08:46:21 +0000
Subject: [PATCH 16/16] force refresh pr
More information about the llvm-commits
mailing list