[Mlir-commits] [clang] [llvm] [mlir] [clang][NVPTX][MLIR][NVVM] Add overloaded fadd intrinsics (PR #217336)
Srinivasa Ravi
llvmlistbot at llvm.org
Tue Aug 25 07:46:38 PDT 2026
https://github.com/Wolfram70 updated https://github.com/llvm/llvm-project/pull/217336
>From 76890b9b487c704ab483abec3c733de059b02df8 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 11 Aug 2026 14:18:11 +0000
Subject: [PATCH 01/12] [clang][NVPTX][MLIR][NVVM] Add overloaded fadd
intrinsics
This change adds the following overloaded fadd intrinsics with
NVPTX codegen:
- `llvm.nvvm.fadd.<rn/rz/rm/rp>`
- `llvm.nvvm.fadd.<rn/rz/rm/rp>.ftz`
- `llvm.nvvm.fadd.<rn/rz/rm/rp>.sat`
- `llvm.nvvm.fadd.<rn/rz/rm/rp>.ftz.sat`
Auto-upgrades the older non-overloaded intrinsics to the new
ones, and updates clang builtins, CIR codegen, and MLIR NVVM ops
to lower to the new intrinsics.
In the interest of completion, this also adds intrinsics support
for lowering some half-precision additions that were omitted
earlier (`f16/f16x2` without saturation, and `bf16/bf16x2`
additions), and also adds support for the `f32x2` type.
PTX Spec Reference: https://docs.nvidia.com/cuda/developer-preview/13.4/parallel-thread-execution/index.html#floating-point-instructions-add
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 54 ++++
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 61 ++++
.../CIR/CodeGenCUDA/builtins-nvvm-math.cu | 24 ++
clang/test/CodeGen/builtins-nvptx.c | 26 +-
llvm/include/llvm/IR/IntrinsicsNVVM.td | 30 +-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 81 +++--
llvm/lib/Analysis/ConstantFolding.cpp | 40 +--
llvm/lib/IR/AutoUpgrade.cpp | 16 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 113 +++++--
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 122 +++++---
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 24 ++
llvm/test/CodeGen/NVPTX/bf16-add.ll | 33 ++
llvm/test/CodeGen/NVPTX/bf16-sub.ll | 35 +++
llvm/test/CodeGen/NVPTX/f16-add-sat.ll | 63 ----
llvm/test/CodeGen/NVPTX/f16-add.ll | 123 ++++++++
llvm/test/CodeGen/NVPTX/f16-sub-sat.ll | 69 -----
llvm/test/CodeGen/NVPTX/f16-sub.ll | 133 ++++++++
llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll | 60 ++++
llvm/test/CodeGen/NVPTX/fp-add-invalid.ll | 47 +++
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 32 +-
llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll | 64 ++++
llvm/test/CodeGen/NVPTX/fp-fold-sub.ll | 20 +-
llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll | 64 ++--
.../InstCombine/NVPTX/nvvm-intrins.ll | 18 +-
.../InstSimplify/const-fold-nvvm-add.ll | 216 ++++++-------
llvm/unittests/IR/IntrinsicsTest.cpp | 2 +-
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 8 -
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 104 ++-----
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 68 +++--
.../Target/LLVMIR/nvvm/addf/addf_invalid.mlir | 10 -
.../Target/LLVMIR/nvvm/addf/addf_vector.mlir | 257 ++++++----------
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 69 +++--
.../Target/LLVMIR/nvvm/subf/subf_invalid.mlir | 10 -
.../Target/LLVMIR/nvvm/subf/subf_vector.mlir | 284 +++++++-----------
34 files changed, 1429 insertions(+), 951 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/bf16-add.ll
create mode 100644 llvm/test/CodeGen/NVPTX/bf16-sub.ll
delete mode 100644 llvm/test/CodeGen/NVPTX/f16-add-sat.ll
create mode 100644 llvm/test/CodeGen/NVPTX/f16-add.ll
delete mode 100644 llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
create mode 100644 llvm/test/CodeGen/NVPTX/f16-sub.ll
create mode 100644 llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
create mode 100644 llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
create mode 100644 llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 2220639876695..f8da1b062a14e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -49,6 +49,20 @@ static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
.getResult();
}
+/// Emit a CIR LLVMIntrinsicCallOp for a binary NVVM intrinsic.
+/// The result type is inferred from the first argument.
+static mlir::Value emitBinaryNVVMIntrinsic(CIRGenFunction &cgf,
+ const CallExpr *expr,
+ llvm::StringRef intrinsicName) {
+ auto &builder = cgf.getBuilder();
+ mlir::Value lhs = cgf.emitScalarExpr(expr->getArg(0));
+ mlir::Value rhs = cgf.emitScalarExpr(expr->getArg(1));
+ return cir::LLVMIntrinsicCallOp::create(
+ builder, cgf.getLoc(expr->getExprLoc()),
+ builder.getStringAttr(intrinsicName), lhs.getType(), {lhs, rhs})
+ .getResult();
+}
+
static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf,
const CallExpr *expr,
cir::AtomicFetchKind kind,
@@ -794,6 +808,46 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx");
case NVPTX::BI__nvvm_ex2_approx_ftz_f:
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
+ case NVPTX::BI__nvvm_add_rn_f:
+ case NVPTX::BI__nvvm_add_rn_d:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn");
+ case NVPTX::BI__nvvm_add_rz_f:
+ case NVPTX::BI__nvvm_add_rz_d:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz");
+ case NVPTX::BI__nvvm_add_rm_f:
+ case NVPTX::BI__nvvm_add_rm_d:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm");
+ case NVPTX::BI__nvvm_add_rp_f:
+ case NVPTX::BI__nvvm_add_rp_d:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp");
+ case NVPTX::BI__nvvm_add_rn_ftz_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.ftz");
+ case NVPTX::BI__nvvm_add_rz_ftz_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.ftz");
+ case NVPTX::BI__nvvm_add_rm_ftz_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.ftz");
+ case NVPTX::BI__nvvm_add_rp_ftz_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.ftz");
+ case NVPTX::BI__nvvm_add_rn_sat_f:
+ case NVPTX::BI__nvvm_add_rn_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_sat_v2f16:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.sat");
+ case NVPTX::BI__nvvm_add_rz_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.sat");
+ case NVPTX::BI__nvvm_add_rm_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.sat");
+ case NVPTX::BI__nvvm_add_rp_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.sat");
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.ftz.sat");
+ case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.ftz.sat");
+ case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.ftz.sat");
+ case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
+ return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.ftz.sat");
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
cgm.errorNYI(expr->getSourceRange(),
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index 64fdae9d8934d..687a2788946ce 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -427,6 +427,13 @@ static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
CGF.EmitScalarExpr(E->getArg(2))});
}
+static Value *MakeBinaryIntrinsic(unsigned IntrinsicID, const CallExpr *E,
+ CodeGenFunction &CGF) {
+ return CGF.Builder.CreateBinaryIntrinsic(IntrinsicID,
+ CGF.EmitScalarExpr(E->getArg(0)),
+ CGF.EmitScalarExpr(E->getArg(1)));
+}
+
} // namespace
Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
@@ -1134,6 +1141,60 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
case NVPTX::BI__nvvm_ex2_approx_ftz_f:
return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_ex2_approx_ftz,
EmitScalarExpr(E->getArg(0)));
+ case NVPTX::BI__nvvm_add_rn_f:
+ case NVPTX::BI__nvvm_add_rn_d:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn, E, *this);
+ case NVPTX::BI__nvvm_add_rz_f:
+ case NVPTX::BI__nvvm_add_rz_d:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz, E, *this);
+ case NVPTX::BI__nvvm_add_rm_f:
+ case NVPTX::BI__nvvm_add_rm_d:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm, E, *this);
+ case NVPTX::BI__nvvm_add_rp_f:
+ case NVPTX::BI__nvvm_add_rp_d:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_ftz, E, *this);
+ case NVPTX::BI__nvvm_add_rz_ftz_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_ftz, E, *this);
+ case NVPTX::BI__nvvm_add_rm_ftz_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_ftz, E, *this);
+ case NVPTX::BI__nvvm_add_rp_ftz_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_ftz, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rm_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rp_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
+ return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_f16:
+ return MakeHalfType(
+ CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_sat, Builder.getHalfTy()),
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_v2f16:
+ return MakeHalfType(
+ CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_sat,
+ FixedVectorType::get(Builder.getHalfTy(), 2)),
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
+ return MakeHalfType(
+ CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat, Builder.getHalfTy()),
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
+ return MakeHalfType(
+ CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat,
+ FixedVectorType::get(Builder.getHalfTy(), 2)),
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
return MakeLdg(*this, E);
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index a2f0edb6d073b..ede4c0605c98b 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -63,3 +63,27 @@ __device__ double test_ex2_approx_d(double x) {
__device__ float test_ex2_approx_ftz_f(float x) {
return __nvvm_ex2_approx_ftz_f(x);
}
+
+// CIR-LABEL: @_Z13test_add_rn_fff
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rn" {{.*}} : (!cir.float, !cir.float) -> !cir.float
+// LLVM-LABEL: @_Z13test_add_rn_fff
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.rn.f32(float
+__device__ float test_add_rn_f(float x, float y) {
+ return __nvvm_add_rn_f(x, y);
+}
+
+// CIR-LABEL: @_Z13test_add_rz_ddd
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rz" {{.*}} : (!cir.double, !cir.double) -> !cir.double
+// LLVM-LABEL: @_Z13test_add_rz_ddd
+// LLVM: call {{.*}}double @llvm.nvvm.fadd.rz.f64(double
+__device__ double test_add_rz_d(double x, double y) {
+ return __nvvm_add_rz_d(x, y);
+}
+
+// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rm.ftz.sat" {{.*}} : (!cir.float, !cir.float) -> !cir.float
+// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.rm.ftz.sat.f32(float
+__device__ float test_add_rm_ftz_sat_f(float x, float y) {
+ return __nvvm_add_rm_ftz_sat_f(x, y);
+}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 87be7b46aad8e..469aff2691a2a 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -245,7 +245,7 @@ __device__ void nvvm_math(float f1, float f2, double d1, double d2) {
float t3 = __nvvm_sqrt_rn_f(f1);
// CHECK: call float @llvm.nvvm.rcp.rn.f
float t4 = __nvvm_rcp_rn_f(f2);
-// CHECK: call float @llvm.nvvm.add.rn.f
+// CHECK: call float @llvm.nvvm.fadd.rn.f32
float t5 = __nvvm_add_rn_f(f1, f2);
// CHECK: call double @llvm.nvvm.fmax.d
@@ -1548,21 +1548,21 @@ __device__ void nvvm_min_max_sm86() {
// CHECK-LABEL: nvvm_add_fma_f32_sat
__device__ void nvvm_add_fma_f32_sat() {
- // CHECK: call float @llvm.nvvm.add.rn.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rn.sat.f32
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rn.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rn.ftz.sat.f32
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rz.sat.f32
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rz.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rz.ftz.sat.f32
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rm.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rm.sat.f32
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rm.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rm.ftz.sat.f32
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rp.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rp.sat.f32
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rp.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.rp.ftz.sat.f32
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1592,13 +1592,13 @@ __device__ void nvvm_add_fma_f32_sat() {
// CHECK-LABEL: nvvm_add_mul_f16_sat
__device__ void nvvm_add_mul_f16_sat() {
- // CHECK: call half @llvm.nvvm.add.rn.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.rn.sat.f16
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.add.rn.ftz.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.rn.ftz.sat.f16
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
- // CHECK: call <2 x half> @llvm.nvvm.add.rn.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 69905899f3007..c3ddb9bf77a7b 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1708,31 +1708,17 @@ let TargetPrefix = "nvvm" in {
}
//
- // Add
+ // FAdd
//
let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative,
- IntrNoCreateUndefOrPoison] in {
- foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in {
- def int_nvvm_add # rnd # ftz # sat # _f : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty]>;
- } // sat
- } // ftz
- def int_nvvm_add # rnd # _d : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty]>;
- }
-
- foreach ftz = ["", "_ftz"] in {
- def int_nvvm_add_rn # ftz # _sat_f16 : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_half_ty], [llvm_half_ty, llvm_half_ty]>;
-
- def int_nvvm_add_rn # ftz # _sat_v2f16 : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_v2f16_ty, llvm_v2f16_ty]>;
-
- } // ftz
- }
+ IntrNoCreateUndefOrPoison] in
+ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def int_nvvm_fadd # rnd # ftz # sat :
+ DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>]>;
//
// Dot Product
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index b38ebc3e2b309..efbb430ae5cd1 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -594,42 +594,75 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd_rm_ftz_sat:
+ case Intrinsic::nvvm_fadd_rn_ftz_sat:
+ case Intrinsic::nvvm_fadd_rp_ftz_sat:
+ case Intrinsic::nvvm_fadd_rz_ftz_sat:
return true;
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_sat:
+ case Intrinsic::nvvm_fadd_rn_sat:
+ case Intrinsic::nvvm_fadd_rp_sat:
+ case Intrinsic::nvvm_fadd_rz_sat:
return false;
}
llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
}
+inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
+ switch (IntrinsicID) {
+ case Intrinsic::nvvm_fadd_rm_sat:
+ case Intrinsic::nvvm_fadd_rn_sat:
+ case Intrinsic::nvvm_fadd_rp_sat:
+ case Intrinsic::nvvm_fadd_rz_sat:
+ case Intrinsic::nvvm_fadd_rm_ftz_sat:
+ case Intrinsic::nvvm_fadd_rn_ftz_sat:
+ case Intrinsic::nvvm_fadd_rp_ftz_sat:
+ case Intrinsic::nvvm_fadd_rz_ftz_sat:
+ return true;
+
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
+ return false;
+ }
+ llvm_unreachable("Checking sat flag for invalid NVVM add intrinsic");
+}
+
inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rm_ftz_f:
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rm_sat:
+ case Intrinsic::nvvm_fadd_rm_ftz_sat:
return APFloat::rmTowardNegative;
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rn_ftz_f:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rn_sat:
+ case Intrinsic::nvvm_fadd_rn_ftz_sat:
return APFloat::rmNearestTiesToEven;
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rp_ftz_f:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rp_sat:
+ case Intrinsic::nvvm_fadd_rp_ftz_sat:
return APFloat::rmTowardPositive;
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rz_ftz_f:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd_rz_sat:
+ case Intrinsic::nvvm_fadd_rz_ftz_sat:
return APFloat::rmTowardZero;
}
llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index c08265be71f33..2ccb1d2b32c92 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1996,18 +1996,14 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
return !IsStrictFP;
// NVVM add intrinsics with explicit rounding modes
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f:
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
// NVVM div intrinsics with explicit rounding modes
case Intrinsic::nvvm_div_rm_d:
@@ -3619,18 +3615,14 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
return ConstantFP::get(Ty, Res);
}
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f: {
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz: {
bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 9865816e36959..1b001d84eda95 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1944,6 +1944,22 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
return NewFn != F;
}
+ // Upgrade the FP add intrinsics, which are overloaded on the operand type
+ // llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
+ // llvm.nvvm.fadd.<rnd>{.ftz}{.sat}.<mangled type>
+ if (Name.starts_with("add.")) {
+ auto [Base, TypeSuffix] = Name.rsplit('.');
+ if (TypeSuffix == "f" || TypeSuffix == "d" || TypeSuffix == "f16" ||
+ TypeSuffix == "v2f16") {
+ IID = Intrinsic::lookupIntrinsicID(("llvm.nvvm.f" + Base).str());
+ if (IID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
+ {F->getReturnType()});
+ return true;
+ }
+ }
+ }
+
// The following nvvm intrinsics correspond exactly to an LLVM idiom, but
// not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
//
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 9df1419d7349d..86f305fb90dd6 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -53,6 +53,7 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Alignment.h"
@@ -7184,22 +7185,42 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
}
}
-static unsigned getF16SubOpc(Intrinsic::ID AddIntrinsicID) {
- switch (AddIntrinsicID) {
+static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID) {
+ const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
+ switch (VT.getScalarType().getSimpleVT().SimpleTy) {
+ case MVT::f16:
+ if (nvvm::FAddShouldSaturate(IID))
+ return IsFTZ ? NVPTXISD::SUB_RN_FTZ_SAT : NVPTXISD::SUB_RN_SAT;
+ return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
+ case MVT::bf16:
+ return NVPTXISD::SUB_RN;
+ case MVT::f32:
+ if (!VT.isVector() || nvvm::FAddShouldSaturate(IID))
+ return 0;
+ switch (nvvm::GetFAddRoundingMode(IID)) {
+ case APFloat::rmNearestTiesToEven:
+ return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
+ case APFloat::rmTowardZero:
+ return IsFTZ ? NVPTXISD::SUB_RZ_FTZ : NVPTXISD::SUB_RZ;
+ case APFloat::rmTowardNegative:
+ return IsFTZ ? NVPTXISD::SUB_RM_FTZ : NVPTXISD::SUB_RM;
+ case APFloat::rmTowardPositive:
+ return IsFTZ ? NVPTXISD::SUB_RP_FTZ : NVPTXISD::SUB_RP;
+ default:
+ llvm_unreachable("Unexpected fadd rounding mode");
+ }
default:
- break;
- case Intrinsic::nvvm_add_rn_sat_f16:
- case Intrinsic::nvvm_add_rn_sat_v2f16:
- return NVPTXISD::SUB_RN_SAT;
- case Intrinsic::nvvm_add_rn_ftz_sat_f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_v2f16:
- return NVPTXISD::SUB_RN_FTZ_SAT;
+ return 0;
}
- llvm_unreachable("Invalid F16 add intrinsic");
}
-static SDValue combineF16AddWithNeg(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID AddIntrinsicID) {
+static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
+ Intrinsic::ID AddIntrinsicID) {
+ const EVT VT = N->getValueType(0);
+ const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID);
+ if (!Opc)
+ return SDValue();
+
SDValue Op1 = N->getOperand(1);
SDValue Op2 = N->getOperand(2);
@@ -7215,24 +7236,74 @@ static SDValue combineF16AddWithNeg(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
- SDLoc DL(N);
- return DAG.getNode(getF16SubOpc(AddIntrinsicID), DL, N->getValueType(0),
- SubOp1, SubOp2);
+ return DAG.getNode(Opc, SDLoc(N), VT, SubOp1, SubOp2);
+}
+
+static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID,
+ const NVPTXSubtarget &STI) {
+ if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(2))
+ return false;
+
+ const bool IsSat = nvvm::FAddShouldSaturate(IID);
+ switch (VT.getScalarType().getSimpleVT().SimpleTy) {
+ case MVT::f16:
+ return nvvm::GetFAddRoundingMode(IID) == APFloat::rmNearestTiesToEven;
+ case MVT::bf16:
+ return nvvm::GetFAddRoundingMode(IID) == APFloat::rmNearestTiesToEven &&
+ !IsSat && !nvvm::FAddShouldFTZ(IID) &&
+ STI.hasNativeBF16Support(ISD::FADD);
+ case MVT::f32:
+ return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
+ case MVT::f64:
+ return !VT.isVector() && !IsSat && !nvvm::FAddShouldFTZ(IID);
+ default:
+ return false;
+ }
+}
+
+static SDValue diagnoseInvalidFAdd(SDNode *N, SelectionDAG &DAG,
+ Intrinsic::ID IID,
+ const NVPTXSubtarget &STI) {
+ const EVT VT = N->getValueType(0);
+ if (isSupportedFAdd(VT, IID, STI))
+ return SDValue();
+
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ Twine(Intrinsic::getBaseName(IID)) + " with operand type " +
+ VT.getEVTString() + " is not supported on this target",
+ SDLoc(N).getDebugLoc()));
+ return DAG.getPOISON(VT);
}
static SDValue combineIntrinsicWOChain(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const NVPTXSubtarget &STI) {
- unsigned IID = N->getConstantOperandVal(0);
+ const Intrinsic::ID IID =
+ static_cast<Intrinsic::ID>(N->getConstantOperandVal(0));
switch (IID) {
default:
break;
- case Intrinsic::nvvm_add_rn_sat_f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_f16:
- case Intrinsic::nvvm_add_rn_sat_v2f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_v2f16:
- return combineF16AddWithNeg(N, DCI.DAG, IID);
+ case Intrinsic::nvvm_fadd_rm:
+ case Intrinsic::nvvm_fadd_rn:
+ case Intrinsic::nvvm_fadd_rp:
+ case Intrinsic::nvvm_fadd_rz:
+ case Intrinsic::nvvm_fadd_rm_ftz:
+ case Intrinsic::nvvm_fadd_rn_ftz:
+ case Intrinsic::nvvm_fadd_rp_ftz:
+ case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd_rm_sat:
+ case Intrinsic::nvvm_fadd_rn_sat:
+ case Intrinsic::nvvm_fadd_rp_sat:
+ case Intrinsic::nvvm_fadd_rz_sat:
+ case Intrinsic::nvvm_fadd_rm_ftz_sat:
+ case Intrinsic::nvvm_fadd_rn_ftz_sat:
+ case Intrinsic::nvvm_fadd_rp_ftz_sat:
+ case Intrinsic::nvvm_fadd_rz_ftz_sat:
+ if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, STI))
+ return V;
+ return combineFAddWithNeg(N, DCI.DAG, IID);
}
return SDValue();
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 5e01b251ff9fd..cd7fcbde978ef 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1494,6 +1494,14 @@ class F_MATH_2<string OpcStr, NVPTXRegClass t_regclass,
[(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1))]>,
Requires<Preds>;
+class F_MATH_2_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP,
+ list<Predicate> Preds = []>
+ : BasicNVPTXInst<(outs t.RC:$dst),
+ (ins t.RC:$src0, t.RC:$src1),
+ OpcStr,
+ [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1))]>,
+ Requires<Preds>;
+
class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,
NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass,
NVPTXRegClass s2_regclass, Intrinsic IntOP, list<Predicate> Preds = []>
@@ -2063,32 +2071,50 @@ let Predicates = [doRsqrtOpt] in {
// Add
//
-def INT_NVVM_ADD_RN_SAT_F16 : F_MATH_2<"add.rn.sat.f16", B16, B16, B16, int_nvvm_add_rn_sat_f16>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16 : F_MATH_2<"add.rn.ftz.sat.f16", B16, B16, B16, int_nvvm_add_rn_ftz_sat_f16>;
-def INT_NVVM_ADD_RN_SAT_F16X2 : F_MATH_2<"add.rn.sat.f16x2", B32, B32, B32, int_nvvm_add_rn_sat_v2f16>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16X2 : F_MATH_2<"add.rn.ftz.sat.f16x2", B32, B32, B32, int_nvvm_add_rn_ftz_sat_v2f16>;
-
-def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2<"add.rn.ftz.f32", B32, B32, B32, int_nvvm_add_rn_ftz_f>;
-def INT_NVVM_ADD_RN_SAT_FTZ_F : F_MATH_2<"add.rn.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rn_ftz_sat_f>;
-def INT_NVVM_ADD_RN_F : F_MATH_2<"add.rn.f32", B32, B32, B32, int_nvvm_add_rn_f>;
-def INT_NVVM_ADD_RN_SAT_F : F_MATH_2<"add.rn.sat.f32", B32, B32, B32, int_nvvm_add_rn_sat_f>;
-def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2<"add.rz.ftz.f32", B32, B32, B32, int_nvvm_add_rz_ftz_f>;
-def INT_NVVM_ADD_RZ_SAT_FTZ_F : F_MATH_2<"add.rz.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rz_ftz_sat_f>;
-def INT_NVVM_ADD_RZ_F : F_MATH_2<"add.rz.f32", B32, B32, B32, int_nvvm_add_rz_f>;
-def INT_NVVM_ADD_RZ_SAT_F : F_MATH_2<"add.rz.sat.f32", B32, B32, B32, int_nvvm_add_rz_sat_f>;
-def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2<"add.rm.ftz.f32", B32, B32, B32, int_nvvm_add_rm_ftz_f>;
-def INT_NVVM_ADD_RM_SAT_FTZ_F : F_MATH_2<"add.rm.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rm_ftz_sat_f>;
-def INT_NVVM_ADD_RM_F : F_MATH_2<"add.rm.f32", B32, B32, B32, int_nvvm_add_rm_f>;
-def INT_NVVM_ADD_RM_SAT_F : F_MATH_2<"add.rm.sat.f32", B32, B32, B32, int_nvvm_add_rm_sat_f>;
-def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2<"add.rp.ftz.f32", B32, B32, B32, int_nvvm_add_rp_ftz_f>;
-def INT_NVVM_ADD_RP_SAT_FTZ_F : F_MATH_2<"add.rp.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rp_ftz_sat_f>;
-def INT_NVVM_ADD_RP_F : F_MATH_2<"add.rp.f32", B32, B32, B32, int_nvvm_add_rp_f>;
-def INT_NVVM_ADD_RP_SAT_F : F_MATH_2<"add.rp.sat.f32", B32, B32, B32, int_nvvm_add_rp_sat_f>;
-
-def INT_NVVM_ADD_RN_D : F_MATH_2<"add.rn.f64", B64, B64, B64, int_nvvm_add_rn_d>;
-def INT_NVVM_ADD_RZ_D : F_MATH_2<"add.rz.f64", B64, B64, B64, int_nvvm_add_rz_d>;
-def INT_NVVM_ADD_RM_D : F_MATH_2<"add.rm.f64", B64, B64, B64, int_nvvm_add_rm_d>;
-def INT_NVVM_ADD_RP_D : F_MATH_2<"add.rp.f64", B64, B64, B64, int_nvvm_add_rp_d>;
+defvar BF16ArithPreds = [hasBF16Math, hasPTX<78>, hasSM<90>];
+
+def INT_NVVM_ADD_RN_F16 : F_MATH_2_TY<"add.rn.f16", F16RT, int_nvvm_fadd_rn>;
+def INT_NVVM_ADD_RN_FTZ_F16 : F_MATH_2_TY<"add.rn.ftz.f16", F16RT, int_nvvm_fadd_rn_ftz>;
+def INT_NVVM_ADD_RN_SAT_F16 : F_MATH_2_TY<"add.rn.sat.f16", F16RT, int_nvvm_fadd_rn_sat>;
+def INT_NVVM_ADD_RN_FTZ_SAT_F16 : F_MATH_2_TY<"add.rn.ftz.sat.f16", F16RT, int_nvvm_fadd_rn_ftz_sat>;
+def INT_NVVM_ADD_RN_F16X2 : F_MATH_2_TY<"add.rn.f16x2", F16X2RT, int_nvvm_fadd_rn>;
+def INT_NVVM_ADD_RN_FTZ_F16X2 : F_MATH_2_TY<"add.rn.ftz.f16x2", F16X2RT, int_nvvm_fadd_rn_ftz>;
+def INT_NVVM_ADD_RN_SAT_F16X2 : F_MATH_2_TY<"add.rn.sat.f16x2", F16X2RT, int_nvvm_fadd_rn_sat>;
+def INT_NVVM_ADD_RN_FTZ_SAT_F16X2 : F_MATH_2_TY<"add.rn.ftz.sat.f16x2", F16X2RT, int_nvvm_fadd_rn_ftz_sat>;
+
+def INT_NVVM_ADD_RN_BF16 :
+ F_MATH_2_TY<"add.rn.bf16", BF16RT, int_nvvm_fadd_rn, BF16ArithPreds>;
+def INT_NVVM_ADD_RN_BF16X2 :
+ F_MATH_2_TY<"add.rn.bf16x2", BF16X2RT, int_nvvm_fadd_rn, BF16ArithPreds>;
+
+def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2_TY<"add.rn.ftz.f32", F32RT, int_nvvm_fadd_rn_ftz>;
+def INT_NVVM_ADD_RN_SAT_FTZ_F : F_MATH_2_TY<"add.rn.sat.ftz.f32", F32RT, int_nvvm_fadd_rn_ftz_sat>;
+def INT_NVVM_ADD_RN_F : F_MATH_2_TY<"add.rn.f32", F32RT, int_nvvm_fadd_rn>;
+def INT_NVVM_ADD_RN_SAT_F : F_MATH_2_TY<"add.rn.sat.f32", F32RT, int_nvvm_fadd_rn_sat>;
+def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2_TY<"add.rz.ftz.f32", F32RT, int_nvvm_fadd_rz_ftz>;
+def INT_NVVM_ADD_RZ_SAT_FTZ_F : F_MATH_2_TY<"add.rz.sat.ftz.f32", F32RT, int_nvvm_fadd_rz_ftz_sat>;
+def INT_NVVM_ADD_RZ_F : F_MATH_2_TY<"add.rz.f32", F32RT, int_nvvm_fadd_rz>;
+def INT_NVVM_ADD_RZ_SAT_F : F_MATH_2_TY<"add.rz.sat.f32", F32RT, int_nvvm_fadd_rz_sat>;
+def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2_TY<"add.rm.ftz.f32", F32RT, int_nvvm_fadd_rm_ftz>;
+def INT_NVVM_ADD_RM_SAT_FTZ_F : F_MATH_2_TY<"add.rm.sat.ftz.f32", F32RT, int_nvvm_fadd_rm_ftz_sat>;
+def INT_NVVM_ADD_RM_F : F_MATH_2_TY<"add.rm.f32", F32RT, int_nvvm_fadd_rm>;
+def INT_NVVM_ADD_RM_SAT_F : F_MATH_2_TY<"add.rm.sat.f32", F32RT, int_nvvm_fadd_rm_sat>;
+def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2_TY<"add.rp.ftz.f32", F32RT, int_nvvm_fadd_rp_ftz>;
+def INT_NVVM_ADD_RP_SAT_FTZ_F : F_MATH_2_TY<"add.rp.sat.ftz.f32", F32RT, int_nvvm_fadd_rp_ftz_sat>;
+def INT_NVVM_ADD_RP_F : F_MATH_2_TY<"add.rp.f32", F32RT, int_nvvm_fadd_rp>;
+def INT_NVVM_ADD_RP_SAT_F : F_MATH_2_TY<"add.rp.sat.f32", F32RT, int_nvvm_fadd_rp_sat>;
+
+def INT_NVVM_ADD_RN_D : F_MATH_2_TY<"add.rn.f64", F64RT, int_nvvm_fadd_rn>;
+def INT_NVVM_ADD_RZ_D : F_MATH_2_TY<"add.rz.f64", F64RT, int_nvvm_fadd_rz>;
+def INT_NVVM_ADD_RM_D : F_MATH_2_TY<"add.rm.f64", F64RT, int_nvvm_fadd_rm>;
+def INT_NVVM_ADD_RP_D : F_MATH_2_TY<"add.rp.f64", F64RT, int_nvvm_fadd_rp>;
+
+foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach ftz = ["", "_ftz"] in
+ def INT_NVVM_ADD # rnd # ftz # _F32X2 :
+ F_MATH_2_TY<!subst("_", ".", "add" # rnd # ftz # "_f32x2"), F32X2RT,
+ !cast<Intrinsic>("int_nvvm_fadd" # rnd # ftz),
+ [hasF32x2Instructions]>;
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach sat = ["", "_sat"] in {
@@ -2097,7 +2123,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
!subst("_", ".", "add" # rnd # sat # "_f32_" # type),
[(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_add" # rnd # sat # "_f")
+ (!cast<Intrinsic>("int_nvvm_fadd" # rnd # sat)
(f32 (fpextend type:$a)),
f32:$b))]>,
Requires<[SM100]>;
@@ -2117,25 +2143,45 @@ let Predicates = [SM100, doNoF32FTZ] in {
// Sub
//
+// These nodes are created by combineFAddWithNeg.
def sub_rn_sat : SDNode<"NVPTXISD::SUB_RN_SAT", SDTFPBinOp>;
def sub_rn_ftz_sat :
SDNode<"NVPTXISD::SUB_RN_FTZ_SAT", SDTFPBinOp>;
-
-class INT_NVVM_SUB_RN<RegTyInfo TyInfo, string variant> :
+
+foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach ftz = ["", "_ftz"] in
+ def sub # rnd # ftz :
+ SDNode<"NVPTXISD::SUB" # !toupper(rnd # ftz), SDTFPBinOp>;
+
+class INT_NVVM_SUB<RegTyInfo TyInfo, string variant> :
BasicNVPTXInst<(outs TyInfo.RC:$dst), (ins TyInfo.RC:$a, TyInfo.RC:$b),
- !subst("_", ".", "sub.rn" # variant # "." # TyInfo.PtxType),
+ !subst("_", ".", "sub" # variant # "." # TyInfo.PtxType),
[(set TyInfo.Ty:$dst,
- (!cast<SDNode>("sub_rn" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
+ (!cast<SDNode>("sub" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
+
+def INT_NVVM_SUB_RN_F16 : INT_NVVM_SUB<F16RT, "_rn">;
+def INT_NVVM_SUB_RN_FTZ_F16 : INT_NVVM_SUB<F16RT, "_rn_ftz">;
+def INT_NVVM_SUB_RN_SAT_F16 : INT_NVVM_SUB<F16RT, "_rn_sat">;
+def INT_NVVM_SUB_RN_FTZ_SAT_F16 : INT_NVVM_SUB<F16RT, "_rn_ftz_sat">;
+def INT_NVVM_SUB_RN_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn">;
+def INT_NVVM_SUB_RN_FTZ_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_ftz">;
+def INT_NVVM_SUB_RN_SAT_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_sat">;
+def INT_NVVM_SUB_RN_FTZ_SAT_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_ftz_sat">;
+
+let Predicates = BF16ArithPreds in {
+ def INT_NVVM_SUB_RN_BF16 : INT_NVVM_SUB<BF16RT, "_rn">;
+ def INT_NVVM_SUB_RN_BF16X2 : INT_NVVM_SUB<BF16X2RT, "_rn">;
+}
-def INT_NVVM_SUB_RN_SAT_F16 : INT_NVVM_SUB_RN<F16RT, "_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16 : INT_NVVM_SUB_RN<F16RT, "_ftz_sat">;
-def INT_NVVM_SUB_RN_SAT_F16X2 : INT_NVVM_SUB_RN<F16X2RT, "_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16X2 : INT_NVVM_SUB_RN<F16X2RT, "_ftz_sat">;
+let Predicates = [hasF32x2Instructions] in
+ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach ftz = ["", "_ftz"] in
+ def INT_NVVM_SUB # rnd # ftz # _F32X2 : INT_NVVM_SUB<F32X2RT, rnd # ftz>;
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_add" # rnd # ftz # sat # "_f");
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # rnd # ftz # sat);
def INT_NVVM_SUB # rnd # ftz # sat # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
@@ -2143,7 +2189,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
}
}
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_add" # rnd # "_d");
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # rnd);
def INT_NVVM_SUB # rnd # _D :
BasicNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
!subst("_", ".", "sub" # rnd # "_f64"),
@@ -2157,7 +2203,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # "_f32_" # type),
[(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_add" # rnd # sat # "_f")
+ (!cast<Intrinsic>("int_nvvm_fadd" # rnd # sat)
(f32 (fpextend type:$a)),
(f32 (fneg f32:$b))))]>,
Requires<[SM100]>;
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index bcbeef260f2bb..8a0c23e0e4f09 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -711,3 +711,27 @@ define void @nvvm_ex2_approx(float %a, double %b, half %c, <2 x half> %d) {
%r4 = call float @llvm.nvvm.ex2.approx.ftz.f(float %a)
ret void
}
+
+define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
+; CHECK: call float @llvm.nvvm.fadd.rn.f32(float %a, float %a)
+; CHECK: call float @llvm.nvvm.fadd.rz.ftz.f32(float %a, float %a)
+; CHECK: call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %a)
+; CHECK: call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %a)
+; CHECK: call double @llvm.nvvm.fadd.rn.f64(double %b, double %b)
+; CHECK: call double @llvm.nvvm.fadd.rz.f64(double %b, double %b)
+; CHECK: call half @llvm.nvvm.fadd.rn.sat.f16(half %c, half %c)
+; CHECK: call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %c, half %c)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %d, <2 x half> %d)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d)
+ %r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
+ %r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
+ %r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
+ %r4 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %a)
+ %r5 = call double @llvm.nvvm.add.rn.d(double %b, double %b)
+ %r6 = call double @llvm.nvvm.add.rz.d(double %b, double %b)
+ %r7 = call half @llvm.nvvm.add.rn.sat.f16(half %c, half %c)
+ %r8 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %c, half %c)
+ %r9 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %d, <2 x half> %d)
+ %r10 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/bf16-add.ll b/llvm/test/CodeGen/NVPTX/bf16-add.ll
new file mode 100644
index 0000000000000..71d90a3ae675c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/bf16-add.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-7.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
+
+define bfloat @add_rn_bf16(bfloat %a, bfloat %b) {
+; CHECK-LABEL: add_rn_bf16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_bf16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_bf16_param_1];
+; CHECK-NEXT: add.rn.bf16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %b)
+ ret bfloat %1
+}
+
+define <2 x bfloat> @add_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; CHECK-LABEL: add_rn_bf16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_bf16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_bf16x2_param_1];
+; CHECK-NEXT: add.rn.bf16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ ret <2 x bfloat> %1
+}
diff --git a/llvm/test/CodeGen/NVPTX/bf16-sub.ll b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
new file mode 100644
index 0000000000000..3d685ebf65cf2
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-7.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
+
+define bfloat @sub_rn_bf16(bfloat %a, bfloat %b) {
+; CHECK-LABEL: sub_rn_bf16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_bf16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_bf16_param_1];
+; CHECK-NEXT: sub.rn.bf16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg bfloat %b
+ %res = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %1)
+ ret bfloat %res
+}
+
+define <2 x bfloat> @sub_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; CHECK-LABEL: sub_rn_bf16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_bf16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_bf16x2_param_1];
+; CHECK-NEXT: sub.rn.bf16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x bfloat> %b
+ %res = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %a, <2 x bfloat> %1)
+ ret <2 x bfloat> %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/f16-add-sat.ll b/llvm/test/CodeGen/NVPTX/f16-add-sat.ll
deleted file mode 100644
index c2ffc126694c4..0000000000000
--- a/llvm/test/CodeGen/NVPTX/f16-add-sat.ll
+++ /dev/null
@@ -1,63 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
-; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
-
-define half @add_rn_sat_f16(half %a, half %b) {
-; CHECK-LABEL: add_rn_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_sat_f16_param_1];
-; CHECK-NEXT: add.rn.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.add.rn.sat.f16(half %a, half %b)
- ret half %1
-}
-
-define <2 x half> @add_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: add_rn_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [add_rn_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [add_rn_sat_f16x2_param_1];
-; CHECK-NEXT: add.rn.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
- ret <2 x half> %1
-}
-
-define half @add_rn_ftz_sat_f16(half %a, half %b) {
-; CHECK-LABEL: add_rn_ftz_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_sat_f16_param_1];
-; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %b)
- ret half %1
-}
-
-define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: add_rn_ftz_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_sat_f16x2_param_1];
-; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
- ret <2 x half> %1
-}
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
new file mode 100644
index 0000000000000..63d7f1e2705d6
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
+; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
+
+define half @add_rn_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_f16_param_1];
+; CHECK-NEXT: add.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.rn.f16(half %a, half %b)
+ ret half %1
+}
+
+define <2 x half> @add_rn_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_f16x2_param_1];
+; CHECK-NEXT: add.rn.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %a, <2 x half> %b)
+ ret <2 x half> %1
+}
+
+define half @add_rn_ftz_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_ftz_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_f16_param_1];
+; CHECK-NEXT: add.rn.ftz.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %a, half %b)
+ ret half %1
+}
+
+define <2 x half> @add_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_ftz_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_f16x2_param_1];
+; CHECK-NEXT: add.rn.ftz.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %a, <2 x half> %b)
+ ret <2 x half> %1
+}
+
+define half @add_rn_sat_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_sat_f16_param_1];
+; CHECK-NEXT: add.rn.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.rn.sat.f16(half %a, half %b)
+ ret half %1
+}
+
+define <2 x half> @add_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_sat_f16x2_param_1];
+; CHECK-NEXT: add.rn.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
+ ret <2 x half> %1
+}
+
+define half @add_rn_ftz_sat_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_ftz_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_sat_f16_param_1];
+; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %a, half %b)
+ ret half %1
+}
+
+define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_ftz_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_sat_f16x2_param_1];
+; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
+ ret <2 x half> %1
+}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll b/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
deleted file mode 100644
index 774ce7ccb2f95..0000000000000
--- a/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
+++ /dev/null
@@ -1,69 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | FileCheck %s
-; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
-; RUN: %if ptxas-isa-6.0 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | %ptxas-verify%}
-
-define half @sub_rn_sat_f16(half %a, half %b) {
-; CHECK-LABEL: sub_rn_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_sat_f16_param_1];
-; CHECK-NEXT: sub.rn.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = fneg half %b
- %res = call half @llvm.nvvm.add.rn.sat.f16(half %a, half %1)
- ret half %res
-}
-
-define <2 x half> @sub_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: sub_rn_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_sat_f16x2_param_1];
-; CHECK-NEXT: sub.rn.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %1)
- ret <2 x half> %res
-}
-
-define half @sub_rn_ftz_sat_f16(half %a, half %b) {
-; CHECK-LABEL: sub_rn_ftz_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_sat_f16_param_1];
-; CHECK-NEXT: sub.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = fneg half %b
- %res = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %1)
- ret half %res
-}
-
-define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: sub_rn_ftz_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_sat_f16x2_param_1];
-; CHECK-NEXT: sub.rn.ftz.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1)
- ret <2 x half> %res
-}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
new file mode 100644
index 0000000000000..e76c033c6705f
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | FileCheck %s
+; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
+; RUN: %if ptxas-isa-6.0 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | %ptxas-verify%}
+
+define half @sub_rn_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_f16_param_1];
+; CHECK-NEXT: sub.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.rn.f16(half %a, half %1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_f16x2_param_1];
+; CHECK-NEXT: sub.rn.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %a, <2 x half> %1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_ftz_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_ftz_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_f16_param_1];
+; CHECK-NEXT: sub.rn.ftz.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.rn.ftz.f16(half %a, half %1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_ftz_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_f16x2_param_1];
+; CHECK-NEXT: sub.rn.ftz.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %a, <2 x half> %1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_sat_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_sat_f16_param_1];
+; CHECK-NEXT: sub.rn.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.rn.sat.f16(half %a, half %1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_sat_f16x2_param_1];
+; CHECK-NEXT: sub.rn.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %a, <2 x half> %1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_ftz_sat_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_ftz_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_sat_f16_param_1];
+; CHECK-NEXT: sub.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %a, half %1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_ftz_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_sat_f16x2_param_1];
+; CHECK-NEXT: sub.rn.ftz.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1)
+ ret <2 x half> %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
new file mode 100644
index 0000000000000..f837c6d5d9a54
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
@@ -0,0 +1,60 @@
+; RUN: llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 | FileCheck %s
+; RUN: %if ptxas-sm_100 && ptxas-isa-8.8 %{ llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 | %ptxas-verify -arch=sm_100 %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define <2 x float> @add_rn(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn(
+; CHECK: add.rn.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz(
+; CHECK: add.rz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rm(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rm(
+; CHECK: add.rm.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rp(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rp(
+; CHECK: add.rp.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rn_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_ftz(
+; CHECK: add.rn.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rz_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_ftz(
+; CHECK: add.rz.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rm_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rm_ftz(
+; CHECK: add.rm.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rp_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rp_ftz(
+; CHECK: add.rp.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
new file mode 100644
index 0000000000000..3114592aaccdc
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
@@ -0,0 +1,47 @@
+; RUN: not llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 2>&1 | FileCheck %s
+; RUN: not llc < %s -mcpu=sm_90 -mattr=+ptx78 -march=nvptx64 2>&1 | FileCheck %s --check-prefix=NOF32X2
+; RUN: not llc < %s -mcpu=sm_80 -mattr=+ptx78 -march=nvptx64 2>&1 | FileCheck %s --check-prefix=NOBF16
+
+target triple = "nvptx64-nvidia-cuda"
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.sat with operand type v2f32 is not supported
+define <2 x float> @sat_f32x2(<2 x float> %a, <2 x float> %b) {
+ %r = call <2 x float> @llvm.nvvm.fadd.rn.sat.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+; NOF32X2: error: {{.*}}llvm.nvvm.fadd.rn with operand type v2f32 is not supported
+define <2 x float> @unsupported_f32x2(<2 x float> %a, <2 x float> %b) {
+ %r = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %b)
+ ret <2 x float> %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.ftz with operand type f64 is not supported
+define double @ftz_f64(double %a, double %b) {
+ %r = call double @llvm.nvvm.fadd.rn.ftz.f64(double %a, double %b)
+ ret double %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rz.sat with operand type f16 is not supported
+define half @rz_f16(half %a, half %b) {
+ %r = call half @llvm.nvvm.fadd.rz.sat.f16(half %a, half %b)
+ ret half %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.ftz with operand type bf16 is not supported
+define bfloat @ftz_bf16(bfloat %a, bfloat %b) {
+ %r = call bfloat @llvm.nvvm.fadd.rn.ftz.bf16(bfloat %a, bfloat %b)
+ ret bfloat %r
+}
+
+; NOBF16: error: {{.*}}llvm.nvvm.fadd.rn with operand type bf16 is not supported
+define bfloat @unsupported_bf16(bfloat %a, bfloat %b) {
+ %r = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %b)
+ ret bfloat %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.rn with operand type v4f32 is not supported
+define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) {
+ %r = call <4 x float> @llvm.nvvm.fadd.rn.v4f32(<4 x float> %a, <4 x float> %b)
+ ret <4 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index c6b3b649aae06..44c09cc177595 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -20,17 +20,17 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: add.rp.sat.ftz.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.add.rn.sat.f(float %a, float %b)
- %r2 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %a, float %r1)
+ %r1 = call float @llvm.nvvm.fadd.rn.sat.f32(float %a, float %b)
+ %r2 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %a, float %r1)
- %r3 = call float @llvm.nvvm.add.rz.sat.f(float %a, float %r2)
- %r4 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %a, float %r3)
+ %r3 = call float @llvm.nvvm.fadd.rz.sat.f32(float %a, float %r2)
+ %r4 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %a, float %r3)
- %r5 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %r4)
- %r6 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %a, float %r5)
+ %r5 = call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %r4)
+ %r6 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %a, float %r5)
- %r7 = call float @llvm.nvvm.add.rp.sat.f(float %a, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %r7)
+ %r7 = call float @llvm.nvvm.fadd.rp.sat.f32(float %a, float %r6)
+ %r8 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %r7)
ret float %r8
}
@@ -54,28 +54,28 @@ define float @sub_sat_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.sat.f(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.rn.sat.f32(float %a, float %f0)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %a, float %f1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rz.sat.f(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.rz.sat.f32(float %a, float %f2)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %a, float %f3)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %f4)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %a, float %f5)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rp.sat.f(float %a, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.rp.sat.f32(float %a, float %f6)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %f7)
ret float %r8
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
new file mode 100644
index 0000000000000..513926de4451d
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s
+; RUN: %if ptxas-sm_100 && ptxas-isa-8.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | %ptxas-verify -arch=sm_100 %}
+
+define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<11>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [sub_f32x2_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [sub_f32x2_param_1];
+; CHECK-NEXT: sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NEXT: sub.rz.f32x2 %rd5, %rd1, %rd4;
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd6, %rd1, %rd5;
+; CHECK-NEXT: sub.rm.f32x2 %rd7, %rd1, %rd6;
+; CHECK-NEXT: sub.rm.ftz.f32x2 %rd8, %rd1, %rd7;
+; CHECK-NEXT: sub.rp.f32x2 %rd9, %rd1, %rd8;
+; CHECK-NEXT: sub.rp.ftz.f32x2 %rd10, %rd1, %rd9;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd10;
+; CHECK-NEXT: ret;
+ %f0 = fneg <2 x float> %b
+ %r1 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %f0)
+
+ %f1 = fneg <2 x float> %r1
+ %r2 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %a, <2 x float> %f1)
+
+ %f2 = fneg <2 x float> %r2
+ %r3 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %a, <2 x float> %f2)
+
+ %f3 = fneg <2 x float> %r3
+ %r4 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %a, <2 x float> %f3)
+
+ %f4 = fneg <2 x float> %r4
+ %r5 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %a, <2 x float> %f4)
+
+ %f5 = fneg <2 x float> %r5
+ %r6 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %a, <2 x float> %f5)
+
+ %f6 = fneg <2 x float> %r6
+ %r7 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %a, <2 x float> %f6)
+
+ %f7 = fneg <2 x float> %r7
+ %r8 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %a, <2 x float> %f7)
+
+ ret <2 x float> %r8
+}
+
+define <2 x float> @sub_f32x2_negated_lhs(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_negated_lhs(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [sub_f32x2_negated_lhs_param_0];
+; CHECK-NEXT: ld.param.b64 %rd2, [sub_f32x2_negated_lhs_param_1];
+; CHECK-NEXT: sub.rz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
+; CHECK-NEXT: ret;
+ %f = fneg <2 x float> %a
+ %r = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %f, <2 x float> %b)
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
index 351f45ccbcc6b..a3084392c7853 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
@@ -20,22 +20,22 @@ define float @sub_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r8;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %a, float %f0)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rn.ftz.f(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %f1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rz.f(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.rz.f32(float %a, float %f2)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %a, float %f3)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rm.f(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.rm.f32(float %a, float %f4)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rm.ftz.f(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %a, float %f5)
ret float %r6
}
@@ -55,16 +55,16 @@ define double @sub_f64(double %a, double %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd6;
; CHECK-NEXT: ret;
%f0 = fneg double %b
- %r1 = call double @llvm.nvvm.add.rn.d(double %a, double %f0)
+ %r1 = call double @llvm.nvvm.fadd.rn.f64(double %a, double %f0)
%f1 = fneg double %r1
- %r2 = call double @llvm.nvvm.add.rz.d(double %a, double %f1)
+ %r2 = call double @llvm.nvvm.fadd.rz.f64(double %a, double %f1)
%f2 = fneg double %r2
- %r3 = call double @llvm.nvvm.add.rm.d(double %a, double %f2)
+ %r3 = call double @llvm.nvvm.fadd.rm.f64(double %a, double %f2)
%f3 = fneg double %r3
- %r4 = call double @llvm.nvvm.add.rp.d(double %a, double %f3)
+ %r4 = call double @llvm.nvvm.fadd.rp.f64(double %a, double %f3)
ret double %r4
}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
index 5f82d2e77a468..aa80dc2185bd5 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
@@ -27,16 +27,16 @@ define float @test_add_f32_f16_1(half %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext half %a to float
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %b)
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %b)
+ %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %r1)
+ %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %r2)
+ %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %r3)
; SAT
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %r4)
+ %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %r5)
+ %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %r6)
+ %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %r7)
ret float %r8
}
@@ -93,16 +93,16 @@ define float @test_add_f32_bf16_1(bfloat %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext bfloat %a to float
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %b)
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %b)
+ %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %r1)
+ %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %r2)
+ %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %r3)
; SAT
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %r4)
+ %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %r5)
+ %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %r6)
+ %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %r7)
ret float %r8
}
@@ -160,29 +160,29 @@ define float @test_sub_f32_f16_1(half %a, float %b) {
%r0 = fpext half %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %f0)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %f1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f2)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rm.f(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f3)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %f4)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %f5)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %f6)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %f7)
ret float %r7
}
@@ -240,29 +240,29 @@ define float @test_sub_f32_bf16_1(bfloat %a, float %b) {
%r0 = fpext bfloat %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %f0)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %f1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f2)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %f3)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %f4)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %f5)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %f6)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %f7)
ret float %r8
}
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index db172c5a34cdc..d0acf650d02c5 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -299,20 +299,20 @@ define float @test_ull2f(i64 %a) #0 {
; CHECK-LABEL: @test_add_rn_d
define double @test_add_rn_d(double %a, double %b) #0 {
-; CHECK: call double @llvm.nvvm.add.rn.d
- %ret = call double @llvm.nvvm.add.rn.d(double %a, double %b)
+; CHECK: call double @llvm.nvvm.fadd.rn.f64
+ %ret = call double @llvm.nvvm.fadd.rn.f64(double %a, double %b)
ret double %ret
}
; CHECK-LABEL: @test_add_rn_f
define float @test_add_rn_f(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.add.rn.f
- %ret = call float @llvm.nvvm.add.rn.f(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.rn.f32
+ %ret = call float @llvm.nvvm.fadd.rn.f32(float %a, float %b)
ret float %ret
}
; CHECK-LABEL: @test_add_rn_f_ftz
define float @test_add_rn_f_ftz(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)
- %ret = call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %b)
+ %ret = call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %b)
ret float %ret
}
@@ -437,9 +437,9 @@ define i32 @test_fshr_clamp_3(i32 %a, i32 %b, i32 %c) {
ret i32 %call
}
-declare double @llvm.nvvm.add.rn.d(double, double)
-declare float @llvm.nvvm.add.rn.f(float, float)
-declare float @llvm.nvvm.add.rn.ftz.f(float, float)
+declare double @llvm.nvvm.fadd.rn.f64(double, double)
+declare float @llvm.nvvm.fadd.rn.f32(float, float)
+declare float @llvm.nvvm.fadd.rn.ftz.f32(float, float)
declare double @llvm.nvvm.ceil.d(double)
declare float @llvm.nvvm.ceil.f(float)
declare float @llvm.nvvm.ceil.ftz.f(float)
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
index b23ae275b71b7..f3129ae0c5f48 100644
--- a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
@@ -13,7 +13,7 @@ define double @test_1_25_minus_2_rm_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rm_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 1.25, double -2.0)
ret double %res
}
@@ -21,7 +21,7 @@ define double @test_1_25_minus_2_rn_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rn_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 1.25, double -2.0)
ret double %res
}
@@ -29,7 +29,7 @@ define double @test_1_25_minus_2_rp_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rp_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 1.25, double -2.0)
ret double %res
}
@@ -37,7 +37,7 @@ define double @test_1_25_minus_2_rz_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rz_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 1.25, double -2.0)
ret double %res
}
@@ -45,7 +45,7 @@ define float @test_1_25_minus_2_rm_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 1.25, float -2.0)
ret float %res
}
@@ -53,7 +53,7 @@ define float @test_1_25_minus_2_rn_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 1.25, float -2.0)
ret float %res
}
@@ -61,7 +61,7 @@ define float @test_1_25_minus_2_rp_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 1.25, float -2.0)
ret float %res
}
@@ -69,7 +69,7 @@ define float @test_1_25_minus_2_rz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -77,7 +77,7 @@ define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -85,7 +85,7 @@ define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -93,7 +93,7 @@ define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -101,7 +101,7 @@ define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.25, float -2.0)
ret float %res
}
@@ -113,109 +113,109 @@ define float @test_1_25_minus_2_rz_ftz_f() {
define double @test_zero_plus_nan_rm_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rm_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rm.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rm.f64(double 0.000000e+00, double +snan(0x4444400000000))
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rm.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 0.0, double 0x7ff4444400000000)
ret double %res
}
define double @test_zero_plus_nan_rn_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rn_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rn.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rn.f64(double 0.000000e+00, double +snan(0x4444400000000))
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rn.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 0.0, double 0x7ff4444400000000)
ret double %res
}
define double @test_zero_plus_nan_rp_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rp_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rp.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rp.f64(double 0.000000e+00, double +snan(0x4444400000000))
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rp.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 0.0, double 0x7ff4444400000000)
ret double %res
}
define double @test_zero_plus_nan_rz_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rz_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rz.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rz.f64(double 0.000000e+00, double +snan(0x4444400000000))
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rz.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 0.0, double 0x7ff4444400000000)
ret double %res
}
define float @test_zero_plus_nan_rm_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rm.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rm.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rm.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rn_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rn.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rn.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rn.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rp_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rp.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rp.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rp.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rm_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rm.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rn_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rn.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rp_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rp.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
define float @test_zero_plus_nan_rz_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rz.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0.0, float 0x7FFF444400000000)
ret float %res
}
@@ -230,7 +230,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rm.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 0x3800000000000000, double 0x3800000000000000)
ret double %res
}
@@ -238,7 +238,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rn.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 0x3800000000000000, double 0x3800000000000000)
ret double %res
}
@@ -246,7 +246,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rp.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 0x3800000000000000, double 0x3800000000000000)
ret double %res
}
@@ -254,7 +254,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rz.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 0x3800000000000000, double 0x3800000000000000)
ret double %res
}
@@ -262,7 +262,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rm.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -270,7 +270,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rn.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -278,7 +278,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rp.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -286,7 +286,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -294,7 +294,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -302,7 +302,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -310,7 +310,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -318,7 +318,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -335,7 +335,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rm.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 0x3810000000000000, double 0xB800000000000000)
ret double %res
}
@@ -343,7 +343,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rn.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 0x3810000000000000, double 0xB800000000000000)
ret double %res
}
@@ -351,7 +351,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rp.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 0x3810000000000000, double 0xB800000000000000)
ret double %res
}
@@ -359,7 +359,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rz.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 0x3810000000000000, double 0xB800000000000000)
ret double %res
}
@@ -367,7 +367,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rm.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -375,7 +375,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rn.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -383,7 +383,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rp.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -391,7 +391,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -399,7 +399,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -407,7 +407,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0x3810000000000000, float 0xB800000000000000)
ret float %res
}
@@ -415,7 +415,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -423,7 +423,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
ret float %res
}
@@ -439,7 +439,7 @@ define float @test_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -447,7 +447,7 @@ define float @test_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -455,7 +455,7 @@ define float @test_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -463,7 +463,7 @@ define float @test_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -471,7 +471,7 @@ define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -479,7 +479,7 @@ define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -487,7 +487,7 @@ define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -495,7 +495,7 @@ define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.0, float 0x3E60000000000000)
ret float %res
}
@@ -511,7 +511,7 @@ define double @test_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 1.0, double 0x3C90000000000000)
ret double %res
}
@@ -519,7 +519,7 @@ define double @test_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 1.0, double 0x3C90000000000000)
ret double %res
}
@@ -527,7 +527,7 @@ define double @test_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0x3FF0000000000001
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 1.0, double 0x3C90000000000000)
ret double %res
}
@@ -535,7 +535,7 @@ define double @test_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 1.0, double 0x3C90000000000000)
ret double %res
}
@@ -551,7 +551,7 @@ define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -559,7 +559,7 @@ define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -567,7 +567,7 @@ define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rp.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -575,7 +575,7 @@ define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -583,7 +583,7 @@ define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -591,7 +591,7 @@ define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -599,7 +599,7 @@ define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -607,7 +607,7 @@ define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float -1.0, float 0x3E60000000000000)
ret float %res
}
@@ -623,7 +623,7 @@ define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rm.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double -1.0, double 0x3C90000000000000)
ret double %res
}
@@ -631,7 +631,7 @@ define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double -1.0, double 0x3C90000000000000)
ret double %res
}
@@ -639,7 +639,7 @@ define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rp.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double -1.0, double 0x3C90000000000000)
ret double %res
}
@@ -647,7 +647,7 @@ define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rz.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double -1.0, double 0x3C90000000000000)
ret double %res
}
@@ -663,7 +663,7 @@ define float @test_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -671,7 +671,7 @@ define float @test_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -679,7 +679,7 @@ define float @test_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -687,7 +687,7 @@ define float @test_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -695,7 +695,7 @@ define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -703,7 +703,7 @@ define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -711,7 +711,7 @@ define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -719,7 +719,7 @@ define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.0, float 0xBE60000000000000)
ret float %res
}
@@ -735,7 +735,7 @@ define double @test_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double 1.0, double 0xBC90000000000000)
ret double %res
}
@@ -743,7 +743,7 @@ define double @test_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double 1.0, double 0xBC90000000000000)
ret double %res
}
@@ -751,7 +751,7 @@ define double @test_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double 1.0, double 0xBC90000000000000)
ret double %res
}
@@ -759,7 +759,7 @@ define double @test_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double 1.0, double 0xBC90000000000000)
ret double %res
}
@@ -775,7 +775,7 @@ define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.add.rm.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -783,7 +783,7 @@ define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -791,7 +791,7 @@ define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -799,7 +799,7 @@ define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -807,7 +807,7 @@ define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -815,7 +815,7 @@ define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -823,7 +823,7 @@ define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -831,7 +831,7 @@ define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float -1.0, float 0xBE60000000000000)
ret float %res
}
@@ -847,7 +847,7 @@ define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0xBFF0000000000001
;
- %res = call double @llvm.nvvm.add.rm.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rm.f64(double -1.0, double 0xBC90000000000000)
ret double %res
}
@@ -855,7 +855,7 @@ define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rn.f64(double -1.0, double 0xBC90000000000000)
ret double %res
}
@@ -863,7 +863,7 @@ define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rp.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rp.f64(double -1.0, double 0xBC90000000000000)
ret double %res
}
@@ -871,6 +871,6 @@ define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rz.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.rz.f64(double -1.0, double 0xBC90000000000000)
ret double %res
}
diff --git a/llvm/unittests/IR/IntrinsicsTest.cpp b/llvm/unittests/IR/IntrinsicsTest.cpp
index 47a1ea7d73906..f982715154bc0 100644
--- a/llvm/unittests/IR/IntrinsicsTest.cpp
+++ b/llvm/unittests/IR/IntrinsicsTest.cpp
@@ -118,7 +118,7 @@ TEST(IntrinsicNameLookup, ClangBuiltinLookup) {
{"__builtin_HEXAGON_A2_tfr", "hexagon", hexagon_A2_tfr},
{"__builtin_lasx_xbz_w", "loongarch", loongarch_lasx_xbz_w},
{"__builtin_mips_bitrev", "mips", mips_bitrev},
- {"__nvvm_add_rn_d", "nvvm", nvvm_add_rn_d},
+ {"__nvvm_mul_rn_d", "nvvm", nvvm_mul_rn_d},
{"__builtin_altivec_dss", "ppc", ppc_altivec_dss},
{"__builtin_riscv_sha512sum1r", "riscv", riscv_sha512sum1r},
{"__builtin_tend", "s390", s390_tend},
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 4dddb33fd050e..cae2eaef3e71a 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -3532,14 +3532,6 @@ static LogicalResult verifyAddSubFOp(OpType op) {
"vector<2xbf16> additions/subtractions");
}
- // FIXME: This is a temporary check disallowing lowering to add.rn.ftz.f16(x2)
- // PTX instructions since the corresponding LLVM intrinsic is missing. This
- // should be removed once the intrinsics for f16 addition (with FTZ only) are
- // available.
- if (opBaseType.isF16() && isFTZ && satMode == NVVM::SaturationMode::NONE)
- return op.emitOpError("FTZ with no saturation is not supported for f16 and "
- "vector<2xf16> additions/subtractions");
-
return success();
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 4201aabc02c3d..8b823fa425206 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -481,87 +481,39 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
llvm::Type *opTypeLLVM = argLHS->getType();
- bool isVectorOp = opTypeLLVM->isVectorTy();
bool isSat = satMode != NVVM::SaturationMode::NONE;
- // FIXME: Add intrinsics for add.rn.ftz.f16x2 and add.rn.ftz.f16 here when
- // they are available.
- static constexpr llvm::Intrinsic::ID f16IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_sat_f16,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f16,
- llvm::Intrinsic::nvvm_add_rn_sat_v2f16,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_v2f16,
- };
-
- static constexpr llvm::Intrinsic::ID f32IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_f,
- llvm::Intrinsic::nvvm_add_rm_f,
- llvm::Intrinsic::nvvm_add_rp_f,
- llvm::Intrinsic::nvvm_add_rz_f,
- llvm::Intrinsic::nvvm_add_rn_sat_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_sat_f,
- llvm::Intrinsic::nvvm_add_rm_sat_f,
- llvm::Intrinsic::nvvm_add_rp_sat_f,
- llvm::Intrinsic::nvvm_add_rz_sat_f,
- llvm::Intrinsic::nvvm_add_rn_ftz_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_ftz_f,
- llvm::Intrinsic::nvvm_add_rm_ftz_f,
- llvm::Intrinsic::nvvm_add_rp_ftz_f,
- llvm::Intrinsic::nvvm_add_rz_ftz_f,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rm_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rp_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rz_ftz_sat_f,
- };
-
- static constexpr llvm::Intrinsic::ID f64IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_d, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_d, llvm::Intrinsic::nvvm_add_rm_d,
- llvm::Intrinsic::nvvm_add_rp_d, llvm::Intrinsic::nvvm_add_rz_d};
-
- auto addIntrinsic = [&](llvm::Intrinsic::ID IID) -> llvm::Value * {
- return createScalarizedIntrinsicCall(builder, IID, opTypeLLVM,
- {argLHS, argRHS}, opTypeLLVM);
- };
-
- // f16 + f16 -> f16 / vector<2xf16> + vector<2xf16> -> vector<2xf16>
- // FIXME: Allow lowering to add.rn.ftz.f16x2 and add.rn.ftz.f16 here when the
- // intrinsics are available.
- if (opTypeLLVM->getScalarType()->isHalfTy()) {
- llvm::Value *result;
- if (isSat) {
- unsigned index = (isVectorOp << 1) | isFTZ;
- result = addIntrinsic(f16IDs[index]);
- } else {
- result = builder.CreateFAdd(argLHS, argRHS);
- }
- mt.mapValue(res, result);
+ static constexpr llvm::Intrinsic::ID addIDs[2][2][5] = {
+ {{llvm::Intrinsic::nvvm_fadd_rn, llvm::Intrinsic::nvvm_fadd_rn,
+ llvm::Intrinsic::nvvm_fadd_rm, llvm::Intrinsic::nvvm_fadd_rp,
+ llvm::Intrinsic::nvvm_fadd_rz},
+ {llvm::Intrinsic::nvvm_fadd_rn_sat, llvm::Intrinsic::nvvm_fadd_rn_sat,
+ llvm::Intrinsic::nvvm_fadd_rm_sat, llvm::Intrinsic::nvvm_fadd_rp_sat,
+ llvm::Intrinsic::nvvm_fadd_rz_sat}},
+ {{llvm::Intrinsic::nvvm_fadd_rn_ftz, llvm::Intrinsic::nvvm_fadd_rn_ftz,
+ llvm::Intrinsic::nvvm_fadd_rm_ftz, llvm::Intrinsic::nvvm_fadd_rp_ftz,
+ llvm::Intrinsic::nvvm_fadd_rz_ftz},
+ {llvm::Intrinsic::nvvm_fadd_rn_ftz_sat,
+ llvm::Intrinsic::nvvm_fadd_rn_ftz_sat,
+ llvm::Intrinsic::nvvm_fadd_rm_ftz_sat,
+ llvm::Intrinsic::nvvm_fadd_rp_ftz_sat,
+ llvm::Intrinsic::nvvm_fadd_rz_ftz_sat}}};
+
+ llvm::Intrinsic::ID id = addIDs[isFTZ][isSat][static_cast<unsigned>(rndMode)];
+
+ // For f64 vector addition, and f32 vector addition with saturation,
+ // we need to scalarize the intrinsic call.
+ llvm::Type *scalarTypeLLVM = opTypeLLVM->getScalarType();
+ if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() ||
+ (isSat && scalarTypeLLVM->isFloatTy()))) {
+ mt.mapValue(res, createScalarizedIntrinsicCall(builder, id, opTypeLLVM,
+ {argLHS, argRHS},
+ scalarTypeLLVM));
return;
}
- // bf16 + bf16 -> bf16 / vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
- if (opTypeLLVM->getScalarType()->isBFloatTy()) {
- mt.mapValue(res, builder.CreateFAdd(argLHS, argRHS));
- return;
- }
-
- // f64 + f64 -> f64 / vector<2xf64> + vector<2xf64> -> vector<2xf64>
- if (opTypeLLVM->getScalarType()->isDoubleTy()) {
- unsigned index = static_cast<unsigned>(rndMode);
- mt.mapValue(res, addIntrinsic(f64IDs[index]));
- return;
- }
-
- // f32 + f32 -> f32 / vector<2xf32> + vector<2xf32> -> vector<2xf32>
- const unsigned numRndModes = 5; // NONE, RM, RN, RP, RZ
- if (opTypeLLVM->getScalarType()->isFloatTy()) {
- unsigned index =
- ((isFTZ << 1) | isSat) * numRndModes + static_cast<unsigned>(rndMode);
- mt.mapValue(res, addIntrinsic(f32IDs[index]));
- return;
- }
+ mt.mapValue(res,
+ createIntrinsicCall(builder, id, opTypeLLVM, {argLHS, argRHS}));
}
void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index 201cfb7e098fc..a5e5a204a36e3 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -3,24 +3,26 @@
// f16 + f16 -> f16
llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fadd_f16_f16(half %0, half %1) {
- // CHECK-NEXT: %3 = fadd half %0, %1
- // CHECK-NEXT: %4 = fadd half %3, %3
- // CHECK-NEXT: %5 = call half @llvm.nvvm.add.rn.sat.f16(half %4, half %4)
- // CHECK-NEXT: %6 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %5, half %5)
- // CHECK-NEXT: ret half %6
+ // CHECK-NEXT: %3 = call half @llvm.nvvm.fadd.rn.f16(half %0, half %1)
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.rn.f16(half %3, half %3)
+ // CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %4, half %4)
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.rn.sat.f16(half %5, half %5)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %6, half %6)
+ // CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
- %f2 = nvvm.addf %f1, %f1 rnd = <rn> : f16
- %f3 = nvvm.addf %f2, %f2 rnd = <rn> sat = <sat> : f16
- %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : f16
- llvm.return %f4 : f16
+ %f2 = nvvm.addf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : f16
+ %f3 = nvvm.addf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : f16
+ %f4 = nvvm.addf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : f16
+ %f5 = nvvm.addf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : f16
+ llvm.return %f5 : f16
}
// bf16 + bf16 -> bf16
llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fadd_bf16_bf16(bfloat %0, bfloat %1) {
- // CHECK-NEXT: %3 = fadd bfloat %0, %1
- // CHECK-NEXT: %4 = fadd bfloat %3, %3
+ // CHECK-NEXT: %3 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %0, bfloat %1)
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %3, bfloat %3)
// CHECK-NEXT: ret bfloat %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : bf16
@@ -31,23 +33,23 @@ llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// f32 + f32 -> f32
llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fadd_f32_f32(float %0, float %1) {
- // CHECK-NEXT: %3 = call float @llvm.nvvm.add.rn.f(float %0, float %1)
- // CHECK-NEXT: %4 = call float @llvm.nvvm.add.rn.f(float %3, float %3)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rn.sat.f(float %4, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.ftz.f(float %5, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %6, float %6)
- // CHECK-NEXT: %8 = call float @llvm.nvvm.add.rm.f(float %7, float %7)
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rm.sat.f(float %8, float %8)
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rm.ftz.f(float %9, float %9)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %10, float %10)
- // CHECK-NEXT: %12 = call float @llvm.nvvm.add.rp.f(float %11, float %11)
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rp.sat.f(float %12, float %12)
- // CHECK-NEXT: %14 = call float @llvm.nvvm.add.rp.ftz.f(float %13, float %13)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %14, float %14)
- // CHECK-NEXT: %16 = call float @llvm.nvvm.add.rz.f(float %15, float %15)
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rz.sat.f(float %16, float %16)
- // CHECK-NEXT: %18 = call float @llvm.nvvm.add.rz.ftz.f(float %17, float %17)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %18, float %18)
+ // CHECK-NEXT: %3 = call float @llvm.nvvm.fadd.rn.f32(float %0, float %1)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.rn.f32(float %3, float %3)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %4, float %4)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %5, float %5)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %6, float %6)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rm.f32(float %7, float %7)
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.rm.sat.f32(float %8, float %8)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %9, float %9)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %10, float %10)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rp.f32(float %11, float %11)
+ // CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.rp.sat.f32(float %12, float %12)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rp.ftz.f32(float %13, float %13)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %14, float %14)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rz.f32(float %15, float %15)
+ // CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.rz.sat.f32(float %16, float %16)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %17, float %17)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %18, float %18)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
@@ -73,11 +75,11 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// f64 + f64 -> f64
llvm.func @fadd_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fadd_f64_f64(double %0, double %1) {
- // CHECK-NEXT: %3 = call double @llvm.nvvm.add.rn.d(double %0, double %1)
- // CHECK-NEXT: %4 = call double @llvm.nvvm.add.rn.d(double %3, double %3)
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rm.d(double %4, double %4)
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rp.d(double %5, double %5)
- // CHECK-NEXT: %7 = call double @llvm.nvvm.add.rz.d(double %6, double %6)
+ // CHECK-NEXT: %3 = call double @llvm.nvvm.fadd.rn.f64(double %0, double %1)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.rn.f64(double %3, double %3)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rm.f64(double %4, double %4)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rp.f64(double %5, double %5)
+ // CHECK-NEXT: %7 = call double @llvm.nvvm.fadd.rz.f64(double %6, double %6)
// CHECK-NEXT: ret double %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
index de9a53c8e95d6..58fb8814903ad 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
@@ -55,13 +55,3 @@ llvm.func @addf_invalid_bf16_sat_ftz(%a : bf16, %b : bf16) -> bf16 {
%f1 = nvvm.addf %a, %b sat = <sat> ftz = true : bf16
llvm.return %f1 : bf16
}
-
-// -----
-
-// FIXME: Remove this test once intrinsics for f16 addition (with FTZ only) are
-// available.
-llvm.func @addf_invalid_f16_ftz_no_sat(%a : f16, %b : f16) -> f16 {
- // expected-error at +1 {{FTZ with no saturation is not supported for f16 and vector<2xf16> additions/subtractions}}
- %f1 = nvvm.addf %a, %b ftz = true : f16
- llvm.return %f1 : f16
-}
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index d8bf3dfcc8fb6..1f7daa6c3c66a 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -3,24 +3,26 @@
// vector<2xf16> + vector<2xf16> -> vector<2xf16>
llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @addf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
- // CHECK-NEXT: %3 = fadd <2 x half> %0, %1
- // CHECK-NEXT: %4 = fadd <2 x half> %3, %3
- // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %4, <2 x half> %4)
- // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %5, <2 x half> %5)
+ // CHECK-NEXT: %3 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %0, <2 x half> %1)
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %3, <2 x half> %3)
+ // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %4, <2 x half> %4)
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %5, <2 x half> %5)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
- %f2 = nvvm.addf %f1, %f1 rnd = <rn> : vector<2xf16>
- %f3 = nvvm.addf %f2, %f2 rnd = <rn> sat = <sat> : vector<2xf16>
- %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
+ %f2 = nvvm.addf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xf16>
+ %f3 = nvvm.addf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : vector<2xf16>
+ %f4 = nvvm.addf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : vector<2xf16>
+ %f5 = nvvm.addf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
// vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @addf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
- // CHECK-NEXT: %3 = fadd <2 x bfloat> %0, %1
- // CHECK-NEXT: %4 = fadd <2 x bfloat> %3, %3
+ // CHECK-NEXT: %3 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1)
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %3, <2 x bfloat> %3)
// CHECK-NEXT: ret <2 x bfloat> %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xbf16>
@@ -31,47 +33,26 @@ llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
// vector<2xf32> + vector<2xf32> -> vector<2xf32>
llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rn.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rn.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rn.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rn.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rn.sat.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rn.sat.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rn.ftz.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rn.ftz.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: %35 = extractelement <2 x float> %34, i32 0
- // CHECK-NEXT: %36 = extractelement <2 x float> %34, i32 0
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> poison, float %37, i32 0
- // CHECK-NEXT: %39 = extractelement <2 x float> %34, i32 1
- // CHECK-NEXT: %40 = extractelement <2 x float> %34, i32 1
- // CHECK-NEXT: %41 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %39, float %40)
- // CHECK-NEXT: %42 = insertelement <2 x float> %38, float %41, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %3, <2 x float> %3)
+ // CHECK-NEXT: %5 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.rn.sat.f32(float %5, float %6)
+ // CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
+ // CHECK-NEXT: %9 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.rn.sat.f32(float %9, float %10)
+ // CHECK-NEXT: %12 = insertelement <2 x float> %8, float %11, i32 1
+ // CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %12, <2 x float> %12)
+ // CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
+ // CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %14, float %15)
+ // CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
+ // CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %18, float %19)
+ // CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
+ // CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rn> : vector<2xf32>
@@ -83,39 +64,25 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rm.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rm.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rm.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rm.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rm.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rm.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rm.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rm.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rm> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rm> sat = <sat> : vector<2xf32>
@@ -126,39 +93,25 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rp.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rp.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rp.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rp.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rp.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rp.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rp.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rp.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rp> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rp> sat = <sat> : vector<2xf32>
@@ -169,39 +122,25 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rz.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rz.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rz.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rz.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rz.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rz.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rz.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rz> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rz> sat = <sat> : vector<2xf32>
@@ -215,19 +154,19 @@ llvm.func @addf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rn(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rn.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rn.f64(double %3, double %4)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rn.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rn.f64(double %7, double %8)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: %11 = extractelement <2 x double> %10, i32 0
// CHECK-NEXT: %12 = extractelement <2 x double> %10, i32 0
- // CHECK-NEXT: %13 = call double @llvm.nvvm.add.rn.d(double %11, double %12)
+ // CHECK-NEXT: %13 = call double @llvm.nvvm.fadd.rn.f64(double %11, double %12)
// CHECK-NEXT: %14 = insertelement <2 x double> poison, double %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x double> %10, i32 1
// CHECK-NEXT: %16 = extractelement <2 x double> %10, i32 1
- // CHECK-NEXT: %17 = call double @llvm.nvvm.add.rn.d(double %15, double %16)
+ // CHECK-NEXT: %17 = call double @llvm.nvvm.fadd.rn.f64(double %15, double %16)
// CHECK-NEXT: %18 = insertelement <2 x double> %14, double %17, i32 1
// CHECK-NEXT: ret <2 x double> %18
// CHECK-NEXT: }
@@ -240,11 +179,11 @@ llvm.func @addf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rm(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rm.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rm.f64(double %3, double %4)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rm.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rm.f64(double %7, double %8)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -256,11 +195,11 @@ llvm.func @addf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rp(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rp.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rp.f64(double %3, double %4)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rp.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rp.f64(double %7, double %8)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -272,11 +211,11 @@ llvm.func @addf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rz(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rz.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rz.f64(double %3, double %4)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rz.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rz.f64(double %7, double %8)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index b2c6163045cb7..a540c0338a59b 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -4,29 +4,32 @@
llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fsub_f16_f16(half %0, half %1) {
// CHECK-NEXT: %3 = fneg half %1
- // CHECK-NEXT: %4 = fadd half %0, %3
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.rn.f16(half %0, half %3)
// CHECK-NEXT: %5 = fneg half %4
- // CHECK-NEXT: %6 = fadd half %4, %5
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.rn.f16(half %4, half %5)
// CHECK-NEXT: %7 = fneg half %6
- // CHECK-NEXT: %8 = call half @llvm.nvvm.add.rn.sat.f16(half %6, half %7)
+ // CHECK-NEXT: %8 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %6, half %7)
// CHECK-NEXT: %9 = fneg half %8
- // CHECK-NEXT: %10 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %8, half %9)
- // CHECK-NEXT: ret half %10
+ // CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.rn.sat.f16(half %8, half %9)
+ // CHECK-NEXT: %11 = fneg half %10
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %10, half %11)
+ // CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
- %f2 = nvvm.subf %f1, %f1 rnd = <rn> : f16
- %f3 = nvvm.subf %f2, %f2 rnd = <rn> sat = <sat> : f16
- %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : f16
- llvm.return %f4 : f16
+ %f2 = nvvm.subf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : f16
+ %f3 = nvvm.subf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : f16
+ %f4 = nvvm.subf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : f16
+ %f5 = nvvm.subf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : f16
+ llvm.return %f5 : f16
}
// bf16 - bf16 -> bf16
llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fsub_bf16_bf16(bfloat %0, bfloat %1) {
// CHECK-NEXT: %3 = fneg bfloat %1
- // CHECK-NEXT: %4 = fadd bfloat %0, %3
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %0, bfloat %3)
// CHECK-NEXT: %5 = fneg bfloat %4
- // CHECK-NEXT: %6 = fadd bfloat %4, %5
+ // CHECK-NEXT: %6 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %4, bfloat %5)
// CHECK-NEXT: ret bfloat %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : bf16
@@ -38,39 +41,39 @@ llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fsub_f32_f32(float %0, float %1) {
// CHECK-NEXT: %3 = fneg float %1
- // CHECK-NEXT: %4 = call float @llvm.nvvm.add.rn.f(float %0, float %3)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.rn.f32(float %0, float %3)
// CHECK-NEXT: %5 = fneg float %4
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.f(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rn.f32(float %4, float %5)
// CHECK-NEXT: %7 = fneg float %6
- // CHECK-NEXT: %8 = call float @llvm.nvvm.add.rn.sat.f(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rn.sat.f32(float %6, float %7)
// CHECK-NEXT: %9 = fneg float %8
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rn.ftz.f(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %8, float %9)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %10, float %11)
// CHECK-NEXT: %13 = fneg float %12
- // CHECK-NEXT: %14 = call float @llvm.nvvm.add.rm.f(float %12, float %13)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rm.f32(float %12, float %13)
// CHECK-NEXT: %15 = fneg float %14
- // CHECK-NEXT: %16 = call float @llvm.nvvm.add.rm.sat.f(float %14, float %15)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rm.sat.f32(float %14, float %15)
// CHECK-NEXT: %17 = fneg float %16
- // CHECK-NEXT: %18 = call float @llvm.nvvm.add.rm.ftz.f(float %16, float %17)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %16, float %17)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %18, float %19)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %18, float %19)
// CHECK-NEXT: %21 = fneg float %20
- // CHECK-NEXT: %22 = call float @llvm.nvvm.add.rp.f(float %20, float %21)
+ // CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.rp.f32(float %20, float %21)
// CHECK-NEXT: %23 = fneg float %22
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rp.sat.f(float %22, float %23)
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fadd.rp.sat.f32(float %22, float %23)
// CHECK-NEXT: %25 = fneg float %24
- // CHECK-NEXT: %26 = call float @llvm.nvvm.add.rp.ftz.f(float %24, float %25)
+ // CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.rp.ftz.f32(float %24, float %25)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %26, float %27)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %26, float %27)
// CHECK-NEXT: %29 = fneg float %28
- // CHECK-NEXT: %30 = call float @llvm.nvvm.add.rz.f(float %28, float %29)
+ // CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.rz.f32(float %28, float %29)
// CHECK-NEXT: %31 = fneg float %30
- // CHECK-NEXT: %32 = call float @llvm.nvvm.add.rz.sat.f(float %30, float %31)
+ // CHECK-NEXT: %32 = call float @llvm.nvvm.fadd.rz.sat.f32(float %30, float %31)
// CHECK-NEXT: %33 = fneg float %32
- // CHECK-NEXT: %34 = call float @llvm.nvvm.add.rz.ftz.f(float %32, float %33)
+ // CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %32, float %33)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %34, float %35)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %34, float %35)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
@@ -97,15 +100,15 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
llvm.func @fsub_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fsub_f64_f64(double %0, double %1) {
// CHECK-NEXT: %3 = fneg double %1
- // CHECK-NEXT: %4 = call double @llvm.nvvm.add.rn.d(double %0, double %3)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.rn.f64(double %0, double %3)
// CHECK-NEXT: %5 = fneg double %4
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rn.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rn.f64(double %4, double %5)
// CHECK-NEXT: %7 = fneg double %6
- // CHECK-NEXT: %8 = call double @llvm.nvvm.add.rm.d(double %6, double %7)
+ // CHECK-NEXT: %8 = call double @llvm.nvvm.fadd.rm.f64(double %6, double %7)
// CHECK-NEXT: %9 = fneg double %8
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rp.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rp.f64(double %8, double %9)
// CHECK-NEXT: %11 = fneg double %10
- // CHECK-NEXT: %12 = call double @llvm.nvvm.add.rz.d(double %10, double %11)
+ // CHECK-NEXT: %12 = call double @llvm.nvvm.fadd.rz.f64(double %10, double %11)
// CHECK-NEXT: ret double %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
index d5cff844ed1d3..38a2ca14bfa90 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
@@ -55,13 +55,3 @@ llvm.func @subf_invalid_bf16_sat_ftz(%a : bf16, %b : bf16) -> bf16 {
%f1 = nvvm.subf %a, %b sat = <sat> ftz = true : bf16
llvm.return %f1 : bf16
}
-
-// -----
-
-// FIXME: Remove this test once intrinsics for f16 addition (with FTZ only) are
-// available.
-llvm.func @subf_invalid_f16_ftz_no_sat(%a : f16, %b : f16) -> f16 {
- // expected-error at +1 {{FTZ with no saturation is not supported for f16 and vector<2xf16> additions/subtractions}}
- %f1 = nvvm.subf %a, %b ftz = true : f16
- llvm.return %f1 : f16
-}
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index 4c0f143806329..b72d3b0ebecde 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -4,19 +4,22 @@
llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @subf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
// CHECK-NEXT: %3 = fneg <2 x half> %1
- // CHECK-NEXT: %4 = fadd <2 x half> %0, %3
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %0, <2 x half> %3)
// CHECK-NEXT: %5 = fneg <2 x half> %4
- // CHECK-NEXT: %6 = fadd <2 x half> %4, %5
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %4, <2 x half> %5)
// CHECK-NEXT: %7 = fneg <2 x half> %6
- // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %6, <2 x half> %7)
+ // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %6, <2 x half> %7)
// CHECK-NEXT: %9 = fneg <2 x half> %8
- // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %8, <2 x half> %9)
+ // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %8, <2 x half> %9)
+ // CHECK-NEXT: %11 = fneg <2 x half> %10
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
- %f2 = nvvm.subf %f1, %f1 rnd = <rn> : vector<2xf16>
- %f3 = nvvm.subf %f2, %f2 rnd = <rn> sat = <sat> : vector<2xf16>
- %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
+ %f2 = nvvm.subf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xf16>
+ %f3 = nvvm.subf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : vector<2xf16>
+ %f4 = nvvm.subf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : vector<2xf16>
+ %f5 = nvvm.subf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
@@ -24,9 +27,9 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @subf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
// CHECK-NEXT: %3 = fneg <2 x bfloat> %1
- // CHECK-NEXT: %4 = fadd <2 x bfloat> %0, %3
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %0, <2 x bfloat> %3)
// CHECK-NEXT: %5 = fneg <2 x bfloat> %4
- // CHECK-NEXT: %6 = fadd <2 x bfloat> %4, %5
+ // CHECK-NEXT: %6 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %4, <2 x bfloat> %5)
// CHECK-NEXT: ret <2 x bfloat> %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xbf16>
@@ -38,51 +41,30 @@ llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rn.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rn.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rn.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rn.sat.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rn.sat.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rn.ftz.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rn.ftz.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: %39 = fneg <2 x float> %38
- // CHECK-NEXT: %40 = extractelement <2 x float> %38, i32 0
- // CHECK-NEXT: %41 = extractelement <2 x float> %39, i32 0
- // CHECK-NEXT: %42 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %40, float %41)
- // CHECK-NEXT: %43 = insertelement <2 x float> poison, float %42, i32 0
- // CHECK-NEXT: %44 = extractelement <2 x float> %38, i32 1
- // CHECK-NEXT: %45 = extractelement <2 x float> %39, i32 1
- // CHECK-NEXT: %46 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %44, float %45)
- // CHECK-NEXT: %47 = insertelement <2 x float> %43, float %46, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %4, <2 x float> %5)
+ // CHECK-NEXT: %7 = fneg <2 x float> %6
+ // CHECK-NEXT: %8 = extractelement <2 x float> %6, i32 0
+ // CHECK-NEXT: %9 = extractelement <2 x float> %7, i32 0
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rn.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %11 = insertelement <2 x float> poison, float %10, i32 0
+ // CHECK-NEXT: %12 = extractelement <2 x float> %6, i32 1
+ // CHECK-NEXT: %13 = extractelement <2 x float> %7, i32 1
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rn.sat.f32(float %12, float %13)
+ // CHECK-NEXT: %15 = insertelement <2 x float> %11, float %14, i32 1
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %15, <2 x float> %16)
+ // CHECK-NEXT: %18 = fneg <2 x float> %17
+ // CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
+ // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %19, float %20)
+ // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
+ // CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
+ // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %23, float %24)
+ // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
+ // CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rn> : vector<2xf32>
@@ -95,42 +77,28 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rm.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rm.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rm.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rm.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rm.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rm.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rm.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rm.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rm> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rm> sat = <sat> : vector<2xf32>
@@ -142,42 +110,28 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rp.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rp.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rp.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rp.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rp.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rp.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rp.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rp> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rp> sat = <sat> : vector<2xf32>
@@ -189,42 +143,28 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rz.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rz.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rz.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rz.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rz.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rz.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rz.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rz.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rz> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rz> sat = <sat> : vector<2xf32>
@@ -239,20 +179,20 @@ llvm.func @subf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rn.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rn.f64(double %4, double %5)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rn.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rn.f64(double %8, double %9)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: %12 = fneg <2 x double> %11
// CHECK-NEXT: %13 = extractelement <2 x double> %11, i32 0
// CHECK-NEXT: %14 = extractelement <2 x double> %12, i32 0
- // CHECK-NEXT: %15 = call double @llvm.nvvm.add.rn.d(double %13, double %14)
+ // CHECK-NEXT: %15 = call double @llvm.nvvm.fadd.rn.f64(double %13, double %14)
// CHECK-NEXT: %16 = insertelement <2 x double> poison, double %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x double> %11, i32 1
// CHECK-NEXT: %18 = extractelement <2 x double> %12, i32 1
- // CHECK-NEXT: %19 = call double @llvm.nvvm.add.rn.d(double %17, double %18)
+ // CHECK-NEXT: %19 = call double @llvm.nvvm.fadd.rn.f64(double %17, double %18)
// CHECK-NEXT: %20 = insertelement <2 x double> %16, double %19, i32 1
// CHECK-NEXT: ret <2 x double> %20
// CHECK-NEXT: }
@@ -266,11 +206,11 @@ llvm.func @subf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rm.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rm.f64(double %4, double %5)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rm.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rm.f64(double %8, double %9)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -283,11 +223,11 @@ llvm.func @subf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rp.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rp.f64(double %4, double %5)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rp.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rp.f64(double %8, double %9)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -300,11 +240,11 @@ llvm.func @subf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rz.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rz.f64(double %4, double %5)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rz.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rz.f64(double %8, double %9)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
>From a355f57176a3d2e789525dd9ac81c6f78c5548d8 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 18 Aug 2026 11:12:52 +0000
Subject: [PATCH 02/12] use immediate argument for rounding mode
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 66 +-
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 85 +--
.../CIR/CodeGenCUDA/builtins-nvvm-math.cu | 12 +-
clang/test/CodeGen/builtins-nvptx.c | 26 +-
llvm/include/llvm/IR/IntrinsicsNVVM.td | 18 +-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 81 +--
llvm/lib/Analysis/ConstantFolding.cpp | 68 +-
llvm/lib/IR/AutoUpgrade.cpp | 47 +-
llvm/lib/IR/NVVMIntrinsicUtils.cpp | 5 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 53 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 130 ++--
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 20 +-
llvm/test/CodeGen/NVPTX/bf16-add.ll | 4 +-
llvm/test/CodeGen/NVPTX/bf16-sub.ll | 4 +-
llvm/test/CodeGen/NVPTX/f16-add.ll | 16 +-
llvm/test/CodeGen/NVPTX/f16-sub.ll | 16 +-
llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll | 16 +-
llvm/test/CodeGen/NVPTX/fp-add-invalid.ll | 28 +-
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 32 +-
llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll | 18 +-
llvm/test/CodeGen/NVPTX/fp-fold-sub.ll | 20 +-
llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll | 64 +-
.../InstCombine/NVPTX/nvvm-intrins.ll | 18 +-
.../InstSimplify/const-fold-nvvm-add.ll | 592 ++++++++++++++----
llvm/test/Verifier/NVPTX/fadd.ll | 16 +
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 39 +-
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 58 +-
.../Target/LLVMIR/nvvm/addf/addf_vector.mlir | 84 +--
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 58 +-
.../Target/LLVMIR/nvvm/subf/subf_vector.mlir | 84 +--
30 files changed, 1096 insertions(+), 682 deletions(-)
create mode 100644 llvm/test/Verifier/NVPTX/fadd.ll
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index f8da1b062a14e..3a3afba581ad0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -49,17 +49,20 @@ static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
.getResult();
}
-/// Emit a CIR LLVMIntrinsicCallOp for a binary NVVM intrinsic.
-/// The result type is inferred from the first argument.
-static mlir::Value emitBinaryNVVMIntrinsic(CIRGenFunction &cgf,
- const CallExpr *expr,
- llvm::StringRef intrinsicName) {
+/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd intrinsic, which takes the
+/// rounding mode as a trailing operand.
+static mlir::Value emitNVVMFAdd(CIRGenFunction &cgf, const CallExpr *expr,
+ llvm::StringRef intrinsicName,
+ llvm::APFloat::roundingMode rm) {
auto &builder = cgf.getBuilder();
+ mlir::Location loc = cgf.getLoc(expr->getExprLoc());
mlir::Value lhs = cgf.emitScalarExpr(expr->getArg(0));
mlir::Value rhs = cgf.emitScalarExpr(expr->getArg(1));
- return cir::LLVMIntrinsicCallOp::create(
- builder, cgf.getLoc(expr->getExprLoc()),
- builder.getStringAttr(intrinsicName), lhs.getType(), {lhs, rhs})
+ mlir::Value rnd =
+ builder.getConstInt(loc, builder.getSInt32Ty(), static_cast<int>(rm));
+ return cir::LLVMIntrinsicCallOp::create(builder, loc,
+ builder.getStringAttr(intrinsicName),
+ lhs.getType(), {lhs, rhs, rnd})
.getResult();
}
@@ -810,44 +813,59 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
case NVPTX::BI__nvvm_add_rn_f:
case NVPTX::BI__nvvm_add_rn_d:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_f:
case NVPTX::BI__nvvm_add_rz_d:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd", llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_f:
case NVPTX::BI__nvvm_add_rm_d:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_f:
case NVPTX::BI__nvvm_add_rp_d:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_add_rn_ftz_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.ftz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_ftz_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.ftz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_ftz_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.ftz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_ftz_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.ftz");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_add_rn_sat_f:
case NVPTX::BI__nvvm_add_rn_sat_f16:
case NVPTX::BI__nvvm_add_rn_sat_v2f16:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rn.ftz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rz.ftz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rm.ftz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return emitBinaryNVVMIntrinsic(*this, expr, "nvvm.fadd.rp.ftz.sat");
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
cgm.errorNYI(expr->getSourceRange(),
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index 687a2788946ce..b8708f9d8ce80 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -395,7 +395,8 @@ static Value *MakeCpAsync(unsigned IntrinsicID, unsigned IntrinsicIDS,
}
static Value *MakeHalfType(Function *Intrinsic, unsigned BuiltinID,
- const CallExpr *E, CodeGenFunction &CGF) {
+ const CallExpr *E, CodeGenFunction &CGF,
+ ArrayRef<Value *> TrailingArgs = {}) {
SmallVector<Value *, 16> Args;
auto *FTy = Intrinsic->getFunctionType();
unsigned ICEArguments = 0;
@@ -410,6 +411,7 @@ static Value *MakeHalfType(Function *Intrinsic, unsigned BuiltinID,
ArgValue = CGF.Builder.CreateBitCast(ArgValue, PTy);
Args.push_back(ArgValue);
}
+ Args.append(TrailingArgs.begin(), TrailingArgs.end());
return CGF.Builder.CreateCall(Intrinsic, Args);
}
@@ -427,11 +429,12 @@ static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
CGF.EmitScalarExpr(E->getArg(2))});
}
-static Value *MakeBinaryIntrinsic(unsigned IntrinsicID, const CallExpr *E,
- CodeGenFunction &CGF) {
- return CGF.Builder.CreateBinaryIntrinsic(IntrinsicID,
- CGF.EmitScalarExpr(E->getArg(0)),
- CGF.EmitScalarExpr(E->getArg(1)));
+static Value *MakeFAdd(unsigned IntrinsicID, APFloat::roundingMode RM,
+ unsigned BuiltinID, const CallExpr *E,
+ CodeGenFunction &CGF) {
+ llvm::Type *Ty = CGF.ConvertType(E->getType());
+ return MakeHalfType(CGF.CGM.getIntrinsic(IntrinsicID, Ty), BuiltinID, E, CGF,
+ {CGF.Builder.getInt32(static_cast<int>(RM))});
}
} // namespace
@@ -1143,58 +1146,60 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
EmitScalarExpr(E->getArg(0)));
case NVPTX::BI__nvvm_add_rn_f:
case NVPTX::BI__nvvm_add_rn_d:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_f:
case NVPTX::BI__nvvm_add_rz_d:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardZero, BuiltinID, E,
+ *this);
case NVPTX::BI__nvvm_add_rm_f:
case NVPTX::BI__nvvm_add_rm_d:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardNegative, BuiltinID,
+ E, *this);
case NVPTX::BI__nvvm_add_rp_f:
case NVPTX::BI__nvvm_add_rp_d:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardPositive, BuiltinID,
+ E, *this);
case NVPTX::BI__nvvm_add_rn_ftz_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_ftz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_ftz_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_ftz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardZero, BuiltinID,
+ E, *this);
case NVPTX::BI__nvvm_add_rm_ftz_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_ftz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_ftz_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_ftz, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rn_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_sat_v2f16:
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardZero, BuiltinID,
+ E, *this);
case NVPTX::BI__nvvm_add_rm_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rz_ftz_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardZero,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rm_ftz_sat, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return MakeBinaryIntrinsic(Intrinsic::nvvm_fadd_rp_ftz_sat, E, *this);
- case NVPTX::BI__nvvm_add_rn_sat_f16:
- return MakeHalfType(
- CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_sat, Builder.getHalfTy()),
- BuiltinID, E, *this);
- case NVPTX::BI__nvvm_add_rn_sat_v2f16:
- return MakeHalfType(
- CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_sat,
- FixedVectorType::get(Builder.getHalfTy(), 2)),
- BuiltinID, E, *this);
- case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
- return MakeHalfType(
- CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat, Builder.getHalfTy()),
- BuiltinID, E, *this);
- case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return MakeHalfType(
- CGM.getIntrinsic(Intrinsic::nvvm_fadd_rn_ftz_sat,
- FixedVectorType::get(Builder.getHalfTy(), 2)),
- BuiltinID, E, *this);
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
return MakeLdg(*this, E);
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index ede4c0605c98b..69df1d376f7f7 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -65,25 +65,25 @@ __device__ float test_ex2_approx_ftz_f(float x) {
}
// CIR-LABEL: @_Z13test_add_rn_fff
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rn" {{.*}} : (!cir.float, !cir.float) -> !cir.float
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
// LLVM-LABEL: @_Z13test_add_rn_fff
-// LLVM: call {{.*}}float @llvm.nvvm.fadd.rn.f32(float
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.f32(float {{.*}}, float {{.*}}, /* rnd=rn */ i32 1)
__device__ float test_add_rn_f(float x, float y) {
return __nvvm_add_rn_f(x, y);
}
// CIR-LABEL: @_Z13test_add_rz_ddd
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rz" {{.*}} : (!cir.double, !cir.double) -> !cir.double
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd" {{.*}} : (!cir.double, !cir.double, !s32i) -> !cir.double
// LLVM-LABEL: @_Z13test_add_rz_ddd
-// LLVM: call {{.*}}double @llvm.nvvm.fadd.rz.f64(double
+// LLVM: call {{.*}}double @llvm.nvvm.fadd.f64(double {{.*}}, double {{.*}}, /* rnd=rz */ i32 0)
__device__ double test_add_rz_d(double x, double y) {
return __nvvm_add_rz_d(x, y);
}
// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.rm.ftz.sat" {{.*}} : (!cir.float, !cir.float) -> !cir.float
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.ftz.sat" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// LLVM: call {{.*}}float @llvm.nvvm.fadd.rm.ftz.sat.f32(float
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.ftz.sat.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
__device__ float test_add_rm_ftz_sat_f(float x, float y) {
return __nvvm_add_rm_ftz_sat_f(x, y);
}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 469aff2691a2a..82c5a9ce9f6b0 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -245,7 +245,7 @@ __device__ void nvvm_math(float f1, float f2, double d1, double d2) {
float t3 = __nvvm_sqrt_rn_f(f1);
// CHECK: call float @llvm.nvvm.rcp.rn.f
float t4 = __nvvm_rcp_rn_f(f2);
-// CHECK: call float @llvm.nvvm.fadd.rn.f32
+// CHECK: call float @llvm.nvvm.fadd.f32({{.*}}i32 1)
float t5 = __nvvm_add_rn_f(f1, f2);
// CHECK: call double @llvm.nvvm.fmax.d
@@ -1548,21 +1548,21 @@ __device__ void nvvm_min_max_sm86() {
// CHECK-LABEL: nvvm_add_fma_f32_sat
__device__ void nvvm_add_fma_f32_sat() {
- // CHECK: call float @llvm.nvvm.fadd.rn.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 1)
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rn.ftz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 0)
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rz.ftz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 0)
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rm.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 3)
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rm.ftz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 3)
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rp.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 2)
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.rp.ftz.sat.f32
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 2)
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1592,13 +1592,13 @@ __device__ void nvvm_add_fma_f32_sat() {
// CHECK-LABEL: nvvm_add_mul_f16_sat
__device__ void nvvm_add_mul_f16_sat() {
- // CHECK: call half @llvm.nvvm.fadd.rn.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.sat.f16({{.*}}i32 1)
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.fadd.rn.ftz.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
- // CHECK: call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index c3ddb9bf77a7b..fc33962adb59b 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1712,13 +1712,17 @@ let TargetPrefix = "nvvm" in {
//
let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative,
- IntrNoCreateUndefOrPoison] in
- foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
- foreach ftz = ["", "_ftz"] in
- foreach sat = ["", "_sat"] in
- def int_nvvm_fadd # rnd # ftz # sat :
- DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
- [LLVMMatchType<0>, LLVMMatchType<0>]>;
+ IntrNoCreateUndefOrPoison, ImmArg<ArgIndex<2>>,
+ Range<ArgIndex<2>, 0, 4>,
+ ArgInfo<ArgIndex<2>,
+ [ArgName<"rnd">,
+ ImmArgPrinter<"printFAddRoundingMode">]>] in
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def int_nvvm_fadd # ftz # sat :
+ DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
+ llvm_i32_ty]>;
//
// Dot Product
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index efbb430ae5cd1..7eee25d5eb175 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -160,6 +160,7 @@ LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
const Constant *ImmArgVal);
LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
const Constant *ImmArgVal);
+LLVM_ABI void printFAddRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
@@ -594,24 +595,12 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz:
- case Intrinsic::nvvm_fadd_rm_ftz_sat:
- case Intrinsic::nvvm_fadd_rn_ftz_sat:
- case Intrinsic::nvvm_fadd_rp_ftz_sat:
- case Intrinsic::nvvm_fadd_rz_ftz_sat:
+ case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fadd_ftz_sat:
return true;
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_sat:
- case Intrinsic::nvvm_fadd_rn_sat:
- case Intrinsic::nvvm_fadd_rp_sat:
- case Intrinsic::nvvm_fadd_rz_sat:
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_sat:
return false;
}
llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
@@ -619,53 +608,35 @@ inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd_rm_sat:
- case Intrinsic::nvvm_fadd_rn_sat:
- case Intrinsic::nvvm_fadd_rp_sat:
- case Intrinsic::nvvm_fadd_rz_sat:
- case Intrinsic::nvvm_fadd_rm_ftz_sat:
- case Intrinsic::nvvm_fadd_rn_ftz_sat:
- case Intrinsic::nvvm_fadd_rp_ftz_sat:
- case Intrinsic::nvvm_fadd_rz_ftz_sat:
+ case Intrinsic::nvvm_fadd_sat:
+ case Intrinsic::nvvm_fadd_ftz_sat:
return true;
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
return false;
}
llvm_unreachable("Checking sat flag for invalid NVVM add intrinsic");
}
-inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rm_sat:
- case Intrinsic::nvvm_fadd_rm_ftz_sat:
- return APFloat::rmTowardNegative;
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rn_sat:
- case Intrinsic::nvvm_fadd_rn_ftz_sat:
- return APFloat::rmNearestTiesToEven;
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rp_sat:
- case Intrinsic::nvvm_fadd_rp_ftz_sat:
- return APFloat::rmTowardPositive;
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rz_ftz:
- case Intrinsic::nvvm_fadd_rz_sat:
- case Intrinsic::nvvm_fadd_rz_ftz_sat:
- return APFloat::rmTowardZero;
+inline APFloat::roundingMode GetFAddRoundingMode(const Value *ImmArgVal) {
+ return static_cast<APFloat::roundingMode>(
+ cast<ConstantInt>(ImmArgVal)->getSExtValue());
+}
+
+inline StringRef GetRoundingModeName(APFloat::roundingMode RM) {
+ switch (RM) {
+ case APFloat::rmNearestTiesToEven:
+ return "rn";
+ case APFloat::rmTowardZero:
+ return "rz";
+ case APFloat::rmTowardNegative:
+ return "rm";
+ case APFloat::rmTowardPositive:
+ return "rp";
+ default:
+ return "";
}
- llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
}
inline bool FMulShouldFTZ(Intrinsic::ID IntrinsicID) {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 2ccb1d2b32c92..cda5171b339a8 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1996,14 +1996,8 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
return !IsStrictFP;
// NVVM add intrinsics with explicit rounding modes
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz:
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
// NVVM div intrinsics with explicit rounding modes
case Intrinsic::nvvm_div_rm_d:
@@ -3615,33 +3609,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
return ConstantFP::get(Ty, Res);
}
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz: {
-
- bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
- APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
- APFloat B = IsFTZ ? FTZPreserveSign(Op2V) : Op2V;
-
- APFloat::roundingMode RoundMode =
- nvvm::GetFAddRoundingMode(IntrinsicID);
-
- APFloat Res = A;
- APFloat::opStatus Status = Res.add(B, RoundMode);
-
- if (!Res.isNaN() &&
- (Status == APFloat::opOK || Status == APFloat::opInexact)) {
- Res = IsFTZ ? FTZPreserveSign(Res) : Res;
- return ConstantFP::get(Ty, Res);
- }
- return nullptr;
- }
-
case Intrinsic::nvvm_mul_rm_f:
case Intrinsic::nvvm_mul_rn_f:
case Intrinsic::nvvm_mul_rp_f:
@@ -4192,6 +4159,31 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
}
}
+ if (IntrinsicID == Intrinsic::nvvm_fadd ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz) {
+ const auto *Op1 = dyn_cast<ConstantFP>(Operands[0]);
+ const auto *Op2 = dyn_cast<ConstantFP>(Operands[1]);
+ if (!Op1 || !Op2)
+ return nullptr;
+
+ bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
+ APFloat A =
+ IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
+ APFloat B =
+ IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
+
+ APFloat Res = A;
+ APFloat::opStatus Status =
+ Res.add(B, nvvm::GetFAddRoundingMode(Operands[2]));
+
+ if (!Res.isNaN() &&
+ (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+ Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+ return ConstantFP::get(Ty, Res);
+ }
+ return nullptr;
+ }
+
if (IntrinsicID == Intrinsic::smul_fix ||
IntrinsicID == Intrinsic::smul_fix_sat) {
const APInt *C0, *C1;
@@ -4473,6 +4465,12 @@ static Constant *ConstantFoldFixedVectorCall(
return ConstantVector::get(Result);
}
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
+ // The rounding mode operand is a scalar, so the lane-wise folding below
+ // does not apply.
+ // TODO: Fold these by passing the rounding mode through to every lane.
+ return nullptr;
default:
break;
}
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 1b001d84eda95..29fadda78f438 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1378,6 +1378,29 @@ static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
return F->getIntrinsicID();
}
+static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
+getNVVMFAddUpgrade(StringRef Modifiers) {
+ std::optional<RoundingMode> RM =
+ StringSwitch<std::optional<RoundingMode>>(Modifiers.take_front(2))
+ .Case("rn", RoundingMode::NearestTiesToEven)
+ .Case("rz", RoundingMode::TowardZero)
+ .Case("rm", RoundingMode::TowardNegative)
+ .Case("rp", RoundingMode::TowardPositive)
+ .Default(std::nullopt);
+ if (!RM)
+ return std::nullopt;
+
+ Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
+ .Case("", Intrinsic::nvvm_fadd)
+ .Case(".ftz", Intrinsic::nvvm_fadd_ftz)
+ .Case(".sat", Intrinsic::nvvm_fadd_sat)
+ .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
+ .Default(Intrinsic::not_intrinsic);
+ if (IID == Intrinsic::not_intrinsic)
+ return std::nullopt;
+ return std::make_pair(IID, *RM);
+}
+
static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
return Name.consume_front("local") || Name.consume_front("shared") ||
Name.consume_front("global") || Name.consume_front("constant") ||
@@ -1945,18 +1968,17 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
}
// Upgrade the FP add intrinsics, which are overloaded on the operand type
+ // and take the rounding mode as an operand:
// llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
- // llvm.nvvm.fadd.<rnd>{.ftz}{.sat}.<mangled type>
+ // llvm.nvvm.fadd{.ftz}{.sat}.<mangled type>
+ // The extra operand means these are expanded in UpgradeIntrinsicCall.
if (Name.starts_with("add.")) {
auto [Base, TypeSuffix] = Name.rsplit('.');
- if (TypeSuffix == "f" || TypeSuffix == "d" || TypeSuffix == "f16" ||
- TypeSuffix == "v2f16") {
- IID = Intrinsic::lookupIntrinsicID(("llvm.nvvm.f" + Base).str());
- if (IID != Intrinsic::not_intrinsic) {
- NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
- {F->getReturnType()});
- return true;
- }
+ if ((TypeSuffix == "f" || TypeSuffix == "d" || TypeSuffix == "f16" ||
+ TypeSuffix == "v2f16") &&
+ getNVVMFAddUpgrade(Base.drop_front(strlen("add.")))) {
+ NewFn = nullptr;
+ return true;
}
}
@@ -3051,6 +3073,13 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
: Intrinsic::nvvm_fabs;
Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
+ } else if (Name.consume_front("add.")) {
+ // nvvm.add.<rnd>[.ftz][.sat].{f,d,f16,v2f16}
+ auto [IID, RM] = *getNVVMFAddUpgrade(Name.rsplit('.').first);
+ Value *A = CI->getArgOperand(0);
+ Rep = Builder.CreateIntrinsic(
+ A->getType(), IID,
+ {A, CI->getArgOperand(1), Builder.getInt32(static_cast<int>(RM))});
} else if (Name.consume_front("ex2.approx.")) {
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index b5cf462dfc470..04d2bffac094e 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -151,6 +151,11 @@ void nvvm::printTensormapSwizzleAtomicity(raw_ostream &OS,
}
}
+void nvvm::printFAddRoundingMode(raw_ostream &OS, const Constant *ImmArgVal) {
+ if (isa<ConstantInt>(ImmArgVal))
+ OS << nvvm::GetRoundingModeName(nvvm::GetFAddRoundingMode(ImmArgVal));
+}
+
void nvvm::printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal) {
if (const auto *CI = dyn_cast<ConstantInt>(ImmArgVal)) {
uint64_t Val = CI->getZExtValue();
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 86f305fb90dd6..eb3d63e9eaa84 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7185,7 +7185,8 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
}
}
-static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID) {
+static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
+ APFloat::roundingMode RM) {
const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
@@ -7197,7 +7198,7 @@ static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID) {
case MVT::f32:
if (!VT.isVector() || nvvm::FAddShouldSaturate(IID))
return 0;
- switch (nvvm::GetFAddRoundingMode(IID)) {
+ switch (RM) {
case APFloat::rmNearestTiesToEven:
return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
case APFloat::rmTowardZero:
@@ -7215,9 +7216,10 @@ static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID) {
}
static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID AddIntrinsicID) {
+ Intrinsic::ID AddIntrinsicID,
+ APFloat::roundingMode RM) {
const EVT VT = N->getValueType(0);
- const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID);
+ const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID, RM);
if (!Opc)
return SDValue();
@@ -7239,7 +7241,7 @@ static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(Opc, SDLoc(N), VT, SubOp1, SubOp2);
}
-static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID,
+static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID, APFloat::roundingMode RM,
const NVPTXSubtarget &STI) {
if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(2))
return false;
@@ -7247,11 +7249,10 @@ static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID,
const bool IsSat = nvvm::FAddShouldSaturate(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
- return nvvm::GetFAddRoundingMode(IID) == APFloat::rmNearestTiesToEven;
+ return RM == APFloat::rmNearestTiesToEven;
case MVT::bf16:
- return nvvm::GetFAddRoundingMode(IID) == APFloat::rmNearestTiesToEven &&
- !IsSat && !nvvm::FAddShouldFTZ(IID) &&
- STI.hasNativeBF16Support(ISD::FADD);
+ return RM == APFloat::rmNearestTiesToEven && !IsSat &&
+ !nvvm::FAddShouldFTZ(IID) && STI.hasNativeBF16Support(ISD::FADD);
case MVT::f32:
return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
case MVT::f64:
@@ -7262,15 +7263,16 @@ static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID,
}
static SDValue diagnoseInvalidFAdd(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID IID,
+ Intrinsic::ID IID, APFloat::roundingMode RM,
const NVPTXSubtarget &STI) {
const EVT VT = N->getValueType(0);
- if (isSupportedFAdd(VT, IID, STI))
+ if (isSupportedFAdd(VT, IID, RM, STI))
return SDValue();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
DAG.getMachineFunction().getFunction(),
- Twine(Intrinsic::getBaseName(IID)) + " with operand type " +
+ Twine(Intrinsic::getBaseName(IID)) + " with rounding mode " +
+ nvvm::GetRoundingModeName(RM) + " and operand type " +
VT.getEVTString() + " is not supported on this target",
SDLoc(N).getDebugLoc()));
return DAG.getPOISON(VT);
@@ -7285,25 +7287,16 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
switch (IID) {
default:
break;
- case Intrinsic::nvvm_fadd_rm:
- case Intrinsic::nvvm_fadd_rn:
- case Intrinsic::nvvm_fadd_rp:
- case Intrinsic::nvvm_fadd_rz:
- case Intrinsic::nvvm_fadd_rm_ftz:
- case Intrinsic::nvvm_fadd_rn_ftz:
- case Intrinsic::nvvm_fadd_rp_ftz:
- case Intrinsic::nvvm_fadd_rz_ftz:
- case Intrinsic::nvvm_fadd_rm_sat:
- case Intrinsic::nvvm_fadd_rn_sat:
- case Intrinsic::nvvm_fadd_rp_sat:
- case Intrinsic::nvvm_fadd_rz_sat:
- case Intrinsic::nvvm_fadd_rm_ftz_sat:
- case Intrinsic::nvvm_fadd_rn_ftz_sat:
- case Intrinsic::nvvm_fadd_rp_ftz_sat:
- case Intrinsic::nvvm_fadd_rz_ftz_sat:
- if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, STI))
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fadd_sat:
+ case Intrinsic::nvvm_fadd_ftz_sat: {
+ const auto RM = static_cast<APFloat::roundingMode>(
+ N->getConstantOperandAPInt(3).getSExtValue());
+ if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, RM, STI))
return V;
- return combineFAddWithNeg(N, DCI.DAG, IID);
+ return combineFAddWithNeg(N, DCI.DAG, IID, RM);
+ }
}
return SDValue();
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index cd7fcbde978ef..0f1098a2ae44c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1494,12 +1494,12 @@ class F_MATH_2<string OpcStr, NVPTXRegClass t_regclass,
[(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1))]>,
Requires<Preds>;
-class F_MATH_2_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP,
- list<Predicate> Preds = []>
+class F_MATH_2_RND_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP, TImmLeaf rnd,
+ list<Predicate> Preds = []>
: BasicNVPTXInst<(outs t.RC:$dst),
(ins t.RC:$src0, t.RC:$src1),
OpcStr,
- [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1))]>,
+ [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1, rnd))]>,
Requires<Preds>;
class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,
@@ -2073,59 +2073,60 @@ let Predicates = [doRsqrtOpt] in {
defvar BF16ArithPreds = [hasBF16Math, hasPTX<78>, hasSM<90>];
-def INT_NVVM_ADD_RN_F16 : F_MATH_2_TY<"add.rn.f16", F16RT, int_nvvm_fadd_rn>;
-def INT_NVVM_ADD_RN_FTZ_F16 : F_MATH_2_TY<"add.rn.ftz.f16", F16RT, int_nvvm_fadd_rn_ftz>;
-def INT_NVVM_ADD_RN_SAT_F16 : F_MATH_2_TY<"add.rn.sat.f16", F16RT, int_nvvm_fadd_rn_sat>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16 : F_MATH_2_TY<"add.rn.ftz.sat.f16", F16RT, int_nvvm_fadd_rn_ftz_sat>;
-def INT_NVVM_ADD_RN_F16X2 : F_MATH_2_TY<"add.rn.f16x2", F16X2RT, int_nvvm_fadd_rn>;
-def INT_NVVM_ADD_RN_FTZ_F16X2 : F_MATH_2_TY<"add.rn.ftz.f16x2", F16X2RT, int_nvvm_fadd_rn_ftz>;
-def INT_NVVM_ADD_RN_SAT_F16X2 : F_MATH_2_TY<"add.rn.sat.f16x2", F16X2RT, int_nvvm_fadd_rn_sat>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16X2 : F_MATH_2_TY<"add.rn.ftz.sat.f16x2", F16X2RT, int_nvvm_fadd_rn_ftz_sat>;
-
-def INT_NVVM_ADD_RN_BF16 :
- F_MATH_2_TY<"add.rn.bf16", BF16RT, int_nvvm_fadd_rn, BF16ArithPreds>;
-def INT_NVVM_ADD_RN_BF16X2 :
- F_MATH_2_TY<"add.rn.bf16x2", BF16X2RT, int_nvvm_fadd_rn, BF16ArithPreds>;
-
-def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2_TY<"add.rn.ftz.f32", F32RT, int_nvvm_fadd_rn_ftz>;
-def INT_NVVM_ADD_RN_SAT_FTZ_F : F_MATH_2_TY<"add.rn.sat.ftz.f32", F32RT, int_nvvm_fadd_rn_ftz_sat>;
-def INT_NVVM_ADD_RN_F : F_MATH_2_TY<"add.rn.f32", F32RT, int_nvvm_fadd_rn>;
-def INT_NVVM_ADD_RN_SAT_F : F_MATH_2_TY<"add.rn.sat.f32", F32RT, int_nvvm_fadd_rn_sat>;
-def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2_TY<"add.rz.ftz.f32", F32RT, int_nvvm_fadd_rz_ftz>;
-def INT_NVVM_ADD_RZ_SAT_FTZ_F : F_MATH_2_TY<"add.rz.sat.ftz.f32", F32RT, int_nvvm_fadd_rz_ftz_sat>;
-def INT_NVVM_ADD_RZ_F : F_MATH_2_TY<"add.rz.f32", F32RT, int_nvvm_fadd_rz>;
-def INT_NVVM_ADD_RZ_SAT_F : F_MATH_2_TY<"add.rz.sat.f32", F32RT, int_nvvm_fadd_rz_sat>;
-def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2_TY<"add.rm.ftz.f32", F32RT, int_nvvm_fadd_rm_ftz>;
-def INT_NVVM_ADD_RM_SAT_FTZ_F : F_MATH_2_TY<"add.rm.sat.ftz.f32", F32RT, int_nvvm_fadd_rm_ftz_sat>;
-def INT_NVVM_ADD_RM_F : F_MATH_2_TY<"add.rm.f32", F32RT, int_nvvm_fadd_rm>;
-def INT_NVVM_ADD_RM_SAT_F : F_MATH_2_TY<"add.rm.sat.f32", F32RT, int_nvvm_fadd_rm_sat>;
-def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2_TY<"add.rp.ftz.f32", F32RT, int_nvvm_fadd_rp_ftz>;
-def INT_NVVM_ADD_RP_SAT_FTZ_F : F_MATH_2_TY<"add.rp.sat.ftz.f32", F32RT, int_nvvm_fadd_rp_ftz_sat>;
-def INT_NVVM_ADD_RP_F : F_MATH_2_TY<"add.rp.f32", F32RT, int_nvvm_fadd_rp>;
-def INT_NVVM_ADD_RP_SAT_F : F_MATH_2_TY<"add.rp.sat.f32", F32RT, int_nvvm_fadd_rp_sat>;
-
-def INT_NVVM_ADD_RN_D : F_MATH_2_TY<"add.rn.f64", F64RT, int_nvvm_fadd_rn>;
-def INT_NVVM_ADD_RZ_D : F_MATH_2_TY<"add.rz.f64", F64RT, int_nvvm_fadd_rz>;
-def INT_NVVM_ADD_RM_D : F_MATH_2_TY<"add.rm.f64", F64RT, int_nvvm_fadd_rm>;
-def INT_NVVM_ADD_RP_D : F_MATH_2_TY<"add.rp.f64", F64RT, int_nvvm_fadd_rp>;
+class RndModeImm<string mode> : TImmLeaf<i32,
+ "return Imm == static_cast<int>(RoundingMode::" # mode # ");">;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+def rnd_rn_imm : RndModeImm<"NearestTiesToEven">;
+def rnd_rz_imm : RndModeImm<"TowardZero">;
+def rnd_rm_imm : RndModeImm<"TowardNegative">;
+def rnd_rp_imm : RndModeImm<"TowardPositive">;
+
+foreach t = [F16RT, F16X2RT] in
foreach ftz = ["", "_ftz"] in
- def INT_NVVM_ADD # rnd # ftz # _F32X2 :
- F_MATH_2_TY<!subst("_", ".", "add" # rnd # ftz # "_f32x2"), F32X2RT,
- !cast<Intrinsic>("int_nvvm_fadd" # rnd # ftz),
- [hasF32x2Instructions]>;
+ foreach sat = ["", "_sat"] in
+ def INT_NVVM_ADD_RN # !toupper(ftz # sat # "_" # t.PtxType) :
+ F_MATH_2_RND_TY<!subst("_", ".", "add_rn" # ftz # sat # "_" # t.PtxType),
+ t, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ rnd_rn_imm>;
+
+foreach t = [BF16RT, BF16X2RT] in
+ def INT_NVVM_ADD_RN_ # !toupper(t.PtxType) :
+ F_MATH_2_RND_TY<"add.rn." # t.PtxType, t, int_nvvm_fadd, rnd_rn_imm,
+ BF16ArithPreds>;
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+ defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+
+ foreach ftz = ["", "_ftz"] in {
+ foreach sat = ["", "_sat"] in
+ def INT_NVVM_ADD # !toupper(rnd # sat # ftz) # _F :
+ F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # sat # ftz # "_f32"),
+ F32RT, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ rnd_imm>;
+
+ def INT_NVVM_ADD # !toupper(rnd # ftz) # _F32X2 :
+ F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # ftz # "_f32x2"), F32X2RT,
+ !cast<Intrinsic>("int_nvvm_fadd" # ftz), rnd_imm,
+ [hasF32x2Instructions]>;
+ }
+
+ def INT_NVVM_ADD # !toupper(rnd) # _D :
+ F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # "_f64"), F64RT,
+ int_nvvm_fadd, rnd_imm>;
+}
+
+foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+ defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
def INT_NVVM_MIXED_ADD # rnd # sat # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
!subst("_", ".", "add" # rnd # sat # "_f32_" # type),
[(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fadd" # rnd # sat)
+ (!cast<Intrinsic>("int_nvvm_fadd" # sat)
(f32 (fpextend type:$a)),
- f32:$b))]>,
+ f32:$b, rnd_imm))]>,
Requires<[SM100]>;
}
}
@@ -2159,19 +2160,15 @@ class INT_NVVM_SUB<RegTyInfo TyInfo, string variant> :
[(set TyInfo.Ty:$dst,
(!cast<SDNode>("sub" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
-def INT_NVVM_SUB_RN_F16 : INT_NVVM_SUB<F16RT, "_rn">;
-def INT_NVVM_SUB_RN_FTZ_F16 : INT_NVVM_SUB<F16RT, "_rn_ftz">;
-def INT_NVVM_SUB_RN_SAT_F16 : INT_NVVM_SUB<F16RT, "_rn_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16 : INT_NVVM_SUB<F16RT, "_rn_ftz_sat">;
-def INT_NVVM_SUB_RN_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn">;
-def INT_NVVM_SUB_RN_FTZ_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_ftz">;
-def INT_NVVM_SUB_RN_SAT_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16X2 : INT_NVVM_SUB<F16X2RT, "_rn_ftz_sat">;
+foreach t = [F16RT, F16X2RT] in
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def INT_NVVM_SUB_RN # !toupper(ftz # sat # "_" # t.PtxType) :
+ INT_NVVM_SUB<t, "_rn" # ftz # sat>;
-let Predicates = BF16ArithPreds in {
- def INT_NVVM_SUB_RN_BF16 : INT_NVVM_SUB<BF16RT, "_rn">;
- def INT_NVVM_SUB_RN_BF16X2 : INT_NVVM_SUB<BF16X2RT, "_rn">;
-}
+let Predicates = BF16ArithPreds in
+ foreach t = [BF16RT, BF16X2RT] in
+ def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, "_rn">;
let Predicates = [hasF32x2Instructions] in
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
@@ -2179,33 +2176,38 @@ let Predicates = [hasF32x2Instructions] in
def INT_NVVM_SUB # rnd # ftz # _F32X2 : INT_NVVM_SUB<F32X2RT, rnd # ftz>;
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+ defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # rnd # ftz # sat);
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat);
def INT_NVVM_SUB # rnd # ftz # sat # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
- [(set f32:$dst, (add_intrin f32:$a, (f32 (fneg f32:$b))))]>;
+ [(set f32:$dst,
+ (add_intrin f32:$a, (f32 (fneg f32:$b)), rnd_imm))]>;
}
}
-
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # rnd);
+
def INT_NVVM_SUB # rnd # _D :
BasicNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
!subst("_", ".", "sub" # rnd # "_f64"),
- [(set f64:$dst, (add_intrin f64:$a, (f64 (fneg f64:$b))))]>;
+ [(set f64:$dst,
+ (int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), rnd_imm))]>;
}
foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+ defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
def INT_NVVM_MIXED_SUB # rnd # sat # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # "_f32_" # type),
[(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fadd" # rnd # sat)
+ (!cast<Intrinsic>("int_nvvm_fadd" # sat)
(f32 (fpextend type:$a)),
- (f32 (fneg f32:$b))))]>,
+ (f32 (fneg f32:$b)), rnd_imm))]>,
Requires<[SM100]>;
}
}
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 8a0c23e0e4f09..2871ddab068bc 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -713,16 +713,16 @@ define void @nvvm_ex2_approx(float %a, double %b, half %c, <2 x half> %d) {
}
define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
-; CHECK: call float @llvm.nvvm.fadd.rn.f32(float %a, float %a)
-; CHECK: call float @llvm.nvvm.fadd.rz.ftz.f32(float %a, float %a)
-; CHECK: call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %a)
-; CHECK: call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %a)
-; CHECK: call double @llvm.nvvm.fadd.rn.f64(double %b, double %b)
-; CHECK: call double @llvm.nvvm.fadd.rz.f64(double %b, double %b)
-; CHECK: call half @llvm.nvvm.fadd.rn.sat.f16(half %c, half %c)
-; CHECK: call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %c, half %c)
-; CHECK: call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %d, <2 x half> %d)
-; CHECK: call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d)
+; CHECK: call float @llvm.nvvm.fadd.f32(float %a, float %a, /* rnd=rn */ i32 1)
+; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %a, /* rnd=rz */ i32 0)
+; CHECK: call float @llvm.nvvm.fadd.sat.f32(float %a, float %a, /* rnd=rm */ i32 3)
+; CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %a, /* rnd=rp */ i32 2)
+; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rn */ i32 1)
+; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rz */ i32 0)
+; CHECK: call half @llvm.nvvm.fadd.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
%r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
%r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
%r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
diff --git a/llvm/test/CodeGen/NVPTX/bf16-add.ll b/llvm/test/CodeGen/NVPTX/bf16-add.ll
index 71d90a3ae675c..374c37b564cf1 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-add.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-add.ll
@@ -13,7 +13,7 @@ define bfloat @add_rn_bf16(bfloat %a, bfloat %b) {
; CHECK-NEXT: add.rn.bf16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %b)
+ %1 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 1)
ret bfloat %1
}
@@ -28,6 +28,6 @@ define <2 x bfloat> @add_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
; CHECK-NEXT: add.rn.bf16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+ %1 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, i32 1)
ret <2 x bfloat> %1
}
diff --git a/llvm/test/CodeGen/NVPTX/bf16-sub.ll b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
index 3d685ebf65cf2..b095a534d3c35 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
@@ -14,7 +14,7 @@ define bfloat @sub_rn_bf16(bfloat %a, bfloat %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg bfloat %b
- %res = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %1)
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %1, i32 1)
ret bfloat %res
}
@@ -30,6 +30,6 @@ define <2 x bfloat> @sub_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x bfloat> %b
- %res = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %a, <2 x bfloat> %1)
+ %res = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %1, i32 1)
ret <2 x bfloat> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
index 63d7f1e2705d6..0b470daf7a535 100644
--- a/llvm/test/CodeGen/NVPTX/f16-add.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -13,7 +13,7 @@ define half @add_rn_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.rn.f16(half %a, half %b)
+ %1 = call half @llvm.nvvm.fadd.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -28,7 +28,7 @@ define <2 x half> @add_rn_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %a, <2 x half> %b)
+ %1 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
@@ -43,7 +43,7 @@ define half @add_rn_ftz_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.ftz.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %a, half %b)
+ %1 = call half @llvm.nvvm.fadd.ftz.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -58,7 +58,7 @@ define <2 x half> @add_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.ftz.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %a, <2 x half> %b)
+ %1 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
@@ -73,7 +73,7 @@ define half @add_rn_sat_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.sat.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.rn.sat.f16(half %a, half %b)
+ %1 = call half @llvm.nvvm.fadd.sat.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -88,7 +88,7 @@ define <2 x half> @add_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.sat.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
+ %1 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
@@ -103,7 +103,7 @@ define half @add_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %a, half %b)
+ %1 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -118,6 +118,6 @@ define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
+ %1 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
index e76c033c6705f..c59875af1d4f9 100644
--- a/llvm/test/CodeGen/NVPTX/f16-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -16,7 +16,7 @@ define half @sub_rn_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.rn.f16(half %a, half %1)
+ %res = call half @llvm.nvvm.fadd.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -32,7 +32,7 @@ define <2 x half> @sub_rn_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %a, <2 x half> %1)
+ %res = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
@@ -48,7 +48,7 @@ define half @sub_rn_ftz_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.rn.ftz.f16(half %a, half %1)
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -64,7 +64,7 @@ define <2 x half> @sub_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %a, <2 x half> %1)
+ %res = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
@@ -80,7 +80,7 @@ define half @sub_rn_sat_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.rn.sat.f16(half %a, half %1)
+ %res = call half @llvm.nvvm.fadd.sat.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -96,7 +96,7 @@ define <2 x half> @sub_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %a, <2 x half> %1)
+ %res = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
@@ -112,7 +112,7 @@ define half @sub_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %a, half %1)
+ %res = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -128,6 +128,6 @@ define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1)
+ %res = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
index f837c6d5d9a54..01f0c6cac37b6 100644
--- a/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
@@ -6,55 +6,55 @@ target triple = "nvptx64-nvidia-cuda"
define <2 x float> @add_rn(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rn(
; CHECK: add.rn.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
ret <2 x float> %r
}
define <2 x float> @add_rz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rz(
; CHECK: add.rz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
ret <2 x float> %r
}
define <2 x float> @add_rm(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rm(
; CHECK: add.rm.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 3)
ret <2 x float> %r
}
define <2 x float> @add_rp(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rp(
; CHECK: add.rp.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 2)
ret <2 x float> %r
}
define <2 x float> @add_rn_ftz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rn_ftz(
; CHECK: add.rn.ftz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
ret <2 x float> %r
}
define <2 x float> @add_rz_ftz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rz_ftz(
; CHECK: add.rz.ftz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
ret <2 x float> %r
}
define <2 x float> @add_rm_ftz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rm_ftz(
; CHECK: add.rm.ftz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 3)
ret <2 x float> %r
}
define <2 x float> @add_rp_ftz(<2 x float> %a, <2 x float> %b) {
; CHECK-LABEL: add_rp_ftz(
; CHECK: add.rp.ftz.f32x2
- %r = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 2)
ret <2 x float> %r
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
index 3114592aaccdc..776935ddd0f91 100644
--- a/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
@@ -4,44 +4,44 @@
target triple = "nvptx64-nvidia-cuda"
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.sat with operand type v2f32 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd.sat with rounding mode rn and operand type v2f32 is not supported
define <2 x float> @sat_f32x2(<2 x float> %a, <2 x float> %b) {
- %r = call <2 x float> @llvm.nvvm.fadd.rn.sat.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.sat.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
ret <2 x float> %r
}
-; NOF32X2: error: {{.*}}llvm.nvvm.fadd.rn with operand type v2f32 is not supported
+; NOF32X2: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type v2f32 is not supported
define <2 x float> @unsupported_f32x2(<2 x float> %a, <2 x float> %b) {
- %r = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
ret <2 x float> %r
}
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.ftz with operand type f64 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd.ftz with rounding mode rn and operand type f64 is not supported
define double @ftz_f64(double %a, double %b) {
- %r = call double @llvm.nvvm.fadd.rn.ftz.f64(double %a, double %b)
+ %r = call double @llvm.nvvm.fadd.ftz.f64(double %a, double %b, i32 1)
ret double %r
}
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rz.sat with operand type f16 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd.sat with rounding mode rz and operand type f16 is not supported
define half @rz_f16(half %a, half %b) {
- %r = call half @llvm.nvvm.fadd.rz.sat.f16(half %a, half %b)
+ %r = call half @llvm.nvvm.fadd.sat.f16(half %a, half %b, i32 0)
ret half %r
}
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rn.ftz with operand type bf16 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd.ftz with rounding mode rn and operand type bf16 is not supported
define bfloat @ftz_bf16(bfloat %a, bfloat %b) {
- %r = call bfloat @llvm.nvvm.fadd.rn.ftz.bf16(bfloat %a, bfloat %b)
+ %r = call bfloat @llvm.nvvm.fadd.ftz.bf16(bfloat %a, bfloat %b, i32 1)
ret bfloat %r
}
-; NOBF16: error: {{.*}}llvm.nvvm.fadd.rn with operand type bf16 is not supported
+; NOBF16: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type bf16 is not supported
define bfloat @unsupported_bf16(bfloat %a, bfloat %b) {
- %r = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %a, bfloat %b)
+ %r = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 1)
ret bfloat %r
}
-; CHECK: error: {{.*}}llvm.nvvm.fadd.rn with operand type v4f32 is not supported
+; CHECK: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type v4f32 is not supported
define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) {
- %r = call <4 x float> @llvm.nvvm.fadd.rn.v4f32(<4 x float> %a, <4 x float> %b)
+ %r = call <4 x float> @llvm.nvvm.fadd.v4f32(<4 x float> %a, <4 x float> %b, i32 1)
ret <4 x float> %r
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index 44c09cc177595..ad0e77d119e65 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -20,17 +20,17 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: add.rp.sat.ftz.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.fadd.rn.sat.f32(float %a, float %b)
- %r2 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %a, float %r1)
+ %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, i32 1)
- %r3 = call float @llvm.nvvm.fadd.rz.sat.f32(float %a, float %r2)
- %r4 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %a, float %r3)
+ %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, i32 0)
- %r5 = call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %r4)
- %r6 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %a, float %r5)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, i32 3)
- %r7 = call float @llvm.nvvm.fadd.rp.sat.f32(float %a, float %r6)
- %r8 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %r7)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, i32 2)
ret float %r8
}
@@ -54,28 +54,28 @@ define float @sub_sat_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.rn.sat.f32(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.rz.sat.f32(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.rm.sat.f32(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, i32 3)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.rp.sat.f32(float %a, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, i32 2)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %a, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, i32 2)
ret float %r8
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
index 513926de4451d..ca854106fa31b 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
@@ -21,28 +21,28 @@ define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd10;
; CHECK-NEXT: ret;
%f0 = fneg <2 x float> %b
- %r1 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %a, <2 x float> %f0)
+ %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f0, i32 1)
%f1 = fneg <2 x float> %r1
- %r2 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %a, <2 x float> %f1)
+ %r2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f1, i32 1)
%f2 = fneg <2 x float> %r2
- %r3 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %a, <2 x float> %f2)
+ %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f2, i32 0)
%f3 = fneg <2 x float> %r3
- %r4 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %a, <2 x float> %f3)
+ %r4 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f3, i32 0)
%f4 = fneg <2 x float> %r4
- %r5 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %a, <2 x float> %f4)
+ %r5 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f4, i32 3)
%f5 = fneg <2 x float> %r5
- %r6 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %a, <2 x float> %f5)
+ %r6 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f5, i32 3)
%f6 = fneg <2 x float> %r6
- %r7 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %a, <2 x float> %f6)
+ %r7 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f6, i32 2)
%f7 = fneg <2 x float> %r7
- %r8 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %a, <2 x float> %f7)
+ %r8 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f7, i32 2)
ret <2 x float> %r8
}
@@ -59,6 +59,6 @@ define <2 x float> @sub_f32x2_negated_lhs(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
; CHECK-NEXT: ret;
%f = fneg <2 x float> %a
- %r = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %f, <2 x float> %b)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %f, <2 x float> %b, i32 0)
ret <2 x float> %r
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
index a3084392c7853..31edae75a48e7 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
@@ -20,22 +20,22 @@ define float @sub_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r8;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.rz.f32(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.rm.f32(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f5, i32 3)
ret float %r6
}
@@ -55,16 +55,16 @@ define double @sub_f64(double %a, double %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd6;
; CHECK-NEXT: ret;
%f0 = fneg double %b
- %r1 = call double @llvm.nvvm.fadd.rn.f64(double %a, double %f0)
+ %r1 = call double @llvm.nvvm.fadd.f64(double %a, double %f0, i32 1)
%f1 = fneg double %r1
- %r2 = call double @llvm.nvvm.fadd.rz.f64(double %a, double %f1)
+ %r2 = call double @llvm.nvvm.fadd.f64(double %a, double %f1, i32 0)
%f2 = fneg double %r2
- %r3 = call double @llvm.nvvm.fadd.rm.f64(double %a, double %f2)
+ %r3 = call double @llvm.nvvm.fadd.f64(double %a, double %f2, i32 3)
%f3 = fneg double %r3
- %r4 = call double @llvm.nvvm.fadd.rp.f64(double %a, double %f3)
+ %r4 = call double @llvm.nvvm.fadd.f64(double %a, double %f3, i32 2)
ret double %r4
}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
index aa80dc2185bd5..9da6c9a39a34e 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
@@ -27,16 +27,16 @@ define float @test_add_f32_f16_1(half %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext half %a to float
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %b)
- %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, i32 0)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, i32 2)
; SAT
- %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, i32 1)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, i32 0)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, i32 3)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, i32 2)
ret float %r8
}
@@ -93,16 +93,16 @@ define float @test_add_f32_bf16_1(bfloat %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext bfloat %a to float
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %b)
- %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, i32 0)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, i32 2)
; SAT
- %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, i32 1)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, i32 0)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, i32 3)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, i32 2)
ret float %r8
}
@@ -160,29 +160,29 @@ define float @test_sub_f32_f16_1(half %a, float %b) {
%r0 = fpext half %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, i32 0)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, i32 3)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, i32 3)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, i32 1)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, i32 0)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, i32 3)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, i32 2)
ret float %r7
}
@@ -240,29 +240,29 @@ define float @test_sub_f32_bf16_1(bfloat %a, float %b) {
%r0 = fpext bfloat %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.rn.f32(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.rz.f32(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, i32 0)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.rm.f32(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, i32 3)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.rp.f32(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, i32 2)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, i32 1)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, i32 0)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.rm.sat.f32(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, i32 3)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, i32 2)
ret float %r8
}
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index d0acf650d02c5..b4b04ef371e94 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -299,20 +299,20 @@ define float @test_ull2f(i64 %a) #0 {
; CHECK-LABEL: @test_add_rn_d
define double @test_add_rn_d(double %a, double %b) #0 {
-; CHECK: call double @llvm.nvvm.fadd.rn.f64
- %ret = call double @llvm.nvvm.fadd.rn.f64(double %a, double %b)
+; CHECK: call double @llvm.nvvm.fadd.f64
+ %ret = call double @llvm.nvvm.fadd.f64(double %a, double %b, /* rnd=rn */ i32 1)
ret double %ret
}
; CHECK-LABEL: @test_add_rn_f
define float @test_add_rn_f(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.fadd.rn.f32
- %ret = call float @llvm.nvvm.fadd.rn.f32(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.f32
+ %ret = call float @llvm.nvvm.fadd.f32(float %a, float %b, /* rnd=rn */ i32 1)
ret float %ret
}
; CHECK-LABEL: @test_add_rn_f_ftz
define float @test_add_rn_f_ftz(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %b)
- %ret = call float @llvm.nvvm.fadd.rn.ftz.f32(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %b, /* rnd=rn */ i32 1)
+ %ret = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %b, /* rnd=rn */ i32 1)
ret float %ret
}
@@ -437,9 +437,9 @@ define i32 @test_fshr_clamp_3(i32 %a, i32 %b, i32 %c) {
ret i32 %call
}
-declare double @llvm.nvvm.fadd.rn.f64(double, double)
-declare float @llvm.nvvm.fadd.rn.f32(float, float)
-declare float @llvm.nvvm.fadd.rn.ftz.f32(float, float)
+declare double @llvm.nvvm.fadd.f64(double, double, i32 immarg)
+declare float @llvm.nvvm.fadd.f32(float, float, i32 immarg)
+declare float @llvm.nvvm.fadd.ftz.f32(float, float, i32 immarg)
declare double @llvm.nvvm.ceil.d(double)
declare float @llvm.nvvm.ceil.f(float)
declare float @llvm.nvvm.ceil.ftz.f(float)
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
index f3129ae0c5f48..02942634327af 100644
--- a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
@@ -13,7 +13,7 @@ define double @test_1_25_minus_2_rm_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rm_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rm */ i32 3)
ret double %res
}
@@ -21,7 +21,7 @@ define double @test_1_25_minus_2_rn_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rn_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rn */ i32 1)
ret double %res
}
@@ -29,7 +29,7 @@ define double @test_1_25_minus_2_rp_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rp_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rp */ i32 2)
ret double %res
}
@@ -37,7 +37,7 @@ define double @test_1_25_minus_2_rz_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rz_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rz */ i32 0)
ret double %res
}
@@ -45,7 +45,7 @@ define float @test_1_25_minus_2_rm_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -53,7 +53,7 @@ define float @test_1_25_minus_2_rn_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -61,7 +61,7 @@ define float @test_1_25_minus_2_rp_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -69,7 +69,7 @@ define float @test_1_25_minus_2_rz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -77,7 +77,7 @@ define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -85,7 +85,7 @@ define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -93,7 +93,7 @@ define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -101,10 +101,74 @@ define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_1_25_minus_2_rm_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rm_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rn_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rn_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rp_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rp_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rz_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rz_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_1_25_minus_2_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rm_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rn_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rp_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rz_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(0.0, NaN) #
;###############################################################
@@ -113,112 +177,184 @@ define float @test_1_25_minus_2_rz_ftz_f() {
define double @test_zero_plus_nan_rm_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rm_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rm.f64(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rm */ i32 3)
ret double %res
}
define double @test_zero_plus_nan_rn_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rn_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rn.f64(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rn */ i32 1)
ret double %res
}
define double @test_zero_plus_nan_rp_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rp_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rp.f64(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rp */ i32 2)
ret double %res
}
define double @test_zero_plus_nan_rz_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rz_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.rz.f64(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rz */ i32 0)
ret double %res
}
define float @test_zero_plus_nan_rm_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rm.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rm */ i32 3)
ret float %res
}
define float @test_zero_plus_nan_rn_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rn.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_zero_plus_nan_rp_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rp.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_zero_plus_nan_rz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rz */ i32 0)
ret float %res
}
define float @test_zero_plus_nan_rm_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rm */ i32 3)
ret float %res
}
define float @test_zero_plus_nan_rn_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_zero_plus_nan_rp_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_zero_plus_nan_rz_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_zero_plus_nan_rm_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rm_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rm */ i32 3)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rn_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rn_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rn */ i32 1)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rp_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rp_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rp */ i32 2)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rz_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rz_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rz */ i32 0)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_zero_plus_nan_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rm_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rm */ i32 3)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rn_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rn */ i32 1)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rp_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rp */ i32 2)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rz_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rz */ i32 0)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(Subnormal, Subnormal) -> Normal #
;###############################################################
@@ -230,7 +366,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -238,7 +374,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -246,7 +382,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -254,7 +390,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -262,7 +398,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -270,7 +406,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -278,7 +414,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -286,7 +422,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -294,7 +430,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -302,7 +438,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -310,7 +446,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -318,10 +454,106 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_subnorm_plus_subnorm_to_normal_rm_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rm_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rn_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rn_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rp_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rp_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rm_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rn_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rp_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rz_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rm_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rn_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rp_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rz_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(Normal, -Subnormal) -> Subnormal #
;###############################################################
@@ -335,7 +567,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -343,7 +575,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -351,7 +583,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -359,7 +591,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -367,7 +599,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -375,7 +607,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -383,7 +615,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -391,7 +623,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -399,7 +631,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -407,7 +639,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -415,7 +647,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -423,10 +655,74 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_normal_minus_subnorm_to_subnorm_rm_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rm_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rn_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rn_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rp_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rp_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rz_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rn_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rp_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rz_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(1.0, 2^(-25)) #
;###############################################################
@@ -439,7 +735,7 @@ define float @test_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -447,7 +743,7 @@ define float @test_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -455,7 +751,7 @@ define float @test_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -463,7 +759,7 @@ define float @test_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -471,7 +767,7 @@ define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -479,7 +775,7 @@ define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -487,7 +783,7 @@ define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -495,7 +791,7 @@ define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -511,7 +807,7 @@ define double @test_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -519,7 +815,7 @@ define double @test_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -527,7 +823,7 @@ define double @test_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0x3FF0000000000001
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -535,10 +831,90 @@ define double @test_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
+;###############################################################
+;# Add(1.0, 2^(-12)) #
+;###############################################################
+; Tests addition of 1.0 and 2^(-12) where the exact result falls between
+; 1.0 and 1.0 + 2^(-10):
+; - RN, RZ, RM: Return 1.0 (rounding to nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-10) (rounding up)
+
+define half @test_1_plus_ulp_rm_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rm_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rn_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rn_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rp_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rp_f16() {
+; CHECK-NEXT: ret half 1.000980e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rz_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rz_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+;###############################################################
+;# Add(1.0, 2^(-9)) #
+;###############################################################
+; Tests addition of 1.0 and 2^(-9) where the exact result falls between
+; 1.0 and 1.0 + 2^(-7):
+; - RN, RZ, RM: Return 1.0 (rounding to nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-7) (rounding up)
+
+define bfloat @test_1_plus_ulp_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rm_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rn_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rp_bf16() {
+; CHECK-NEXT: ret bfloat 1.007810e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rz_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(-1.0, 2^(-25)) #
;###############################################################
@@ -551,7 +927,7 @@ define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -559,7 +935,7 @@ define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -567,7 +943,7 @@ define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -575,7 +951,7 @@ define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -583,7 +959,7 @@ define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -591,7 +967,7 @@ define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -599,7 +975,7 @@ define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -607,7 +983,7 @@ define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -623,7 +999,7 @@ define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -631,7 +1007,7 @@ define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -639,7 +1015,7 @@ define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -647,7 +1023,7 @@ define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -663,7 +1039,7 @@ define float @test_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -671,7 +1047,7 @@ define float @test_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -679,7 +1055,7 @@ define float @test_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -687,7 +1063,7 @@ define float @test_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -695,7 +1071,7 @@ define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -703,7 +1079,7 @@ define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -711,7 +1087,7 @@ define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -719,7 +1095,7 @@ define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -735,7 +1111,7 @@ define double @test_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -743,7 +1119,7 @@ define double @test_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -751,7 +1127,7 @@ define double @test_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -759,7 +1135,7 @@ define double @test_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -775,7 +1151,7 @@ define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.fadd.rm.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -783,7 +1159,7 @@ define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -791,7 +1167,7 @@ define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -799,7 +1175,7 @@ define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -807,7 +1183,7 @@ define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.fadd.rm.ftz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -815,7 +1191,7 @@ define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rn.ftz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -823,7 +1199,7 @@ define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rp.ftz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -831,7 +1207,7 @@ define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fadd.rz.ftz.f32(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -847,7 +1223,7 @@ define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0xBFF0000000000001
;
- %res = call double @llvm.nvvm.fadd.rm.f64(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -855,7 +1231,7 @@ define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rn.f64(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -863,7 +1239,7 @@ define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rp.f64(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -871,6 +1247,6 @@ define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fadd.rz.f64(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
diff --git a/llvm/test/Verifier/NVPTX/fadd.ll b/llvm/test/Verifier/NVPTX/fadd.ll
new file mode 100644
index 0000000000000..d3a03abf6f1f2
--- /dev/null
+++ b/llvm/test/Verifier/NVPTX/fadd.ll
@@ -0,0 +1,16 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+declare float @llvm.nvvm.fadd.f32(float, float, i32 immarg)
+
+define void @test_fadd_rounding_mode(float %a) {
+ ; CHECK: immarg value 4 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 4)
+
+ ; CHECK: immarg value 7 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 7)
+
+ ; CHECK: immarg value -1 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 -1)
+
+ ret void
+}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 8b823fa425206..b3c953de574b9 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -465,7 +465,9 @@ createScalarizedIntrinsicCall(llvm::IRBuilderBase &builder,
llvm::SmallVector<llvm::Value *> scalarArgs;
for (llvm::Value *op : operands)
scalarArgs.push_back(
- builder.CreateExtractElement(op, builder.getInt32(i)));
+ op->getType()->isVectorTy()
+ ? builder.CreateExtractElement(op, builder.getInt32(i))
+ : op);
llvm::Value *res = createIntrinsicCall(builder, IID, retType, scalarArgs);
result = builder.CreateInsertElement(result, res, builder.getInt32(i));
}
@@ -483,23 +485,18 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
llvm::Type *opTypeLLVM = argLHS->getType();
bool isSat = satMode != NVVM::SaturationMode::NONE;
- static constexpr llvm::Intrinsic::ID addIDs[2][2][5] = {
- {{llvm::Intrinsic::nvvm_fadd_rn, llvm::Intrinsic::nvvm_fadd_rn,
- llvm::Intrinsic::nvvm_fadd_rm, llvm::Intrinsic::nvvm_fadd_rp,
- llvm::Intrinsic::nvvm_fadd_rz},
- {llvm::Intrinsic::nvvm_fadd_rn_sat, llvm::Intrinsic::nvvm_fadd_rn_sat,
- llvm::Intrinsic::nvvm_fadd_rm_sat, llvm::Intrinsic::nvvm_fadd_rp_sat,
- llvm::Intrinsic::nvvm_fadd_rz_sat}},
- {{llvm::Intrinsic::nvvm_fadd_rn_ftz, llvm::Intrinsic::nvvm_fadd_rn_ftz,
- llvm::Intrinsic::nvvm_fadd_rm_ftz, llvm::Intrinsic::nvvm_fadd_rp_ftz,
- llvm::Intrinsic::nvvm_fadd_rz_ftz},
- {llvm::Intrinsic::nvvm_fadd_rn_ftz_sat,
- llvm::Intrinsic::nvvm_fadd_rn_ftz_sat,
- llvm::Intrinsic::nvvm_fadd_rm_ftz_sat,
- llvm::Intrinsic::nvvm_fadd_rp_ftz_sat,
- llvm::Intrinsic::nvvm_fadd_rz_ftz_sat}}};
-
- llvm::Intrinsic::ID id = addIDs[isFTZ][isSat][static_cast<unsigned>(rndMode)];
+ static constexpr llvm::Intrinsic::ID addIDs[2][2] = {
+ {llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
+ {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
+
+ static constexpr llvm::RoundingMode roundingModes[5] = {
+ llvm::RoundingMode::NearestTiesToEven,
+ llvm::RoundingMode::NearestTiesToEven, llvm::RoundingMode::TowardNegative,
+ llvm::RoundingMode::TowardPositive, llvm::RoundingMode::TowardZero};
+
+ llvm::Intrinsic::ID id = addIDs[isFTZ][isSat];
+ llvm::Value *rnd = builder.getInt32(
+ static_cast<int>(roundingModes[static_cast<unsigned>(rndMode)]));
// For f64 vector addition, and f32 vector addition with saturation,
// we need to scalarize the intrinsic call.
@@ -507,13 +504,13 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() ||
(isSat && scalarTypeLLVM->isFloatTy()))) {
mt.mapValue(res, createScalarizedIntrinsicCall(builder, id, opTypeLLVM,
- {argLHS, argRHS},
+ {argLHS, argRHS, rnd},
scalarTypeLLVM));
return;
}
- mt.mapValue(res,
- createIntrinsicCall(builder, id, opTypeLLVM, {argLHS, argRHS}));
+ mt.mapValue(
+ res, createIntrinsicCall(builder, id, opTypeLLVM, {argLHS, argRHS, rnd}));
}
void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index a5e5a204a36e3..a376351ac20ec 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -3,11 +3,11 @@
// f16 + f16 -> f16
llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fadd_f16_f16(half %0, half %1) {
- // CHECK-NEXT: %3 = call half @llvm.nvvm.fadd.rn.f16(half %0, half %1)
- // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.rn.f16(half %3, half %3)
- // CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %4, half %4)
- // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.rn.sat.f16(half %5, half %5)
- // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %6, half %6)
+ // CHECK-NEXT: %3 = call half @llvm.nvvm.fadd.f16(half %0, half %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %3, half %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.ftz.f16(half %4, half %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.sat.f16(half %5, half %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %6, half %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
@@ -21,8 +21,8 @@ llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// bf16 + bf16 -> bf16
llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fadd_bf16_bf16(bfloat %0, bfloat %1) {
- // CHECK-NEXT: %3 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %0, bfloat %1)
- // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %3, bfloat %3)
+ // CHECK-NEXT: %3 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %0, bfloat %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %3, bfloat %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret bfloat %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : bf16
@@ -33,23 +33,23 @@ llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// f32 + f32 -> f32
llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fadd_f32_f32(float %0, float %1) {
- // CHECK-NEXT: %3 = call float @llvm.nvvm.fadd.rn.f32(float %0, float %1)
- // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.rn.f32(float %3, float %3)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.rn.sat.f32(float %4, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %5, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %6, float %6)
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rm.f32(float %7, float %7)
- // CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.rm.sat.f32(float %8, float %8)
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %9, float %9)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %10, float %10)
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rp.f32(float %11, float %11)
- // CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.rp.sat.f32(float %12, float %12)
- // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rp.ftz.f32(float %13, float %13)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %14, float %14)
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rz.f32(float %15, float %15)
- // CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.rz.sat.f32(float %16, float %16)
- // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %17, float %17)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %18, float %18)
+ // CHECK-NEXT: %3 = call float @llvm.nvvm.fadd.f32(float %0, float %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %3, float %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.ftz.f32(float %5, float %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %6, float %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.f32(float %7, float %7, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %8, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %9, float %9, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %10, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.f32(float %11, float %11, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %12, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.ftz.f32(float %13, float %13, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.f32(float %15, float %15, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.sat.f32(float %16, float %16, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %17, float %17, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
@@ -75,11 +75,11 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// f64 + f64 -> f64
llvm.func @fadd_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fadd_f64_f64(double %0, double %1) {
- // CHECK-NEXT: %3 = call double @llvm.nvvm.fadd.rn.f64(double %0, double %1)
- // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.rn.f64(double %3, double %3)
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rm.f64(double %4, double %4)
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rp.f64(double %5, double %5)
- // CHECK-NEXT: %7 = call double @llvm.nvvm.fadd.rz.f64(double %6, double %6)
+ // CHECK-NEXT: %3 = call double @llvm.nvvm.fadd.f64(double %0, double %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.f64(double %3, double %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %4, double %4, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %5, double %5, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %7 = call double @llvm.nvvm.fadd.f64(double %6, double %6, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret double %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index 1f7daa6c3c66a..d3421ac025f2b 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -3,11 +3,11 @@
// vector<2xf16> + vector<2xf16> -> vector<2xf16>
llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @addf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
- // CHECK-NEXT: %3 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %0, <2 x half> %1)
- // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %3, <2 x half> %3)
- // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %4, <2 x half> %4)
- // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %5, <2 x half> %5)
- // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6)
+ // CHECK-NEXT: %3 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %0, <2 x half> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %3, <2 x half> %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %4, <2 x half> %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %5, <2 x half> %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
@@ -21,8 +21,8 @@ llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @addf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
- // CHECK-NEXT: %3 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1)
- // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %3, <2 x bfloat> %3)
+ // CHECK-NEXT: %3 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %3, <2 x bfloat> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x bfloat> %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xbf16>
@@ -33,24 +33,24 @@ llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
// vector<2xf32> + vector<2xf32> -> vector<2xf32>
llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %0, <2 x float> %1)
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %3, <2 x float> %3)
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %3, <2 x float> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = extractelement <2 x float> %4, i32 0
// CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.rn.sat.f32(float %5, float %6)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.sat.f32(float %5, float %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
// CHECK-NEXT: %9 = extractelement <2 x float> %4, i32 1
// CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
- // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.rn.sat.f32(float %9, float %10)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.sat.f32(float %9, float %10, /* rnd=rn */ i32 1)
// CHECK-NEXT: %12 = insertelement <2 x float> %8, float %11, i32 1
- // CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %12, <2 x float> %12)
+ // CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %12, <2 x float> %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %14, float %15)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %15, /* rnd=rn */ i32 1)
// CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
// CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %18, float %19)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rn */ i32 1)
// CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
// CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
@@ -64,23 +64,23 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rm */ i32 3)
// CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
// CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rm.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rm */ i32 3)
// CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
// CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rm.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rm */ i32 3)
// CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rm */ i32 3)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rm */ i32 3)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -93,23 +93,23 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rp */ i32 2)
// CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
// CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rp.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rp */ i32 2)
// CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
// CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rp.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rp */ i32 2)
// CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -122,23 +122,23 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %0, <2 x float> %1)
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rz */ i32 0)
// CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
// CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rz.sat.f32(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rz */ i32 0)
// CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
// CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rz.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rz */ i32 0)
// CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %11, <2 x float> %11)
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %13, float %14)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rz */ i32 0)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -154,19 +154,19 @@ llvm.func @addf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rn(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rn.f64(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rn.f64(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rn */ i32 1)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: %11 = extractelement <2 x double> %10, i32 0
// CHECK-NEXT: %12 = extractelement <2 x double> %10, i32 0
- // CHECK-NEXT: %13 = call double @llvm.nvvm.fadd.rn.f64(double %11, double %12)
+ // CHECK-NEXT: %13 = call double @llvm.nvvm.fadd.f64(double %11, double %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = insertelement <2 x double> poison, double %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x double> %10, i32 1
// CHECK-NEXT: %16 = extractelement <2 x double> %10, i32 1
- // CHECK-NEXT: %17 = call double @llvm.nvvm.fadd.rn.f64(double %15, double %16)
+ // CHECK-NEXT: %17 = call double @llvm.nvvm.fadd.f64(double %15, double %16, /* rnd=rn */ i32 1)
// CHECK-NEXT: %18 = insertelement <2 x double> %14, double %17, i32 1
// CHECK-NEXT: ret <2 x double> %18
// CHECK-NEXT: }
@@ -179,11 +179,11 @@ llvm.func @addf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rm(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rm.f64(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rm */ i32 3)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rm.f64(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rm */ i32 3)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -195,11 +195,11 @@ llvm.func @addf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rp(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rp.f64(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rp */ i32 2)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rp.f64(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rp */ i32 2)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -211,11 +211,11 @@ llvm.func @addf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rz(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.rz.f64(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rz */ i32 0)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.rz.f64(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rz */ i32 0)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index a540c0338a59b..eaf33ae64ee76 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -4,15 +4,15 @@
llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fsub_f16_f16(half %0, half %1) {
// CHECK-NEXT: %3 = fneg half %1
- // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.rn.f16(half %0, half %3)
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %0, half %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg half %4
- // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.rn.f16(half %4, half %5)
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.f16(half %4, half %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg half %6
- // CHECK-NEXT: %8 = call half @llvm.nvvm.fadd.rn.ftz.f16(half %6, half %7)
+ // CHECK-NEXT: %8 = call half @llvm.nvvm.fadd.ftz.f16(half %6, half %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg half %8
- // CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.rn.sat.f16(half %8, half %9)
+ // CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.sat.f16(half %8, half %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg half %10
- // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.rn.ftz.sat.f16(half %10, half %11)
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %10, half %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
@@ -27,9 +27,9 @@ llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fsub_bf16_bf16(bfloat %0, bfloat %1) {
// CHECK-NEXT: %3 = fneg bfloat %1
- // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %0, bfloat %3)
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %0, bfloat %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg bfloat %4
- // CHECK-NEXT: %6 = call bfloat @llvm.nvvm.fadd.rn.bf16(bfloat %4, bfloat %5)
+ // CHECK-NEXT: %6 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %4, bfloat %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret bfloat %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : bf16
@@ -41,39 +41,39 @@ llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fsub_f32_f32(float %0, float %1) {
// CHECK-NEXT: %3 = fneg float %1
- // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.rn.f32(float %0, float %3)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %0, float %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg float %4
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.rn.f32(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.f32(float %4, float %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg float %6
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rn.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg float %8
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rn.ftz.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: %13 = fneg float %12
- // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rm.f32(float %12, float %13)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.f32(float %12, float %13, /* rnd=rm */ i32 3)
// CHECK-NEXT: %15 = fneg float %14
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.rm.sat.f32(float %14, float %15)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.sat.f32(float %14, float %15, /* rnd=rm */ i32 3)
// CHECK-NEXT: %17 = fneg float %16
- // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.rm.ftz.f32(float %16, float %17)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %16, float %17, /* rnd=rm */ i32 3)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %18, float %19)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rm */ i32 3)
// CHECK-NEXT: %21 = fneg float %20
- // CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.rp.f32(float %20, float %21)
+ // CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.f32(float %20, float %21, /* rnd=rp */ i32 2)
// CHECK-NEXT: %23 = fneg float %22
- // CHECK-NEXT: %24 = call float @llvm.nvvm.fadd.rp.sat.f32(float %22, float %23)
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fadd.sat.f32(float %22, float %23, /* rnd=rp */ i32 2)
// CHECK-NEXT: %25 = fneg float %24
- // CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.rp.ftz.f32(float %24, float %25)
+ // CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.ftz.f32(float %24, float %25, /* rnd=rp */ i32 2)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %26, float %27)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %26, float %27, /* rnd=rp */ i32 2)
// CHECK-NEXT: %29 = fneg float %28
- // CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.rz.f32(float %28, float %29)
+ // CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.f32(float %28, float %29, /* rnd=rz */ i32 0)
// CHECK-NEXT: %31 = fneg float %30
- // CHECK-NEXT: %32 = call float @llvm.nvvm.fadd.rz.sat.f32(float %30, float %31)
+ // CHECK-NEXT: %32 = call float @llvm.nvvm.fadd.sat.f32(float %30, float %31, /* rnd=rz */ i32 0)
// CHECK-NEXT: %33 = fneg float %32
- // CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.rz.ftz.f32(float %32, float %33)
+ // CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.ftz.f32(float %32, float %33, /* rnd=rz */ i32 0)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %34, float %35)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %34, float %35, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
@@ -100,15 +100,15 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
llvm.func @fsub_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fsub_f64_f64(double %0, double %1) {
// CHECK-NEXT: %3 = fneg double %1
- // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.rn.f64(double %0, double %3)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.f64(double %0, double %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg double %4
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rn.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg double %6
- // CHECK-NEXT: %8 = call double @llvm.nvvm.fadd.rm.f64(double %6, double %7)
+ // CHECK-NEXT: %8 = call double @llvm.nvvm.fadd.f64(double %6, double %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = fneg double %8
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rp.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rp */ i32 2)
// CHECK-NEXT: %11 = fneg double %10
- // CHECK-NEXT: %12 = call double @llvm.nvvm.fadd.rz.f64(double %10, double %11)
+ // CHECK-NEXT: %12 = call double @llvm.nvvm.fadd.f64(double %10, double %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret double %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index b72d3b0ebecde..78109d5ef9e7d 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -4,15 +4,15 @@
llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @subf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
// CHECK-NEXT: %3 = fneg <2 x half> %1
- // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %0, <2 x half> %3)
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %0, <2 x half> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg <2 x half> %4
- // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.rn.v2f16(<2 x half> %4, <2 x half> %5)
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %4, <2 x half> %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg <2 x half> %6
- // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.v2f16(<2 x half> %6, <2 x half> %7)
+ // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %6, <2 x half> %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg <2 x half> %8
- // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.rn.sat.v2f16(<2 x half> %8, <2 x half> %9)
+ // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %8, <2 x half> %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg <2 x half> %10
- // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.rn.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11)
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
@@ -27,9 +27,9 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @subf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
// CHECK-NEXT: %3 = fneg <2 x bfloat> %1
- // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %0, <2 x bfloat> %3)
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %0, <2 x bfloat> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg <2 x bfloat> %4
- // CHECK-NEXT: %6 = call <2 x bfloat> @llvm.nvvm.fadd.rn.v2bf16(<2 x bfloat> %4, <2 x bfloat> %5)
+ // CHECK-NEXT: %6 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %4, <2 x bfloat> %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x bfloat> %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xbf16>
@@ -41,28 +41,28 @@ llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg <2 x float> %4
- // CHECK-NEXT: %6 = call <2 x float> @llvm.nvvm.fadd.rn.v2f32(<2 x float> %4, <2 x float> %5)
+ // CHECK-NEXT: %6 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %4, <2 x float> %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg <2 x float> %6
// CHECK-NEXT: %8 = extractelement <2 x float> %6, i32 0
// CHECK-NEXT: %9 = extractelement <2 x float> %7, i32 0
- // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.rn.sat.f32(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = insertelement <2 x float> poison, float %10, i32 0
// CHECK-NEXT: %12 = extractelement <2 x float> %6, i32 1
// CHECK-NEXT: %13 = extractelement <2 x float> %7, i32 1
- // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.rn.sat.f32(float %12, float %13)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %13, /* rnd=rn */ i32 1)
// CHECK-NEXT: %15 = insertelement <2 x float> %11, float %14, i32 1
// CHECK-NEXT: %16 = fneg <2 x float> %15
- // CHECK-NEXT: %17 = call <2 x float> @llvm.nvvm.fadd.rn.ftz.v2f32(<2 x float> %15, <2 x float> %16)
+ // CHECK-NEXT: %17 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %15, <2 x float> %16, /* rnd=rn */ i32 1)
// CHECK-NEXT: %18 = fneg <2 x float> %17
// CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
// CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %19, float %20)
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %19, float %20, /* rnd=rn */ i32 1)
// CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
// CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
// CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.rn.ftz.sat.f32(float %23, float %24)
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %23, float %24, /* rnd=rn */ i32 1)
// CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
// CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
@@ -77,26 +77,26 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rm.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rm */ i32 3)
// CHECK-NEXT: %5 = fneg <2 x float> %4
// CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
// CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rm.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
// CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
// CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rm.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rm */ i32 3)
// CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
// CHECK-NEXT: %14 = fneg <2 x float> %13
- // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rm.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rm */ i32 3)
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rm.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rm */ i32 3)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -110,26 +110,26 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rp.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rp */ i32 2)
// CHECK-NEXT: %5 = fneg <2 x float> %4
// CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
// CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rp.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rp */ i32 2)
// CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
// CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
// CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rp.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
// CHECK-NEXT: %14 = fneg <2 x float> %13
- // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rp.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rp.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rp */ i32 2)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -143,26 +143,26 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.rz.v2f32(<2 x float> %0, <2 x float> %3)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rz */ i32 0)
// CHECK-NEXT: %5 = fneg <2 x float> %4
// CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
// CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
- // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.rz.sat.f32(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rz */ i32 0)
// CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
// CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
// CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.rz.sat.f32(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
// CHECK-NEXT: %14 = fneg <2 x float> %13
- // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.rz.ftz.v2f32(<2 x float> %13, <2 x float> %14)
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rz */ i32 0)
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %17, float %18)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.rz.ftz.sat.f32(float %21, float %22)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rz */ i32 0)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -179,20 +179,20 @@ llvm.func @subf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rn.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rn.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: %12 = fneg <2 x double> %11
// CHECK-NEXT: %13 = extractelement <2 x double> %11, i32 0
// CHECK-NEXT: %14 = extractelement <2 x double> %12, i32 0
- // CHECK-NEXT: %15 = call double @llvm.nvvm.fadd.rn.f64(double %13, double %14)
+ // CHECK-NEXT: %15 = call double @llvm.nvvm.fadd.f64(double %13, double %14, /* rnd=rn */ i32 1)
// CHECK-NEXT: %16 = insertelement <2 x double> poison, double %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x double> %11, i32 1
// CHECK-NEXT: %18 = extractelement <2 x double> %12, i32 1
- // CHECK-NEXT: %19 = call double @llvm.nvvm.fadd.rn.f64(double %17, double %18)
+ // CHECK-NEXT: %19 = call double @llvm.nvvm.fadd.f64(double %17, double %18, /* rnd=rn */ i32 1)
// CHECK-NEXT: %20 = insertelement <2 x double> %16, double %19, i32 1
// CHECK-NEXT: ret <2 x double> %20
// CHECK-NEXT: }
@@ -206,11 +206,11 @@ llvm.func @subf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rm.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rm */ i32 3)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rm.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rm */ i32 3)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -223,11 +223,11 @@ llvm.func @subf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rp.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rp */ i32 2)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rp.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rp */ i32 2)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -240,11 +240,11 @@ llvm.func @subf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.rz.f64(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rz */ i32 0)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.rz.f64(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rz */ i32 0)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
>From 56f040d549d8b2e77121c42436c733652f272b97 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 18 Aug 2026 11:23:15 +0000
Subject: [PATCH 03/12] fix some constant fold tests and add some missing tests
---
.../InstSimplify/const-fold-nvvm-add.ll | 186 +++++++++++++++++-
1 file changed, 177 insertions(+), 9 deletions(-)
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
index 02942634327af..edc4b2233f235 100644
--- a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
@@ -557,11 +557,10 @@ define bfloat @test_subnorm_plus_subnorm_to_normal_rz_bf16() {
;###############################################################
;# Add(Normal, -Subnormal) -> Subnormal #
;###############################################################
-; Tests addition of 2^-126 (the smallest normal number) and -(2^127).
-; - Without FTZ: The result is correctly computed as a subnormal (2^127)
-; - With FTZ: The result is flushed to zero.
-; This verifies that the output is also flushed to zero, as we'd end up
-; with 2^-126 if we only flushed the inputs.
+; Tests addition of 2^-126 (the smallest normal number) and -(2^-127).
+; - Without FTZ: The result is correctly computed as a subnormal (2^-127)
+; - With FTZ: The subnormal input is flushed to zero, so the result is the
+; normal input (2^-126)
define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
@@ -645,17 +644,17 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
-; CHECK-NEXT: ret float 0.000000e+00
+; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
-; CHECK-NEXT: ret float 0.000000e+00
+; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -691,6 +690,38 @@ define half @test_normal_minus_subnorm_to_subnorm_rz_f16() {
ret half %res
}
+define half @test_normal_minus_subnorm_to_subnorm_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rm_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rn_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rp_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rz_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
; CHECK-NEXT: ret bfloat 5.877470e-39
@@ -723,6 +754,143 @@ define bfloat @test_normal_minus_subnorm_to_subnorm_rz_bf16() {
ret bfloat %res
}
+;###############################################################
+;# Add(Normal, -Normal) -> Subnormal #
+;###############################################################
+; Tests addition of 1.5*(2^-126) and -(2^-126), where both inputs are normal
+; but the exact result is subnormal.
+; - Without FTZ: The result is the exact difference (2^-127)
+; - With FTZ: The result is flushed to zero. Flushing the inputs alone would
+; leave it untouched, as neither input is subnormal.
+
+define float @test_normal_minus_normal_to_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rm_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rm */ i32 3)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rn_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rn */ i32 1)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rp_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rp */ i32 2)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rz_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rz */ i32 0)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rm_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rm */ i32 3)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rn_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rn */ i32 1)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rp_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rp */ i32 2)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rz_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rz */ i32 0)
+ ret float %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rm_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rm_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rn_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rn_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rp_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rp_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rz_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rm_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rn_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rp_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rz_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
;###############################################################
;# Add(1.0, 2^(-25)) #
;###############################################################
>From 7fe1dba4c0ae76af6a61fd02b2b28742f7d8b15d Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 19 Aug 2026 11:58:22 +0000
Subject: [PATCH 04/12] move ftz modifier to the end of the intrinsic name
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 8 ++++----
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 8 ++++----
.../test/CIR/CodeGenCUDA/builtins-nvvm-math.cu | 4 ++--
clang/test/CodeGen/builtins-nvptx.c | 12 ++++++------
llvm/include/llvm/IR/IntrinsicsNVVM.td | 6 +++---
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 4 ++--
llvm/lib/IR/AutoUpgrade.cpp | 4 ++--
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 8 ++++----
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 6 +++---
llvm/test/CodeGen/NVPTX/f16-add.ll | 4 ++--
llvm/test/CodeGen/NVPTX/f16-sub.ll | 4 ++--
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 16 ++++++++--------
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 2 +-
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 10 +++++-----
.../Target/LLVMIR/nvvm/addf/addf_vector.mlir | 18 +++++++++---------
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 10 +++++-----
.../Target/LLVMIR/nvvm/subf/subf_vector.mlir | 18 +++++++++---------
18 files changed, 72 insertions(+), 72 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 3a3afba581ad0..68eb0cf1206f1 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -855,16 +855,16 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index b8708f9d8ce80..a0577a349885b 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -1189,16 +1189,16 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmNearestTiesToEven,
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmNearestTiesToEven,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardZero,
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardZero,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardNegative,
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardNegative,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardPositive,
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardPositive,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index 69df1d376f7f7..c2f4d19322cfe 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -81,9 +81,9 @@ __device__ double test_add_rz_d(double x, double y) {
}
// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.ftz.sat" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.sat.ftz" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// LLVM: call {{.*}}float @llvm.nvvm.fadd.ftz.sat.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.sat.ftz.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
__device__ float test_add_rm_ftz_sat_f(float x, float y) {
return __nvvm_add_rm_ftz_sat_f(x, y);
}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 82c5a9ce9f6b0..53fc5aa8d6d8c 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -1550,19 +1550,19 @@ __device__ void nvvm_min_max_sm86() {
__device__ void nvvm_add_fma_f32_sat() {
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 1)
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 1)
+ // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 0)
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 0)
+ // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 0)
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 3)
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 3)
+ // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 3)
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 2)
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 2)
+ // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 2)
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1594,11 +1594,11 @@ __device__ void nvvm_add_fma_f32_sat() {
__device__ void nvvm_add_mul_f16_sat() {
// CHECK: call half @llvm.nvvm.fadd.sat.f16({{.*}}i32 1)
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16({{.*}}i32 1)
+ // CHECK: call half @llvm.nvvm.fadd.sat.ftz.f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
// CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16({{.*}}i32 1)
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index fc33962adb59b..729aff85b283d 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1717,9 +1717,9 @@ let TargetPrefix = "nvvm" in {
ArgInfo<ArgIndex<2>,
[ArgName<"rnd">,
ImmArgPrinter<"printFAddRoundingMode">]>] in
- foreach ftz = ["", "_ftz"] in
- foreach sat = ["", "_sat"] in
- def int_nvvm_fadd # ftz # sat :
+ foreach sat = ["", "_sat"] in
+ foreach ftz = ["", "_ftz"] in
+ def int_nvvm_fadd # sat # ftz :
DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
[LLVMMatchType<0>, LLVMMatchType<0>,
llvm_i32_ty]>;
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index 7eee25d5eb175..b52a277e8de60 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -596,7 +596,7 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_ftz:
- case Intrinsic::nvvm_fadd_ftz_sat:
+ case Intrinsic::nvvm_fadd_sat_ftz:
return true;
case Intrinsic::nvvm_fadd:
@@ -609,7 +609,7 @@ inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_ftz_sat:
+ case Intrinsic::nvvm_fadd_sat_ftz:
return true;
case Intrinsic::nvvm_fadd:
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 29fadda78f438..17e8e46a67358 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1394,7 +1394,7 @@ getNVVMFAddUpgrade(StringRef Modifiers) {
.Case("", Intrinsic::nvvm_fadd)
.Case(".ftz", Intrinsic::nvvm_fadd_ftz)
.Case(".sat", Intrinsic::nvvm_fadd_sat)
- .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
+ .Case(".ftz.sat", Intrinsic::nvvm_fadd_sat_ftz)
.Default(Intrinsic::not_intrinsic);
if (IID == Intrinsic::not_intrinsic)
return std::nullopt;
@@ -1970,7 +1970,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
// Upgrade the FP add intrinsics, which are overloaded on the operand type
// and take the rounding mode as an operand:
// llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
- // llvm.nvvm.fadd{.ftz}{.sat}.<mangled type>
+ // llvm.nvvm.fadd{.sat}{.ftz}.<mangled type>
// The extra operand means these are expanded in UpgradeIntrinsicCall.
if (Name.starts_with("add.")) {
auto [Base, TypeSuffix] = Name.rsplit('.');
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index eb3d63e9eaa84..d2e0fe1975745 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7290,7 +7290,7 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_ftz_sat: {
+ case Intrinsic::nvvm_fadd_sat_ftz: {
const auto RM = static_cast<APFloat::roundingMode>(
N->getConstantOperandAPInt(3).getSExtValue());
if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, RM, STI))
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 0f1098a2ae44c..637bf1fa033cb 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2071,7 +2071,7 @@ let Predicates = [doRsqrtOpt] in {
// Add
//
-defvar BF16ArithPreds = [hasBF16Math, hasPTX<78>, hasSM<90>];
+defvar BF16ArithPreds = [hasBF16Math, PTX78, SM90];
class RndModeImm<string mode> : TImmLeaf<i32,
"return Imm == static_cast<int>(RoundingMode::" # mode # ");">;
@@ -2086,7 +2086,7 @@ foreach t = [F16RT, F16X2RT] in
foreach sat = ["", "_sat"] in
def INT_NVVM_ADD_RN # !toupper(ftz # sat # "_" # t.PtxType) :
F_MATH_2_RND_TY<!subst("_", ".", "add_rn" # ftz # sat # "_" # t.PtxType),
- t, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ t, !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz),
rnd_rn_imm>;
foreach t = [BF16RT, BF16X2RT] in
@@ -2101,7 +2101,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach sat = ["", "_sat"] in
def INT_NVVM_ADD # !toupper(rnd # sat # ftz) # _F :
F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # sat # ftz # "_f32"),
- F32RT, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ F32RT, !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz),
rnd_imm>;
def INT_NVVM_ADD # !toupper(rnd # ftz) # _F32X2 :
@@ -2180,7 +2180,7 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat);
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz);
def INT_NVVM_SUB # rnd # ftz # sat # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
!subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 2871ddab068bc..96fc033a03f30 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -716,13 +716,13 @@ define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
; CHECK: call float @llvm.nvvm.fadd.f32(float %a, float %a, /* rnd=rn */ i32 1)
; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %a, /* rnd=rz */ i32 0)
; CHECK: call float @llvm.nvvm.fadd.sat.f32(float %a, float %a, /* rnd=rm */ i32 3)
-; CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %a, /* rnd=rp */ i32 2)
+; CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %a, /* rnd=rp */ i32 2)
; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rn */ i32 1)
; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rz */ i32 0)
; CHECK: call half @llvm.nvvm.fadd.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
-; CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fadd.sat.ftz.f16(half %c, half %c, /* rnd=rn */ i32 1)
; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
-; CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
%r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
%r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
%r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
index 0b470daf7a535..3a3c2c66cf180 100644
--- a/llvm/test/CodeGen/NVPTX/f16-add.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -103,7 +103,7 @@ define half @add_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %b, i32 1)
+ %1 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -118,6 +118,6 @@ define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
+ %1 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
index c59875af1d4f9..e3d32ac1a52ec 100644
--- a/llvm/test/CodeGen/NVPTX/f16-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -112,7 +112,7 @@ define half @sub_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %1, i32 1)
+ %res = call half @llvm.nvvm.fadd.sat.ftz.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -128,6 +128,6 @@ define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index ad0e77d119e65..061155f995472 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -21,16 +21,16 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, i32 1)
- %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r1, i32 1)
%r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r3, i32 0)
%r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, i32 3)
- %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r5, i32 3)
%r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, i32 2)
- %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r7, i32 2)
ret float %r8
}
@@ -57,25 +57,25 @@ define float @sub_sat_f32(float %a, float %b) {
%r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
%r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
%r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f5, i32 3)
%f6 = fneg float %r6
%r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, i32 2)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f7, i32 2)
ret float %r8
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index b3c953de574b9..1cfa2eb1712db 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -487,7 +487,7 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
static constexpr llvm::Intrinsic::ID addIDs[2][2] = {
{llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
- {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
+ {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_sat_ftz}};
static constexpr llvm::RoundingMode roundingModes[5] = {
llvm::RoundingMode::NearestTiesToEven,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index a376351ac20ec..d3eea29fe372d 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -7,7 +7,7 @@ llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %3, half %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.ftz.f16(half %4, half %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.sat.f16(half %5, half %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %6, half %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %6, half %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
@@ -37,19 +37,19 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %3, float %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.ftz.f32(float %5, float %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %6, float %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %6, float %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.f32(float %7, float %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %8, /* rnd=rm */ i32 3)
// CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %9, float %9, /* rnd=rm */ i32 3)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %10, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %10, float %10, /* rnd=rm */ i32 3)
// CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.f32(float %11, float %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %12, /* rnd=rp */ i32 2)
// CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.ftz.f32(float %13, float %13, /* rnd=rp */ i32 2)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %14, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.f32(float %15, float %15, /* rnd=rz */ i32 0)
// CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.sat.f32(float %16, float %16, /* rnd=rz */ i32 0)
// CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %17, float %17, /* rnd=rz */ i32 0)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index d3421ac025f2b..7b892cf1c691f 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -7,7 +7,7 @@ llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %3, <2 x half> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %4, <2 x half> %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %5, <2 x half> %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
@@ -46,11 +46,11 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %12, <2 x float> %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %15, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %14, float %15, /* rnd=rn */ i32 1)
// CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
// CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %19, /* rnd=rn */ i32 1)
// CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
// CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
@@ -76,11 +76,11 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rm */ i32 3)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rm */ i32 3)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -105,11 +105,11 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -134,11 +134,11 @@ llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rz */ i32 0)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index eaf33ae64ee76..cb0d21945ddbc 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -12,7 +12,7 @@ llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: %9 = fneg half %8
// CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.sat.f16(half %8, half %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg half %10
- // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %10, half %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %10, half %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
@@ -49,7 +49,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %9 = fneg float %8
// CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %10, float %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: %13 = fneg float %12
// CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.f32(float %12, float %13, /* rnd=rm */ i32 3)
// CHECK-NEXT: %15 = fneg float %14
@@ -57,7 +57,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %17 = fneg float %16
// CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %16, float %17, /* rnd=rm */ i32 3)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %19, /* rnd=rm */ i32 3)
// CHECK-NEXT: %21 = fneg float %20
// CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.f32(float %20, float %21, /* rnd=rp */ i32 2)
// CHECK-NEXT: %23 = fneg float %22
@@ -65,7 +65,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %25 = fneg float %24
// CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.ftz.f32(float %24, float %25, /* rnd=rp */ i32 2)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %26, float %27, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %26, float %27, /* rnd=rp */ i32 2)
// CHECK-NEXT: %29 = fneg float %28
// CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.f32(float %28, float %29, /* rnd=rz */ i32 0)
// CHECK-NEXT: %31 = fneg float %30
@@ -73,7 +73,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %33 = fneg float %32
// CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.ftz.f32(float %32, float %33, /* rnd=rz */ i32 0)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %34, float %35, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %34, float %35, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index 78109d5ef9e7d..c184da42b2b97 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -12,7 +12,7 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: %9 = fneg <2 x half> %8
// CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %8, <2 x half> %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg <2 x half> %10
- // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
@@ -58,11 +58,11 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %18 = fneg <2 x float> %17
// CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
// CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %19, float %20, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %19, float %20, /* rnd=rn */ i32 1)
// CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
// CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
// CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %23, float %24, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %23, float %24, /* rnd=rn */ i32 1)
// CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
// CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
@@ -92,11 +92,11 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rm */ i32 3)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -125,11 +125,11 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rp */ i32 2)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -158,11 +158,11 @@ llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rz */ i32 0)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
>From 860fdd1fced1a88e42b4f2b5c45c3cd0b18c8bcf Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 19 Aug 2026 12:49:56 +0000
Subject: [PATCH 05/12] fix tests
---
llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll | 12 ++++++------
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 8 ++++----
mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir | 8 ++++----
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 8 ++++----
mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir | 8 ++++----
5 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
index ca854106fa31b..7aebe89a1c1fd 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
@@ -8,8 +8,8 @@ define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: .reg .b64 %rd<11>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [sub_f32x2_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [sub_f32x2_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_param_1];
; CHECK-NEXT: sub.rn.f32x2 %rd3, %rd1, %rd2;
; CHECK-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
; CHECK-NEXT: sub.rz.f32x2 %rd5, %rd1, %rd4;
@@ -18,7 +18,7 @@ define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: sub.rm.ftz.f32x2 %rd8, %rd1, %rd7;
; CHECK-NEXT: sub.rp.f32x2 %rd9, %rd1, %rd8;
; CHECK-NEXT: sub.rp.ftz.f32x2 %rd10, %rd1, %rd9;
-; CHECK-NEXT: st.param.b64 [func_retval0], %rd10;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd10;
; CHECK-NEXT: ret;
%f0 = fneg <2 x float> %b
%r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f0, i32 1)
@@ -53,10 +53,10 @@ define <2 x float> @sub_f32x2_negated_lhs(<2 x float> %a, <2 x float> %b) {
; CHECK-NEXT: .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b64 %rd1, [sub_f32x2_negated_lhs_param_0];
-; CHECK-NEXT: ld.param.b64 %rd2, [sub_f32x2_negated_lhs_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_negated_lhs_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_negated_lhs_param_1];
; CHECK-NEXT: sub.rz.f32x2 %rd3, %rd2, %rd1;
-; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd3;
; CHECK-NEXT: ret;
%f = fneg <2 x float> %a
%r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %f, <2 x float> %b, i32 0)
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index d3eea29fe372d..c9bfde9ebc88a 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -11,10 +11,10 @@ llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
- %f2 = nvvm.addf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : f16
- %f3 = nvvm.addf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : f16
- %f4 = nvvm.addf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : f16
- %f5 = nvvm.addf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : f16
+ %f2 = nvvm.addf %f1, %f1 rnd = <rn> : f16
+ %f3 = nvvm.addf %f2, %f2 rnd = <rn> ftz = true : f16
+ %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> : f16
+ %f5 = nvvm.addf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : f16
llvm.return %f5 : f16
}
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index 7b892cf1c691f..bb83fd11b467e 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -11,10 +11,10 @@ llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
- %f2 = nvvm.addf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xf16>
- %f3 = nvvm.addf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : vector<2xf16>
- %f4 = nvvm.addf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : vector<2xf16>
- %f5 = nvvm.addf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : vector<2xf16>
+ %f2 = nvvm.addf %f1, %f1 rnd = <rn> : vector<2xf16>
+ %f3 = nvvm.addf %f2, %f2 rnd = <rn> ftz = true : vector<2xf16>
+ %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> : vector<2xf16>
+ %f5 = nvvm.addf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index cb0d21945ddbc..d269ba88fe9fd 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -16,10 +16,10 @@ llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
- %f2 = nvvm.subf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : f16
- %f3 = nvvm.subf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : f16
- %f4 = nvvm.subf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : f16
- %f5 = nvvm.subf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : f16
+ %f2 = nvvm.subf %f1, %f1 rnd = <rn> : f16
+ %f3 = nvvm.subf %f2, %f2 rnd = <rn> ftz = true : f16
+ %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> : f16
+ %f5 = nvvm.subf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : f16
llvm.return %f5 : f16
}
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index c184da42b2b97..890d63a8921e9 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -16,10 +16,10 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
- %f2 = nvvm.subf %f1, %f1 {rnd = #nvvm.fp_rnd_mode<rn>} : vector<2xf16>
- %f3 = nvvm.subf %f2, %f2 {rnd = #nvvm.fp_rnd_mode<rn>, ftz=true} : vector<2xf16>
- %f4 = nvvm.subf %f3, %f3 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>} : vector<2xf16>
- %f5 = nvvm.subf %f4, %f4 {rnd = #nvvm.fp_rnd_mode<rn>, sat = #nvvm.sat_mode<sat>, ftz=true} : vector<2xf16>
+ %f2 = nvvm.subf %f1, %f1 rnd = <rn> : vector<2xf16>
+ %f3 = nvvm.subf %f2, %f2 rnd = <rn> ftz = true : vector<2xf16>
+ %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> : vector<2xf16>
+ %f5 = nvvm.subf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
>From a294f310640ff39d71b135327c8d2cabf76a2975 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 19 Aug 2026 13:08:34 +0000
Subject: [PATCH 06/12] add docs
---
llvm/docs/NVPTXUsage.md | 83 ++++++++++++++++++++++++++++++++++++-----
1 file changed, 73 insertions(+), 10 deletions(-)
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index efad04b638494..f5d4e99c15777 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1174,6 +1174,33 @@ For more information, see [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread
### Arithmetic Intrinsics
+Some of these intrinsics take the rounding mode as an `i32` immediate operand
+instead of encoding it in the intrinsic name. The accepted values match the
+`llvm::RoundingMode` enumeration and are described in the following table:
+
+(fp-rounding-modes)=
+
+```{list-table} Floating-Point Rounding Modes
+:widths: 15 15 70
+:header-rows: 1
+
+ * - Value
+ - Rounding Mode
+ - Description
+ * - 0
+ - `rz`
+ - Round towards zero
+ * - 1
+ - `rn`
+ - Round to nearest, with ties to even
+ * - 2
+ - `rp`
+ - Round towards positive infinity
+ * - 3
+ - `rm`
+ - Round towards negative infinity
+```
+
#### '`llvm.nvvm.fabs.*`' Intrinsic
##### Syntax:
@@ -1273,29 +1300,65 @@ used in the '`llvm.nvvm.idp4a.[us].u`' variants, while sign-extension is used
with '`llvm.nvvm.idp4a.[us].s`' variants. The dot product of these 4-element
vectors is added to `%c` to produce the return.
-#### '`llvm.nvvm.add.*`' Half-precision Intrinsics
+#### '`llvm.nvvm.fadd.*`' Intrinsics
##### Syntax:
-```llvm
-declare half @llvm.nvvm.add.rn.sat.f16(half %a, half %b)
-declare <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
+This is an overloaded intrinsic. The '`.sat`' and '`.ftz`' modifiers are
+optional.
-declare half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %b)
-declare <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
+```llvm
+declare half @llvm.nvvm.fadd{.sat}{.ftz}.f16(half %a, half %b, i32 immarg %rnd)
+declare <2 x half> @llvm.nvvm.fadd{.sat}{.ftz}.v2f16(<2 x half> %a, <2 x half> %b, i32 immarg %rnd)
+declare bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 immarg %rnd)
+declare <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, i32 immarg %rnd)
+declare float @llvm.nvvm.fadd{.sat}{.ftz}.f32(float %a, float %b, i32 immarg %rnd)
+declare <2 x float> @llvm.nvvm.fadd{.ftz}.v2f32(<2 x float> %a, <2 x float> %b, i32 immarg %rnd)
+declare double @llvm.nvvm.fadd.f64(double %a, double %b, i32 immarg %rnd)
```
##### Overview:
-The '`llvm.nvvm.add.*`' intrinsics perform an addition operation with the
-specified rounding mode and modifiers.
+The '`llvm.nvvm.fadd.*`' intrinsics add `%a` and `%b` using the rounding mode
+selected by `%rnd` and the modifiers present in the intrinsic name. They
+correspond directly to the `add` PTX instruction.
##### Semantics:
-The '`.sat`' modifier performs a saturating addition where the result is
-clamped to `[0.0, 1.0]` and `NaN` results are flushed to `+0.0f`.
+`%rnd` selects the rounding mode applied to the result, see
+{ref}`fp-rounding-modes`.
+
The '`.ftz`' modifier flushes subnormal inputs and results to sign-preserving
zero.
+The '`.sat`' modifier performs a saturating addition where the result is
+clamped to `[0.0, 1.0]` and `NaN` results are flushed to `+0.0f`.
+
+Not every combination of operand type, rounding mode and modifier maps to a
+PTX instruction. The supported combinations are:
+
+```{list-table}
+:widths: 25 25 25 25
+:header-rows: 1
+
+ * - Operand Type
+ - Rounding Modes
+ - Modifiers
+ * - `half`, `<2 x half>`
+ - `rn`
+ - `.sat`, `.ftz`
+ * - `bfloat`, `<2 x bfloat>`
+ - `rn`
+ - None
+ * - `float`
+ - `rn`, `rz`, `rp`, `rm`
+ - `.sat`, `.ftz`
+ * - `<2 x float>`
+ - `rn`, `rz`, `rp`, `rm`
+ - `.ftz`
+ * - `double`
+ - `rn`, `rz`, `rp`, `rm`
+ - None
+```
#### '`llvm.nvvm.mul.*`' Half-precision Intrinsics
>From 6e604d57dc09e51c7f1dfbd3531acf2b064597e3 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 19 Aug 2026 13:17:02 +0000
Subject: [PATCH 07/12] fix docs
---
llvm/docs/NVPTXUsage.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index f5d4e99c15777..f32db5e0f4695 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1337,7 +1337,7 @@ Not every combination of operand type, rounding mode and modifier maps to a
PTX instruction. The supported combinations are:
```{list-table}
-:widths: 25 25 25 25
+:widths: 25 25 25
:header-rows: 1
* - Operand Type
>From 8ef199bf111d1a3de8c5f2c65e6acdbcdeb3da89 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 21 Aug 2026 16:38:51 +0000
Subject: [PATCH 08/12] revert to older order of modifiers
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 8 ++---
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 8 ++---
.../CIR/CodeGenCUDA/builtins-nvvm-math.cu | 4 +--
clang/test/CodeGen/builtins-nvptx.c | 12 +++----
llvm/docs/NVPTXUsage.md | 12 +++----
llvm/include/llvm/IR/IntrinsicsNVVM.td | 6 ++--
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 4 +--
llvm/lib/IR/AutoUpgrade.cpp | 4 +--
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 12 +++----
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 6 ++--
llvm/test/CodeGen/NVPTX/f16-add.ll | 4 +--
llvm/test/CodeGen/NVPTX/f16-sub.ll | 4 +--
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 32 +++++++++----------
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 2 +-
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir | 10 +++---
.../Target/LLVMIR/nvvm/addf/addf_vector.mlir | 18 +++++------
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir | 10 +++---
.../Target/LLVMIR/nvvm/subf/subf_vector.mlir | 18 +++++------
19 files changed, 88 insertions(+), 88 deletions(-)
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 68eb0cf1206f1..3a3afba581ad0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -855,16 +855,16 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
llvm::APFloat::rmTowardZero);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
llvm::APFloat::rmTowardNegative);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat.ftz",
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index a0577a349885b..b8708f9d8ce80 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -1189,16 +1189,16 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
- return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmNearestTiesToEven,
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmNearestTiesToEven,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardZero,
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardZero,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardNegative,
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardNegative,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
- return MakeFAdd(Intrinsic::nvvm_fadd_sat_ftz, APFloat::rmTowardPositive,
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardPositive,
BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index c2f4d19322cfe..69df1d376f7f7 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -81,9 +81,9 @@ __device__ double test_add_rz_d(double x, double y) {
}
// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// CIR: cir.call_llvm_intrinsic "nvvm.fadd.sat.ftz" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.ftz.sat" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
-// LLVM: call {{.*}}float @llvm.nvvm.fadd.sat.ftz.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.ftz.sat.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
__device__ float test_add_rm_ftz_sat_f(float x, float y) {
return __nvvm_add_rm_ftz_sat_f(x, y);
}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 53fc5aa8d6d8c..82c5a9ce9f6b0 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -1550,19 +1550,19 @@ __device__ void nvvm_min_max_sm86() {
__device__ void nvvm_add_fma_f32_sat() {
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 1)
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 1)
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 0)
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 0)
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 0)
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 3)
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 3)
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 3)
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 2)
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32({{.*}}i32 2)
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 2)
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1594,11 +1594,11 @@ __device__ void nvvm_add_fma_f32_sat() {
__device__ void nvvm_add_mul_f16_sat() {
// CHECK: call half @llvm.nvvm.fadd.sat.f16({{.*}}i32 1)
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.fadd.sat.ftz.f16({{.*}}i32 1)
+ // CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
// CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16({{.*}}i32 1)
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index f32db5e0f4695..8631a86751005 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1304,15 +1304,15 @@ vectors is added to `%c` to produce the return.
##### Syntax:
-This is an overloaded intrinsic. The '`.sat`' and '`.ftz`' modifiers are
+This is an overloaded intrinsic. The '`.ftz`' and '`.sat`' modifiers are
optional.
```llvm
-declare half @llvm.nvvm.fadd{.sat}{.ftz}.f16(half %a, half %b, i32 immarg %rnd)
-declare <2 x half> @llvm.nvvm.fadd{.sat}{.ftz}.v2f16(<2 x half> %a, <2 x half> %b, i32 immarg %rnd)
+declare half @llvm.nvvm.fadd{.ftz}{.sat}.f16(half %a, half %b, i32 immarg %rnd)
+declare <2 x half> @llvm.nvvm.fadd{.ftz}{.sat}.v2f16(<2 x half> %a, <2 x half> %b, i32 immarg %rnd)
declare bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 immarg %rnd)
declare <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, i32 immarg %rnd)
-declare float @llvm.nvvm.fadd{.sat}{.ftz}.f32(float %a, float %b, i32 immarg %rnd)
+declare float @llvm.nvvm.fadd{.ftz}{.sat}.f32(float %a, float %b, i32 immarg %rnd)
declare <2 x float> @llvm.nvvm.fadd{.ftz}.v2f32(<2 x float> %a, <2 x float> %b, i32 immarg %rnd)
declare double @llvm.nvvm.fadd.f64(double %a, double %b, i32 immarg %rnd)
```
@@ -1345,13 +1345,13 @@ PTX instruction. The supported combinations are:
- Modifiers
* - `half`, `<2 x half>`
- `rn`
- - `.sat`, `.ftz`
+ - `.ftz`, `.sat`
* - `bfloat`, `<2 x bfloat>`
- `rn`
- None
* - `float`
- `rn`, `rz`, `rp`, `rm`
- - `.sat`, `.ftz`
+ - `.ftz`, `.sat`
* - `<2 x float>`
- `rn`, `rz`, `rp`, `rm`
- `.ftz`
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 729aff85b283d..fc33962adb59b 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1717,9 +1717,9 @@ let TargetPrefix = "nvvm" in {
ArgInfo<ArgIndex<2>,
[ArgName<"rnd">,
ImmArgPrinter<"printFAddRoundingMode">]>] in
- foreach sat = ["", "_sat"] in
- foreach ftz = ["", "_ftz"] in
- def int_nvvm_fadd # sat # ftz :
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def int_nvvm_fadd # ftz # sat :
DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
[LLVMMatchType<0>, LLVMMatchType<0>,
llvm_i32_ty]>;
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index b52a277e8de60..7eee25d5eb175 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -596,7 +596,7 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_ftz:
- case Intrinsic::nvvm_fadd_sat_ftz:
+ case Intrinsic::nvvm_fadd_ftz_sat:
return true;
case Intrinsic::nvvm_fadd:
@@ -609,7 +609,7 @@ inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_sat_ftz:
+ case Intrinsic::nvvm_fadd_ftz_sat:
return true;
case Intrinsic::nvvm_fadd:
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 17e8e46a67358..29fadda78f438 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1394,7 +1394,7 @@ getNVVMFAddUpgrade(StringRef Modifiers) {
.Case("", Intrinsic::nvvm_fadd)
.Case(".ftz", Intrinsic::nvvm_fadd_ftz)
.Case(".sat", Intrinsic::nvvm_fadd_sat)
- .Case(".ftz.sat", Intrinsic::nvvm_fadd_sat_ftz)
+ .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
.Default(Intrinsic::not_intrinsic);
if (IID == Intrinsic::not_intrinsic)
return std::nullopt;
@@ -1970,7 +1970,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
// Upgrade the FP add intrinsics, which are overloaded on the operand type
// and take the rounding mode as an operand:
// llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
- // llvm.nvvm.fadd{.sat}{.ftz}.<mangled type>
+ // llvm.nvvm.fadd{.ftz}{.sat}.<mangled type>
// The extra operand means these are expanded in UpgradeIntrinsicCall.
if (Name.starts_with("add.")) {
auto [Base, TypeSuffix] = Name.rsplit('.');
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index d2e0fe1975745..eb3d63e9eaa84 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7290,7 +7290,7 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_sat_ftz: {
+ case Intrinsic::nvvm_fadd_ftz_sat: {
const auto RM = static_cast<APFloat::roundingMode>(
N->getConstantOperandAPInt(3).getSExtValue());
if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, RM, STI))
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 637bf1fa033cb..023238c7bd386 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2086,7 +2086,7 @@ foreach t = [F16RT, F16X2RT] in
foreach sat = ["", "_sat"] in
def INT_NVVM_ADD_RN # !toupper(ftz # sat # "_" # t.PtxType) :
F_MATH_2_RND_TY<!subst("_", ".", "add_rn" # ftz # sat # "_" # t.PtxType),
- t, !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz),
+ t, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
rnd_rn_imm>;
foreach t = [BF16RT, BF16X2RT] in
@@ -2099,9 +2099,9 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in
- def INT_NVVM_ADD # !toupper(rnd # sat # ftz) # _F :
- F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # sat # ftz # "_f32"),
- F32RT, !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz),
+ def INT_NVVM_ADD # !toupper(rnd # ftz # sat) # _F :
+ F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # ftz # sat # "_f32"),
+ F32RT, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
rnd_imm>;
def INT_NVVM_ADD # !toupper(rnd # ftz) # _F32X2 :
@@ -2180,10 +2180,10 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # sat # ftz);
+ defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat);
def INT_NVVM_SUB # rnd # ftz # sat # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
- !subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
+ !subst("_", ".", "sub" # rnd # ftz # sat # "_f32"),
[(set f32:$dst,
(add_intrin f32:$a, (f32 (fneg f32:$b)), rnd_imm))]>;
}
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 96fc033a03f30..2871ddab068bc 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -716,13 +716,13 @@ define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
; CHECK: call float @llvm.nvvm.fadd.f32(float %a, float %a, /* rnd=rn */ i32 1)
; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %a, /* rnd=rz */ i32 0)
; CHECK: call float @llvm.nvvm.fadd.sat.f32(float %a, float %a, /* rnd=rm */ i32 3)
-; CHECK: call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %a, /* rnd=rp */ i32 2)
+; CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %a, /* rnd=rp */ i32 2)
; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rn */ i32 1)
; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rz */ i32 0)
; CHECK: call half @llvm.nvvm.fadd.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
-; CHECK: call half @llvm.nvvm.fadd.sat.ftz.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
-; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
%r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
%r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
%r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
index 3a3c2c66cf180..0b470daf7a535 100644
--- a/llvm/test/CodeGen/NVPTX/f16-add.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -103,7 +103,7 @@ define half @add_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %a, half %b, i32 1)
+ %1 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %b, i32 1)
ret half %1
}
@@ -118,6 +118,6 @@ define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
+ %1 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
ret <2 x half> %1
}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
index e3d32ac1a52ec..c59875af1d4f9 100644
--- a/llvm/test/CodeGen/NVPTX/f16-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -112,7 +112,7 @@ define half @sub_rn_ftz_sat_f16(half %a, half %b) {
; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
; CHECK-NEXT: ret;
%1 = fneg half %b
- %res = call half @llvm.nvvm.fadd.sat.ftz.f16(half %a, half %1, i32 1)
+ %res = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %1, i32 1)
ret half %res
}
@@ -128,6 +128,6 @@ define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
; CHECK-NEXT: ret;
%1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index 061155f995472..0b93da7af17d0 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -11,26 +11,26 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: ld.param.b32 %r1, [add_sat_f32_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [add_sat_f32_param_1];
; CHECK-NEXT: add.rn.sat.f32 %r3, %r1, %r2;
-; CHECK-NEXT: add.rn.sat.ftz.f32 %r4, %r1, %r3;
+; CHECK-NEXT: add.rn.ftz.sat.f32 %r4, %r1, %r3;
; CHECK-NEXT: add.rz.sat.f32 %r5, %r1, %r4;
-; CHECK-NEXT: add.rz.sat.ftz.f32 %r6, %r1, %r5;
+; CHECK-NEXT: add.rz.ftz.sat.f32 %r6, %r1, %r5;
; CHECK-NEXT: add.rm.sat.f32 %r7, %r1, %r6;
-; CHECK-NEXT: add.rm.sat.ftz.f32 %r8, %r1, %r7;
+; CHECK-NEXT: add.rm.ftz.sat.f32 %r8, %r1, %r7;
; CHECK-NEXT: add.rp.sat.f32 %r9, %r1, %r8;
-; CHECK-NEXT: add.rp.sat.ftz.f32 %r10, %r1, %r9;
+; CHECK-NEXT: add.rp.ftz.sat.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, i32 1)
- %r2 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, i32 1)
%r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, i32 0)
%r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, i32 3)
- %r6 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, i32 3)
%r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, i32 2)
- %r8 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %r7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, i32 2)
ret float %r8
}
@@ -44,38 +44,38 @@ define float @sub_sat_f32(float %a, float %b) {
; CHECK-NEXT: ld.param.b32 %r1, [sub_sat_f32_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [sub_sat_f32_param_1];
; CHECK-NEXT: sub.rn.sat.f32 %r3, %r1, %r2;
-; CHECK-NEXT: sub.rn.sat.ftz.f32 %r4, %r1, %r3;
+; CHECK-NEXT: sub.rn.ftz.sat.f32 %r4, %r1, %r3;
; CHECK-NEXT: sub.rz.sat.f32 %r5, %r1, %r4;
-; CHECK-NEXT: sub.rz.sat.ftz.f32 %r6, %r1, %r5;
+; CHECK-NEXT: sub.rz.ftz.sat.f32 %r6, %r1, %r5;
; CHECK-NEXT: sub.rm.sat.f32 %r7, %r1, %r6;
-; CHECK-NEXT: sub.rm.sat.ftz.f32 %r8, %r1, %r7;
+; CHECK-NEXT: sub.rm.ftz.sat.f32 %r8, %r1, %r7;
; CHECK-NEXT: sub.rp.sat.f32 %r9, %r1, %r8;
-; CHECK-NEXT: sub.rp.sat.ftz.f32 %r10, %r1, %r9;
+; CHECK-NEXT: sub.rp.ftz.sat.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%f0 = fneg float %b
%r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
%r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
%r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, i32 3)
%f6 = fneg float %r6
%r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, i32 2)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %a, float %f7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, i32 2)
ret float %r8
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 1cfa2eb1712db..b3c953de574b9 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -487,7 +487,7 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
static constexpr llvm::Intrinsic::ID addIDs[2][2] = {
{llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
- {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_sat_ftz}};
+ {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
static constexpr llvm::RoundingMode roundingModes[5] = {
llvm::RoundingMode::NearestTiesToEven,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index c9bfde9ebc88a..17c7e3f36406b 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -7,7 +7,7 @@ llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %3, half %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.ftz.f16(half %4, half %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.sat.f16(half %5, half %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %6, half %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %6, half %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
@@ -37,19 +37,19 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %3, float %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.ftz.f32(float %5, float %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %6, float %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %6, float %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.f32(float %7, float %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %8, /* rnd=rm */ i32 3)
// CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %9, float %9, /* rnd=rm */ i32 3)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %10, float %10, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %10, /* rnd=rm */ i32 3)
// CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.f32(float %11, float %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %12, /* rnd=rp */ i32 2)
// CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.ftz.f32(float %13, float %13, /* rnd=rp */ i32 2)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %14, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.f32(float %15, float %15, /* rnd=rz */ i32 0)
// CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.sat.f32(float %16, float %16, /* rnd=rz */ i32 0)
// CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %17, float %17, /* rnd=rz */ i32 0)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index bb83fd11b467e..504b0e5cae6bf 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -7,7 +7,7 @@ llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %3, <2 x half> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %4, <2 x half> %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %5, <2 x half> %5, /* rnd=rn */ i32 1)
- // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
@@ -46,11 +46,11 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %12, <2 x float> %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %14, float %15, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %15, /* rnd=rn */ i32 1)
// CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
// CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %19, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rn */ i32 1)
// CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
// CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
@@ -76,11 +76,11 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rm */ i32 3)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rm */ i32 3)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -105,11 +105,11 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rp */ i32 2)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rp */ i32 2)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
@@ -134,11 +134,11 @@ llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
// CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %13, float %14, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rz */ i32 0)
// CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
// CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
// CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index d269ba88fe9fd..7226b0e172d75 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -12,7 +12,7 @@ llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-NEXT: %9 = fneg half %8
// CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.sat.f16(half %8, half %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg half %10
- // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.sat.ftz.f16(half %10, half %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %10, half %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
@@ -49,7 +49,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %9 = fneg float %8
// CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %10, float %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: %13 = fneg float %12
// CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.f32(float %12, float %13, /* rnd=rm */ i32 3)
// CHECK-NEXT: %15 = fneg float %14
@@ -57,7 +57,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %17 = fneg float %16
// CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %16, float %17, /* rnd=rm */ i32 3)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %18, float %19, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rm */ i32 3)
// CHECK-NEXT: %21 = fneg float %20
// CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.f32(float %20, float %21, /* rnd=rp */ i32 2)
// CHECK-NEXT: %23 = fneg float %22
@@ -65,7 +65,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %25 = fneg float %24
// CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.ftz.f32(float %24, float %25, /* rnd=rp */ i32 2)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %26, float %27, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %26, float %27, /* rnd=rp */ i32 2)
// CHECK-NEXT: %29 = fneg float %28
// CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.f32(float %28, float %29, /* rnd=rz */ i32 0)
// CHECK-NEXT: %31 = fneg float %30
@@ -73,7 +73,7 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-NEXT: %33 = fneg float %32
// CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.ftz.f32(float %32, float %33, /* rnd=rz */ i32 0)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %34, float %35, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %34, float %35, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index 890d63a8921e9..ee4ab35a0132f 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -12,7 +12,7 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
// CHECK-NEXT: %9 = fneg <2 x half> %8
// CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %8, <2 x half> %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg <2 x half> %10
- // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.sat.ftz.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
@@ -58,11 +58,11 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %18 = fneg <2 x float> %17
// CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
// CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %19, float %20, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %19, float %20, /* rnd=rn */ i32 1)
// CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
// CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
// CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %23, float %24, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %23, float %24, /* rnd=rn */ i32 1)
// CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
// CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
@@ -92,11 +92,11 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rm */ i32 3)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -125,11 +125,11 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rp */ i32 2)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
@@ -158,11 +158,11 @@ llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
// CHECK-NEXT: %16 = fneg <2 x float> %15
// CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
// CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
- // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
// CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
// CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
- // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.sat.ftz.f32(float %21, float %22, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rz */ i32 0)
// CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
// CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
>From cc7d54171f962a47f6b93df9e1d76737663dffb6 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Mon, 24 Aug 2026 15:40:10 +0000
Subject: [PATCH 09/12] address comments
---
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 31 +++++------
llvm/lib/IR/AutoUpgrade.cpp | 50 ++++++++---------
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 36 +++++++------
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 54 +++++++++++--------
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 25 ++++++---
5 files changed, 107 insertions(+), 89 deletions(-)
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index 7eee25d5eb175..1cfdd921b0f2b 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -15,6 +15,7 @@
#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
#define LLVM_IR_NVVMINTRINSICUTILS_H
+#include <cassert>
#include <stdint.h>
#include "llvm/ADT/APFloat.h"
@@ -593,30 +594,30 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
return DenormalMode::getIEEE();
}
-inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
+inline bool IsFAddIntrinsic(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
+ case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fadd_sat:
case Intrinsic::nvvm_fadd_ftz_sat:
return true;
-
- case Intrinsic::nvvm_fadd:
- case Intrinsic::nvvm_fadd_sat:
+ default:
return false;
}
- llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
}
-inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_ftz_sat:
- return true;
+inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
+ assert(IsFAddIntrinsic(IntrinsicID) &&
+ "Checking FTZ flag for invalid NVVM add intrinsic");
+ return IntrinsicID == Intrinsic::nvvm_fadd_ftz ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz_sat;
+}
- case Intrinsic::nvvm_fadd:
- case Intrinsic::nvvm_fadd_ftz:
- return false;
- }
- llvm_unreachable("Checking sat flag for invalid NVVM add intrinsic");
+inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
+ assert(IsFAddIntrinsic(IntrinsicID) &&
+ "Checking sat flag for invalid NVVM add intrinsic");
+ return IntrinsicID == Intrinsic::nvvm_fadd_sat ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz_sat;
}
inline APFloat::roundingMode GetFAddRoundingMode(const Value *ImmArgVal) {
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 29fadda78f438..0afb42085c734 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1379,15 +1379,19 @@ static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
}
static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
-getNVVMFAddUpgrade(StringRef Modifiers) {
- std::optional<RoundingMode> RM =
- StringSwitch<std::optional<RoundingMode>>(Modifiers.take_front(2))
- .Case("rn", RoundingMode::NearestTiesToEven)
- .Case("rz", RoundingMode::TowardZero)
- .Case("rm", RoundingMode::TowardNegative)
- .Case("rp", RoundingMode::TowardPositive)
+getNVVMFAddUpgrade(StringRef Name) {
+ auto [Modifiers, Type] = Name.rsplit('.');
+ if (Type != "f" && Type != "d" && Type != "f16" && Type != "v2f16")
+ return std::nullopt;
+
+ std::optional<llvm::RoundingMode> RoundingMode =
+ StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
+ .Case("rn", llvm::RoundingMode::NearestTiesToEven)
+ .Case("rz", llvm::RoundingMode::TowardZero)
+ .Case("rm", llvm::RoundingMode::TowardNegative)
+ .Case("rp", llvm::RoundingMode::TowardPositive)
.Default(std::nullopt);
- if (!RM)
+ if (!RoundingMode)
return std::nullopt;
Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
@@ -1398,7 +1402,8 @@ getNVVMFAddUpgrade(StringRef Modifiers) {
.Default(Intrinsic::not_intrinsic);
if (IID == Intrinsic::not_intrinsic)
return std::nullopt;
- return std::make_pair(IID, *RM);
+
+ return std::make_pair(IID, *RoundingMode);
}
static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
@@ -1967,21 +1972,6 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
return NewFn != F;
}
- // Upgrade the FP add intrinsics, which are overloaded on the operand type
- // and take the rounding mode as an operand:
- // llvm.nvvm.add.<rnd>{.ftz}{.sat}.<type> =>
- // llvm.nvvm.fadd{.ftz}{.sat}.<mangled type>
- // The extra operand means these are expanded in UpgradeIntrinsicCall.
- if (Name.starts_with("add.")) {
- auto [Base, TypeSuffix] = Name.rsplit('.');
- if ((TypeSuffix == "f" || TypeSuffix == "d" || TypeSuffix == "f16" ||
- TypeSuffix == "v2f16") &&
- getNVVMFAddUpgrade(Base.drop_front(strlen("add.")))) {
- NewFn = nullptr;
- return true;
- }
- }
-
// The following nvvm intrinsics correspond exactly to an LLVM idiom, but
// not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
//
@@ -1994,6 +1984,9 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
else if (Name.consume_front("fabs."))
// nvvm.fabs.{f,ftz.f,d}
Expand = Name == "f" || Name == "ftz.f" || Name == "d";
+ else if (Name.consume_front("add."))
+ // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
+ Expand = getNVVMFAddUpgrade(Name).has_value();
else if (Name.consume_front("ex2.approx."))
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Expand =
@@ -3074,12 +3067,15 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
: Intrinsic::nvvm_fabs;
Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
} else if (Name.consume_front("add.")) {
- // nvvm.add.<rnd>[.ftz][.sat].{f,d,f16,v2f16}
- auto [IID, RM] = *getNVVMFAddUpgrade(Name.rsplit('.').first);
+ // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
+ auto FAdd = getNVVMFAddUpgrade(Name);
+ assert(FAdd && "unsupported nvvm.add.* intrinsic");
+ auto [IID, RoundingMode] = *FAdd;
Value *A = CI->getArgOperand(0);
Rep = Builder.CreateIntrinsic(
A->getType(), IID,
- {A, CI->getArgOperand(1), Builder.getInt32(static_cast<int>(RM))});
+ {A, CI->getArgOperand(1),
+ Builder.getInt32(static_cast<int>(RoundingMode))});
} else if (Name.consume_front("ex2.approx.")) {
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index eb3d63e9eaa84..15f6e07a52e1e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7186,7 +7186,7 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
}
static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
- APFloat::roundingMode RM) {
+ APFloat::roundingMode RoundingMode) {
const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
@@ -7198,7 +7198,7 @@ static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
case MVT::f32:
if (!VT.isVector() || nvvm::FAddShouldSaturate(IID))
return 0;
- switch (RM) {
+ switch (RoundingMode) {
case APFloat::rmNearestTiesToEven:
return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
case APFloat::rmTowardZero:
@@ -7217,9 +7217,9 @@ static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
Intrinsic::ID AddIntrinsicID,
- APFloat::roundingMode RM) {
+ APFloat::roundingMode RoundingMode) {
const EVT VT = N->getValueType(0);
- const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID, RM);
+ const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID, RoundingMode);
if (!Opc)
return SDValue();
@@ -7241,38 +7241,40 @@ static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(Opc, SDLoc(N), VT, SubOp1, SubOp2);
}
-static bool isSupportedFAdd(EVT VT, Intrinsic::ID IID, APFloat::roundingMode RM,
- const NVPTXSubtarget &STI) {
+static bool isSupportedFAdd(EVT VT, const NVPTXSubtarget &STI,
+ Intrinsic::ID IID,
+ APFloat::roundingMode RoundingMode) {
if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(2))
return false;
+ const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
+ const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
const bool IsSat = nvvm::FAddShouldSaturate(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
- return RM == APFloat::rmNearestTiesToEven;
+ return IsRN;
case MVT::bf16:
- return RM == APFloat::rmNearestTiesToEven && !IsSat &&
- !nvvm::FAddShouldFTZ(IID) && STI.hasNativeBF16Support(ISD::FADD);
+ return IsRN && !IsSat && !IsFTZ && STI.hasNativeBF16Support(ISD::FADD);
case MVT::f32:
return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
case MVT::f64:
- return !VT.isVector() && !IsSat && !nvvm::FAddShouldFTZ(IID);
+ return !VT.isVector() && !IsSat && !IsFTZ;
default:
return false;
}
}
static SDValue diagnoseInvalidFAdd(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID IID, APFloat::roundingMode RM,
- const NVPTXSubtarget &STI) {
+ const NVPTXSubtarget &STI, Intrinsic::ID IID,
+ APFloat::roundingMode RoundingMode) {
const EVT VT = N->getValueType(0);
- if (isSupportedFAdd(VT, IID, RM, STI))
+ if (isSupportedFAdd(VT, STI, IID, RoundingMode))
return SDValue();
DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
DAG.getMachineFunction().getFunction(),
Twine(Intrinsic::getBaseName(IID)) + " with rounding mode " +
- nvvm::GetRoundingModeName(RM) + " and operand type " +
+ nvvm::GetRoundingModeName(RoundingMode) + " and operand type " +
VT.getEVTString() + " is not supported on this target",
SDLoc(N).getDebugLoc()));
return DAG.getPOISON(VT);
@@ -7291,11 +7293,11 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
case Intrinsic::nvvm_fadd_ftz:
case Intrinsic::nvvm_fadd_sat:
case Intrinsic::nvvm_fadd_ftz_sat: {
- const auto RM = static_cast<APFloat::roundingMode>(
+ const auto RoundingMode = static_cast<APFloat::roundingMode>(
N->getConstantOperandAPInt(3).getSExtValue());
- if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, IID, RM, STI))
+ if (SDValue V = diagnoseInvalidFAdd(N, DCI.DAG, STI, IID, RoundingMode))
return V;
- return combineFAddWithNeg(N, DCI.DAG, IID, RM);
+ return combineFAddWithNeg(N, DCI.DAG, IID, RoundingMode);
}
}
return SDValue();
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 023238c7bd386..a08569d1c883c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1861,6 +1861,8 @@ def : Pat<(int_nvvm_cos_approx_f f32:$a), (COS_APPROX_f32 $a, NoFTZ)>;
// Fma
//
+defvar FPRoundingModes = ["_rn", "_rz", "_rm", "_rp"];
+
class FMA_TUPLE<string V, Intrinsic I, NVPTXRegClass RC,
list<Predicate> Preds = []> {
string Variant = V;
@@ -1935,7 +1937,7 @@ multiclass FMA_INST {
defm INT_NVVM_FMA : FMA_INST;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
+foreach rnd = FPRoundingModes in {
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
def INT_NVVM_MIXED_FMA # rnd # sat # _f32_ # type :
@@ -2073,29 +2075,35 @@ let Predicates = [doRsqrtOpt] in {
defvar BF16ArithPreds = [hasBF16Math, PTX78, SM90];
-class RndModeImm<string mode> : TImmLeaf<i32,
+class FPRndModeImm<string mode> : TImmLeaf<i32,
"return Imm == static_cast<int>(RoundingMode::" # mode # ");">;
-def rnd_rn_imm : RndModeImm<"NearestTiesToEven">;
-def rnd_rz_imm : RndModeImm<"TowardZero">;
-def rnd_rm_imm : RndModeImm<"TowardNegative">;
-def rnd_rp_imm : RndModeImm<"TowardPositive">;
+def fp_rnd_rn_imm : FPRndModeImm<"NearestTiesToEven">;
+def fp_rnd_rz_imm : FPRndModeImm<"TowardZero">;
+def fp_rnd_rm_imm : FPRndModeImm<"TowardNegative">;
+def fp_rnd_rp_imm : FPRndModeImm<"TowardPositive">;
foreach t = [F16RT, F16X2RT] in
- foreach ftz = ["", "_ftz"] in
- foreach sat = ["", "_sat"] in
- def INT_NVVM_ADD_RN # !toupper(ftz # sat # "_" # t.PtxType) :
- F_MATH_2_RND_TY<!subst("_", ".", "add_rn" # ftz # sat # "_" # t.PtxType),
- t, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
- rnd_rn_imm>;
+ foreach ftz = ["", "ftz"] in
+ foreach sat = ["", "sat"] in {
+ // drop empty strings from the modifier list
+ defvar mods = !filter(m, [ftz, sat], !not(!empty(m)));
+ defvar suffix = !listconcat(mods, [t.PtxType]);
+ defvar add_intrin =
+ !cast<Intrinsic>(!interleave(!listconcat(["int_nvvm_fadd"], mods), "_"));
+
+ def INT_NVVM_ADD_RN_ # !toupper(!interleave(suffix, "_")) :
+ F_MATH_2_RND_TY<!interleave(!listconcat(["add", "rn"], suffix), "."), t,
+ add_intrin, fp_rnd_rn_imm>;
+ }
foreach t = [BF16RT, BF16X2RT] in
def INT_NVVM_ADD_RN_ # !toupper(t.PtxType) :
- F_MATH_2_RND_TY<"add.rn." # t.PtxType, t, int_nvvm_fadd, rnd_rn_imm,
+ F_MATH_2_RND_TY<"add.rn." # t.PtxType, t, int_nvvm_fadd, fp_rnd_rn_imm,
BF16ArithPreds>;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in
@@ -2115,8 +2123,8 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
int_nvvm_fadd, rnd_imm>;
}
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
@@ -2149,7 +2157,7 @@ def sub_rn_sat : SDNode<"NVPTXISD::SUB_RN_SAT", SDTFPBinOp>;
def sub_rn_ftz_sat :
SDNode<"NVPTXISD::SUB_RN_FTZ_SAT", SDTFPBinOp>;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+foreach rnd = FPRoundingModes in
foreach ftz = ["", "_ftz"] in
def sub # rnd # ftz :
SDNode<"NVPTXISD::SUB" # !toupper(rnd # ftz), SDTFPBinOp>;
@@ -2171,12 +2179,12 @@ let Predicates = BF16ArithPreds in
def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, "_rn">;
let Predicates = [hasF32x2Instructions] in
- foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in
+ foreach rnd = FPRoundingModes in
foreach ftz = ["", "_ftz"] in
def INT_NVVM_SUB # rnd # ftz # _F32X2 : INT_NVVM_SUB<F32X2RT, rnd # ftz>;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
foreach ftz = ["", "_ftz"] in {
foreach sat = ["", "_sat"] in {
@@ -2196,8 +2204,8 @@ foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
(int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), rnd_imm))]>;
}
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- defvar rnd_imm = !cast<TImmLeaf>("rnd" # rnd # "_imm");
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
foreach sat = ["", "_sat"] in {
foreach type = [f16, bf16] in {
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index b3c953de574b9..ba6eecb58144d 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -448,6 +448,22 @@ getFenceProxySyncRestrictID(NVVM::MemOrderKind order) {
nvvm_fence_proxy_async_generic_release_sync_restrict_space_cta_scope_cluster;
}
+static llvm::RoundingMode getLLVMRoundingMode(NVVM::FPRoundingMode rndMode) {
+ switch (rndMode) {
+ case NVVM::FPRoundingMode::NONE:
+ case NVVM::FPRoundingMode::RN:
+ return llvm::RoundingMode::NearestTiesToEven;
+ case NVVM::FPRoundingMode::RM:
+ return llvm::RoundingMode::TowardNegative;
+ case NVVM::FPRoundingMode::RP:
+ return llvm::RoundingMode::TowardPositive;
+ case NVVM::FPRoundingMode::RZ:
+ return llvm::RoundingMode::TowardZero;
+ default:
+ llvm_unreachable("unsupported rounding mode for nvvm fp arithmetic");
+ }
+}
+
// Calls an LLVM intrinsic on the given operands. For f32/f64 vector types,
// the intrinsic is called per-element and the results are packed back into a
// vector. If retType is non-null, it is forwarded as the return-type
@@ -489,14 +505,9 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
{llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
{llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
- static constexpr llvm::RoundingMode roundingModes[5] = {
- llvm::RoundingMode::NearestTiesToEven,
- llvm::RoundingMode::NearestTiesToEven, llvm::RoundingMode::TowardNegative,
- llvm::RoundingMode::TowardPositive, llvm::RoundingMode::TowardZero};
-
llvm::Intrinsic::ID id = addIDs[isFTZ][isSat];
- llvm::Value *rnd = builder.getInt32(
- static_cast<int>(roundingModes[static_cast<unsigned>(rndMode)]));
+ llvm::Value *rnd =
+ builder.getInt32(static_cast<int>(getLLVMRoundingMode(rndMode)));
// For f64 vector addition, and f32 vector addition with saturation,
// we need to scalarize the intrinsic call.
>From a0d62383d7acc9f872afd6a44a10c911a8a355a4 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 25 Aug 2026 13:08:42 +0000
Subject: [PATCH 10/12] address comments
---
llvm/lib/IR/AutoUpgrade.cpp | 2 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 35 +++++++++----------
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 9 ++---
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 13 ++++---
4 files changed, 29 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 0afb42085c734..9dfed852f7931 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1381,7 +1381,7 @@ static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
getNVVMFAddUpgrade(StringRef Name) {
auto [Modifiers, Type] = Name.rsplit('.');
- if (Type != "f" && Type != "d" && Type != "f16" && Type != "v2f16")
+ if (!is_contained({"f", "d", "f16", "v2f16"}, Type))
return std::nullopt;
std::optional<llvm::RoundingMode> RoundingMode =
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 15f6e07a52e1e..62699f895c9eb 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7188,28 +7188,27 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
APFloat::roundingMode RoundingMode) {
const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
+ const bool IsSat = nvvm::FAddShouldSaturate(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
- case MVT::f16:
- if (nvvm::FAddShouldSaturate(IID))
- return IsFTZ ? NVPTXISD::SUB_RN_FTZ_SAT : NVPTXISD::SUB_RN_SAT;
- return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
+ case MVT::f16: {
+ static constexpr unsigned SubRNOpcodes[2][2] = {
+ {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_SAT},
+ {NVPTXISD::SUB_RN_FTZ, NVPTXISD::SUB_RN_FTZ_SAT}};
+ return SubRNOpcodes[IsFTZ][IsSat];
+ }
case MVT::bf16:
return NVPTXISD::SUB_RN;
- case MVT::f32:
- if (!VT.isVector() || nvvm::FAddShouldSaturate(IID))
+ case MVT::f32: {
+ // for f32x2 inputs
+ if (!VT.isVector() || IsSat)
return 0;
- switch (RoundingMode) {
- case APFloat::rmNearestTiesToEven:
- return IsFTZ ? NVPTXISD::SUB_RN_FTZ : NVPTXISD::SUB_RN;
- case APFloat::rmTowardZero:
- return IsFTZ ? NVPTXISD::SUB_RZ_FTZ : NVPTXISD::SUB_RZ;
- case APFloat::rmTowardNegative:
- return IsFTZ ? NVPTXISD::SUB_RM_FTZ : NVPTXISD::SUB_RM;
- case APFloat::rmTowardPositive:
- return IsFTZ ? NVPTXISD::SUB_RP_FTZ : NVPTXISD::SUB_RP;
- default:
- llvm_unreachable("Unexpected fadd rounding mode");
- }
+ static constexpr unsigned SubF32x2Opcodes[4][2] = {
+ {NVPTXISD::SUB_RZ, NVPTXISD::SUB_RZ_FTZ}, // RZ
+ {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_FTZ}, // RN
+ {NVPTXISD::SUB_RP, NVPTXISD::SUB_RP_FTZ}, // RP
+ {NVPTXISD::SUB_RM, NVPTXISD::SUB_RM_FTZ}}; // RM
+ return SubF32x2Opcodes[static_cast<unsigned>(RoundingMode)][IsFTZ];
+ }
default:
return 0;
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index a08569d1c883c..acea8f149ef3d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2086,14 +2086,11 @@ def fp_rnd_rp_imm : FPRndModeImm<"TowardPositive">;
foreach t = [F16RT, F16X2RT] in
foreach ftz = ["", "ftz"] in
foreach sat = ["", "sat"] in {
- // drop empty strings from the modifier list
- defvar mods = !filter(m, [ftz, sat], !not(!empty(m)));
- defvar suffix = !listconcat(mods, [t.PtxType]);
defvar add_intrin =
- !cast<Intrinsic>(!interleave(!listconcat(["int_nvvm_fadd"], mods), "_"));
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret);
- def INT_NVVM_ADD_RN_ # !toupper(!interleave(suffix, "_")) :
- F_MATH_2_RND_TY<!interleave(!listconcat(["add", "rn"], suffix), "."), t,
+ def INT_NVVM_ADD_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
+ F_MATH_2_RND_TY<StrJoin<".", ["add", "rn", ftz, sat, t.PtxType]>.ret, t,
add_intrin, fp_rnd_rn_imm>;
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index ba6eecb58144d..612425754e620 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -448,9 +448,9 @@ getFenceProxySyncRestrictID(NVVM::MemOrderKind order) {
nvvm_fence_proxy_async_generic_release_sync_restrict_space_cta_scope_cluster;
}
-static llvm::RoundingMode getLLVMRoundingMode(NVVM::FPRoundingMode rndMode) {
+static llvm::RoundingMode
+getLLVMRoundingModeForFPArith(NVVM::FPRoundingMode rndMode) {
switch (rndMode) {
- case NVVM::FPRoundingMode::NONE:
case NVVM::FPRoundingMode::RN:
return llvm::RoundingMode::NearestTiesToEven;
case NVVM::FPRoundingMode::RM:
@@ -460,7 +460,10 @@ static llvm::RoundingMode getLLVMRoundingMode(NVVM::FPRoundingMode rndMode) {
case NVVM::FPRoundingMode::RZ:
return llvm::RoundingMode::TowardZero;
default:
- llvm_unreachable("unsupported rounding mode for nvvm fp arithmetic");
+ // default rounding mode is RN
+ assert(rndMode == NVVM::FPRoundingMode::NONE &&
+ "unsupported rounding mode for nvvm fp arithmetic");
+ return llvm::RoundingMode::NearestTiesToEven;
}
}
@@ -506,8 +509,8 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
{llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
llvm::Intrinsic::ID id = addIDs[isFTZ][isSat];
- llvm::Value *rnd =
- builder.getInt32(static_cast<int>(getLLVMRoundingMode(rndMode)));
+ llvm::Value *rnd = builder.getInt32(
+ static_cast<int>(getLLVMRoundingModeForFPArith(rndMode)));
// For f64 vector addition, and f32 vector addition with saturation,
// we need to scalarize the intrinsic call.
>From 6bf4f5f1cb5d9e42f9205fc5e421c9604a833c86 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 25 Aug 2026 14:26:58 +0000
Subject: [PATCH 11/12] address comments
---
llvm/include/llvm/IR/IntrinsicsNVVM.td | 2 +-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 31 ++--------------
llvm/lib/Analysis/ConstantFolding.cpp | 40 ++++++++++-----------
llvm/lib/IR/NVVMIntrinsicUtils.cpp | 4 +--
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 12 ++++---
5 files changed, 31 insertions(+), 58 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index fc33962adb59b..48a4fbadfa3e2 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1716,7 +1716,7 @@ let TargetPrefix = "nvvm" in {
Range<ArgIndex<2>, 0, 4>,
ArgInfo<ArgIndex<2>,
[ArgName<"rnd">,
- ImmArgPrinter<"printFAddRoundingMode">]>] in
+ ImmArgPrinter<"printFPRoundingMode">]>] in
foreach ftz = ["", "_ftz"] in
foreach sat = ["", "_sat"] in
def int_nvvm_fadd # ftz # sat :
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index 1cfdd921b0f2b..dff5afa5e8746 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -15,7 +15,6 @@
#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
#define LLVM_IR_NVVMINTRINSICUTILS_H
-#include <cassert>
#include <stdint.h>
#include "llvm/ADT/APFloat.h"
@@ -161,7 +160,7 @@ LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
const Constant *ImmArgVal);
LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
const Constant *ImmArgVal);
-LLVM_ABI void printFAddRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
+LLVM_ABI void printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
@@ -594,33 +593,7 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
return DenormalMode::getIEEE();
}
-inline bool IsFAddIntrinsic(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_fadd:
- case Intrinsic::nvvm_fadd_ftz:
- case Intrinsic::nvvm_fadd_sat:
- case Intrinsic::nvvm_fadd_ftz_sat:
- return true;
- default:
- return false;
- }
-}
-
-inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
- assert(IsFAddIntrinsic(IntrinsicID) &&
- "Checking FTZ flag for invalid NVVM add intrinsic");
- return IntrinsicID == Intrinsic::nvvm_fadd_ftz ||
- IntrinsicID == Intrinsic::nvvm_fadd_ftz_sat;
-}
-
-inline bool FAddShouldSaturate(Intrinsic::ID IntrinsicID) {
- assert(IsFAddIntrinsic(IntrinsicID) &&
- "Checking sat flag for invalid NVVM add intrinsic");
- return IntrinsicID == Intrinsic::nvvm_fadd_sat ||
- IntrinsicID == Intrinsic::nvvm_fadd_ftz_sat;
-}
-
-inline APFloat::roundingMode GetFAddRoundingMode(const Value *ImmArgVal) {
+inline APFloat::roundingMode GetRoundingModeFromImmArg(const Value *ImmArgVal) {
return static_cast<APFloat::roundingMode>(
cast<ConstantInt>(ImmArgVal)->getSExtValue());
}
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index cda5171b339a8..1b59616727dea 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -4156,32 +4156,28 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
}
}
}
- }
- }
-
- if (IntrinsicID == Intrinsic::nvvm_fadd ||
- IntrinsicID == Intrinsic::nvvm_fadd_ftz) {
- const auto *Op1 = dyn_cast<ConstantFP>(Operands[0]);
- const auto *Op2 = dyn_cast<ConstantFP>(Operands[1]);
- if (!Op1 || !Op2)
- return nullptr;
- bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
- APFloat A =
- IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
- APFloat B =
- IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
+ // TODO: Add constant folding for the _sat variants.
+ if (IntrinsicID == Intrinsic::nvvm_fadd ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz) {
+ bool IsFTZ = IntrinsicID == Intrinsic::nvvm_fadd_ftz;
+ APFloat A =
+ IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
+ APFloat B =
+ IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
- APFloat Res = A;
- APFloat::opStatus Status =
- Res.add(B, nvvm::GetFAddRoundingMode(Operands[2]));
+ APFloat Res = A;
+ APFloat::opStatus Status =
+ Res.add(B, nvvm::GetRoundingModeFromImmArg(Operands[2]));
- if (!Res.isNaN() &&
- (Status == APFloat::opOK || Status == APFloat::opInexact)) {
- Res = IsFTZ ? FTZPreserveSign(Res) : Res;
- return ConstantFP::get(Ty, Res);
+ if (!Res.isNaN() &&
+ (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+ Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+ return ConstantFP::get(Ty, Res);
+ }
+ return nullptr;
+ }
}
- return nullptr;
}
if (IntrinsicID == Intrinsic::smul_fix ||
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index 04d2bffac094e..a464e40560115 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -151,9 +151,9 @@ void nvvm::printTensormapSwizzleAtomicity(raw_ostream &OS,
}
}
-void nvvm::printFAddRoundingMode(raw_ostream &OS, const Constant *ImmArgVal) {
+void nvvm::printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal) {
if (isa<ConstantInt>(ImmArgVal))
- OS << nvvm::GetRoundingModeName(nvvm::GetFAddRoundingMode(ImmArgVal));
+ OS << nvvm::GetRoundingModeName(nvvm::GetRoundingModeFromImmArg(ImmArgVal));
}
void nvvm::printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 62699f895c9eb..a57b5d0e02313 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7187,8 +7187,10 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
APFloat::roundingMode RoundingMode) {
- const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
- const bool IsSat = nvvm::FAddShouldSaturate(IID);
+ const bool IsFTZ =
+ IID == Intrinsic::nvvm_fadd_ftz || IID == Intrinsic::nvvm_fadd_ftz_sat;
+ const bool IsSat =
+ IID == Intrinsic::nvvm_fadd_sat || IID == Intrinsic::nvvm_fadd_ftz_sat;
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16: {
static constexpr unsigned SubRNOpcodes[2][2] = {
@@ -7247,8 +7249,10 @@ static bool isSupportedFAdd(EVT VT, const NVPTXSubtarget &STI,
return false;
const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
- const bool IsFTZ = nvvm::FAddShouldFTZ(IID);
- const bool IsSat = nvvm::FAddShouldSaturate(IID);
+ const bool IsFTZ =
+ IID == Intrinsic::nvvm_fadd_ftz || IID == Intrinsic::nvvm_fadd_ftz_sat;
+ const bool IsSat =
+ IID == Intrinsic::nvvm_fadd_sat || IID == Intrinsic::nvvm_fadd_ftz_sat;
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
return IsRN;
>From afe201e2059ec9c325ba5b126c5b613adae8f4cf Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 25 Aug 2026 14:45:56 +0000
Subject: [PATCH 12/12] use StrJoin instead of subst
---
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 130 +++++++++++------------
1 file changed, 65 insertions(+), 65 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index acea8f149ef3d..670a1426fc8a0 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1861,7 +1861,7 @@ def : Pat<(int_nvvm_cos_approx_f f32:$a), (COS_APPROX_f32 $a, NoFTZ)>;
// Fma
//
-defvar FPRoundingModes = ["_rn", "_rz", "_rm", "_rp"];
+defvar FPRoundingModes = ["rn", "rz", "rm", "rp"];
class FMA_TUPLE<string V, Intrinsic I, NVPTXRegClass RC,
list<Predicate> Preds = []> {
@@ -1937,21 +1937,18 @@ multiclass FMA_INST {
defm INT_NVVM_FMA : FMA_INST;
-foreach rnd = FPRoundingModes in {
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_FMA # rnd # sat # _f32_ # type :
+foreach rnd = FPRoundingModes in
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_FMA_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B16:$b, B32:$c),
- !subst("_", ".", "fma" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fma" # rnd # sat # "_f")
+ StrJoin<".", ["fma", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", rnd, sat, "f"]>.ret)
(f32 (fpextend type:$a)),
(f32 (fpextend type:$b)),
f32:$c))]>,
Requires<[SM100]>;
- }
- }
-}
// Pattern for llvm.fma.f32 intrinsic when there is no FTZ flag
let Predicates = [SM100, doNoF32FTZ] in {
@@ -2100,41 +2097,41 @@ foreach t = [BF16RT, BF16X2RT] in
BF16ArithPreds>;
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
-
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in
- def INT_NVVM_ADD # !toupper(rnd # ftz # sat) # _F :
- F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # ftz # sat # "_f32"),
- F32RT, !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat),
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
+
+ foreach ftz = ["", "ftz"] in {
+ foreach sat = ["", "sat"] in
+ def INT_NVVM_ADD_ # !toupper(StrJoin<"_", [rnd, ftz, sat]>.ret) # _F :
+ F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, ftz, sat, "f32"]>.ret, F32RT,
+ !cast<Intrinsic>(
+ StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret),
rnd_imm>;
- def INT_NVVM_ADD # !toupper(rnd # ftz) # _F32X2 :
- F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # ftz # "_f32x2"), F32X2RT,
- !cast<Intrinsic>("int_nvvm_fadd" # ftz), rnd_imm,
- [hasF32x2Instructions]>;
+ def INT_NVVM_ADD_ # !toupper(StrJoin<"_", [rnd, ftz]>.ret) # _F32X2 :
+ F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, ftz, "f32x2"]>.ret, F32X2RT,
+ !cast<Intrinsic>(
+ StrJoin<"_", ["int_nvvm_fadd", ftz]>.ret),
+ rnd_imm, [hasF32x2Instructions]>;
}
- def INT_NVVM_ADD # !toupper(rnd) # _D :
- F_MATH_2_RND_TY<!subst("_", ".", "add" # rnd # "_f64"), F64RT,
- int_nvvm_fadd, rnd_imm>;
+ def INT_NVVM_ADD_ # !toupper(rnd) # _D :
+ F_MATH_2_RND_TY<StrJoin<".", ["add", rnd, "f64"]>.ret, F64RT, int_nvvm_fadd,
+ rnd_imm>;
}
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_ADD # rnd # sat # _f32_ # type :
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_ADD_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
- !subst("_", ".", "add" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fadd" # sat)
+ StrJoin<".", ["add", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", sat]>.ret)
(f32 (fpextend type:$a)),
f32:$b, rnd_imm))]>,
Requires<[SM100]>;
- }
- }
}
// Pattern for fadd when there is no FTZ flag
@@ -2155,67 +2152,70 @@ def sub_rn_ftz_sat :
SDNode<"NVPTXISD::SUB_RN_FTZ_SAT", SDTFPBinOp>;
foreach rnd = FPRoundingModes in
- foreach ftz = ["", "_ftz"] in
- def sub # rnd # ftz :
- SDNode<"NVPTXISD::SUB" # !toupper(rnd # ftz), SDTFPBinOp>;
+ foreach ftz = ["", "ftz"] in {
+ defvar variant = StrJoin<"_", [rnd, ftz]>.ret;
+ def sub_ # variant :
+ SDNode<"NVPTXISD::SUB_" # !toupper(variant), SDTFPBinOp>;
+ }
-class INT_NVVM_SUB<RegTyInfo TyInfo, string variant> :
+class INT_NVVM_SUB<RegTyInfo TyInfo, list<string> variant> :
BasicNVPTXInst<(outs TyInfo.RC:$dst), (ins TyInfo.RC:$a, TyInfo.RC:$b),
- !subst("_", ".", "sub" # variant # "." # TyInfo.PtxType),
- [(set TyInfo.Ty:$dst,
- (!cast<SDNode>("sub" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
+ StrJoin<".", !listconcat(["sub"], variant, [TyInfo.PtxType])>.ret,
+ [(set TyInfo.Ty:$dst,
+ (!cast<SDNode>(StrJoin<"_", !listconcat(["sub"], variant)>.ret)
+ TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
foreach t = [F16RT, F16X2RT] in
- foreach ftz = ["", "_ftz"] in
- foreach sat = ["", "_sat"] in
- def INT_NVVM_SUB_RN # !toupper(ftz # sat # "_" # t.PtxType) :
- INT_NVVM_SUB<t, "_rn" # ftz # sat>;
+ foreach ftz = ["", "ftz"] in
+ foreach sat = ["", "sat"] in
+ def INT_NVVM_SUB_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
+ INT_NVVM_SUB<t, ["rn", ftz, sat]>;
let Predicates = BF16ArithPreds in
foreach t = [BF16RT, BF16X2RT] in
- def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, "_rn">;
+ def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, ["rn"]>;
let Predicates = [hasF32x2Instructions] in
foreach rnd = FPRoundingModes in
- foreach ftz = ["", "_ftz"] in
- def INT_NVVM_SUB # rnd # ftz # _F32X2 : INT_NVVM_SUB<F32X2RT, rnd # ftz>;
+ foreach ftz = ["", "ftz"] in
+ def INT_NVVM_SUB_ # StrJoin<"_", [rnd, ftz]>.ret # _F32X2 :
+ INT_NVVM_SUB<F32X2RT, [rnd, ftz]>;
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_fadd" # ftz # sat);
- def INT_NVVM_SUB # rnd # ftz # sat # _F :
+ foreach ftz = ["", "ftz"] in {
+ foreach sat = ["", "sat"] in {
+ defvar add_intrin =
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret);
+ def INT_NVVM_SUB_ # StrJoin<"_", [rnd, ftz, sat]>.ret # _F :
BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
- !subst("_", ".", "sub" # rnd # ftz # sat # "_f32"),
+ StrJoin<".", ["sub", rnd, ftz, sat, "f32"]>.ret,
[(set f32:$dst,
(add_intrin f32:$a, (f32 (fneg f32:$b)), rnd_imm))]>;
}
}
- def INT_NVVM_SUB # rnd # _D :
+ def INT_NVVM_SUB_ # rnd # _D :
BasicNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
- !subst("_", ".", "sub" # rnd # "_f64"),
+ StrJoin<".", ["sub", rnd, "f64"]>.ret,
[(set f64:$dst,
(int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), rnd_imm))]>;
}
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>("fp_rnd" # rnd # "_imm");
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_SUB # rnd # sat # _f32_ # type :
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_SUB_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
- !subst("_", ".", "sub" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fadd" # sat)
+ StrJoin<".", ["sub", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", sat]>.ret)
(f32 (fpextend type:$a)),
(f32 (fneg f32:$b)), rnd_imm))]>,
Requires<[SM100]>;
- }
- }
}
// Pattern for fsub when there is no FTZ flag
More information about the Mlir-commits
mailing list