[clang] afb42af - [clang][NVPTX][MLIR][NVVM] Add overloaded fadd intrinsics (#221681)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Sep 7 04:21:35 PDT 2026
Author: Srinivasa Ravi
Date: 2026-09-07T16:51:27+05:30
New Revision: afb42af9cc18d0dd08ccac3577310e049095700b
URL: https://github.com/llvm/llvm-project/commit/afb42af9cc18d0dd08ccac3577310e049095700b
DIFF: https://github.com/llvm/llvm-project/commit/afb42af9cc18d0dd08ccac3577310e049095700b.diff
LOG: [clang][NVPTX][MLIR][NVVM] Add overloaded fadd intrinsics (#221681)
This change adds the following overloaded `fadd` intrinsics with
NVPTX codegen:
- `llvm.nvvm.fadd`
- `llvm.nvvm.fadd.ftz`
- `llvm.nvvm.fadd.sat`
- `llvm.nvvm.fadd.ftz.sat`
The rounding mode is passed in as an `i32` immediate operand.
Auto-upgrades the older non-overloaded intrinsics to the new
ones, and updates clang builtins, CIR codegen, and MLIR NVVM ops
to lower to the new intrinsics.
In the interest of completion, this also:
- Adds intrinsics support for lowering some half-precision additions
that were omitted earlier (`f16/f16x2` without saturation, and
`bf16/bf16x2` additions), and support for the `f32x2` type.
- Extends the DAG combine pattern for lowering `fneg + add-intrinsic`
to `sub` instructions to all supported types.
- Adds tests for constant folding of these intrinsics with the newly
supported scalar types.
PTX Spec Reference:
https://docs.nvidia.com/cuda/developer-preview/13.4/parallel-thread-execution/index.html#floating-point-instructions-add
Assisted-by: Claude Opus 5
Added:
llvm/test/CodeGen/NVPTX/bf16-add.ll
llvm/test/CodeGen/NVPTX/bf16-sub.ll
llvm/test/CodeGen/NVPTX/f16-add.ll
llvm/test/CodeGen/NVPTX/f16-sub.ll
llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
llvm/test/Verifier/NVPTX/fadd.ll
Modified:
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
clang/test/CodeGen/builtins-nvptx.c
llvm/docs/NVPTXUsage.md
llvm/include/llvm/IR/IntrinsicsNVVM.td
llvm/include/llvm/IR/NVVMIntrinsicUtils.h
llvm/lib/Analysis/ConstantFolding.cpp
llvm/lib/IR/AutoUpgrade.cpp
llvm/lib/IR/NVVMIntrinsicUtils.cpp
llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
llvm/unittests/IR/IntrinsicsTest.cpp
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
Removed:
llvm/test/CodeGen/NVPTX/f16-add-sat.ll
llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
################################################################################
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 5f284c11c32727..0b19fb18cd430b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -69,6 +69,23 @@ static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
.getResult();
}
+/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd intrinsic, which takes the
+/// rounding mode as a trailing operand.
+static mlir::Value emitNVVMFAdd(CIRGenFunction &cgf, const CallExpr *expr,
+ llvm::StringRef intrinsicName,
+ llvm::APFloat::roundingMode rm) {
+ auto &builder = cgf.getBuilder();
+ mlir::Location loc = cgf.getLoc(expr->getExprLoc());
+ mlir::Value lhs = cgf.emitScalarExpr(expr->getArg(0));
+ mlir::Value rhs = cgf.emitScalarExpr(expr->getArg(1));
+ mlir::Value rnd =
+ builder.getConstInt(loc, builder.getSInt32Ty(), static_cast<int>(rm));
+ return cir::LLVMIntrinsicCallOp::create(builder, loc,
+ builder.getStringAttr(intrinsicName),
+ lhs.getType(), {lhs, rhs, rnd})
+ .getResult();
+}
+
static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr,
llvm::StringRef intrinsicName,
bool returnsPred) {
@@ -822,6 +839,61 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx");
case NVPTX::BI__nvvm_ex2_approx_ftz_f:
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
+ case NVPTX::BI__nvvm_add_rn_f:
+ case NVPTX::BI__nvvm_add_rn_d:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_add_rz_f:
+ case NVPTX::BI__nvvm_add_rz_d:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd", llvm::APFloat::rmTowardZero);
+ case NVPTX::BI__nvvm_add_rm_f:
+ case NVPTX::BI__nvvm_add_rm_d:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmTowardNegative);
+ case NVPTX::BI__nvvm_add_rp_f:
+ case NVPTX::BI__nvvm_add_rp_d:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd",
+ llvm::APFloat::rmTowardPositive);
+ case NVPTX::BI__nvvm_add_rn_ftz_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_add_rz_ftz_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardZero);
+ case NVPTX::BI__nvvm_add_rm_ftz_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardNegative);
+ case NVPTX::BI__nvvm_add_rp_ftz_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz",
+ llvm::APFloat::rmTowardPositive);
+ case NVPTX::BI__nvvm_add_rn_sat_f:
+ case NVPTX::BI__nvvm_add_rn_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_sat_v2f16:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_add_rz_sat_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardZero);
+ case NVPTX::BI__nvvm_add_rm_sat_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardNegative);
+ case NVPTX::BI__nvvm_add_rp_sat_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat",
+ llvm::APFloat::rmTowardPositive);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardZero);
+ case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardNegative);
+ case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
+ return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
cgm.errorNYI(expr->getSourceRange(),
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index e3ef3ae488a0c9..06c5069d6f984b 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -433,6 +433,14 @@ static Value *MakeFMAOOB(unsigned IntrinsicID, llvm::Type *Ty,
CGF.EmitScalarExpr(E->getArg(2))});
}
+static Value *MakeFAdd(unsigned IntrinsicID, APFloat::roundingMode RM,
+ unsigned BuiltinID, const CallExpr *E,
+ CodeGenFunction &CGF) {
+ llvm::Type *Ty = CGF.ConvertType(E->getType());
+ return MakeHalfType(CGF.CGM.getIntrinsic(IntrinsicID, Ty), BuiltinID, E, CGF,
+ {CGF.Builder.getInt32(static_cast<int>(RM))});
+}
+
} // namespace
Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
@@ -1181,6 +1189,62 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
case NVPTX::BI__nvvm_ex2_approx_ftz_f:
return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_ex2_approx_ftz,
EmitScalarExpr(E->getArg(0)));
+ case NVPTX::BI__nvvm_add_rn_f:
+ case NVPTX::BI__nvvm_add_rn_d:
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rz_f:
+ case NVPTX::BI__nvvm_add_rz_d:
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardZero, BuiltinID, E,
+ *this);
+ case NVPTX::BI__nvvm_add_rm_f:
+ case NVPTX::BI__nvvm_add_rm_d:
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardNegative, BuiltinID,
+ E, *this);
+ case NVPTX::BI__nvvm_add_rp_f:
+ case NVPTX::BI__nvvm_add_rp_d:
+ return MakeFAdd(Intrinsic::nvvm_fadd, APFloat::rmTowardPositive, BuiltinID,
+ E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rz_ftz_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardZero, BuiltinID,
+ E, *this);
+ case NVPTX::BI__nvvm_add_rm_ftz_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rp_ftz_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rn_sat_f:
+ case NVPTX::BI__nvvm_add_rn_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_sat_v2f16:
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rz_sat_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardZero, BuiltinID,
+ E, *this);
+ case NVPTX::BI__nvvm_add_rm_sat_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rp_sat_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_sat, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_f16:
+ case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rz_ftz_sat_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardZero,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rm_ftz_sat_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_add_rp_ftz_sat_f:
+ return MakeFAdd(Intrinsic::nvvm_fadd_ftz_sat, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_ldg_h:
case NVPTX::BI__nvvm_ldg_h2:
return MakeLdg(*this, E);
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index a2f0edb6d073be..69df1d376f7f73 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -63,3 +63,27 @@ __device__ double test_ex2_approx_d(double x) {
__device__ float test_ex2_approx_ftz_f(float x) {
return __nvvm_ex2_approx_ftz_f(x);
}
+
+// CIR-LABEL: @_Z13test_add_rn_fff
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
+// LLVM-LABEL: @_Z13test_add_rn_fff
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.f32(float {{.*}}, float {{.*}}, /* rnd=rn */ i32 1)
+__device__ float test_add_rn_f(float x, float y) {
+ return __nvvm_add_rn_f(x, y);
+}
+
+// CIR-LABEL: @_Z13test_add_rz_ddd
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd" {{.*}} : (!cir.double, !cir.double, !s32i) -> !cir.double
+// LLVM-LABEL: @_Z13test_add_rz_ddd
+// LLVM: call {{.*}}double @llvm.nvvm.fadd.f64(double {{.*}}, double {{.*}}, /* rnd=rz */ i32 0)
+__device__ double test_add_rz_d(double x, double y) {
+ return __nvvm_add_rz_d(x, y);
+}
+
+// CIR-LABEL: @_Z21test_add_rm_ftz_sat_fff
+// CIR: cir.call_llvm_intrinsic "nvvm.fadd.ftz.sat" {{.*}} : (!cir.float, !cir.float, !s32i) -> !cir.float
+// LLVM-LABEL: @_Z21test_add_rm_ftz_sat_fff
+// LLVM: call {{.*}}float @llvm.nvvm.fadd.ftz.sat.f32(float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
+__device__ float test_add_rm_ftz_sat_f(float x, float y) {
+ return __nvvm_add_rm_ftz_sat_f(x, y);
+}
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index c72634025dbc70..bed1498236b061 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -251,7 +251,7 @@ __device__ void nvvm_math(float f1, float f2, double d1, double d2) {
float t3 = __nvvm_sqrt_rn_f(f1);
// CHECK: call float @llvm.nvvm.rcp.rn.f
float t4 = __nvvm_rcp_rn_f(f2);
-// CHECK: call float @llvm.nvvm.add.rn.f
+// CHECK: call float @llvm.nvvm.fadd.f32({{.*}}i32 1)
float t5 = __nvvm_add_rn_f(f1, f2);
// CHECK: call double @llvm.nvvm.fmax.d
@@ -1628,21 +1628,21 @@ __device__ void nvvm_min_max_sm86() {
// CHECK-LABEL: nvvm_add_fma_f32_sat
__device__ void nvvm_add_fma_f32_sat() {
- // CHECK: call float @llvm.nvvm.add.rn.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 1)
__nvvm_add_rn_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rn.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 0)
__nvvm_add_rz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rz.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 0)
__nvvm_add_rz_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rm.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 3)
__nvvm_add_rm_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rm.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 3)
__nvvm_add_rm_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rp.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.sat.f32({{.*}}i32 2)
__nvvm_add_rp_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.add.rp.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 2)
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
// CHECK: call float @llvm.nvvm.fma.rn.sat.f
@@ -1672,13 +1672,13 @@ __device__ void nvvm_add_fma_f32_sat() {
// CHECK-LABEL: nvvm_add_mul_f16_sat
__device__ void nvvm_add_mul_f16_sat() {
- // CHECK: call half @llvm.nvvm.add.rn.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.sat.f16({{.*}}i32 1)
__nvvm_add_rn_sat_f16(F16, F16_2);
- // CHECK: call half @llvm.nvvm.add.rn.ftz.sat.f16
+ // CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_f16(F16, F16_2);
- // CHECK: call <2 x half> @llvm.nvvm.add.rn.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_sat_v2f16(F16X2, F16X2_2);
- // CHECK: call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16
+ // CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16({{.*}}i32 1)
__nvvm_add_rn_ftz_sat_v2f16(F16X2, F16X2_2);
// CHECK: call half @llvm.nvvm.mul.rn.sat.f16
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index e05af780126915..d2ee19c42a602c 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1185,6 +1185,33 @@ For more information, see [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread
### Arithmetic Intrinsics
+Some of these intrinsics take the rounding mode as an `i32` immediate operand
+instead of encoding it in the intrinsic name. The accepted values match the
+`llvm::RoundingMode` enumeration and are described in the following table:
+
+(fp-rounding-modes)=
+
+```{list-table} Floating-Point Rounding Modes
+:widths: 15 15 70
+:header-rows: 1
+
+ * - Value
+ - Rounding Mode
+ - Description
+ * - 0
+ - `rz`
+ - Round towards zero
+ * - 1
+ - `rn`
+ - Round to nearest, with ties to even
+ * - 2
+ - `rp`
+ - Round towards positive infinity
+ * - 3
+ - `rm`
+ - Round towards negative infinity
+```
+
#### '`llvm.nvvm.fabs.*`' Intrinsic
##### Syntax:
@@ -1284,29 +1311,65 @@ used in the '`llvm.nvvm.idp4a.[us].u`' variants, while sign-extension is used
with '`llvm.nvvm.idp4a.[us].s`' variants. The dot product of these 4-element
vectors is added to `%c` to produce the return.
-#### '`llvm.nvvm.add.*`' Half-precision Intrinsics
+#### '`llvm.nvvm.fadd.*`' Intrinsics
##### Syntax:
-```llvm
-declare half @llvm.nvvm.add.rn.sat.f16(half %a, half %b)
-declare <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
+This is an overloaded intrinsic. The '`.ftz`' and '`.sat`' modifiers are
+optional.
-declare half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %b)
-declare <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
+```llvm
+declare half @llvm.nvvm.fadd{.ftz}{.sat}.f16(half %a, half %b, i32 immarg %rnd)
+declare <2 x half> @llvm.nvvm.fadd{.ftz}{.sat}.v2f16(<2 x half> %a, <2 x half> %b, i32 immarg %rnd)
+declare bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 immarg %rnd)
+declare <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, i32 immarg %rnd)
+declare float @llvm.nvvm.fadd{.ftz}{.sat}.f32(float %a, float %b, i32 immarg %rnd)
+declare <2 x float> @llvm.nvvm.fadd{.ftz}.v2f32(<2 x float> %a, <2 x float> %b, i32 immarg %rnd)
+declare double @llvm.nvvm.fadd.f64(double %a, double %b, i32 immarg %rnd)
```
##### Overview:
-The '`llvm.nvvm.add.*`' intrinsics perform an addition operation with the
-specified rounding mode and modifiers.
+The '`llvm.nvvm.fadd.*`' intrinsics add `%a` and `%b` using the rounding mode
+selected by `%rnd` and the modifiers present in the intrinsic name. They
+correspond directly to the `add` PTX instruction.
##### Semantics:
-The '`.sat`' modifier performs a saturating addition where the result is
-clamped to `[0.0, 1.0]` and `NaN` results are flushed to `+0.0f`.
+`%rnd` selects the rounding mode applied to the result, see
+{ref}`fp-rounding-modes`.
+
The '`.ftz`' modifier flushes subnormal inputs and results to sign-preserving
zero.
+The '`.sat`' modifier performs a saturating addition where the result is
+clamped to `[0.0, 1.0]` and `NaN` results are flushed to `+0.0f`.
+
+Not every combination of operand type, rounding mode and modifier maps to a
+PTX instruction. The supported combinations are:
+
+```{list-table}
+:widths: 25 25 25
+:header-rows: 1
+
+ * - Operand Type
+ - Rounding Modes
+ - Modifiers
+ * - `half`, `<2 x half>`
+ - `rn`
+ - `.ftz`, `.sat`
+ * - `bfloat`, `<2 x bfloat>`
+ - `rn`
+ - None
+ * - `float`
+ - `rn`, `rz`, `rp`, `rm`
+ - `.ftz`, `.sat`
+ * - `<2 x float>`
+ - `rn`, `rz`, `rp`, `rm`
+ - `.ftz`
+ * - `double`
+ - `rn`, `rz`, `rp`, `rm`
+ - None
+```
#### '`llvm.nvvm.mul.*`' Half-precision Intrinsics
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index a85b7de7693d64..c50609f9fc5eeb 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1810,31 +1810,21 @@ let TargetPrefix = "nvvm" in {
}
//
- // Add
+ // FAdd
//
let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative,
- IntrNoCreateUndefOrPoison] in {
- foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in {
- def int_nvvm_add # rnd # ftz # sat # _f : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty]>;
- } // sat
- } // ftz
- def int_nvvm_add # rnd # _d : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty]>;
- }
-
- foreach ftz = ["", "_ftz"] in {
- def int_nvvm_add_rn # ftz # _sat_f16 : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_half_ty], [llvm_half_ty, llvm_half_ty]>;
-
- def int_nvvm_add_rn # ftz # _sat_v2f16 : NVVMBuiltin,
- DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_v2f16_ty, llvm_v2f16_ty]>;
-
- } // ftz
- }
+ IntrNoCreateUndefOrPoison, ImmArg<ArgIndex<2>>,
+ Range<ArgIndex<2>, 0, 4>,
+ ArgInfo<ArgIndex<2>,
+ [ArgName<"rnd">,
+ ImmArgPrinter<"printFPRoundingMode">]>] in
+ foreach ftz = ["", "_ftz"] in
+ foreach sat = ["", "_sat"] in
+ def int_nvvm_fadd # ftz # sat :
+ DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
+ llvm_i32_ty]>;
//
// Dot Product
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index a6f7555be838a3..950f852ba820bd 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -178,6 +178,7 @@ LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
const Constant *ImmArgVal);
LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
const Constant *ImmArgVal);
+LLVM_ABI void printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal);
inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
@@ -610,47 +611,24 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
return DenormalMode::getIEEE();
}
-inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f:
- return true;
-
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
- return false;
- }
- llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
+inline APFloat::roundingMode GetRoundingModeFromImmArg(const Value *ImmArgVal) {
+ return static_cast<APFloat::roundingMode>(
+ cast<ConstantInt>(ImmArgVal)->getSExtValue());
}
-inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rm_ftz_f:
- return APFloat::rmTowardNegative;
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rn_ftz_f:
- return APFloat::rmNearestTiesToEven;
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rp_ftz_f:
- return APFloat::rmTowardPositive;
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rz_ftz_f:
- return APFloat::rmTowardZero;
+inline StringRef GetRoundingModeName(APFloat::roundingMode RM) {
+ switch (RM) {
+ case APFloat::rmNearestTiesToEven:
+ return "rn";
+ case APFloat::rmTowardZero:
+ return "rz";
+ case APFloat::rmTowardNegative:
+ return "rm";
+ case APFloat::rmTowardPositive:
+ return "rp";
+ default:
+ return "";
}
- llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
}
inline bool FMulShouldFTZ(Intrinsic::ID IntrinsicID) {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index c819e188cac959..7eb44a398ebcad 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1995,18 +1995,8 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
return !IsStrictFP;
// NVVM add intrinsics with explicit rounding modes
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f:
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
// NVVM div intrinsics with explicit rounding modes
case Intrinsic::nvvm_div_rm_d:
@@ -3618,37 +3608,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
return ConstantFP::get(Ty, Res);
}
- case Intrinsic::nvvm_add_rm_f:
- case Intrinsic::nvvm_add_rn_f:
- case Intrinsic::nvvm_add_rp_f:
- case Intrinsic::nvvm_add_rz_f:
- case Intrinsic::nvvm_add_rm_d:
- case Intrinsic::nvvm_add_rn_d:
- case Intrinsic::nvvm_add_rp_d:
- case Intrinsic::nvvm_add_rz_d:
- case Intrinsic::nvvm_add_rm_ftz_f:
- case Intrinsic::nvvm_add_rn_ftz_f:
- case Intrinsic::nvvm_add_rp_ftz_f:
- case Intrinsic::nvvm_add_rz_ftz_f: {
-
- bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
- APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
- APFloat B = IsFTZ ? FTZPreserveSign(Op2V) : Op2V;
-
- APFloat::roundingMode RoundMode =
- nvvm::GetFAddRoundingMode(IntrinsicID);
-
- APFloat Res = A;
- APFloat::opStatus Status = Res.add(B, RoundMode);
-
- if (!Res.isNaN() &&
- (Status == APFloat::opOK || Status == APFloat::opInexact)) {
- Res = IsFTZ ? FTZPreserveSign(Res) : Res;
- return ConstantFP::get(Ty, Res);
- }
- return nullptr;
- }
-
case Intrinsic::nvvm_mul_rm_f:
case Intrinsic::nvvm_mul_rn_f:
case Intrinsic::nvvm_mul_rp_f:
@@ -4196,6 +4155,27 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
}
}
}
+
+ // TODO: Add constant folding for the _sat variants.
+ if (IntrinsicID == Intrinsic::nvvm_fadd ||
+ IntrinsicID == Intrinsic::nvvm_fadd_ftz) {
+ bool IsFTZ = IntrinsicID == Intrinsic::nvvm_fadd_ftz;
+ APFloat A =
+ IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
+ APFloat B =
+ IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
+
+ APFloat Res = A;
+ APFloat::opStatus Status =
+ Res.add(B, nvvm::GetRoundingModeFromImmArg(Operands[2]));
+
+ if (!Res.isNaN() &&
+ (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+ Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+ return ConstantFP::get(Ty, Res);
+ }
+ return nullptr;
+ }
}
}
@@ -4482,6 +4462,12 @@ static Constant *ConstantFoldFixedVectorCall(
return ConstantVector::get(Result);
}
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
+ // The rounding mode operand is a scalar, so the lane-wise folding below
+ // does not apply.
+ // TODO: Fold these by passing the rounding mode through to every lane.
+ return nullptr;
default:
break;
}
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 378a0b5ef53f99..a19b4ef1a78abe 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1421,6 +1421,34 @@ static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
return F->getIntrinsicID();
}
+static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
+getNVVMFAddUpgrade(StringRef Name) {
+ auto [Modifiers, Type] = Name.rsplit('.');
+ if (!is_contained({"f", "d", "f16", "v2f16"}, Type))
+ return std::nullopt;
+
+ std::optional<llvm::RoundingMode> RoundingMode =
+ StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
+ .Case("rn", llvm::RoundingMode::NearestTiesToEven)
+ .Case("rz", llvm::RoundingMode::TowardZero)
+ .Case("rm", llvm::RoundingMode::TowardNegative)
+ .Case("rp", llvm::RoundingMode::TowardPositive)
+ .Default(std::nullopt);
+ if (!RoundingMode)
+ return std::nullopt;
+
+ Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
+ .Case("", Intrinsic::nvvm_fadd)
+ .Case(".ftz", Intrinsic::nvvm_fadd_ftz)
+ .Case(".sat", Intrinsic::nvvm_fadd_sat)
+ .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
+ .Default(Intrinsic::not_intrinsic);
+ if (IID == Intrinsic::not_intrinsic)
+ return std::nullopt;
+
+ return std::make_pair(IID, *RoundingMode);
+}
+
static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
return Name.consume_front("local") || Name.consume_front("shared") ||
Name.consume_front("global") || Name.consume_front("constant") ||
@@ -2005,6 +2033,9 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
else if (Name.consume_front("fabs."))
// nvvm.fabs.{f,ftz.f,d}
Expand = Name == "f" || Name == "ftz.f" || Name == "d";
+ else if (Name.consume_front("add."))
+ // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
+ Expand = getNVVMFAddUpgrade(Name).has_value();
else if (Name.consume_front("ex2.approx."))
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Expand =
@@ -3084,6 +3115,16 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
: Intrinsic::nvvm_fabs;
Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
+ } else if (Name.consume_front("add.")) {
+ // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
+ auto FAdd = getNVVMFAddUpgrade(Name);
+ assert(FAdd && "unsupported nvvm.add.* intrinsic");
+ auto [IID, RoundingMode] = *FAdd;
+ Value *A = CI->getArgOperand(0);
+ Rep = Builder.CreateIntrinsic(
+ A->getType(), IID,
+ {A, CI->getArgOperand(1),
+ Builder.getInt32(static_cast<int>(RoundingMode))});
} else if (Name.consume_front("ex2.approx.")) {
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
diff --git a/llvm/lib/IR/NVVMIntrinsicUtils.cpp b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
index b67f70d2568dc9..8069cf7eb6f1e5 100644
--- a/llvm/lib/IR/NVVMIntrinsicUtils.cpp
+++ b/llvm/lib/IR/NVVMIntrinsicUtils.cpp
@@ -161,6 +161,11 @@ void nvvm::printTensormapSwizzleAtomicity(raw_ostream &OS,
}
}
+void nvvm::printFPRoundingMode(raw_ostream &OS, const Constant *ImmArgVal) {
+ if (isa<ConstantInt>(ImmArgVal))
+ OS << nvvm::GetRoundingModeName(nvvm::GetRoundingModeFromImmArg(ImmArgVal));
+}
+
void nvvm::printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal) {
if (const auto *CI = dyn_cast<ConstantInt>(ImmArgVal)) {
uint64_t Val = CI->getZExtValue();
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index 0875875611f2c0..19855fd647eca4 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -150,6 +150,16 @@ void NVPTXInstPrinter::printCvtMode(const MCInst *MI, int OpNum,
llvm_unreachable("Invalid conversion modifier");
}
+void NVPTXInstPrinter::printFPRoundingMode(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &,
+ raw_ostream &O) {
+ const auto RM =
+ static_cast<APFloat::roundingMode>(MI->getOperand(OpNum).getImm());
+ const StringRef Name = nvvm::GetRoundingModeName(RM);
+ assert(!Name.empty() && "Invalid FP rounding mode");
+ O << Name;
+}
+
void NVPTXInstPrinter::printFTZFlag(const MCInst *MI, int OpNum,
const MCSubtargetInfo &, raw_ostream &O) {
const MCOperand &MO = MI->getOperand(OpNum);
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index f6a086e6a33790..d98b9998a91798 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -42,6 +42,8 @@ class NVPTXInstPrinter : public MCInstPrinter {
raw_ostream &O, StringRef Modifier = {});
void printCmpMode(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
+ void printFPRoundingMode(const MCInst *MI, int OpNum,
+ const MCSubtargetInfo &STI, raw_ostream &O);
void printAtomicCode(const MCInst *MI, int OpNum, const MCSubtargetInfo &STI,
raw_ostream &O, StringRef Modifier = {});
void printEvictionAndPrefetchHint(const MCInst *MI, int OpNum,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 70fb1aa11b5d6e..a946ceaa35d819 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -53,6 +53,7 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCContext.h"
@@ -7179,22 +7180,45 @@ static SDValue sinkProxyReg(SDValue R, SDValue Chain,
}
}
-static unsigned getF16SubOpc(Intrinsic::ID AddIntrinsicID) {
- switch (AddIntrinsicID) {
+static unsigned getFAddWithNegOpcode(EVT VT, Intrinsic::ID IID,
+ APFloat::roundingMode RoundingMode) {
+ const bool IsFTZ =
+ IID == Intrinsic::nvvm_fadd_ftz || IID == Intrinsic::nvvm_fadd_ftz_sat;
+ const bool IsSat =
+ IID == Intrinsic::nvvm_fadd_sat || IID == Intrinsic::nvvm_fadd_ftz_sat;
+ switch (VT.getScalarType().getSimpleVT().SimpleTy) {
+ case MVT::f16: {
+ static constexpr unsigned SubRNOpcodes[2][2] = {
+ {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_SAT},
+ {NVPTXISD::SUB_RN_FTZ, NVPTXISD::SUB_RN_FTZ_SAT}};
+ return SubRNOpcodes[IsFTZ][IsSat];
+ }
+ case MVT::bf16:
+ return NVPTXISD::SUB_RN;
+ case MVT::f32: {
+ // for f32x2 inputs
+ if (!VT.isVector() || IsSat)
+ return 0;
+ static constexpr unsigned SubF32x2Opcodes[4][2] = {
+ {NVPTXISD::SUB_RZ, NVPTXISD::SUB_RZ_FTZ}, // RZ
+ {NVPTXISD::SUB_RN, NVPTXISD::SUB_RN_FTZ}, // RN
+ {NVPTXISD::SUB_RP, NVPTXISD::SUB_RP_FTZ}, // RP
+ {NVPTXISD::SUB_RM, NVPTXISD::SUB_RM_FTZ}}; // RM
+ return SubF32x2Opcodes[static_cast<unsigned>(RoundingMode)][IsFTZ];
+ }
default:
- break;
- case Intrinsic::nvvm_add_rn_sat_f16:
- case Intrinsic::nvvm_add_rn_sat_v2f16:
- return NVPTXISD::SUB_RN_SAT;
- case Intrinsic::nvvm_add_rn_ftz_sat_f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_v2f16:
- return NVPTXISD::SUB_RN_FTZ_SAT;
+ return 0;
}
- llvm_unreachable("Invalid F16 add intrinsic");
}
-static SDValue combineF16AddWithNeg(SDNode *N, SelectionDAG &DAG,
- Intrinsic::ID AddIntrinsicID) {
+static SDValue combineFAddWithNeg(SDNode *N, SelectionDAG &DAG,
+ Intrinsic::ID AddIntrinsicID,
+ APFloat::roundingMode RoundingMode) {
+ const EVT VT = N->getValueType(0);
+ const unsigned Opc = getFAddWithNegOpcode(VT, AddIntrinsicID, RoundingMode);
+ if (!Opc)
+ return SDValue();
+
SDValue Op1 = N->getOperand(1);
SDValue Op2 = N->getOperand(2);
@@ -7210,24 +7234,69 @@ static SDValue combineF16AddWithNeg(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
- SDLoc DL(N);
- return DAG.getNode(getF16SubOpc(AddIntrinsicID), DL, N->getValueType(0),
- SubOp1, SubOp2);
+ return DAG.getNode(Opc, SDLoc(N), VT, SubOp1, SubOp2);
+}
+
+// TODO: Remove the type-legality checks here once
+// https://github.com/llvm/llvm-project/pull/172442 lands, adding support for
+// explicit type constraints for overloaded intrinsics in tablegen.
+static bool isSupportedFAdd(EVT VT, const NVPTXSubtarget &STI,
+ Intrinsic::ID IID,
+ APFloat::roundingMode RoundingMode) {
+ if (VT.isVector() && VT.getVectorElementCount() != ElementCount::getFixed(2))
+ return false;
+
+ const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
+ const bool IsFTZ =
+ IID == Intrinsic::nvvm_fadd_ftz || IID == Intrinsic::nvvm_fadd_ftz_sat;
+ const bool IsSat =
+ IID == Intrinsic::nvvm_fadd_sat || IID == Intrinsic::nvvm_fadd_ftz_sat;
+ switch (VT.getScalarType().getSimpleVT().SimpleTy) {
+ case MVT::f16:
+ return IsRN;
+ case MVT::bf16:
+ return IsRN && !IsSat && !IsFTZ && STI.hasNativeBF16Support(ISD::FADD);
+ case MVT::f32:
+ return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
+ case MVT::f64:
+ return !VT.isVector() && !IsSat && !IsFTZ;
+ default:
+ return false;
+ }
+}
+
+static SDValue diagnoseUnsupportedFAdd(SDNode *N, SelectionDAG &DAG,
+ Intrinsic::ID IID,
+ APFloat::roundingMode RoundingMode) {
+ const EVT VT = N->getValueType(0);
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ DAG.getMachineFunction().getFunction(),
+ Twine(Intrinsic::getBaseName(IID)) + " with rounding mode " +
+ nvvm::GetRoundingModeName(RoundingMode) + " and operand type " +
+ VT.getEVTString() + " is not supported on this target",
+ SDLoc(N).getDebugLoc()));
+ return DAG.getPOISON(VT);
}
static SDValue combineIntrinsicWOChain(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const NVPTXSubtarget &STI) {
- unsigned IID = N->getConstantOperandVal(0);
+ const Intrinsic::ID IID =
+ static_cast<Intrinsic::ID>(N->getConstantOperandVal(0));
switch (IID) {
default:
break;
- case Intrinsic::nvvm_add_rn_sat_f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_f16:
- case Intrinsic::nvvm_add_rn_sat_v2f16:
- case Intrinsic::nvvm_add_rn_ftz_sat_v2f16:
- return combineF16AddWithNeg(N, DCI.DAG, IID);
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fadd_sat:
+ case Intrinsic::nvvm_fadd_ftz_sat: {
+ const auto RoundingMode = static_cast<APFloat::roundingMode>(
+ N->getConstantOperandAPInt(3).getSExtValue());
+ if (!isSupportedFAdd(N->getValueType(0), STI, IID, RoundingMode))
+ return diagnoseUnsupportedFAdd(N, DCI.DAG, IID, RoundingMode);
+ return combineFAddWithNeg(N, DCI.DAG, IID, RoundingMode);
+ }
}
return SDValue();
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 53d682468f7f53..846bcc05574648 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -114,6 +114,10 @@ def PrmtMode : Operand<i32> {
let PrintMethod = "printPrmtMode";
}
+def FPRoundingMode : Operand<i32> {
+ let PrintMethod = "printFPRoundingMode";
+}
+
class NVPTXAddressSpace<string enum, string name, string suffix = "." # name> {
string Suffix = suffix;
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index b7801b3496bcb4..f41a1fffc187d3 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1687,6 +1687,22 @@ class F_MATH_2<string OpcStr, NVPTXRegClass t_regclass,
[(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1))]>,
Requires<Preds>;
+class F_MATH_2_RND_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP, TImmLeaf rnd,
+ list<Predicate> Preds = []>
+ : BasicNVPTXInst<(outs t.RC:$dst),
+ (ins t.RC:$src0, t.RC:$src1),
+ OpcStr,
+ [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1, rnd))]>,
+ Requires<Preds>;
+
+class F_MATH_2_RNDOP_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP,
+ list<Predicate> Preds = []>
+ : BasicFlagsNVPTXInst<(outs t.RC:$dst),
+ (ins t.RC:$src0, t.RC:$src1), (ins FPRoundingMode:$rnd),
+ OpcStr,
+ [(set t.Ty:$dst, (IntOP t.Ty:$src0, t.Ty:$src1, timm:$rnd))]>,
+ Requires<Preds>;
+
class F_MATH_3<string OpcStr, NVPTXRegClass t_regclass,
NVPTXRegClass s0_regclass, NVPTXRegClass s1_regclass,
NVPTXRegClass s2_regclass, Intrinsic IntOP, list<Predicate> Preds = []>
@@ -2046,6 +2062,8 @@ def : Pat<(int_nvvm_cos_approx_f f32:$a), (COS_APPROX_f32 $a, NoFTZ)>;
// Fma
//
+defvar FPRoundingModes = ["rn", "rz", "rm", "rp"];
+
class FMA_TUPLE<string V, Intrinsic I, NVPTXRegClass RC,
list<Predicate> Preds = []> {
string Variant = V;
@@ -2120,21 +2138,18 @@ multiclass FMA_INST {
defm INT_NVVM_FMA : FMA_INST;
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_FMA # rnd # sat # _f32_ # type :
+foreach rnd = FPRoundingModes in
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_FMA_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B16:$b, B32:$c),
- !subst("_", ".", "fma" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_fma" # rnd # sat # "_f")
+ StrJoin<".", ["fma", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", rnd, sat, "f"]>.ret)
(f32 (fpextend type:$a)),
(f32 (fpextend type:$b)),
f32:$c))]>,
Requires<[SM100]>;
- }
- }
-}
// Pattern for llvm.fma.f32 intrinsic when there is no FTZ flag
let Predicates = [SM100, doNoF32FTZ] in {
@@ -2256,46 +2271,65 @@ let Predicates = [doRsqrtOpt] in {
// Add
//
-def INT_NVVM_ADD_RN_SAT_F16 : F_MATH_2<"add.rn.sat.f16", B16, B16, B16, int_nvvm_add_rn_sat_f16>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16 : F_MATH_2<"add.rn.ftz.sat.f16", B16, B16, B16, int_nvvm_add_rn_ftz_sat_f16>;
-def INT_NVVM_ADD_RN_SAT_F16X2 : F_MATH_2<"add.rn.sat.f16x2", B32, B32, B32, int_nvvm_add_rn_sat_v2f16>;
-def INT_NVVM_ADD_RN_FTZ_SAT_F16X2 : F_MATH_2<"add.rn.ftz.sat.f16x2", B32, B32, B32, int_nvvm_add_rn_ftz_sat_v2f16>;
-
-def INT_NVVM_ADD_RN_FTZ_F : F_MATH_2<"add.rn.ftz.f32", B32, B32, B32, int_nvvm_add_rn_ftz_f>;
-def INT_NVVM_ADD_RN_SAT_FTZ_F : F_MATH_2<"add.rn.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rn_ftz_sat_f>;
-def INT_NVVM_ADD_RN_F : F_MATH_2<"add.rn.f32", B32, B32, B32, int_nvvm_add_rn_f>;
-def INT_NVVM_ADD_RN_SAT_F : F_MATH_2<"add.rn.sat.f32", B32, B32, B32, int_nvvm_add_rn_sat_f>;
-def INT_NVVM_ADD_RZ_FTZ_F : F_MATH_2<"add.rz.ftz.f32", B32, B32, B32, int_nvvm_add_rz_ftz_f>;
-def INT_NVVM_ADD_RZ_SAT_FTZ_F : F_MATH_2<"add.rz.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rz_ftz_sat_f>;
-def INT_NVVM_ADD_RZ_F : F_MATH_2<"add.rz.f32", B32, B32, B32, int_nvvm_add_rz_f>;
-def INT_NVVM_ADD_RZ_SAT_F : F_MATH_2<"add.rz.sat.f32", B32, B32, B32, int_nvvm_add_rz_sat_f>;
-def INT_NVVM_ADD_RM_FTZ_F : F_MATH_2<"add.rm.ftz.f32", B32, B32, B32, int_nvvm_add_rm_ftz_f>;
-def INT_NVVM_ADD_RM_SAT_FTZ_F : F_MATH_2<"add.rm.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rm_ftz_sat_f>;
-def INT_NVVM_ADD_RM_F : F_MATH_2<"add.rm.f32", B32, B32, B32, int_nvvm_add_rm_f>;
-def INT_NVVM_ADD_RM_SAT_F : F_MATH_2<"add.rm.sat.f32", B32, B32, B32, int_nvvm_add_rm_sat_f>;
-def INT_NVVM_ADD_RP_FTZ_F : F_MATH_2<"add.rp.ftz.f32", B32, B32, B32, int_nvvm_add_rp_ftz_f>;
-def INT_NVVM_ADD_RP_SAT_FTZ_F : F_MATH_2<"add.rp.sat.ftz.f32", B32, B32, B32, int_nvvm_add_rp_ftz_sat_f>;
-def INT_NVVM_ADD_RP_F : F_MATH_2<"add.rp.f32", B32, B32, B32, int_nvvm_add_rp_f>;
-def INT_NVVM_ADD_RP_SAT_F : F_MATH_2<"add.rp.sat.f32", B32, B32, B32, int_nvvm_add_rp_sat_f>;
-
-def INT_NVVM_ADD_RN_D : F_MATH_2<"add.rn.f64", B64, B64, B64, int_nvvm_add_rn_d>;
-def INT_NVVM_ADD_RZ_D : F_MATH_2<"add.rz.f64", B64, B64, B64, int_nvvm_add_rz_d>;
-def INT_NVVM_ADD_RM_D : F_MATH_2<"add.rm.f64", B64, B64, B64, int_nvvm_add_rm_d>;
-def INT_NVVM_ADD_RP_D : F_MATH_2<"add.rp.f64", B64, B64, B64, int_nvvm_add_rp_d>;
-
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_ADD # rnd # sat # _f32_ # type :
+defvar BF16ArithPreds = [hasBF16Math, PTX78, SM90];
+
+class FPRndModeImm<string mode> : TImmLeaf<i32,
+ "return Imm == static_cast<int>(RoundingMode::" # mode # ");">;
+
+def fp_rnd_rn_imm : FPRndModeImm<"NearestTiesToEven">;
+def fp_rnd_rz_imm : FPRndModeImm<"TowardZero">;
+def fp_rnd_rm_imm : FPRndModeImm<"TowardNegative">;
+def fp_rnd_rp_imm : FPRndModeImm<"TowardPositive">;
+
+// f16/f16x2
+foreach t = [F16RT, F16X2RT] in
+ foreach ftz = ["", "ftz"] in
+ foreach sat = ["", "sat"] in
+ def INT_NVVM_ADD_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
+ F_MATH_2_RND_TY<StrJoin<".", ["add.rn", ftz, sat, t.PtxType]>.ret, t,
+ !cast<Intrinsic>(
+ StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret),
+ fp_rnd_rn_imm>;
+
+// bf16/bf16x2
+foreach t = [BF16RT, BF16X2RT] in
+ def INT_NVVM_ADD_RN_ # !toupper(t.PtxType) :
+ F_MATH_2_RND_TY<"add.rn." # t.PtxType, t, int_nvvm_fadd, fp_rnd_rn_imm,
+ BF16ArithPreds>;
+
+// f32/f32x2/f64
+foreach ftz = ["", "ftz"] in {
+ foreach sat = ["", "sat"] in
+ def StrJoin<"_", ["INT_NVVM_ADD", !toupper(ftz), !toupper(sat), "F"]>.ret :
+ F_MATH_2_RNDOP_TY<StrJoin<".", ["add.${rnd}", ftz, sat, "f32"]>.ret,
+ F32RT,
+ !cast<Intrinsic>(
+ StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret)>;
+
+ def StrJoin<"_", ["INT_NVVM_ADD", !toupper(ftz), "F32X2"]>.ret :
+ F_MATH_2_RNDOP_TY<StrJoin<".", ["add.${rnd}", ftz, "f32x2"]>.ret, F32X2RT,
+ !cast<Intrinsic>(
+ StrJoin<"_", ["int_nvvm_fadd", ftz]>.ret),
+ [hasF32x2Instructions]>;
+}
+
+def INT_NVVM_ADD_D :
+ F_MATH_2_RNDOP_TY<"add.${rnd}.f64", F64RT, int_nvvm_fadd>;
+
+// mixed precision
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
+
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_ADD_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
- !subst("_", ".", "add" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_add" # rnd # sat # "_f")
+ StrJoin<".", ["add", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", sat]>.ret)
(f32 (fpextend type:$a)),
- f32:$b))]>,
+ f32:$b, rnd_imm))]>,
Requires<[SM100]>;
- }
- }
}
// Pattern for fadd when there is no FTZ flag
@@ -2310,52 +2344,76 @@ let Predicates = [SM100, doNoF32FTZ] in {
// Sub
//
+// These nodes are created by combineFAddWithNeg.
def sub_rn_sat : SDNode<"NVPTXISD::SUB_RN_SAT", SDTFPBinOp>;
def sub_rn_ftz_sat :
SDNode<"NVPTXISD::SUB_RN_FTZ_SAT", SDTFPBinOp>;
-
-class INT_NVVM_SUB_RN<RegTyInfo TyInfo, string variant> :
+
+foreach rnd = FPRoundingModes in
+ foreach ftz = ["", "ftz"] in {
+ defvar variant = StrJoin<"_", [rnd, ftz]>.ret;
+ def sub_ # variant :
+ SDNode<"NVPTXISD::SUB_" # !toupper(variant), SDTFPBinOp>;
+ }
+
+class INT_NVVM_SUB<RegTyInfo TyInfo, list<string> variant> :
BasicNVPTXInst<(outs TyInfo.RC:$dst), (ins TyInfo.RC:$a, TyInfo.RC:$b),
- !subst("_", ".", "sub.rn" # variant # "." # TyInfo.PtxType),
- [(set TyInfo.Ty:$dst,
- (!cast<SDNode>("sub_rn" # variant) TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
-
-def INT_NVVM_SUB_RN_SAT_F16 : INT_NVVM_SUB_RN<F16RT, "_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16 : INT_NVVM_SUB_RN<F16RT, "_ftz_sat">;
-def INT_NVVM_SUB_RN_SAT_F16X2 : INT_NVVM_SUB_RN<F16X2RT, "_sat">;
-def INT_NVVM_SUB_RN_FTZ_SAT_F16X2 : INT_NVVM_SUB_RN<F16X2RT, "_ftz_sat">;
-
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in {
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_add" # rnd # ftz # sat # "_f");
- def INT_NVVM_SUB # rnd # ftz # sat # _F :
- BasicNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
- !subst("_", ".", "sub" # rnd # sat # ftz # "_f32"),
- [(set f32:$dst, (add_intrin f32:$a, (f32 (fneg f32:$b))))]>;
- }
+ StrJoin<".", !listconcat(["sub"], variant, [TyInfo.PtxType])>.ret,
+ [(set TyInfo.Ty:$dst,
+ (!cast<SDNode>(StrJoin<"_", !listconcat(["sub"], variant)>.ret)
+ TyInfo.Ty:$a, TyInfo.Ty:$b))]>;
+
+// f16/f16x2
+foreach t = [F16RT, F16X2RT] in
+ foreach ftz = ["", "ftz"] in
+ foreach sat = ["", "sat"] in
+ def INT_NVVM_SUB_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
+ INT_NVVM_SUB<t, ["rn", ftz, sat]>;
+
+// bf16/bf16x2
+let Predicates = BF16ArithPreds in
+ foreach t = [BF16RT, BF16X2RT] in
+ def INT_NVVM_SUB_RN_ # !toupper(t.PtxType) : INT_NVVM_SUB<t, ["rn"]>;
+
+// f32x2
+let Predicates = [hasF32x2Instructions] in
+ foreach rnd = FPRoundingModes in
+ foreach ftz = ["", "ftz"] in
+ def INT_NVVM_SUB_ # StrJoin<"_", [rnd, ftz]>.ret # _F32X2 :
+ INT_NVVM_SUB<F32X2RT, [rnd, ftz]>;
+
+// f32/f64
+foreach ftz = ["", "ftz"] in
+ foreach sat = ["", "sat"] in {
+ defvar add_intrin =
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz, sat]>.ret);
+ def StrJoin<"_", ["INT_NVVM_SUB", !toupper(ftz), !toupper(sat), "F"]>.ret :
+ BasicFlagsNVPTXInst<(outs B32:$dst), (ins B32:$a, B32:$b),
+ (ins FPRoundingMode:$rnd),
+ StrJoin<".", ["sub.${rnd}", ftz, sat, "f32"]>.ret,
+ [(set f32:$dst,
+ (add_intrin f32:$a, (f32 (fneg f32:$b)), timm:$rnd))]>;
}
-
- defvar add_intrin = !cast<Intrinsic>("int_nvvm_add" # rnd # "_d");
- def INT_NVVM_SUB # rnd # _D :
- BasicNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
- !subst("_", ".", "sub" # rnd # "_f64"),
- [(set f64:$dst, (add_intrin f64:$a, (f64 (fneg f64:$b))))]>;
-}
-foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- foreach sat = ["", "_sat"] in {
- foreach type = [f16, bf16] in {
- def INT_NVVM_MIXED_SUB # rnd # sat # _f32_ # type :
+def INT_NVVM_SUB_D :
+ BasicFlagsNVPTXInst<(outs B64:$dst), (ins B64:$a, B64:$b),
+ (ins FPRoundingMode:$rnd), "sub.${rnd}.f64",
+ [(set f64:$dst, (int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), timm:$rnd))]>;
+
+// mixed precision
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
+
+ foreach sat = ["", "sat"] in
+ foreach type = [f16, bf16] in
+ def INT_NVVM_MIXED_SUB_ # StrJoin<"_", [rnd, sat]>.ret # _f32_ # type :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B32:$b),
- !subst("_", ".", "sub" # rnd # sat # "_f32_" # type),
- [(set f32:$dst,
- (!cast<Intrinsic>("int_nvvm_add" # rnd # sat # "_f")
+ StrJoin<".", ["sub", rnd, sat, "f32", type.LLVMName]>.ret,
+ [(set f32:$dst,
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", sat]>.ret)
(f32 (fpextend type:$a)),
- (f32 (fneg f32:$b))))]>,
+ (f32 (fneg f32:$b)), rnd_imm))]>,
Requires<[SM100]>;
- }
- }
}
// Pattern for fsub when there is no FTZ flag
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index bcbeef260f2bb7..2871ddab068bca 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -711,3 +711,27 @@ define void @nvvm_ex2_approx(float %a, double %b, half %c, <2 x half> %d) {
%r4 = call float @llvm.nvvm.ex2.approx.ftz.f(float %a)
ret void
}
+
+define void @nvvm_add(float %a, double %b, half %c, <2 x half> %d) {
+; CHECK: call float @llvm.nvvm.fadd.f32(float %a, float %a, /* rnd=rn */ i32 1)
+; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %a, /* rnd=rz */ i32 0)
+; CHECK: call float @llvm.nvvm.fadd.sat.f32(float %a, float %a, /* rnd=rm */ i32 3)
+; CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %a, /* rnd=rp */ i32 2)
+; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rn */ i32 1)
+; CHECK: call double @llvm.nvvm.fadd.f64(double %b, double %b, /* rnd=rz */ i32 0)
+; CHECK: call half @llvm.nvvm.fadd.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fadd.ftz.sat.f16(half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+ %r1 = call float @llvm.nvvm.add.rn.f(float %a, float %a)
+ %r2 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %a)
+ %r3 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %a)
+ %r4 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %a)
+ %r5 = call double @llvm.nvvm.add.rn.d(double %b, double %b)
+ %r6 = call double @llvm.nvvm.add.rz.d(double %b, double %b)
+ %r7 = call half @llvm.nvvm.add.rn.sat.f16(half %c, half %c)
+ %r8 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %c, half %c)
+ %r9 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %d, <2 x half> %d)
+ %r10 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d)
+ ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/bf16-add.ll b/llvm/test/CodeGen/NVPTX/bf16-add.ll
new file mode 100644
index 00000000000000..374c37b564cf17
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/bf16-add.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-7.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
+
+define bfloat @add_rn_bf16(bfloat %a, bfloat %b) {
+; CHECK-LABEL: add_rn_bf16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_bf16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_bf16_param_1];
+; CHECK-NEXT: add.rn.bf16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 1)
+ ret bfloat %1
+}
+
+define <2 x bfloat> @add_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; CHECK-LABEL: add_rn_bf16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_bf16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_bf16x2_param_1];
+; CHECK-NEXT: add.rn.bf16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, i32 1)
+ ret <2 x bfloat> %1
+}
diff --git a/llvm/test/CodeGen/NVPTX/bf16-sub.ll b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
new file mode 100644
index 00000000000000..b095a534d3c355
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/bf16-sub.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck %s
+; RUN: %if ptxas-sm_90 && ptxas-isa-7.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
+
+define bfloat @sub_rn_bf16(bfloat %a, bfloat %b) {
+; CHECK-LABEL: sub_rn_bf16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_bf16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_bf16_param_1];
+; CHECK-NEXT: sub.rn.bf16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg bfloat %b
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %1, i32 1)
+ ret bfloat %res
+}
+
+define <2 x bfloat> @sub_rn_bf16x2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; CHECK-LABEL: sub_rn_bf16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_bf16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_bf16x2_param_1];
+; CHECK-NEXT: sub.rn.bf16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x bfloat> %b
+ %res = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %1, i32 1)
+ ret <2 x bfloat> %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/f16-add-sat.ll b/llvm/test/CodeGen/NVPTX/f16-add-sat.ll
deleted file mode 100644
index c2ffc126694c4a..00000000000000
--- a/llvm/test/CodeGen/NVPTX/f16-add-sat.ll
+++ /dev/null
@@ -1,63 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
-; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
-
-define half @add_rn_sat_f16(half %a, half %b) {
-; CHECK-LABEL: add_rn_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_sat_f16_param_1];
-; CHECK-NEXT: add.rn.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.add.rn.sat.f16(half %a, half %b)
- ret half %1
-}
-
-define <2 x half> @add_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: add_rn_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [add_rn_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [add_rn_sat_f16x2_param_1];
-; CHECK-NEXT: add.rn.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %b)
- ret <2 x half> %1
-}
-
-define half @add_rn_ftz_sat_f16(half %a, half %b) {
-; CHECK-LABEL: add_rn_ftz_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_sat_f16_param_1];
-; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %b)
- ret half %1
-}
-
-define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: add_rn_ftz_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_sat_f16x2_param_1];
-; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b)
- ret <2 x half> %1
-}
diff --git a/llvm/test/CodeGen/NVPTX/f16-add.ll b/llvm/test/CodeGen/NVPTX/f16-add.ll
new file mode 100644
index 00000000000000..0b470daf7a5359
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/f16-add.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
+; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
+
+define half @add_rn_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_f16_param_1];
+; CHECK-NEXT: add.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.f16(half %a, half %b, i32 1)
+ ret half %1
+}
+
+define <2 x half> @add_rn_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_f16x2_param_1];
+; CHECK-NEXT: add.rn.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
+ ret <2 x half> %1
+}
+
+define half @add_rn_ftz_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_ftz_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_f16_param_1];
+; CHECK-NEXT: add.rn.ftz.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.ftz.f16(half %a, half %b, i32 1)
+ ret half %1
+}
+
+define <2 x half> @add_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_ftz_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_f16x2_param_1];
+; CHECK-NEXT: add.rn.ftz.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
+ ret <2 x half> %1
+}
+
+define half @add_rn_sat_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_sat_f16_param_1];
+; CHECK-NEXT: add.rn.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.sat.f16(half %a, half %b, i32 1)
+ ret half %1
+}
+
+define <2 x half> @add_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_sat_f16x2_param_1];
+; CHECK-NEXT: add.rn.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
+ ret <2 x half> %1
+}
+
+define half @add_rn_ftz_sat_f16(half %a, half %b) {
+; CHECK-LABEL: add_rn_ftz_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [add_rn_ftz_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [add_rn_ftz_sat_f16_param_1];
+; CHECK-NEXT: add.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %b, i32 1)
+ ret half %1
+}
+
+define <2 x half> @add_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: add_rn_ftz_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [add_rn_ftz_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [add_rn_ftz_sat_f16x2_param_1];
+; CHECK-NEXT: add.rn.ftz.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %b, i32 1)
+ ret <2 x half> %1
+}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll b/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
deleted file mode 100644
index 774ce7ccb2f958..00000000000000
--- a/llvm/test/CodeGen/NVPTX/f16-sub-sat.ll
+++ /dev/null
@@ -1,69 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | FileCheck %s
-; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
-; RUN: %if ptxas-isa-6.0 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | %ptxas-verify%}
-
-define half @sub_rn_sat_f16(half %a, half %b) {
-; CHECK-LABEL: sub_rn_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_sat_f16_param_1];
-; CHECK-NEXT: sub.rn.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = fneg half %b
- %res = call half @llvm.nvvm.add.rn.sat.f16(half %a, half %1)
- ret half %res
-}
-
-define <2 x half> @sub_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: sub_rn_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_sat_f16x2_param_1];
-; CHECK-NEXT: sub.rn.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %a, <2 x half> %1)
- ret <2 x half> %res
-}
-
-define half @sub_rn_ftz_sat_f16(half %a, half %b) {
-; CHECK-LABEL: sub_rn_ftz_sat_f16(
-; CHECK: {
-; CHECK-NEXT: .reg .b16 %rs<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_sat_f16_param_0];
-; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_sat_f16_param_1];
-; CHECK-NEXT: sub.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT: ret;
- %1 = fneg half %b
- %res = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %a, half %1)
- ret half %res
-}
-
-define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: sub_rn_ftz_sat_f16x2(
-; CHECK: {
-; CHECK-NEXT: .reg .b32 %r<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT: // %bb.0:
-; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_sat_f16x2_param_0];
-; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_sat_f16x2_param_1];
-; CHECK-NEXT: sub.rn.ftz.sat.f16x2 %r3, %r1, %r2;
-; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
-; CHECK-NEXT: ret;
- %1 = fneg <2 x half> %b
- %res = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1)
- ret <2 x half> %res
-}
diff --git a/llvm/test/CodeGen/NVPTX/f16-sub.ll b/llvm/test/CodeGen/NVPTX/f16-sub.ll
new file mode 100644
index 00000000000000..c59875af1d4f94
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/f16-sub.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | FileCheck %s
+; RUN: %if ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify%}
+; RUN: %if ptxas-isa-6.0 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx60 | %ptxas-verify%}
+
+define half @sub_rn_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_f16_param_1];
+; CHECK-NEXT: sub.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.f16(half %a, half %1, i32 1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_f16x2_param_1];
+; CHECK-NEXT: sub.rn.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_ftz_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_ftz_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_f16_param_1];
+; CHECK-NEXT: sub.rn.ftz.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half %a, half %1, i32 1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_ftz_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_ftz_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_f16x2_param_1];
+; CHECK-NEXT: sub.rn.ftz.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_sat_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_sat_f16_param_1];
+; CHECK-NEXT: sub.rn.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.sat.f16(half %a, half %1, i32 1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_sat_f16x2_param_1];
+; CHECK-NEXT: sub.rn.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
+ ret <2 x half> %res
+}
+
+define half @sub_rn_ftz_sat_f16(half %a, half %b) {
+; CHECK-LABEL: sub_rn_ftz_sat_f16(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [sub_rn_ftz_sat_f16_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [sub_rn_ftz_sat_f16_param_1];
+; CHECK-NEXT: sub.rn.ftz.sat.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT: ret;
+ %1 = fneg half %b
+ %res = call half @llvm.nvvm.fadd.ftz.sat.f16(half %a, half %1, i32 1)
+ ret half %res
+}
+
+define <2 x half> @sub_rn_ftz_sat_f16x2(<2 x half> %a, <2 x half> %b) {
+; CHECK-LABEL: sub_rn_ftz_sat_f16x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [sub_rn_ftz_sat_f16x2_param_0];
+; CHECK-NEXT: ld.param.b32 %r2, [sub_rn_ftz_sat_f16x2_param_1];
+; CHECK-NEXT: sub.rn.ftz.sat.f16x2 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %1 = fneg <2 x half> %b
+ %res = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %a, <2 x half> %1, i32 1)
+ ret <2 x half> %res
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
new file mode 100644
index 00000000000000..01f0c6cac37b6a
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-add-f32x2.ll
@@ -0,0 +1,60 @@
+; RUN: llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 | FileCheck %s
+; RUN: %if ptxas-sm_100 && ptxas-isa-8.8 %{ llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 | %ptxas-verify -arch=sm_100 %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define <2 x float> @add_rn(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn(
+; CHECK: add.rn.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz(
+; CHECK: add.rz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rm(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rm(
+; CHECK: add.rm.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 3)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rp(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rp(
+; CHECK: add.rp.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 2)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rn_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_ftz(
+; CHECK: add.rn.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rz_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_ftz(
+; CHECK: add.rz.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rm_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rm_ftz(
+; CHECK: add.rm.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 3)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_rp_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rp_ftz(
+; CHECK: add.rp.ftz.f32x2
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 2)
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
new file mode 100644
index 00000000000000..776935ddd0f910
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-add-invalid.ll
@@ -0,0 +1,47 @@
+; RUN: not llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 2>&1 | FileCheck %s
+; RUN: not llc < %s -mcpu=sm_90 -mattr=+ptx78 -march=nvptx64 2>&1 | FileCheck %s --check-prefix=NOF32X2
+; RUN: not llc < %s -mcpu=sm_80 -mattr=+ptx78 -march=nvptx64 2>&1 | FileCheck %s --check-prefix=NOBF16
+
+target triple = "nvptx64-nvidia-cuda"
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.sat with rounding mode rn and operand type v2f32 is not supported
+define <2 x float> @sat_f32x2(<2 x float> %a, <2 x float> %b) {
+ %r = call <2 x float> @llvm.nvvm.fadd.sat.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+; NOF32X2: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type v2f32 is not supported
+define <2 x float> @unsupported_f32x2(<2 x float> %a, <2 x float> %b) {
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.ftz with rounding mode rn and operand type f64 is not supported
+define double @ftz_f64(double %a, double %b) {
+ %r = call double @llvm.nvvm.fadd.ftz.f64(double %a, double %b, i32 1)
+ ret double %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.sat with rounding mode rz and operand type f16 is not supported
+define half @rz_f16(half %a, half %b) {
+ %r = call half @llvm.nvvm.fadd.sat.f16(half %a, half %b, i32 0)
+ ret half %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd.ftz with rounding mode rn and operand type bf16 is not supported
+define bfloat @ftz_bf16(bfloat %a, bfloat %b) {
+ %r = call bfloat @llvm.nvvm.fadd.ftz.bf16(bfloat %a, bfloat %b, i32 1)
+ ret bfloat %r
+}
+
+; NOBF16: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type bf16 is not supported
+define bfloat @unsupported_bf16(bfloat %a, bfloat %b) {
+ %r = call bfloat @llvm.nvvm.fadd.bf16(bfloat %a, bfloat %b, i32 1)
+ ret bfloat %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fadd with rounding mode rn and operand type v4f32 is not supported
+define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) {
+ %r = call <4 x float> @llvm.nvvm.fadd.v4f32(<4 x float> %a, <4 x float> %b, i32 1)
+ ret <4 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index c6b3b649aae066..0b93da7af17d02 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -11,26 +11,26 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: ld.param.b32 %r1, [add_sat_f32_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [add_sat_f32_param_1];
; CHECK-NEXT: add.rn.sat.f32 %r3, %r1, %r2;
-; CHECK-NEXT: add.rn.sat.ftz.f32 %r4, %r1, %r3;
+; CHECK-NEXT: add.rn.ftz.sat.f32 %r4, %r1, %r3;
; CHECK-NEXT: add.rz.sat.f32 %r5, %r1, %r4;
-; CHECK-NEXT: add.rz.sat.ftz.f32 %r6, %r1, %r5;
+; CHECK-NEXT: add.rz.ftz.sat.f32 %r6, %r1, %r5;
; CHECK-NEXT: add.rm.sat.f32 %r7, %r1, %r6;
-; CHECK-NEXT: add.rm.sat.ftz.f32 %r8, %r1, %r7;
+; CHECK-NEXT: add.rm.ftz.sat.f32 %r8, %r1, %r7;
; CHECK-NEXT: add.rp.sat.f32 %r9, %r1, %r8;
-; CHECK-NEXT: add.rp.sat.ftz.f32 %r10, %r1, %r9;
+; CHECK-NEXT: add.rp.ftz.sat.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.add.rn.sat.f(float %a, float %b)
- %r2 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %a, float %r1)
+ %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, i32 1)
- %r3 = call float @llvm.nvvm.add.rz.sat.f(float %a, float %r2)
- %r4 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %a, float %r3)
+ %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, i32 0)
- %r5 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %r4)
- %r6 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %a, float %r5)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, i32 3)
- %r7 = call float @llvm.nvvm.add.rp.sat.f(float %a, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %r7)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, i32 2)
ret float %r8
}
@@ -44,38 +44,38 @@ define float @sub_sat_f32(float %a, float %b) {
; CHECK-NEXT: ld.param.b32 %r1, [sub_sat_f32_param_0];
; CHECK-NEXT: ld.param.b32 %r2, [sub_sat_f32_param_1];
; CHECK-NEXT: sub.rn.sat.f32 %r3, %r1, %r2;
-; CHECK-NEXT: sub.rn.sat.ftz.f32 %r4, %r1, %r3;
+; CHECK-NEXT: sub.rn.ftz.sat.f32 %r4, %r1, %r3;
; CHECK-NEXT: sub.rz.sat.f32 %r5, %r1, %r4;
-; CHECK-NEXT: sub.rz.sat.ftz.f32 %r6, %r1, %r5;
+; CHECK-NEXT: sub.rz.ftz.sat.f32 %r6, %r1, %r5;
; CHECK-NEXT: sub.rm.sat.f32 %r7, %r1, %r6;
-; CHECK-NEXT: sub.rm.sat.ftz.f32 %r8, %r1, %r7;
+; CHECK-NEXT: sub.rm.ftz.sat.f32 %r8, %r1, %r7;
; CHECK-NEXT: sub.rp.sat.f32 %r9, %r1, %r8;
-; CHECK-NEXT: sub.rp.sat.ftz.f32 %r10, %r1, %r9;
+; CHECK-NEXT: sub.rp.ftz.sat.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.sat.f(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rz.sat.f(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rm.sat.f(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, i32 3)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rp.sat.f(float %a, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, i32 2)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %a, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, i32 2)
ret float %r8
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
new file mode 100644
index 00000000000000..7aebe89a1c1fdc
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub-f32x2.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | FileCheck %s
+; RUN: %if ptxas-sm_100 && ptxas-isa-8.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100 -mattr=+ptx88 | %ptxas-verify -arch=sm_100 %}
+
+define <2 x float> @sub_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<11>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_param_1];
+; CHECK-NEXT: sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NEXT: sub.rz.f32x2 %rd5, %rd1, %rd4;
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd6, %rd1, %rd5;
+; CHECK-NEXT: sub.rm.f32x2 %rd7, %rd1, %rd6;
+; CHECK-NEXT: sub.rm.ftz.f32x2 %rd8, %rd1, %rd7;
+; CHECK-NEXT: sub.rp.f32x2 %rd9, %rd1, %rd8;
+; CHECK-NEXT: sub.rp.ftz.f32x2 %rd10, %rd1, %rd9;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd10;
+; CHECK-NEXT: ret;
+ %f0 = fneg <2 x float> %b
+ %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f0, i32 1)
+
+ %f1 = fneg <2 x float> %r1
+ %r2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f1, i32 1)
+
+ %f2 = fneg <2 x float> %r2
+ %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f2, i32 0)
+
+ %f3 = fneg <2 x float> %r3
+ %r4 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f3, i32 0)
+
+ %f4 = fneg <2 x float> %r4
+ %r5 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f4, i32 3)
+
+ %f5 = fneg <2 x float> %r5
+ %r6 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f5, i32 3)
+
+ %f6 = fneg <2 x float> %r6
+ %r7 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %f6, i32 2)
+
+ %f7 = fneg <2 x float> %r7
+ %r8 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %f7, i32 2)
+
+ ret <2 x float> %r8
+}
+
+define <2 x float> @sub_f32x2_negated_lhs(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_negated_lhs(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_negated_lhs_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_negated_lhs_param_1];
+; CHECK-NEXT: sub.rz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NEXT: ret;
+ %f = fneg <2 x float> %a
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %f, <2 x float> %b, i32 0)
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
index 351f45ccbcc6ba..31edae75a48e77 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fold-sub.ll
@@ -20,22 +20,22 @@ define float @sub_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r8;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %a, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %a, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rn.ftz.f(float %a, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f1, i32 1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rz.f(float %a, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %a, float %f2, i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rz.ftz.f(float %a, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f3, i32 0)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rm.f(float %a, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.f32(float %a, float %f4, i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rm.ftz.f(float %a, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %f5, i32 3)
ret float %r6
}
@@ -55,16 +55,16 @@ define double @sub_f64(double %a, double %b) {
; CHECK-NEXT: st.param.b64 [func_retval0], %rd6;
; CHECK-NEXT: ret;
%f0 = fneg double %b
- %r1 = call double @llvm.nvvm.add.rn.d(double %a, double %f0)
+ %r1 = call double @llvm.nvvm.fadd.f64(double %a, double %f0, i32 1)
%f1 = fneg double %r1
- %r2 = call double @llvm.nvvm.add.rz.d(double %a, double %f1)
+ %r2 = call double @llvm.nvvm.fadd.f64(double %a, double %f1, i32 0)
%f2 = fneg double %r2
- %r3 = call double @llvm.nvvm.add.rm.d(double %a, double %f2)
+ %r3 = call double @llvm.nvvm.fadd.f64(double %a, double %f2, i32 3)
%f3 = fneg double %r3
- %r4 = call double @llvm.nvvm.add.rp.d(double %a, double %f3)
+ %r4 = call double @llvm.nvvm.fadd.f64(double %a, double %f3, i32 2)
ret double %r4
}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
index 5f82d2e77a4683..9da6c9a39a34e7 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
@@ -27,16 +27,16 @@ define float @test_add_f32_f16_1(half %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext half %a to float
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %b)
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, i32 0)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, i32 2)
; SAT
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, i32 1)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, i32 0)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, i32 3)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, i32 2)
ret float %r8
}
@@ -93,16 +93,16 @@ define float @test_add_f32_bf16_1(bfloat %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext bfloat %a to float
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %b)
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %r1)
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %r2)
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %r3)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, i32 0)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, i32 2)
; SAT
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %r4)
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %r5)
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %r6)
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %r7)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, i32 1)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, i32 0)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, i32 3)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, i32 2)
ret float %r8
}
@@ -160,29 +160,29 @@ define float @test_sub_f32_f16_1(half %a, float %b) {
%r0 = fpext half %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, i32 0)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, i32 3)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rm.f(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, i32 3)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, i32 1)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, i32 0)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, i32 3)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, i32 2)
ret float %r7
}
@@ -240,29 +240,29 @@ define float @test_sub_f32_bf16_1(bfloat %a, float %b) {
%r0 = fpext bfloat %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.add.rn.f(float %r0, float %f0)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.add.rz.f(float %r0, float %f1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, i32 0)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.add.rm.f(float %r0, float %f2)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, i32 3)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.add.rp.f(float %r0, float %f3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, i32 2)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.add.rn.sat.f(float %r0, float %f4)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, i32 1)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.add.rz.sat.f(float %r0, float %f5)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, i32 0)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.add.rm.sat.f(float %r0, float %f6)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, i32 3)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.add.rp.sat.f(float %r0, float %f7)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, i32 2)
ret float %r8
}
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index db172c5a34cdca..b4b04ef371e94f 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -299,20 +299,20 @@ define float @test_ull2f(i64 %a) #0 {
; CHECK-LABEL: @test_add_rn_d
define double @test_add_rn_d(double %a, double %b) #0 {
-; CHECK: call double @llvm.nvvm.add.rn.d
- %ret = call double @llvm.nvvm.add.rn.d(double %a, double %b)
+; CHECK: call double @llvm.nvvm.fadd.f64
+ %ret = call double @llvm.nvvm.fadd.f64(double %a, double %b, /* rnd=rn */ i32 1)
ret double %ret
}
; CHECK-LABEL: @test_add_rn_f
define float @test_add_rn_f(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.add.rn.f
- %ret = call float @llvm.nvvm.add.rn.f(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.f32
+ %ret = call float @llvm.nvvm.fadd.f32(float %a, float %b, /* rnd=rn */ i32 1)
ret float %ret
}
; CHECK-LABEL: @test_add_rn_f_ftz
define float @test_add_rn_f_ftz(float %a, float %b) #0 {
-; CHECK: call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)
- %ret = call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)
+; CHECK: call float @llvm.nvvm.fadd.ftz.f32(float %a, float %b, /* rnd=rn */ i32 1)
+ %ret = call float @llvm.nvvm.fadd.ftz.f32(float %a, float %b, /* rnd=rn */ i32 1)
ret float %ret
}
@@ -437,9 +437,9 @@ define i32 @test_fshr_clamp_3(i32 %a, i32 %b, i32 %c) {
ret i32 %call
}
-declare double @llvm.nvvm.add.rn.d(double, double)
-declare float @llvm.nvvm.add.rn.f(float, float)
-declare float @llvm.nvvm.add.rn.ftz.f(float, float)
+declare double @llvm.nvvm.fadd.f64(double, double, i32 immarg)
+declare float @llvm.nvvm.fadd.f32(float, float, i32 immarg)
+declare float @llvm.nvvm.fadd.ftz.f32(float, float, i32 immarg)
declare double @llvm.nvvm.ceil.d(double)
declare float @llvm.nvvm.ceil.f(float)
declare float @llvm.nvvm.ceil.ftz.f(float)
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
index b23ae275b71b78..edc4b2233f2352 100644
--- a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
@@ -13,7 +13,7 @@ define double @test_1_25_minus_2_rm_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rm_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rm */ i32 3)
ret double %res
}
@@ -21,7 +21,7 @@ define double @test_1_25_minus_2_rn_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rn_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rn */ i32 1)
ret double %res
}
@@ -29,7 +29,7 @@ define double @test_1_25_minus_2_rp_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rp_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rp */ i32 2)
ret double %res
}
@@ -37,7 +37,7 @@ define double @test_1_25_minus_2_rz_d() {
; CHECK-LABEL: define double @test_1_25_minus_2_rz_d() {
; CHECK-NEXT: ret double -7.500000e-01
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.25, double -2.0)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.25, double -2.0, /* rnd=rz */ i32 0)
ret double %res
}
@@ -45,7 +45,7 @@ define float @test_1_25_minus_2_rm_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -53,7 +53,7 @@ define float @test_1_25_minus_2_rn_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -61,7 +61,7 @@ define float @test_1_25_minus_2_rp_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -69,7 +69,7 @@ define float @test_1_25_minus_2_rz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.25, float -2.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -77,7 +77,7 @@ define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rm_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -85,7 +85,7 @@ define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rn_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -93,7 +93,7 @@ define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rp_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -101,10 +101,74 @@ define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_25_minus_2_rz_ftz_f() {
; CHECK-NEXT: ret float -7.500000e-01
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.25, float -2.0)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.25, float -2.0, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_1_25_minus_2_rm_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rm_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rn_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rn_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rp_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rp_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_1_25_minus_2_rz_f16() {
+; CHECK-LABEL: define half @test_1_25_minus_2_rz_f16() {
+; CHECK-NEXT: ret half -7.500000e-01
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.25, half -2.0, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_1_25_minus_2_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rm_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rn_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rp_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_1_25_minus_2_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_1_25_minus_2_rz_bf16() {
+; CHECK-NEXT: ret bfloat -7.500000e-01
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.25, bfloat -2.0, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(0.0, NaN) #
;###############################################################
@@ -113,112 +177,184 @@ define float @test_1_25_minus_2_rz_ftz_f() {
define double @test_zero_plus_nan_rm_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rm_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rm.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rm.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rm */ i32 3)
ret double %res
}
define double @test_zero_plus_nan_rn_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rn_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rn.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rn.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rn */ i32 1)
ret double %res
}
define double @test_zero_plus_nan_rp_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rp_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rp.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rp.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rp */ i32 2)
ret double %res
}
define double @test_zero_plus_nan_rz_d() {
; CHECK-LABEL: define double @test_zero_plus_nan_rz_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.add.rz.d(double 0.000000e+00, double +snan(0x4444400000000))
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fadd.f64(double 0.000000e+00, double +snan(0x4444400000000), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.add.rz.d(double 0.0, double 0x7ff4444400000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0.0, double 0x7ff4444400000000, /* rnd=rz */ i32 0)
ret double %res
}
define float @test_zero_plus_nan_rm_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rm.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rm.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rm */ i32 3)
ret float %res
}
define float @test_zero_plus_nan_rn_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rn.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rn.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_zero_plus_nan_rp_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rp.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rp.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_zero_plus_nan_rz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rz */ i32 0)
ret float %res
}
define float @test_zero_plus_nan_rm_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rm_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rm.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rm */ i32 3)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rm */ i32 3)
ret float %res
}
define float @test_zero_plus_nan_rn_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rn_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rn.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rn */ i32 1)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_zero_plus_nan_rp_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rp_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rp.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rp */ i32 2)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_zero_plus_nan_rz_ftz_f() {
; CHECK-LABEL: define float @test_zero_plus_nan_rz_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.add.rz.ftz.f(float 0.000000e+00, float +nan(0x3A2220))
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fadd.ftz.f32(float 0.000000e+00, float +nan(0x3A2220), /* rnd=rz */ i32 0)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0.0, float 0x7FFF444400000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0.0, float 0x7FFF444400000000, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_zero_plus_nan_rm_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rm_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rm */ i32 3)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rn_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rn_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rn */ i32 1)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rp_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rp_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rp */ i32 2)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_zero_plus_nan_rz_f16() {
+; CHECK-LABEL: define half @test_zero_plus_nan_rz_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fadd.f16(half 0.000000e+00, half +qnan, /* rnd=rz */ i32 0)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0.0, half 0xH7E00, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_zero_plus_nan_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rm_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rm */ i32 3)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rn_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rn */ i32 1)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rp_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rp */ i32 2)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_zero_plus_nan_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_zero_plus_nan_rz_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.000000e+00, bfloat +qnan, /* rnd=rz */ i32 0)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0.0, bfloat 0xR7FC0, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(Subnormal, Subnormal) -> Normal #
;###############################################################
@@ -230,7 +366,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rm.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -238,7 +374,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rn.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -246,7 +382,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rp.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -254,7 +390,7 @@ define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
; CHECK-NEXT: ret double f0x3810000000000000
;
- %res = call double @llvm.nvvm.add.rz.d(double 0x3800000000000000, double 0x3800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3800000000000000, double 0x3800000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -262,7 +398,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rm.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -270,7 +406,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rn.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -278,7 +414,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rp.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -286,7 +422,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -294,7 +430,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -302,7 +438,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -310,7 +446,7 @@ define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -318,24 +454,119 @@ define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3800000000000000, float 0x3800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
+define half @test_subnorm_plus_subnorm_to_normal_rm_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rm_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rn_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rn_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rp_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rp_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0200, half 0xH0200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rm_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rn_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rp_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_subnorm_plus_subnorm_to_normal_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_subnorm_plus_subnorm_to_normal_rz_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0200, half 0xH0200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rm_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rn_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rp_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_subnorm_plus_subnorm_to_normal_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_subnorm_plus_subnorm_to_normal_rz_bf16() {
+; CHECK-NEXT: ret bfloat 1.175490e-38
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0040, bfloat 0xR0040, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(Normal, -Subnormal) -> Subnormal #
;###############################################################
-; Tests addition of 2^-126 (the smallest normal number) and -(2^127).
-; - Without FTZ: The result is correctly computed as a subnormal (2^127)
-; - With FTZ: The result is flushed to zero.
-; This verifies that the output is also flushed to zero, as we'd end up
-; with 2^-126 if we only flushed the inputs.
+; Tests addition of 2^-126 (the smallest normal number) and -(2^-127).
+; - Without FTZ: The result is correctly computed as a subnormal (2^-127)
+; - With FTZ: The subnormal input is flushed to zero, so the result is the
+; normal input (2^-126)
define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rm.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -343,7 +574,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rn.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -351,7 +582,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rp.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -359,7 +590,7 @@ define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
; CHECK-NEXT: ret double f0x3800000000000000
;
- %res = call double @llvm.nvvm.add.rz.d(double 0x3810000000000000, double 0xB800000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 0x3810000000000000, double 0xB800000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -367,7 +598,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rm.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -375,7 +606,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rn.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -383,7 +614,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rp.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -391,7 +622,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.add.rz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -399,7 +630,7 @@ define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -407,26 +638,259 @@ define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 0x3810000000000000, float 0xB800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
-; CHECK-NEXT: ret float 0.000000e+00
+; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
+; CHECK-NEXT: ret float f0x00800000
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3810000000000000, float 0xB800000000000000, /* rnd=rz */ i32 0)
+ ret float %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rm_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rm_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rn_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rn_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rp_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rp_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rz_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0400, half 0xH8200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rm_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rn_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rp_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_subnorm_to_subnorm_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_subnorm_to_subnorm_rz_ftz_f16() {
+; CHECK-NEXT: ret half 6.103520e-05
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0400, half 0xH8200, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rm_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rn_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rp_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_normal_minus_subnorm_to_subnorm_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_normal_minus_subnorm_to_subnorm_rz_bf16() {
+; CHECK-NEXT: ret bfloat 5.877470e-39
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 0xR0080, bfloat 0xR8040, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
+;###############################################################
+;# Add(Normal, -Normal) -> Subnormal #
+;###############################################################
+; Tests addition of 1.5*(2^-126) and -(2^-126), where both inputs are normal
+; but the exact result is subnormal.
+; - Without FTZ: The result is the exact
diff erence (2^-127)
+; - With FTZ: The result is flushed to zero. Flushing the inputs alone would
+; leave it untouched, as neither input is subnormal.
+
+define float @test_normal_minus_normal_to_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rm_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rm */ i32 3)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rn_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rn */ i32 1)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rp_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rp */ i32 2)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rz_f() {
+; CHECK-NEXT: ret float f0x00400000
+;
+ %res = call float @llvm.nvvm.fadd.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rz */ i32 0)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rm_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rm */ i32 3)
ret float %res
}
+define float @test_normal_minus_normal_to_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rn_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rn */ i32 1)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rp_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rp */ i32 2)
+ ret float %res
+}
+
+define float @test_normal_minus_normal_to_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_normal_to_subnorm_rz_ftz_f() {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 0x3818000000000000, float 0xB810000000000000, /* rnd=rz */ i32 0)
+ ret float %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rm_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rm_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rn_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rn_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rp_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rp_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rz_f16() {
+; CHECK-NEXT: ret half 3.051760e-05
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 0xH0600, half 0xH8400, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rm_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rm_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rn_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rp_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_normal_minus_normal_to_subnorm_rz_ftz_f16() {
+; CHECK-LABEL: define half @test_normal_minus_normal_to_subnorm_rz_ftz_f16() {
+; CHECK-NEXT: ret half 0.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.ftz.f16(half 0xH0600, half 0xH8400, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
;###############################################################
;# Add(1.0, 2^(-25)) #
;###############################################################
@@ -439,7 +903,7 @@ define float @test_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -447,7 +911,7 @@ define float @test_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -455,7 +919,7 @@ define float @test_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -463,7 +927,7 @@ define float @test_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -471,7 +935,7 @@ define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -479,7 +943,7 @@ define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -487,7 +951,7 @@ define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -495,7 +959,7 @@ define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -511,7 +975,7 @@ define double @test_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -519,7 +983,7 @@ define double @test_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -527,7 +991,7 @@ define double @test_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0x3FF0000000000001
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -535,10 +999,90 @@ define double @test_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0x3C90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
+;###############################################################
+;# Add(1.0, 2^(-12)) #
+;###############################################################
+; Tests addition of 1.0 and 2^(-12) where the exact result falls between
+; 1.0 and 1.0 + 2^(-10):
+; - RN, RZ, RM: Return 1.0 (rounding to nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-10) (rounding up)
+
+define half @test_1_plus_ulp_rm_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rm_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rn_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rn_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rp_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rp_f16() {
+; CHECK-NEXT: ret half 1.000980e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_1_plus_ulp_rz_f16() {
+; CHECK-LABEL: define half @test_1_plus_ulp_rz_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fadd.f16(half 1.0, half 0xH0C00, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+;###############################################################
+;# Add(1.0, 2^(-9)) #
+;###############################################################
+; Tests addition of 1.0 and 2^(-9) where the exact result falls between
+; 1.0 and 1.0 + 2^(-7):
+; - RN, RZ, RM: Return 1.0 (rounding to nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-7) (rounding up)
+
+define bfloat @test_1_plus_ulp_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rm_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rn_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rp_bf16() {
+; CHECK-NEXT: ret bfloat 1.007810e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_1_plus_ulp_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_1_plus_ulp_rz_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fadd.bf16(bfloat 1.0, bfloat 0xR3B00, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
;###############################################################
;# Add(-1.0, 2^(-25)) #
;###############################################################
@@ -551,7 +1095,7 @@ define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -559,7 +1103,7 @@ define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -567,7 +1111,7 @@ define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rp.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -575,7 +1119,7 @@ define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -583,7 +1127,7 @@ define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -591,7 +1135,7 @@ define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -599,7 +1143,7 @@ define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -607,7 +1151,7 @@ define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -623,7 +1167,7 @@ define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rm.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -631,7 +1175,7 @@ define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -639,7 +1183,7 @@ define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rp.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -647,7 +1191,7 @@ define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rz.d(double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0x3C90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -663,7 +1207,7 @@ define float @test_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rm.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -671,7 +1215,7 @@ define float @test_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -679,7 +1223,7 @@ define float @test_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -687,7 +1231,7 @@ define float @test_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float 1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -695,7 +1239,7 @@ define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -703,7 +1247,7 @@ define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -711,7 +1255,7 @@ define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -719,7 +1263,7 @@ define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float 1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -735,7 +1279,7 @@ define double @test_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rm.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -743,7 +1287,7 @@ define double @test_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -751,7 +1295,7 @@ define double @test_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.add.rp.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -759,7 +1303,7 @@ define double @test_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.add.rz.d(double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double 1.0, double 0xBC90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -775,7 +1319,7 @@ define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.add.rm.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -783,7 +1327,7 @@ define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -791,7 +1335,7 @@ define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -799,7 +1343,7 @@ define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.f32(float -1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -807,7 +1351,7 @@ define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.add.rm.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -815,7 +1359,7 @@ define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rn.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -823,7 +1367,7 @@ define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rp.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -831,7 +1375,7 @@ define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.add.rz.ftz.f(float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fadd.ftz.f32(float -1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -847,7 +1391,7 @@ define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0xBFF0000000000001
;
- %res = call double @llvm.nvvm.add.rm.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -855,7 +1399,7 @@ define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rn.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -863,7 +1407,7 @@ define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rp.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -871,6 +1415,6 @@ define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_neg_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.add.rz.d(double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fadd.f64(double -1.0, double 0xBC90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
diff --git a/llvm/test/Verifier/NVPTX/fadd.ll b/llvm/test/Verifier/NVPTX/fadd.ll
new file mode 100644
index 00000000000000..d3a03abf6f1f24
--- /dev/null
+++ b/llvm/test/Verifier/NVPTX/fadd.ll
@@ -0,0 +1,16 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+declare float @llvm.nvvm.fadd.f32(float, float, i32 immarg)
+
+define void @test_fadd_rounding_mode(float %a) {
+ ; CHECK: immarg value 4 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 4)
+
+ ; CHECK: immarg value 7 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 7)
+
+ ; CHECK: immarg value -1 for arg 2 out of range [0,4)
+ call float @llvm.nvvm.fadd.f32(float %a, float %a, i32 -1)
+
+ ret void
+}
diff --git a/llvm/unittests/IR/IntrinsicsTest.cpp b/llvm/unittests/IR/IntrinsicsTest.cpp
index 47a1ea7d739062..f982715154bc05 100644
--- a/llvm/unittests/IR/IntrinsicsTest.cpp
+++ b/llvm/unittests/IR/IntrinsicsTest.cpp
@@ -118,7 +118,7 @@ TEST(IntrinsicNameLookup, ClangBuiltinLookup) {
{"__builtin_HEXAGON_A2_tfr", "hexagon", hexagon_A2_tfr},
{"__builtin_lasx_xbz_w", "loongarch", loongarch_lasx_xbz_w},
{"__builtin_mips_bitrev", "mips", mips_bitrev},
- {"__nvvm_add_rn_d", "nvvm", nvvm_add_rn_d},
+ {"__nvvm_mul_rn_d", "nvvm", nvvm_mul_rn_d},
{"__builtin_altivec_dss", "ppc", ppc_altivec_dss},
{"__builtin_riscv_sha512sum1r", "riscv", riscv_sha512sum1r},
{"__builtin_tend", "s390", s390_tend},
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 843344703d0a5b..43280c4a6f00af 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -3609,14 +3609,6 @@ static LogicalResult verifyAddSubFOp(OpType op) {
"vector<2xbf16> additions/subtractions");
}
- // FIXME: This is a temporary check disallowing lowering to add.rn.ftz.f16(x2)
- // PTX instructions since the corresponding LLVM intrinsic is missing. This
- // should be removed once the intrinsics for f16 addition (with FTZ only) are
- // available.
- if (opBaseType.isF16() && isFTZ && satMode == NVVM::SaturationMode::NONE)
- return op.emitOpError("FTZ with no saturation is not supported for f16 and "
- "vector<2xf16> additions/subtractions");
-
return success();
}
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 4201aabc02c3da..612425754e6201 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -448,6 +448,25 @@ getFenceProxySyncRestrictID(NVVM::MemOrderKind order) {
nvvm_fence_proxy_async_generic_release_sync_restrict_space_cta_scope_cluster;
}
+static llvm::RoundingMode
+getLLVMRoundingModeForFPArith(NVVM::FPRoundingMode rndMode) {
+ switch (rndMode) {
+ case NVVM::FPRoundingMode::RN:
+ return llvm::RoundingMode::NearestTiesToEven;
+ case NVVM::FPRoundingMode::RM:
+ return llvm::RoundingMode::TowardNegative;
+ case NVVM::FPRoundingMode::RP:
+ return llvm::RoundingMode::TowardPositive;
+ case NVVM::FPRoundingMode::RZ:
+ return llvm::RoundingMode::TowardZero;
+ default:
+ // default rounding mode is RN
+ assert(rndMode == NVVM::FPRoundingMode::NONE &&
+ "unsupported rounding mode for nvvm fp arithmetic");
+ return llvm::RoundingMode::NearestTiesToEven;
+ }
+}
+
// Calls an LLVM intrinsic on the given operands. For f32/f64 vector types,
// the intrinsic is called per-element and the results are packed back into a
// vector. If retType is non-null, it is forwarded as the return-type
@@ -465,7 +484,9 @@ createScalarizedIntrinsicCall(llvm::IRBuilderBase &builder,
llvm::SmallVector<llvm::Value *> scalarArgs;
for (llvm::Value *op : operands)
scalarArgs.push_back(
- builder.CreateExtractElement(op, builder.getInt32(i)));
+ op->getType()->isVectorTy()
+ ? builder.CreateExtractElement(op, builder.getInt32(i))
+ : op);
llvm::Value *res = createIntrinsicCall(builder, IID, retType, scalarArgs);
result = builder.CreateInsertElement(result, res, builder.getInt32(i));
}
@@ -481,87 +502,29 @@ void NVVM::AddFOp::lowerAddFToLLVMIR(llvm::Value *argLHS, llvm::Value *argRHS,
LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
llvm::Type *opTypeLLVM = argLHS->getType();
- bool isVectorOp = opTypeLLVM->isVectorTy();
bool isSat = satMode != NVVM::SaturationMode::NONE;
- // FIXME: Add intrinsics for add.rn.ftz.f16x2 and add.rn.ftz.f16 here when
- // they are available.
- static constexpr llvm::Intrinsic::ID f16IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_sat_f16,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f16,
- llvm::Intrinsic::nvvm_add_rn_sat_v2f16,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_v2f16,
- };
-
- static constexpr llvm::Intrinsic::ID f32IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_f,
- llvm::Intrinsic::nvvm_add_rm_f,
- llvm::Intrinsic::nvvm_add_rp_f,
- llvm::Intrinsic::nvvm_add_rz_f,
- llvm::Intrinsic::nvvm_add_rn_sat_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_sat_f,
- llvm::Intrinsic::nvvm_add_rm_sat_f,
- llvm::Intrinsic::nvvm_add_rp_sat_f,
- llvm::Intrinsic::nvvm_add_rz_sat_f,
- llvm::Intrinsic::nvvm_add_rn_ftz_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_ftz_f,
- llvm::Intrinsic::nvvm_add_rm_ftz_f,
- llvm::Intrinsic::nvvm_add_rp_ftz_f,
- llvm::Intrinsic::nvvm_add_rz_ftz_f,
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rm_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rp_ftz_sat_f,
- llvm::Intrinsic::nvvm_add_rz_ftz_sat_f,
- };
-
- static constexpr llvm::Intrinsic::ID f64IDs[] = {
- llvm::Intrinsic::nvvm_add_rn_d, // default rounding mode RN
- llvm::Intrinsic::nvvm_add_rn_d, llvm::Intrinsic::nvvm_add_rm_d,
- llvm::Intrinsic::nvvm_add_rp_d, llvm::Intrinsic::nvvm_add_rz_d};
-
- auto addIntrinsic = [&](llvm::Intrinsic::ID IID) -> llvm::Value * {
- return createScalarizedIntrinsicCall(builder, IID, opTypeLLVM,
- {argLHS, argRHS}, opTypeLLVM);
- };
-
- // f16 + f16 -> f16 / vector<2xf16> + vector<2xf16> -> vector<2xf16>
- // FIXME: Allow lowering to add.rn.ftz.f16x2 and add.rn.ftz.f16 here when the
- // intrinsics are available.
- if (opTypeLLVM->getScalarType()->isHalfTy()) {
- llvm::Value *result;
- if (isSat) {
- unsigned index = (isVectorOp << 1) | isFTZ;
- result = addIntrinsic(f16IDs[index]);
- } else {
- result = builder.CreateFAdd(argLHS, argRHS);
- }
- mt.mapValue(res, result);
- return;
- }
-
- // bf16 + bf16 -> bf16 / vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
- if (opTypeLLVM->getScalarType()->isBFloatTy()) {
- mt.mapValue(res, builder.CreateFAdd(argLHS, argRHS));
- return;
- }
-
- // f64 + f64 -> f64 / vector<2xf64> + vector<2xf64> -> vector<2xf64>
- if (opTypeLLVM->getScalarType()->isDoubleTy()) {
- unsigned index = static_cast<unsigned>(rndMode);
- mt.mapValue(res, addIntrinsic(f64IDs[index]));
+ static constexpr llvm::Intrinsic::ID addIDs[2][2] = {
+ {llvm::Intrinsic::nvvm_fadd, llvm::Intrinsic::nvvm_fadd_sat},
+ {llvm::Intrinsic::nvvm_fadd_ftz, llvm::Intrinsic::nvvm_fadd_ftz_sat}};
+
+ llvm::Intrinsic::ID id = addIDs[isFTZ][isSat];
+ llvm::Value *rnd = builder.getInt32(
+ static_cast<int>(getLLVMRoundingModeForFPArith(rndMode)));
+
+ // For f64 vector addition, and f32 vector addition with saturation,
+ // we need to scalarize the intrinsic call.
+ llvm::Type *scalarTypeLLVM = opTypeLLVM->getScalarType();
+ if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() ||
+ (isSat && scalarTypeLLVM->isFloatTy()))) {
+ mt.mapValue(res, createScalarizedIntrinsicCall(builder, id, opTypeLLVM,
+ {argLHS, argRHS, rnd},
+ scalarTypeLLVM));
return;
}
- // f32 + f32 -> f32 / vector<2xf32> + vector<2xf32> -> vector<2xf32>
- const unsigned numRndModes = 5; // NONE, RM, RN, RP, RZ
- if (opTypeLLVM->getScalarType()->isFloatTy()) {
- unsigned index =
- ((isFTZ << 1) | isSat) * numRndModes + static_cast<unsigned>(rndMode);
- mt.mapValue(res, addIntrinsic(f32IDs[index]));
- return;
- }
+ mt.mapValue(
+ res, createIntrinsicCall(builder, id, opTypeLLVM, {argLHS, argRHS, rnd}));
}
void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
index 201cfb7e098fc7..17c7e3f36406b0 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf.mlir
@@ -3,24 +3,26 @@
// f16 + f16 -> f16
llvm.func @fadd_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fadd_f16_f16(half %0, half %1) {
- // CHECK-NEXT: %3 = fadd half %0, %1
- // CHECK-NEXT: %4 = fadd half %3, %3
- // CHECK-NEXT: %5 = call half @llvm.nvvm.add.rn.sat.f16(half %4, half %4)
- // CHECK-NEXT: %6 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %5, half %5)
- // CHECK-NEXT: ret half %6
+ // CHECK-NEXT: %3 = call half @llvm.nvvm.fadd.f16(half %0, half %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %3, half %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call half @llvm.nvvm.fadd.ftz.f16(half %4, half %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.sat.f16(half %5, half %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %6, half %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: ret half %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f16
%f2 = nvvm.addf %f1, %f1 rnd = <rn> : f16
- %f3 = nvvm.addf %f2, %f2 rnd = <rn> sat = <sat> : f16
- %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : f16
- llvm.return %f4 : f16
+ %f3 = nvvm.addf %f2, %f2 rnd = <rn> ftz = true : f16
+ %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> : f16
+ %f5 = nvvm.addf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : f16
+ llvm.return %f5 : f16
}
// bf16 + bf16 -> bf16
llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fadd_bf16_bf16(bfloat %0, bfloat %1) {
- // CHECK-NEXT: %3 = fadd bfloat %0, %1
- // CHECK-NEXT: %4 = fadd bfloat %3, %3
+ // CHECK-NEXT: %3 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %0, bfloat %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %3, bfloat %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret bfloat %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : bf16
@@ -31,23 +33,23 @@ llvm.func @fadd_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// f32 + f32 -> f32
llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fadd_f32_f32(float %0, float %1) {
- // CHECK-NEXT: %3 = call float @llvm.nvvm.add.rn.f(float %0, float %1)
- // CHECK-NEXT: %4 = call float @llvm.nvvm.add.rn.f(float %3, float %3)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rn.sat.f(float %4, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.ftz.f(float %5, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %6, float %6)
- // CHECK-NEXT: %8 = call float @llvm.nvvm.add.rm.f(float %7, float %7)
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rm.sat.f(float %8, float %8)
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rm.ftz.f(float %9, float %9)
- // CHECK-NEXT: %11 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %10, float %10)
- // CHECK-NEXT: %12 = call float @llvm.nvvm.add.rp.f(float %11, float %11)
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rp.sat.f(float %12, float %12)
- // CHECK-NEXT: %14 = call float @llvm.nvvm.add.rp.ftz.f(float %13, float %13)
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %14, float %14)
- // CHECK-NEXT: %16 = call float @llvm.nvvm.add.rz.f(float %15, float %15)
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rz.sat.f(float %16, float %16)
- // CHECK-NEXT: %18 = call float @llvm.nvvm.add.rz.ftz.f(float %17, float %17)
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %18, float %18)
+ // CHECK-NEXT: %3 = call float @llvm.nvvm.fadd.f32(float %0, float %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %3, float %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.ftz.f32(float %5, float %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %6, float %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.f32(float %7, float %7, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %8, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %9, float %9, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %10, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.f32(float %11, float %11, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %13 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %12, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.ftz.f32(float %13, float %13, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.f32(float %15, float %15, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %17 = call float @llvm.nvvm.fadd.sat.f32(float %16, float %16, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %17, float %17, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %18, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %19
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f32
@@ -73,11 +75,11 @@ llvm.func @fadd_f32_f32(%a : f32, %b : f32) -> f32 {
// f64 + f64 -> f64
llvm.func @fadd_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fadd_f64_f64(double %0, double %1) {
- // CHECK-NEXT: %3 = call double @llvm.nvvm.add.rn.d(double %0, double %1)
- // CHECK-NEXT: %4 = call double @llvm.nvvm.add.rn.d(double %3, double %3)
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rm.d(double %4, double %4)
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rp.d(double %5, double %5)
- // CHECK-NEXT: %7 = call double @llvm.nvvm.add.rz.d(double %6, double %6)
+ // CHECK-NEXT: %3 = call double @llvm.nvvm.fadd.f64(double %0, double %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.f64(double %3, double %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %4, double %4, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %5, double %5, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %7 = call double @llvm.nvvm.fadd.f64(double %6, double %6, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret double %7
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
index de9a53c8e95d68..58fb8814903add 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_invalid.mlir
@@ -55,13 +55,3 @@ llvm.func @addf_invalid_bf16_sat_ftz(%a : bf16, %b : bf16) -> bf16 {
%f1 = nvvm.addf %a, %b sat = <sat> ftz = true : bf16
llvm.return %f1 : bf16
}
-
-// -----
-
-// FIXME: Remove this test once intrinsics for f16 addition (with FTZ only) are
-// available.
-llvm.func @addf_invalid_f16_ftz_no_sat(%a : f16, %b : f16) -> f16 {
- // expected-error at +1 {{FTZ with no saturation is not supported for f16 and vector<2xf16> additions/subtractions}}
- %f1 = nvvm.addf %a, %b ftz = true : f16
- llvm.return %f1 : f16
-}
diff --git a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
index d8bf3dfcc8fb69..504b0e5cae6bf0 100644
--- a/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/addf/addf_vector.mlir
@@ -3,24 +3,26 @@
// vector<2xf16> + vector<2xf16> -> vector<2xf16>
llvm.func @addf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @addf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
- // CHECK-NEXT: %3 = fadd <2 x half> %0, %1
- // CHECK-NEXT: %4 = fadd <2 x half> %3, %3
- // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %4, <2 x half> %4)
- // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %5, <2 x half> %5)
+ // CHECK-NEXT: %3 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %0, <2 x half> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %3, <2 x half> %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %4, <2 x half> %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %5, <2 x half> %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %6, <2 x half> %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %3
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf16>
%f2 = nvvm.addf %f1, %f1 rnd = <rn> : vector<2xf16>
- %f3 = nvvm.addf %f2, %f2 rnd = <rn> sat = <sat> : vector<2xf16>
- %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
+ %f3 = nvvm.addf %f2, %f2 rnd = <rn> ftz = true : vector<2xf16>
+ %f4 = nvvm.addf %f3, %f3 rnd = <rn> sat = <sat> : vector<2xf16>
+ %f5 = nvvm.addf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
// vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @addf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
- // CHECK-NEXT: %3 = fadd <2 x bfloat> %0, %1
- // CHECK-NEXT: %4 = fadd <2 x bfloat> %3, %3
+ // CHECK-NEXT: %3 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %3, <2 x bfloat> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x bfloat> %4
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xbf16>
@@ -31,47 +33,26 @@ llvm.func @addf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
// vector<2xf32> + vector<2xf32> -> vector<2xf32>
llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rn.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rn.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rn.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rn.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rn.sat.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rn.sat.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rn.ftz.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rn.ftz.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: %35 = extractelement <2 x float> %34, i32 0
- // CHECK-NEXT: %36 = extractelement <2 x float> %34, i32 0
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> poison, float %37, i32 0
- // CHECK-NEXT: %39 = extractelement <2 x float> %34, i32 1
- // CHECK-NEXT: %40 = extractelement <2 x float> %34, i32 1
- // CHECK-NEXT: %41 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %39, float %40)
- // CHECK-NEXT: %42 = insertelement <2 x float> %38, float %41, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %3, <2 x float> %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fadd.sat.f32(float %5, float %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
+ // CHECK-NEXT: %9 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = call float @llvm.nvvm.fadd.sat.f32(float %9, float %10, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %12 = insertelement <2 x float> %8, float %11, i32 1
+ // CHECK-NEXT: %13 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %12, <2 x float> %12, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %14 = extractelement <2 x float> %13, i32 0
+ // CHECK-NEXT: %15 = extractelement <2 x float> %13, i32 0
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %14, float %15, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %17 = insertelement <2 x float> poison, float %16, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %13, i32 1
+ // CHECK-NEXT: %19 = extractelement <2 x float> %13, i32 1
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %21 = insertelement <2 x float> %17, float %20, i32 1
+ // CHECK-NEXT: ret <2 x float> %13
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rn> : vector<2xf32>
@@ -83,39 +64,25 @@ llvm.func @addf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rm.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rm.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rm.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rm.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rm.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rm.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rm> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rm> sat = <sat> : vector<2xf32>
@@ -126,39 +93,25 @@ llvm.func @addf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rp.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rp.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rp.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rp.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rp.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rp.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rp> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rp> sat = <sat> : vector<2xf32>
@@ -169,39 +122,25 @@ llvm.func @addf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @addf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @addf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
- // CHECK-NEXT: %3 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %4 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %5 = call float @llvm.nvvm.add.rz.f(float %3, float %4)
- // CHECK-NEXT: %6 = insertelement <2 x float> poison, float %5, i32 0
- // CHECK-NEXT: %7 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %8 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %9 = call float @llvm.nvvm.add.rz.f(float %7, float %8)
- // CHECK-NEXT: %10 = insertelement <2 x float> %6, float %9, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %12 = extractelement <2 x float> %10, i32 0
- // CHECK-NEXT: %13 = call float @llvm.nvvm.add.rz.sat.f(float %11, float %12)
- // CHECK-NEXT: %14 = insertelement <2 x float> poison, float %13, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %16 = extractelement <2 x float> %10, i32 1
- // CHECK-NEXT: %17 = call float @llvm.nvvm.add.rz.sat.f(float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> %14, float %17, i32 1
- // CHECK-NEXT: %19 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
- // CHECK-NEXT: %21 = call float @llvm.nvvm.add.rz.ftz.f(float %19, float %20)
- // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
- // CHECK-NEXT: %25 = call float @llvm.nvvm.add.rz.ftz.f(float %23, float %24)
- // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %28 = extractelement <2 x float> %26, i32 0
- // CHECK-NEXT: %29 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %27, float %28)
- // CHECK-NEXT: %30 = insertelement <2 x float> poison, float %29, i32 0
- // CHECK-NEXT: %31 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %32 = extractelement <2 x float> %26, i32 1
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> %30, float %33, i32 1
- // CHECK-NEXT: ret <2 x float> %34
+ // CHECK-NEXT: %3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %1, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %4 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.sat.f32(float %4, float %5, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
+ // CHECK-NEXT: %12 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %11, <2 x float> %11, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %13 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
+ // CHECK-NEXT: %15 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %13, float %14, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
+ // CHECK-NEXT: ret <2 x float> %20
// CHECK-NEXT: }
%f1 = nvvm.addf %a, %b rnd = <rz> : vector<2xf32>
%f2 = nvvm.addf %f1, %f1 rnd = <rz> sat = <sat> : vector<2xf32>
@@ -215,19 +154,19 @@ llvm.func @addf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rn(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rn.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rn.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rn */ i32 1)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: %11 = extractelement <2 x double> %10, i32 0
// CHECK-NEXT: %12 = extractelement <2 x double> %10, i32 0
- // CHECK-NEXT: %13 = call double @llvm.nvvm.add.rn.d(double %11, double %12)
+ // CHECK-NEXT: %13 = call double @llvm.nvvm.fadd.f64(double %11, double %12, /* rnd=rn */ i32 1)
// CHECK-NEXT: %14 = insertelement <2 x double> poison, double %13, i32 0
// CHECK-NEXT: %15 = extractelement <2 x double> %10, i32 1
// CHECK-NEXT: %16 = extractelement <2 x double> %10, i32 1
- // CHECK-NEXT: %17 = call double @llvm.nvvm.add.rn.d(double %15, double %16)
+ // CHECK-NEXT: %17 = call double @llvm.nvvm.fadd.f64(double %15, double %16, /* rnd=rn */ i32 1)
// CHECK-NEXT: %18 = insertelement <2 x double> %14, double %17, i32 1
// CHECK-NEXT: ret <2 x double> %18
// CHECK-NEXT: }
@@ -240,11 +179,11 @@ llvm.func @addf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rm(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rm.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rm */ i32 3)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rm.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rm */ i32 3)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -256,11 +195,11 @@ llvm.func @addf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rp(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rp.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rp */ i32 2)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rp.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rp */ i32 2)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
@@ -272,11 +211,11 @@ llvm.func @addf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-LABEL: define <2 x double> @addf_vector_f64_f64_rz(<2 x double> %0, <2 x double> %1) {
// CHECK-NEXT: %3 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %4 = extractelement <2 x double> %1, i32 0
- // CHECK-NEXT: %5 = call double @llvm.nvvm.add.rz.d(double %3, double %4)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fadd.f64(double %3, double %4, /* rnd=rz */ i32 0)
// CHECK-NEXT: %6 = insertelement <2 x double> poison, double %5, i32 0
// CHECK-NEXT: %7 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %8 = extractelement <2 x double> %1, i32 1
- // CHECK-NEXT: %9 = call double @llvm.nvvm.add.rz.d(double %7, double %8)
+ // CHECK-NEXT: %9 = call double @llvm.nvvm.fadd.f64(double %7, double %8, /* rnd=rz */ i32 0)
// CHECK-NEXT: %10 = insertelement <2 x double> %6, double %9, i32 1
// CHECK-NEXT: ret <2 x double> %10
// CHECK-NEXT: }
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
index b2c6163045cb77..7226b0e172d75f 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf.mlir
@@ -4,29 +4,32 @@
llvm.func @fsub_f16_f16(%a : f16, %b : f16) -> f16 {
// CHECK-LABEL: define half @fsub_f16_f16(half %0, half %1) {
// CHECK-NEXT: %3 = fneg half %1
- // CHECK-NEXT: %4 = fadd half %0, %3
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fadd.f16(half %0, half %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg half %4
- // CHECK-NEXT: %6 = fadd half %4, %5
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fadd.f16(half %4, half %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg half %6
- // CHECK-NEXT: %8 = call half @llvm.nvvm.add.rn.sat.f16(half %6, half %7)
+ // CHECK-NEXT: %8 = call half @llvm.nvvm.fadd.ftz.f16(half %6, half %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg half %8
- // CHECK-NEXT: %10 = call half @llvm.nvvm.add.rn.ftz.sat.f16(half %8, half %9)
- // CHECK-NEXT: ret half %10
+ // CHECK-NEXT: %10 = call half @llvm.nvvm.fadd.sat.f16(half %8, half %9, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %11 = fneg half %10
+ // CHECK-NEXT: %12 = call half @llvm.nvvm.fadd.ftz.sat.f16(half %10, half %11, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: ret half %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f16
%f2 = nvvm.subf %f1, %f1 rnd = <rn> : f16
- %f3 = nvvm.subf %f2, %f2 rnd = <rn> sat = <sat> : f16
- %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : f16
- llvm.return %f4 : f16
+ %f3 = nvvm.subf %f2, %f2 rnd = <rn> ftz = true : f16
+ %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> : f16
+ %f5 = nvvm.subf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : f16
+ llvm.return %f5 : f16
}
// bf16 - bf16 -> bf16
llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fsub_bf16_bf16(bfloat %0, bfloat %1) {
// CHECK-NEXT: %3 = fneg bfloat %1
- // CHECK-NEXT: %4 = fadd bfloat %0, %3
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %0, bfloat %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg bfloat %4
- // CHECK-NEXT: %6 = fadd bfloat %4, %5
+ // CHECK-NEXT: %6 = call bfloat @llvm.nvvm.fadd.bf16(bfloat %4, bfloat %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret bfloat %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : bf16
@@ -38,39 +41,39 @@ llvm.func @fsub_bf16_bf16(%a : bf16, %b : bf16) -> bf16 {
llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
// CHECK-LABEL: define float @fsub_f32_f32(float %0, float %1) {
// CHECK-NEXT: %3 = fneg float %1
- // CHECK-NEXT: %4 = call float @llvm.nvvm.add.rn.f(float %0, float %3)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fadd.f32(float %0, float %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg float %4
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.f(float %4, float %5)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fadd.f32(float %4, float %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg float %6
- // CHECK-NEXT: %8 = call float @llvm.nvvm.add.rn.sat.f(float %6, float %7)
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg float %8
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rn.ftz.f(float %8, float %9)
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.ftz.f32(float %8, float %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = fneg float %10
- // CHECK-NEXT: %12 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %10, float %11)
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %10, float %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: %13 = fneg float %12
- // CHECK-NEXT: %14 = call float @llvm.nvvm.add.rm.f(float %12, float %13)
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.f32(float %12, float %13, /* rnd=rm */ i32 3)
// CHECK-NEXT: %15 = fneg float %14
- // CHECK-NEXT: %16 = call float @llvm.nvvm.add.rm.sat.f(float %14, float %15)
+ // CHECK-NEXT: %16 = call float @llvm.nvvm.fadd.sat.f32(float %14, float %15, /* rnd=rm */ i32 3)
// CHECK-NEXT: %17 = fneg float %16
- // CHECK-NEXT: %18 = call float @llvm.nvvm.add.rm.ftz.f(float %16, float %17)
+ // CHECK-NEXT: %18 = call float @llvm.nvvm.fadd.ftz.f32(float %16, float %17, /* rnd=rm */ i32 3)
// CHECK-NEXT: %19 = fneg float %18
- // CHECK-NEXT: %20 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %18, float %19)
+ // CHECK-NEXT: %20 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %18, float %19, /* rnd=rm */ i32 3)
// CHECK-NEXT: %21 = fneg float %20
- // CHECK-NEXT: %22 = call float @llvm.nvvm.add.rp.f(float %20, float %21)
+ // CHECK-NEXT: %22 = call float @llvm.nvvm.fadd.f32(float %20, float %21, /* rnd=rp */ i32 2)
// CHECK-NEXT: %23 = fneg float %22
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rp.sat.f(float %22, float %23)
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fadd.sat.f32(float %22, float %23, /* rnd=rp */ i32 2)
// CHECK-NEXT: %25 = fneg float %24
- // CHECK-NEXT: %26 = call float @llvm.nvvm.add.rp.ftz.f(float %24, float %25)
+ // CHECK-NEXT: %26 = call float @llvm.nvvm.fadd.ftz.f32(float %24, float %25, /* rnd=rp */ i32 2)
// CHECK-NEXT: %27 = fneg float %26
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %26, float %27)
+ // CHECK-NEXT: %28 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %26, float %27, /* rnd=rp */ i32 2)
// CHECK-NEXT: %29 = fneg float %28
- // CHECK-NEXT: %30 = call float @llvm.nvvm.add.rz.f(float %28, float %29)
+ // CHECK-NEXT: %30 = call float @llvm.nvvm.fadd.f32(float %28, float %29, /* rnd=rz */ i32 0)
// CHECK-NEXT: %31 = fneg float %30
- // CHECK-NEXT: %32 = call float @llvm.nvvm.add.rz.sat.f(float %30, float %31)
+ // CHECK-NEXT: %32 = call float @llvm.nvvm.fadd.sat.f32(float %30, float %31, /* rnd=rz */ i32 0)
// CHECK-NEXT: %33 = fneg float %32
- // CHECK-NEXT: %34 = call float @llvm.nvvm.add.rz.ftz.f(float %32, float %33)
+ // CHECK-NEXT: %34 = call float @llvm.nvvm.fadd.ftz.f32(float %32, float %33, /* rnd=rz */ i32 0)
// CHECK-NEXT: %35 = fneg float %34
- // CHECK-NEXT: %36 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %34, float %35)
+ // CHECK-NEXT: %36 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %34, float %35, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %36
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f32
@@ -97,15 +100,15 @@ llvm.func @fsub_f32_f32(%a : f32, %b : f32) -> f32 {
llvm.func @fsub_f64_f64(%a : f64, %b : f64) -> f64 {
// CHECK-LABEL: define double @fsub_f64_f64(double %0, double %1) {
// CHECK-NEXT: %3 = fneg double %1
- // CHECK-NEXT: %4 = call double @llvm.nvvm.add.rn.d(double %0, double %3)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fadd.f64(double %0, double %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg double %4
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rn.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg double %6
- // CHECK-NEXT: %8 = call double @llvm.nvvm.add.rm.d(double %6, double %7)
+ // CHECK-NEXT: %8 = call double @llvm.nvvm.fadd.f64(double %6, double %7, /* rnd=rm */ i32 3)
// CHECK-NEXT: %9 = fneg double %8
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rp.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rp */ i32 2)
// CHECK-NEXT: %11 = fneg double %10
- // CHECK-NEXT: %12 = call double @llvm.nvvm.add.rz.d(double %10, double %11)
+ // CHECK-NEXT: %12 = call double @llvm.nvvm.fadd.f64(double %10, double %11, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret double %12
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
index d5cff844ed1d33..38a2ca14bfa907 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_invalid.mlir
@@ -55,13 +55,3 @@ llvm.func @subf_invalid_bf16_sat_ftz(%a : bf16, %b : bf16) -> bf16 {
%f1 = nvvm.subf %a, %b sat = <sat> ftz = true : bf16
llvm.return %f1 : bf16
}
-
-// -----
-
-// FIXME: Remove this test once intrinsics for f16 addition (with FTZ only) are
-// available.
-llvm.func @subf_invalid_f16_ftz_no_sat(%a : f16, %b : f16) -> f16 {
- // expected-error at +1 {{FTZ with no saturation is not supported for f16 and vector<2xf16> additions/subtractions}}
- %f1 = nvvm.subf %a, %b ftz = true : f16
- llvm.return %f1 : f16
-}
diff --git a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
index 4c0f143806329b..ee4ab35a0132fd 100644
--- a/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/subf/subf_vector.mlir
@@ -4,19 +4,22 @@
llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @subf_vector_f16_f16(<2 x half> %0, <2 x half> %1) {
// CHECK-NEXT: %3 = fneg <2 x half> %1
- // CHECK-NEXT: %4 = fadd <2 x half> %0, %3
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %0, <2 x half> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg <2 x half> %4
- // CHECK-NEXT: %6 = fadd <2 x half> %4, %5
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fadd.v2f16(<2 x half> %4, <2 x half> %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = fneg <2 x half> %6
- // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.add.rn.sat.v2f16(<2 x half> %6, <2 x half> %7)
+ // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fadd.ftz.v2f16(<2 x half> %6, <2 x half> %7, /* rnd=rn */ i32 1)
// CHECK-NEXT: %9 = fneg <2 x half> %8
- // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.add.rn.ftz.sat.v2f16(<2 x half> %8, <2 x half> %9)
+ // CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fadd.sat.v2f16(<2 x half> %8, <2 x half> %9, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %11 = fneg <2 x half> %10
+ // CHECK-NEXT: %12 = call <2 x half> @llvm.nvvm.fadd.ftz.sat.v2f16(<2 x half> %10, <2 x half> %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x half> %4
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf16>
%f2 = nvvm.subf %f1, %f1 rnd = <rn> : vector<2xf16>
- %f3 = nvvm.subf %f2, %f2 rnd = <rn> sat = <sat> : vector<2xf16>
- %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
+ %f3 = nvvm.subf %f2, %f2 rnd = <rn> ftz = true : vector<2xf16>
+ %f4 = nvvm.subf %f3, %f3 rnd = <rn> sat = <sat> : vector<2xf16>
+ %f5 = nvvm.subf %f4, %f4 rnd = <rn> sat = <sat> ftz = true : vector<2xf16>
llvm.return %f1 : vector<2xf16>
}
@@ -24,9 +27,9 @@ llvm.func @subf_vector_f16_f16(%a : vector<2xf16>, %b : vector<2xf16>) -> vector
llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @subf_vector_bf16_bf16(<2 x bfloat> %0, <2 x bfloat> %1) {
// CHECK-NEXT: %3 = fneg <2 x bfloat> %1
- // CHECK-NEXT: %4 = fadd <2 x bfloat> %0, %3
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %0, <2 x bfloat> %3, /* rnd=rn */ i32 1)
// CHECK-NEXT: %5 = fneg <2 x bfloat> %4
- // CHECK-NEXT: %6 = fadd <2 x bfloat> %4, %5
+ // CHECK-NEXT: %6 = call <2 x bfloat> @llvm.nvvm.fadd.v2bf16(<2 x bfloat> %4, <2 x bfloat> %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret <2 x bfloat> %6
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xbf16>
@@ -38,51 +41,30 @@ llvm.func @subf_vector_bf16_bf16(%a : vector<2xbf16>, %b : vector<2xbf16>) -> ve
llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rn(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rn.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rn.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rn.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rn.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rn.sat.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rn.sat.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rn.ftz.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rn.ftz.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: %39 = fneg <2 x float> %38
- // CHECK-NEXT: %40 = extractelement <2 x float> %38, i32 0
- // CHECK-NEXT: %41 = extractelement <2 x float> %39, i32 0
- // CHECK-NEXT: %42 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %40, float %41)
- // CHECK-NEXT: %43 = insertelement <2 x float> poison, float %42, i32 0
- // CHECK-NEXT: %44 = extractelement <2 x float> %38, i32 1
- // CHECK-NEXT: %45 = extractelement <2 x float> %39, i32 1
- // CHECK-NEXT: %46 = call float @llvm.nvvm.add.rn.ftz.sat.f(float %44, float %45)
- // CHECK-NEXT: %47 = insertelement <2 x float> %43, float %46, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %4, <2 x float> %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = fneg <2 x float> %6
+ // CHECK-NEXT: %8 = extractelement <2 x float> %6, i32 0
+ // CHECK-NEXT: %9 = extractelement <2 x float> %7, i32 0
+ // CHECK-NEXT: %10 = call float @llvm.nvvm.fadd.sat.f32(float %8, float %9, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %11 = insertelement <2 x float> poison, float %10, i32 0
+ // CHECK-NEXT: %12 = extractelement <2 x float> %6, i32 1
+ // CHECK-NEXT: %13 = extractelement <2 x float> %7, i32 1
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fadd.sat.f32(float %12, float %13, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %15 = insertelement <2 x float> %11, float %14, i32 1
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %15, <2 x float> %16, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %18 = fneg <2 x float> %17
+ // CHECK-NEXT: %19 = extractelement <2 x float> %17, i32 0
+ // CHECK-NEXT: %20 = extractelement <2 x float> %18, i32 0
+ // CHECK-NEXT: %21 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %19, float %20, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %22 = insertelement <2 x float> poison, float %21, i32 0
+ // CHECK-NEXT: %23 = extractelement <2 x float> %17, i32 1
+ // CHECK-NEXT: %24 = extractelement <2 x float> %18, i32 1
+ // CHECK-NEXT: %25 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %23, float %24, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %26 = insertelement <2 x float> %22, float %25, i32 1
+ // CHECK-NEXT: ret <2 x float> %17
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rn> : vector<2xf32>
@@ -95,42 +77,28 @@ llvm.func @subf_vector_f32_f32_rn(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rm(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rm.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rm.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rm.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rm.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rm.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rm.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rm.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rm> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rm> sat = <sat> : vector<2xf32>
@@ -142,42 +110,28 @@ llvm.func @subf_vector_f32_f32_rm(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rp(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rp.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rp.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rp.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rp.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rp.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rp.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rp.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rp> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rp> sat = <sat> : vector<2xf32>
@@ -189,42 +143,28 @@ llvm.func @subf_vector_f32_f32_rp(%a : vector<2xf32>, %b : vector<2xf32>) -> vec
llvm.func @subf_vector_f32_f32_rz(%a : vector<2xf32>, %b : vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @subf_vector_f32_f32_rz(<2 x float> %0, <2 x float> %1) {
// CHECK-NEXT: %3 = fneg <2 x float> %1
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %3, i32 0
- // CHECK-NEXT: %6 = call float @llvm.nvvm.add.rz.f(float %4, float %5)
- // CHECK-NEXT: %7 = insertelement <2 x float> poison, float %6, i32 0
- // CHECK-NEXT: %8 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %9 = extractelement <2 x float> %3, i32 1
- // CHECK-NEXT: %10 = call float @llvm.nvvm.add.rz.f(float %8, float %9)
- // CHECK-NEXT: %11 = insertelement <2 x float> %7, float %10, i32 1
- // CHECK-NEXT: %12 = fneg <2 x float> %11
- // CHECK-NEXT: %13 = extractelement <2 x float> %11, i32 0
- // CHECK-NEXT: %14 = extractelement <2 x float> %12, i32 0
- // CHECK-NEXT: %15 = call float @llvm.nvvm.add.rz.sat.f(float %13, float %14)
- // CHECK-NEXT: %16 = insertelement <2 x float> poison, float %15, i32 0
- // CHECK-NEXT: %17 = extractelement <2 x float> %11, i32 1
- // CHECK-NEXT: %18 = extractelement <2 x float> %12, i32 1
- // CHECK-NEXT: %19 = call float @llvm.nvvm.add.rz.sat.f(float %17, float %18)
- // CHECK-NEXT: %20 = insertelement <2 x float> %16, float %19, i32 1
- // CHECK-NEXT: %21 = fneg <2 x float> %20
- // CHECK-NEXT: %22 = extractelement <2 x float> %20, i32 0
- // CHECK-NEXT: %23 = extractelement <2 x float> %21, i32 0
- // CHECK-NEXT: %24 = call float @llvm.nvvm.add.rz.ftz.f(float %22, float %23)
- // CHECK-NEXT: %25 = insertelement <2 x float> poison, float %24, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %20, i32 1
- // CHECK-NEXT: %27 = extractelement <2 x float> %21, i32 1
- // CHECK-NEXT: %28 = call float @llvm.nvvm.add.rz.ftz.f(float %26, float %27)
- // CHECK-NEXT: %29 = insertelement <2 x float> %25, float %28, i32 1
- // CHECK-NEXT: %30 = fneg <2 x float> %29
- // CHECK-NEXT: %31 = extractelement <2 x float> %29, i32 0
- // CHECK-NEXT: %32 = extractelement <2 x float> %30, i32 0
- // CHECK-NEXT: %33 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %31, float %32)
- // CHECK-NEXT: %34 = insertelement <2 x float> poison, float %33, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %29, i32 1
- // CHECK-NEXT: %36 = extractelement <2 x float> %30, i32 1
- // CHECK-NEXT: %37 = call float @llvm.nvvm.add.rz.ftz.sat.f(float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> %34, float %37, i32 1
- // CHECK-NEXT: ret <2 x float> %38
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %0, <2 x float> %3, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %5 = fneg <2 x float> %4
+ // CHECK-NEXT: %6 = extractelement <2 x float> %4, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %8 = call float @llvm.nvvm.fadd.sat.f32(float %6, float %7, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %9 = insertelement <2 x float> poison, float %8, i32 0
+ // CHECK-NEXT: %10 = extractelement <2 x float> %4, i32 1
+ // CHECK-NEXT: %11 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %12 = call float @llvm.nvvm.fadd.sat.f32(float %10, float %11, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %13 = insertelement <2 x float> %9, float %12, i32 1
+ // CHECK-NEXT: %14 = fneg <2 x float> %13
+ // CHECK-NEXT: %15 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %13, <2 x float> %14, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %16 = fneg <2 x float> %15
+ // CHECK-NEXT: %17 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %16, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %16, i32 1
+ // CHECK-NEXT: %23 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %21, float %22, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %24 = insertelement <2 x float> %20, float %23, i32 1
+ // CHECK-NEXT: ret <2 x float> %24
// CHECK-NEXT: }
%f1 = nvvm.subf %a, %b rnd = <rz> : vector<2xf32>
%f2 = nvvm.subf %f1, %f1 rnd = <rz> sat = <sat> : vector<2xf32>
@@ -239,20 +179,20 @@ llvm.func @subf_vector_f64_f64_rn(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rn.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rn */ i32 1)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rn.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rn */ i32 1)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: %12 = fneg <2 x double> %11
// CHECK-NEXT: %13 = extractelement <2 x double> %11, i32 0
// CHECK-NEXT: %14 = extractelement <2 x double> %12, i32 0
- // CHECK-NEXT: %15 = call double @llvm.nvvm.add.rn.d(double %13, double %14)
+ // CHECK-NEXT: %15 = call double @llvm.nvvm.fadd.f64(double %13, double %14, /* rnd=rn */ i32 1)
// CHECK-NEXT: %16 = insertelement <2 x double> poison, double %15, i32 0
// CHECK-NEXT: %17 = extractelement <2 x double> %11, i32 1
// CHECK-NEXT: %18 = extractelement <2 x double> %12, i32 1
- // CHECK-NEXT: %19 = call double @llvm.nvvm.add.rn.d(double %17, double %18)
+ // CHECK-NEXT: %19 = call double @llvm.nvvm.fadd.f64(double %17, double %18, /* rnd=rn */ i32 1)
// CHECK-NEXT: %20 = insertelement <2 x double> %16, double %19, i32 1
// CHECK-NEXT: ret <2 x double> %20
// CHECK-NEXT: }
@@ -266,11 +206,11 @@ llvm.func @subf_vector_f64_f64_rm(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rm.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rm */ i32 3)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rm.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rm */ i32 3)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -283,11 +223,11 @@ llvm.func @subf_vector_f64_f64_rp(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rp.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rp */ i32 2)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rp.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rp */ i32 2)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
@@ -300,11 +240,11 @@ llvm.func @subf_vector_f64_f64_rz(%a : vector<2xf64>, %b : vector<2xf64>) -> vec
// CHECK-NEXT: %3 = fneg <2 x double> %1
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %3, i32 0
- // CHECK-NEXT: %6 = call double @llvm.nvvm.add.rz.d(double %4, double %5)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fadd.f64(double %4, double %5, /* rnd=rz */ i32 0)
// CHECK-NEXT: %7 = insertelement <2 x double> poison, double %6, i32 0
// CHECK-NEXT: %8 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %9 = extractelement <2 x double> %3, i32 1
- // CHECK-NEXT: %10 = call double @llvm.nvvm.add.rz.d(double %8, double %9)
+ // CHECK-NEXT: %10 = call double @llvm.nvvm.fadd.f64(double %8, double %9, /* rnd=rz */ i32 0)
// CHECK-NEXT: %11 = insertelement <2 x double> %7, double %10, i32 1
// CHECK-NEXT: ret <2 x double> %11
// CHECK-NEXT: }
More information about the cfe-commits
mailing list