[Mlir-commits] [clang] [llvm] [mlir] [clang][NVPTX][MLIR][NVVM] Add overloaded fma intrinsics (PR #232672)
Srinivasa Ravi
llvmlistbot at llvm.org
Fri Oct 16 03:23:39 PDT 2026
https://github.com/Wolfram70 created https://github.com/llvm/llvm-project/pull/232672
This change adds the following overloaded `fma` intrinsics with
NVPTX codegen:
- `llvm.nvvm.fma`
- `llvm.nvvm.fma.ftz`
- `llvm.nvvm.fma.sat`
- `llvm.nvvm.fma.ftz.sat`
- `llvm.nvvm.fma.relu`
- `llvm.nvvm.fma.ftz.relu`
The rounding mode is passed in as an `i32` immediate operand.
Auto-upgrades the older non-overloaded intrinsics (including the
legacy variants that used integer storage types for the bf16
operands) to the new ones, and updates clang builtins, CIR codegen,
and MLIR NVVM ops to lower to the new intrinsics. The
`llvm.nvvm.fma.rn.oob{.relu}` intrinsics are left unchanged.
In the interest of completion, this also:
- Adds support for the `f32x2` type (`fma.rnd{.ftz}.f32x2`,
`sm_100+`).
- Adds tests for constant folding of these intrinsics with the newly
supported scalar types.
PTX Spec References:
https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#floating-point-instructions-fma
https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#half-precision-floating-point-instructions-fma
Assisted-by: Claude Opus 5.5
>From d46bdf31841b6e3d015eaeb7042093bf9afcfad3 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Mon, 12 Oct 2026 07:20:50 +0000
Subject: [PATCH 1/4] [clang][NVPTX][MLIR][NVVM] Add overloaded fma intrinsics
This change adds the following overloaded `fma` intrinsics with
NVPTX codegen:
- `llvm.nvvm.fma`
- `llvm.nvvm.fma.ftz`
- `llvm.nvvm.fma.sat`
- `llvm.nvvm.fma.ftz.sat`
- `llvm.nvvm.fma.relu`
- `llvm.nvvm.fma.ftz.relu`
The rounding mode is passed in as an `i32` immediate operand.
Auto-upgrades the older non-overloaded intrinsics (including the
legacy variants that used integer storage types for the bf16
operands) to the new ones, and updates clang builtins, CIR codegen,
and MLIR NVVM ops to lower to the new intrinsics. The
`llvm.nvvm.fma.rn.oob{.relu}` intrinsics are left unchanged.
In the interest of completion, this also:
- Adds support for the `f32x2` type (`fma.rnd{.ftz}.f32x2`,
sm_100+).
- Adds tests for constant folding of these intrinsics with the newly
supported scalar types.
PTX Spec References:
https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#floating-point-instructions-fma
https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#half-precision-floating-point-instructions-fma
Assisted-by: Claude Opus 5.5
---
clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp | 111 ++--
clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp | 84 ++-
.../CIR/CodeGenCUDA/builtins-nvvm-half.cu | 48 +-
.../CIR/CodeGenCUDA/builtins-nvvm-math.cu | 32 ++
.../builtins-nvptx-native-half-type-native.c | 22 +-
.../CodeGen/builtins-nvptx-native-half-type.c | 24 +-
clang/test/CodeGen/builtins-nvptx.c | 24 +-
llvm/docs/NVPTXUsage.md | 69 ++-
llvm/include/llvm/IR/IntrinsicsNVVM.td | 53 +-
llvm/include/llvm/IR/NVVMIntrinsicUtils.h | 85 ++-
llvm/lib/Analysis/ConstantFolding.cpp | 93 ++--
llvm/lib/IR/AutoUpgrade.cpp | 68 ++-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 18 +-
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 121 +++--
.../Target/NVPTX/NVPTXTargetTransformInfo.cpp | 42 +-
.../Assembler/auto_upgrade_nvvm_intrinsics.ll | 57 ++
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 16 +-
llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll | 133 +++++
llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll | 65 +++
.../CodeGen/NVPTX/math-intrins-sm53-ptx42.ll | 32 +-
.../math-intrins-sm80-ptx70-instcombine.ll | 32 +-
.../CodeGen/NVPTX/math-intrins-sm80-ptx70.ll | 32 +-
llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll | 32 +-
.../InstCombine/NVPTX/nvvm-intrins.ll | 16 +-
.../InstSimplify/const-fold-nvvm-fma.ll | 488 ++++++++++++++----
llvm/test/Verifier/NVPTX/fma.ll | 16 +
.../Verifier/invalid_intrinsic_signatures.ll | 4 +-
.../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 121 ++---
mlir/test/Target/LLVMIR/nvvm/fma/fma.mlir | 56 +-
.../Target/LLVMIR/nvvm/fma/fma_vector.mlir | 288 ++++-------
30 files changed, 1453 insertions(+), 829 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll
create mode 100644 llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll
create mode 100644 llvm/test/Verifier/NVPTX/fma.ll
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
index 146dad6b0ec413..fc984df8e7d67b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltinNVPTX.cpp
@@ -85,6 +85,24 @@ static mlir::Value emitNVVMFPArith(CIRGenFunction &cgf, const CallExpr *expr,
.getResult();
}
+/// Emit a CIR LLVMIntrinsicCallOp for an NVVM fma intrinsic, which takes the
+/// rounding mode as a trailing operand.
+static mlir::Value emitNVVMFMA(CIRGenFunction &cgf, const CallExpr *expr,
+ llvm::StringRef intrinsicName,
+ llvm::APFloat::roundingMode rm) {
+ auto &builder = cgf.getBuilder();
+ mlir::Location loc = cgf.getLoc(expr->getExprLoc());
+ mlir::Value a = cgf.emitScalarExpr(expr->getArg(0));
+ mlir::Value b = cgf.emitScalarExpr(expr->getArg(1));
+ mlir::Value c = cgf.emitScalarExpr(expr->getArg(2));
+ mlir::Value rnd =
+ builder.getConstInt(loc, builder.getSInt32Ty(), static_cast<int>(rm));
+ return cir::LLVMIntrinsicCallOp::create(builder, loc,
+ builder.getStringAttr(intrinsicName),
+ a.getType(), {a, b, c, rnd})
+ .getResult();
+}
+
static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr,
llvm::StringRef intrinsicName,
bool returnsPred) {
@@ -552,43 +570,76 @@ CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
getContext().BuiltinInfo.getName(builtinId));
return mlir::Value{};
case NVPTX::BI__nvvm_fma_rn_f16:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.f16")
- .getValue();
case NVPTX::BI__nvvm_fma_rn_f16x2:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.f16x2")
- .getValue();
+ case NVPTX::BI__nvvm_fma_rn_bf16:
+ case NVPTX::BI__nvvm_fma_rn_bf16x2:
+ case NVPTX::BI__nvvm_fma_rn_f:
+ case NVPTX::BI__nvvm_fma_rn_d:
+ return emitNVVMFMA(*this, expr, "nvvm.fma",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_fma_rz_f:
+ case NVPTX::BI__nvvm_fma_rz_d:
+ return emitNVVMFMA(*this, expr, "nvvm.fma", llvm::APFloat::rmTowardZero);
+ case NVPTX::BI__nvvm_fma_rm_f:
+ case NVPTX::BI__nvvm_fma_rm_d:
+ return emitNVVMFMA(*this, expr, "nvvm.fma",
+ llvm::APFloat::rmTowardNegative);
+ case NVPTX::BI__nvvm_fma_rp_f:
+ case NVPTX::BI__nvvm_fma_rp_d:
+ return emitNVVMFMA(*this, expr, "nvvm.fma",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_fma_rn_ftz_f16:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.f16")
- .getValue();
case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.f16x2")
- .getValue();
- case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.relu.f16")
- .getValue();
- case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
- return emitBuiltinWithOneOverloadedType<3>(expr,
- "nvvm.fma.rn.ftz.relu.f16x2")
- .getValue();
+ case NVPTX::BI__nvvm_fma_rn_ftz_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_fma_rz_ftz_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz",
+ llvm::APFloat::rmTowardZero);
+ case NVPTX::BI__nvvm_fma_rm_ftz_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz",
+ llvm::APFloat::rmTowardNegative);
+ case NVPTX::BI__nvvm_fma_rp_ftz_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz",
+ llvm::APFloat::rmTowardPositive);
+ case NVPTX::BI__nvvm_fma_rn_sat_f16:
+ case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
+ case NVPTX::BI__nvvm_fma_rn_sat_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.sat",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_fma_rz_sat_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.sat",
+ llvm::APFloat::rmTowardZero);
+ case NVPTX::BI__nvvm_fma_rm_sat_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.sat",
+ llvm::APFloat::rmTowardNegative);
+ case NVPTX::BI__nvvm_fma_rp_sat_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.sat",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.sat.f16")
- .getValue();
case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
- return emitBuiltinWithOneOverloadedType<3>(expr,
- "nvvm.fma.rn.ftz.sat.f16x2")
- .getValue();
+ case NVPTX::BI__nvvm_fma_rn_ftz_sat_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.sat",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_fma_rz_ftz_sat_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.sat",
+ llvm::APFloat::rmTowardZero);
+ case NVPTX::BI__nvvm_fma_rm_ftz_sat_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.sat",
+ llvm::APFloat::rmTowardNegative);
+ case NVPTX::BI__nvvm_fma_rp_ftz_sat_f:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.sat",
+ llvm::APFloat::rmTowardPositive);
case NVPTX::BI__nvvm_fma_rn_relu_f16:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.relu.f16")
- .getValue();
case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.relu.f16x2")
- .getValue();
- case NVPTX::BI__nvvm_fma_rn_sat_f16:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.sat.f16")
- .getValue();
- case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
- return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.sat.f16x2")
- .getValue();
+ case NVPTX::BI__nvvm_fma_rn_relu_bf16:
+ case NVPTX::BI__nvvm_fma_rn_relu_bf16x2:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.relu",
+ llvm::APFloat::rmNearestTiesToEven);
+ case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
+ case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
+ return emitNVVMFMA(*this, expr, "nvvm.fma.ftz.relu",
+ llvm::APFloat::rmNearestTiesToEven);
case NVPTX::BI__nvvm_fma_rn_oob_f16:
case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
case NVPTX::BI__nvvm_fma_rn_oob_bf16:
diff --git a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
index a513fcc12953ba..691d181d89bb8a 100644
--- a/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/NVPTX.cpp
@@ -1180,33 +1180,77 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
#undef PZO_CVT
case NVPTX::BI__nvvm_fma_rn_f16:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *this);
case NVPTX::BI__nvvm_fma_rn_f16x2:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rn_bf16:
+ case NVPTX::BI__nvvm_fma_rn_bf16x2:
+ case NVPTX::BI__nvvm_fma_rn_f:
+ case NVPTX::BI__nvvm_fma_rn_d:
+ return MakeFPArith(Intrinsic::nvvm_fma, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rz_f:
+ case NVPTX::BI__nvvm_fma_rz_d:
+ return MakeFPArith(Intrinsic::nvvm_fma, APFloat::rmTowardZero, BuiltinID, E,
+ *this);
+ case NVPTX::BI__nvvm_fma_rm_f:
+ case NVPTX::BI__nvvm_fma_rm_d:
+ return MakeFPArith(Intrinsic::nvvm_fma, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rp_f:
+ case NVPTX::BI__nvvm_fma_rp_d:
+ return MakeFPArith(Intrinsic::nvvm_fma, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_fma_rn_ftz_f16:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *this);
case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *this);
- case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
- *this);
- case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
- *this);
+ case NVPTX::BI__nvvm_fma_rn_ftz_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rz_ftz_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz, APFloat::rmTowardZero,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rm_ftz_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rp_ftz_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rn_sat_f16:
+ case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
+ case NVPTX::BI__nvvm_fma_rn_sat_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_sat, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rz_sat_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_sat, APFloat::rmTowardZero,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rm_sat_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_sat, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rp_sat_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_sat, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
- *this);
case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
- *this);
+ case NVPTX::BI__nvvm_fma_rn_ftz_sat_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz_sat,
+ APFloat::rmNearestTiesToEven, BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rz_ftz_sat_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz_sat, APFloat::rmTowardZero,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rm_ftz_sat_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz_sat, APFloat::rmTowardNegative,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rp_ftz_sat_f:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz_sat, APFloat::rmTowardPositive,
+ BuiltinID, E, *this);
case NVPTX::BI__nvvm_fma_rn_relu_f16:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *this);
case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *this);
- case NVPTX::BI__nvvm_fma_rn_sat_f16:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *this);
- case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
- return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rn_relu_bf16:
+ case NVPTX::BI__nvvm_fma_rn_relu_bf16x2:
+ return MakeFPArith(Intrinsic::nvvm_fma_relu, APFloat::rmNearestTiesToEven,
+ BuiltinID, E, *this);
+ case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
+ case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
+ return MakeFPArith(Intrinsic::nvvm_fma_ftz_relu,
+ APFloat::rmNearestTiesToEven, BuiltinID, E, *this);
case NVPTX::BI__nvvm_fma_rn_oob_f16:
return MakeFMAOOB(Intrinsic::nvvm_fma_rn_oob, Builder.getHalfTy(), E,
*this);
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-half.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-half.cu
index d8dd88616d830c..98d902f8aa9c31 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-half.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-half.cu
@@ -39,97 +39,97 @@ extern "C" __device__ f16x2 test_ex2_approx_f16x2(f16x2 a0) {
}
// CIR-LABEL: @test_fma_rn_f16
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.f16" {{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16
+// CIR: cir.call_llvm_intrinsic "nvvm.fma" {{.*}} : (!cir.f16, !cir.f16, !cir.f16, !s32i) -> !cir.f16
// LLVM-LABEL: @test_fma_rn_f16(
-// LLVM: call{{.*}} half @llvm.nvvm.fma.rn.f16(half %{{.*}}, half %{{.*}}, half %{{.*}})
+// LLVM: call{{.*}} half @llvm.nvvm.fma.f16(half %{{.*}}, half %{{.*}}, half %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ _Float16 test_fma_rn_f16(_Float16 a0, _Float16 a1, _Float16 a2) {
return __nvvm_fma_rn_f16(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_f16x2
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.f16x2" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>) -> !cir.vector<2 x !cir.f16>
+// CIR: cir.call_llvm_intrinsic "nvvm.fma" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !s32i) -> !cir.vector<2 x !cir.f16>
// LLVM-LABEL: @test_fma_rn_f16x2(
-// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.rn.f16x2(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ f16x2 test_fma_rn_f16x2(f16x2 a0, f16x2 a1, f16x2 a2) {
return __nvvm_fma_rn_f16x2(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_ftz_f16
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.ftz.f16" {{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.ftz" {{.*}} : (!cir.f16, !cir.f16, !cir.f16, !s32i) -> !cir.f16
// LLVM-LABEL: @test_fma_rn_ftz_f16(
-// LLVM: call{{.*}} half @llvm.nvvm.fma.rn.ftz.f16(half %{{.*}}, half %{{.*}}, half %{{.*}})
+// LLVM: call{{.*}} half @llvm.nvvm.fma.ftz.f16(half %{{.*}}, half %{{.*}}, half %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ _Float16 test_fma_rn_ftz_f16(_Float16 a0, _Float16 a1, _Float16 a2) {
return __nvvm_fma_rn_ftz_f16(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_ftz_f16x2
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.ftz.f16x2" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>) -> !cir.vector<2 x !cir.f16>
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.ftz" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !s32i) -> !cir.vector<2 x !cir.f16>
// LLVM-LABEL: @test_fma_rn_ftz_f16x2(
-// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ f16x2 test_fma_rn_ftz_f16x2(f16x2 a0, f16x2 a1, f16x2 a2) {
return __nvvm_fma_rn_ftz_f16x2(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_ftz_relu_f16
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.ftz.relu.f16" {{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.ftz.relu" {{.*}} : (!cir.f16, !cir.f16, !cir.f16, !s32i) -> !cir.f16
// LLVM-LABEL: @test_fma_rn_ftz_relu_f16(
-// LLVM: call{{.*}} half @llvm.nvvm.fma.rn.ftz.relu.f16(half %{{.*}}, half %{{.*}}, half %{{.*}})
+// LLVM: call{{.*}} half @llvm.nvvm.fma.ftz.relu.f16(half %{{.*}}, half %{{.*}}, half %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ _Float16 test_fma_rn_ftz_relu_f16(_Float16 a0, _Float16 a1, _Float16 a2) {
return __nvvm_fma_rn_ftz_relu_f16(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_ftz_relu_f16x2
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.ftz.relu.f16x2" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>) -> !cir.vector<2 x !cir.f16>
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.ftz.relu" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !s32i) -> !cir.vector<2 x !cir.f16>
// LLVM-LABEL: @test_fma_rn_ftz_relu_f16x2(
-// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.rn.ftz.relu.f16x2(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.ftz.relu.v2f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ f16x2 test_fma_rn_ftz_relu_f16x2(f16x2 a0, f16x2 a1, f16x2 a2) {
return __nvvm_fma_rn_ftz_relu_f16x2(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_ftz_sat_f16
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.ftz.sat.f16" {{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.ftz.sat" {{.*}} : (!cir.f16, !cir.f16, !cir.f16, !s32i) -> !cir.f16
// LLVM-LABEL: @test_fma_rn_ftz_sat_f16(
-// LLVM: call{{.*}} half @llvm.nvvm.fma.rn.ftz.sat.f16(half %{{.*}}, half %{{.*}}, half %{{.*}})
+// LLVM: call{{.*}} half @llvm.nvvm.fma.ftz.sat.f16(half %{{.*}}, half %{{.*}}, half %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ _Float16 test_fma_rn_ftz_sat_f16(_Float16 a0, _Float16 a1, _Float16 a2) {
return __nvvm_fma_rn_ftz_sat_f16(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_ftz_sat_f16x2
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.ftz.sat.f16x2" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>) -> !cir.vector<2 x !cir.f16>
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.ftz.sat" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !s32i) -> !cir.vector<2 x !cir.f16>
// LLVM-LABEL: @test_fma_rn_ftz_sat_f16x2(
-// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ f16x2 test_fma_rn_ftz_sat_f16x2(f16x2 a0, f16x2 a1, f16x2 a2) {
return __nvvm_fma_rn_ftz_sat_f16x2(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_relu_f16
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.relu.f16" {{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.relu" {{.*}} : (!cir.f16, !cir.f16, !cir.f16, !s32i) -> !cir.f16
// LLVM-LABEL: @test_fma_rn_relu_f16(
-// LLVM: call{{.*}} half @llvm.nvvm.fma.rn.relu.f16(half %{{.*}}, half %{{.*}}, half %{{.*}})
+// LLVM: call{{.*}} half @llvm.nvvm.fma.relu.f16(half %{{.*}}, half %{{.*}}, half %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ _Float16 test_fma_rn_relu_f16(_Float16 a0, _Float16 a1, _Float16 a2) {
return __nvvm_fma_rn_relu_f16(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_relu_f16x2
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.relu.f16x2" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>) -> !cir.vector<2 x !cir.f16>
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.relu" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !s32i) -> !cir.vector<2 x !cir.f16>
// LLVM-LABEL: @test_fma_rn_relu_f16x2(
-// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.rn.relu.f16x2(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.relu.v2f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ f16x2 test_fma_rn_relu_f16x2(f16x2 a0, f16x2 a1, f16x2 a2) {
return __nvvm_fma_rn_relu_f16x2(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_sat_f16
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.sat.f16" {{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.sat" {{.*}} : (!cir.f16, !cir.f16, !cir.f16, !s32i) -> !cir.f16
// LLVM-LABEL: @test_fma_rn_sat_f16(
-// LLVM: call{{.*}} half @llvm.nvvm.fma.rn.sat.f16(half %{{.*}}, half %{{.*}}, half %{{.*}})
+// LLVM: call{{.*}} half @llvm.nvvm.fma.sat.f16(half %{{.*}}, half %{{.*}}, half %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ _Float16 test_fma_rn_sat_f16(_Float16 a0, _Float16 a1, _Float16 a2) {
return __nvvm_fma_rn_sat_f16(a0, a1, a2);
}
// CIR-LABEL: @test_fma_rn_sat_f16x2
-// CIR: cir.call_llvm_intrinsic "nvvm.fma.rn.sat.f16x2" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>) -> !cir.vector<2 x !cir.f16>
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.sat" {{.*}} : (!cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !cir.vector<2 x !cir.f16>, !s32i) -> !cir.vector<2 x !cir.f16>
// LLVM-LABEL: @test_fma_rn_sat_f16x2(
-// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.rn.sat.f16x2(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}})
+// LLVM: call{{.*}} <2 x half> @llvm.nvvm.fma.sat.v2f16(<2 x half> %{{.*}}, <2 x half> %{{.*}}, <2 x half> %{{.*}}, /* rnd=rn */ i32 1)
extern "C" __device__ f16x2 test_fma_rn_sat_f16x2(f16x2 a0, f16x2 a1, f16x2 a2) {
return __nvvm_fma_rn_sat_f16x2(a0, a1, a2);
}
diff --git a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
index 9660598f680d77..1b3f5c79bb44a7 100644
--- a/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
+++ b/clang/test/CIR/CodeGenCUDA/builtins-nvvm-math.cu
@@ -111,3 +111,35 @@ __device__ double test_mul_rz_d(double x, double y) {
__device__ float test_mul_rp_ftz_f(float x, float y) {
return __nvvm_mul_rp_ftz_f(x, y);
}
+
+// CIR-LABEL: @_Z13test_fma_rn_ffff
+// CIR: cir.call_llvm_intrinsic "nvvm.fma" {{.*}} : (!cir.float, !cir.float, !cir.float, !s32i) -> !cir.float
+// LLVM-LABEL: @_Z13test_fma_rn_ffff
+// LLVM: call {{.*}}float @llvm.nvvm.fma.f32(float {{.*}}, float {{.*}}, float {{.*}}, /* rnd=rn */ i32 1)
+__device__ float test_fma_rn_f(float x, float y, float z) {
+ return __nvvm_fma_rn_f(x, y, z);
+}
+
+// CIR-LABEL: @_Z13test_fma_rz_dddd
+// CIR: cir.call_llvm_intrinsic "nvvm.fma" {{.*}} : (!cir.double, !cir.double, !cir.double, !s32i) -> !cir.double
+// LLVM-LABEL: @_Z13test_fma_rz_dddd
+// LLVM: call {{.*}}double @llvm.nvvm.fma.f64(double {{.*}}, double {{.*}}, double {{.*}}, /* rnd=rz */ i32 0)
+__device__ double test_fma_rz_d(double x, double y, double z) {
+ return __nvvm_fma_rz_d(x, y, z);
+}
+
+// CIR-LABEL: @_Z21test_fma_rm_ftz_sat_ffff
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.ftz.sat" {{.*}} : (!cir.float, !cir.float, !cir.float, !s32i) -> !cir.float
+// LLVM-LABEL: @_Z21test_fma_rm_ftz_sat_ffff
+// LLVM: call {{.*}}float @llvm.nvvm.fma.ftz.sat.f32(float {{.*}}, float {{.*}}, float {{.*}}, /* rnd=rm */ i32 3)
+__device__ float test_fma_rm_ftz_sat_f(float x, float y, float z) {
+ return __nvvm_fma_rm_ftz_sat_f(x, y, z);
+}
+
+// CIR-LABEL: @_Z17test_fma_rp_sat_ffff
+// CIR: cir.call_llvm_intrinsic "nvvm.fma.sat" {{.*}} : (!cir.float, !cir.float, !cir.float, !s32i) -> !cir.float
+// LLVM-LABEL: @_Z17test_fma_rp_sat_ffff
+// LLVM: call {{.*}}float @llvm.nvvm.fma.sat.f32(float {{.*}}, float {{.*}}, float {{.*}}, /* rnd=rp */ i32 2)
+__device__ float test_fma_rp_sat_f(float x, float y, float z) {
+ return __nvvm_fma_rp_sat_f(x, y, z);
+}
diff --git a/clang/test/CodeGen/builtins-nvptx-native-half-type-native.c b/clang/test/CodeGen/builtins-nvptx-native-half-type-native.c
index 60a35f4fe0c370..7d6a06379124c4 100644
--- a/clang/test/CodeGen/builtins-nvptx-native-half-type-native.c
+++ b/clang/test/CodeGen/builtins-nvptx-native-half-type-native.c
@@ -9,17 +9,17 @@ typedef __fp16 __fp16v2 __attribute__((ext_vector_type(2)));
// CHECK: call half @llvm.nvvm.ex2.approx.f16(half {{.*}})
// CHECK: call <2 x half> @llvm.nvvm.ex2.approx.v2f16(<2 x half> {{.*}})
-// CHECK: call half @llvm.nvvm.fma.rn.relu.f16(half {{.*}}, half {{.*}}, half {{.*}})
-// CHECK: call half @llvm.nvvm.fma.rn.ftz.relu.f16(half {{.*}}, half {{.*}}, half {{.*}})
-// CHECK: call <2 x half> @llvm.nvvm.fma.rn.relu.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}}, <2 x half> {{.*}})
-// CHECK: call <2 x half> @llvm.nvvm.fma.rn.ftz.relu.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}}, <2 x half> {{.*}})
-// CHECK: call half @llvm.nvvm.fma.rn.ftz.f16(half {{.*}}, half {{.*}}, half {{.*}})
-// CHECK: call half @llvm.nvvm.fma.rn.sat.f16(half {{.*}}, half {{.*}}, half {{.*}})
-// CHECK: call half @llvm.nvvm.fma.rn.ftz.sat.f16(half {{.*}}, half {{.*}}, half {{.*}})
-// CHECK: call <2 x half> @llvm.nvvm.fma.rn.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}}, <2 x half> {{.*}})
-// CHECK: call <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}}, <2 x half> {{.*}})
-// CHECK: call <2 x half> @llvm.nvvm.fma.rn.sat.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}}, <2 x half> {{.*}})
-// CHECK: call <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2(<2 x half> {{.*}}, <2 x half> {{.*}}, <2 x half> {{.*}})
+// CHECK: call half @llvm.nvvm.fma.relu.f16({{.*}}i32 1)
+// CHECK: call half @llvm.nvvm.fma.ftz.relu.f16({{.*}}i32 1)
+// CHECK: call <2 x half> @llvm.nvvm.fma.relu.v2f16({{.*}}i32 1)
+// CHECK: call <2 x half> @llvm.nvvm.fma.ftz.relu.v2f16({{.*}}i32 1)
+// CHECK: call half @llvm.nvvm.fma.ftz.f16({{.*}}i32 1)
+// CHECK: call half @llvm.nvvm.fma.sat.f16({{.*}}i32 1)
+// CHECK: call half @llvm.nvvm.fma.ftz.sat.f16({{.*}}i32 1)
+// CHECK: call <2 x half> @llvm.nvvm.fma.v2f16({{.*}}i32 1)
+// CHECK: call <2 x half> @llvm.nvvm.fma.ftz.v2f16({{.*}}i32 1)
+// CHECK: call <2 x half> @llvm.nvvm.fma.sat.v2f16({{.*}}i32 1)
+// CHECK: call <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16({{.*}}i32 1)
// CHECK: call half @llvm.nvvm.fmin.f16(half {{.*}}, half {{.*}})
// CHECK: call half @llvm.nvvm.fmin.ftz.f16(half {{.*}}, half {{.*}})
// CHECK: call half @llvm.nvvm.fmin.nan.f16(half {{.*}}, half {{.*}})
diff --git a/clang/test/CodeGen/builtins-nvptx-native-half-type.c b/clang/test/CodeGen/builtins-nvptx-native-half-type.c
index 6092054feb96f0..5f221c3151f369 100644
--- a/clang/test/CodeGen/builtins-nvptx-native-half-type.c
+++ b/clang/test/CodeGen/builtins-nvptx-native-half-type.c
@@ -90,15 +90,15 @@ __device__ void nvvm_min_max_sm80() {
// CHECK-LABEL: nvvm_fma_f16_f16x2_sm80
__device__ void nvvm_fma_f16_f16x2_sm80() {
#if __CUDA_ARCH__ >= 800
- // CHECK_PTX70_SM80: call half @llvm.nvvm.fma.rn.relu.f16
+ // CHECK_PTX70_SM80: call half @llvm.nvvm.fma.relu.f16({{.*}}i32 1)
__nvvm_fma_rn_relu_f16(0.1f16, 0.1f16, 0.1f16);
- // CHECK_PTX70_SM80: call half @llvm.nvvm.fma.rn.ftz.relu.f16
+ // CHECK_PTX70_SM80: call half @llvm.nvvm.fma.ftz.relu.f16({{.*}}i32 1)
__nvvm_fma_rn_ftz_relu_f16(0.1f16, 0.1f16, 0.1f16);
- // CHECK_PTX70_SM80: call <2 x half> @llvm.nvvm.fma.rn.relu.f16x2
+ // CHECK_PTX70_SM80: call <2 x half> @llvm.nvvm.fma.relu.v2f16({{.*}}i32 1)
__nvvm_fma_rn_relu_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
{0.1f16, 0.7f16});
- // CHECK_PTX70_SM80: call <2 x half> @llvm.nvvm.fma.rn.ftz.relu.f16x2
+ // CHECK_PTX70_SM80: call <2 x half> @llvm.nvvm.fma.ftz.relu.v2f16({{.*}}i32 1)
__nvvm_fma_rn_ftz_relu_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
{0.1f16, 0.7f16});
#endif
@@ -108,25 +108,25 @@ __device__ void nvvm_fma_f16_f16x2_sm80() {
// CHECK-LABEL: nvvm_fma_f16_f16x2_sm53
__device__ void nvvm_fma_f16_f16x2_sm53() {
#if __CUDA_ARCH__ >= 530
- // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.rn.f16
+ // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.f16({{.*}}i32 1)
__nvvm_fma_rn_f16(0.1f16, 0.1f16, 0.1f16);
- // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.rn.ftz.f16
+ // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.ftz.f16({{.*}}i32 1)
__nvvm_fma_rn_ftz_f16(0.1f16, 0.1f16, 0.1f16);
- // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.rn.sat.f16
+ // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.sat.f16({{.*}}i32 1)
__nvvm_fma_rn_sat_f16(0.1f16, 0.1f16, 0.1f16);
- // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.rn.ftz.sat.f16
+ // CHECK_PTX65_SM53: call half @llvm.nvvm.fma.ftz.sat.f16({{.*}}i32 1)
__nvvm_fma_rn_ftz_sat_f16(0.1f16, 0.1f16, 0.1f16);
- // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.rn.f16x2
+ // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.v2f16({{.*}}i32 1)
__nvvm_fma_rn_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
{0.1f16, 0.7f16});
- // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2
+ // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.ftz.v2f16({{.*}}i32 1)
__nvvm_fma_rn_ftz_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
{0.1f16, 0.7f16});
- // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.rn.sat.f16x2
+ // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.sat.v2f16({{.*}}i32 1)
__nvvm_fma_rn_sat_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
{0.1f16, 0.7f16});
- // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2
+ // CHECK_PTX65_SM53: call <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16({{.*}}i32 1)
__nvvm_fma_rn_ftz_sat_f16x2({0.1f16, 0.7f16}, {0.1f16, 0.7f16},
{0.1f16, 0.7f16});
#endif
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index b7126a82b8064b..996d57461a5a30 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -1651,13 +1651,13 @@ __device__ void nvvm_min_max_sm80() {
// CHECK-LABEL: nvvm_fma_bf16_bf16x2_sm80
__device__ void nvvm_fma_bf16_bf16x2_sm80() {
#if __CUDA_ARCH__ >= 800
- // CHECK_PTX70_SM80: call bfloat @llvm.nvvm.fma.rn.bf16
+ // CHECK_PTX70_SM80: call bfloat @llvm.nvvm.fma.bf16({{.*}}i32 1)
__nvvm_fma_rn_bf16(BF16, BF16_2, BF16_2);
- // CHECK_PTX70_SM80: call bfloat @llvm.nvvm.fma.rn.relu.bf16
+ // CHECK_PTX70_SM80: call bfloat @llvm.nvvm.fma.relu.bf16({{.*}}i32 1)
__nvvm_fma_rn_relu_bf16(BF16, BF16_2, BF16_2);
- // CHECK_PTX70_SM80: call <2 x bfloat> @llvm.nvvm.fma.rn.bf16x2
+ // CHECK_PTX70_SM80: call <2 x bfloat> @llvm.nvvm.fma.v2bf16({{.*}}i32 1)
__nvvm_fma_rn_bf16x2(BF16X2, BF16X2_2, BF16X2_2);
- // CHECK_PTX70_SM80: call <2 x bfloat> @llvm.nvvm.fma.rn.relu.bf16x2
+ // CHECK_PTX70_SM80: call <2 x bfloat> @llvm.nvvm.fma.relu.v2bf16({{.*}}i32 1)
__nvvm_fma_rn_relu_bf16x2(BF16X2, BF16X2_2, BF16X2_2);
#endif
// CHECK: ret void
@@ -1723,21 +1723,21 @@ __device__ void nvvm_add_fma_f32_sat() {
// CHECK: call float @llvm.nvvm.fadd.ftz.sat.f32({{.*}}i32 2)
__nvvm_add_rp_ftz_sat_f(1.0f, 2.0f);
- // CHECK: call float @llvm.nvvm.fma.rn.sat.f
+ // CHECK: call float @llvm.nvvm.fma.sat.f32({{.*}}i32 1)
__nvvm_fma_rn_sat_f(1.0f, 2.0f, 3.0f);
- // CHECK: call float @llvm.nvvm.fma.rn.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fma.ftz.sat.f32({{.*}}i32 1)
__nvvm_fma_rn_ftz_sat_f(1.0f, 2.0f, 3.0f);
- // CHECK: call float @llvm.nvvm.fma.rz.sat.f
+ // CHECK: call float @llvm.nvvm.fma.sat.f32({{.*}}i32 0)
__nvvm_fma_rz_sat_f(1.0f, 2.0f, 3.0f);
- // CHECK: call float @llvm.nvvm.fma.rz.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fma.ftz.sat.f32({{.*}}i32 0)
__nvvm_fma_rz_ftz_sat_f(1.0f, 2.0f, 3.0f);
- // CHECK: call float @llvm.nvvm.fma.rm.sat.f
+ // CHECK: call float @llvm.nvvm.fma.sat.f32({{.*}}i32 3)
__nvvm_fma_rm_sat_f(1.0f, 2.0f, 3.0f);
- // CHECK: call float @llvm.nvvm.fma.rm.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fma.ftz.sat.f32({{.*}}i32 3)
__nvvm_fma_rm_ftz_sat_f(1.0f, 2.0f, 3.0f);
- // CHECK: call float @llvm.nvvm.fma.rp.sat.f
+ // CHECK: call float @llvm.nvvm.fma.sat.f32({{.*}}i32 2)
__nvvm_fma_rp_sat_f(1.0f, 2.0f, 3.0f);
- // CHECK: call float @llvm.nvvm.fma.rp.ftz.sat.f
+ // CHECK: call float @llvm.nvvm.fma.ftz.sat.f32({{.*}}i32 2)
__nvvm_fma_rp_ftz_sat_f(1.0f, 2.0f, 3.0f);
// CHECK: ret void
diff --git a/llvm/docs/NVPTXUsage.md b/llvm/docs/NVPTXUsage.md
index a752e4c0894c90..39af1d828ac56e 100644
--- a/llvm/docs/NVPTXUsage.md
+++ b/llvm/docs/NVPTXUsage.md
@@ -1610,23 +1610,21 @@ PTX instruction. The supported combinations are:
- None
```
-#### '`llvm.nvvm.fma.*`' Half-precision Intrinsics
+#### '`llvm.nvvm.fma.*`' Intrinsics
##### Syntax:
-```llvm
-declare half @llvm.nvvm.fma.rn{.ftz}.f16(half %a, half %b, half %c)
-declare <2 x half> @llvm.nvvm.fma.rn{.ftz}.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
-declare bfloat @llvm.nvvm.fma.rn.bf16(bfloat %a, bfloat %b, bfloat %c)
-declare <2 x bfloat> @llvm.nvvm.fma.rn.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
-
-declare half @llvm.nvvm.fma.rn{.ftz}.sat.f16(half %a, half %b, half %c)
-declare <2 x half> @llvm.nvvm.fma.rn{.ftz}.sat.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+These are overloaded intrinsics. The '`.ftz`' modifier and one of the '`.sat`'
+or '`.relu`' modifiers are optional.
-declare half @llvm.nvvm.fma.rn{.ftz}.relu.f16(half %a, half %b, half %c)
-declare <2 x half> @llvm.nvvm.fma.rn{.ftz}.relu.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
-declare bfloat @llvm.nvvm.fma.rn.relu.bf16(bfloat %a, bfloat %b, bfloat %c)
-declare <2 x bfloat> @llvm.nvvm.fma.rn.relu.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+```llvm
+declare half @llvm.nvvm.fma{.ftz}{.sat|.relu}.f16(half %a, half %b, half %c, i32 immarg %rnd)
+declare <2 x half> @llvm.nvvm.fma{.ftz}{.sat|.relu}.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c, i32 immarg %rnd)
+declare bfloat @llvm.nvvm.fma{.relu}.bf16(bfloat %a, bfloat %b, bfloat %c, i32 immarg %rnd)
+declare <2 x bfloat> @llvm.nvvm.fma{.relu}.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c, i32 immarg %rnd)
+declare float @llvm.nvvm.fma{.ftz}{.sat}.f32(float %a, float %b, float %c, i32 immarg %rnd)
+declare <2 x float> @llvm.nvvm.fma{.ftz}.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 immarg %rnd)
+declare double @llvm.nvvm.fma.f64(double %a, double %b, double %c, i32 immarg %rnd)
declare half @llvm.nvvm.fma.rn.oob{.relu}.f16(half %a, half %b, half %c)
declare <2 x half> @llvm.nvvm.fma.rn.oob{.relu}.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c)
@@ -1636,19 +1634,54 @@ declare <2 x bfloat> @llvm.nvvm.fma.rn.oob{.relu}.v2bf16(<2 x bfloat> %a, <2 x b
##### Overview:
-The '`llvm.nvvm.fma.*`' intrinsics perform a fused multiply-add with no loss
-of precision in the intermediate product and addition.
+The '`llvm.nvvm.fma.*`' intrinsics perform a fused multiply-add of `%a`, `%b`
+and `%c` with no loss of precision in the intermediate product and addition,
+using the rounding mode selected by `%rnd` and the modifiers present in the
+intrinsic name. They correspond directly to the `fma` PTX instruction.
##### Semantics:
-The '`.sat`' modifier performs a saturating operation where the result is
-clamped to `[0.0, 1.0]` and `NaN` results are flushed to `+0.0f`.
+`%rnd` selects the rounding mode applied to the result, see
+{ref}`fp-rounding-modes`.
+
The '`.ftz`' modifier flushes subnormal inputs and results to sign-preserving
zero.
+The '`.sat`' modifier performs a saturating operation where the result is
+clamped to `[0.0, 1.0]` and `NaN` results are flushed to `+0.0f`.
The '`.relu`' modifier clamps the result to `0` if negative and `NaN`
results are flushed to canonical `NaN`.
The '`.oob`' modifier clamps the result to `0` if either of the operands is
-an `OOB NaN` (defined under [Tensors](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensors)) value.
+an `OOB NaN` (defined under [Tensors](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensors)) value. The
+'`llvm.nvvm.fma.rn.oob{.relu}`' intrinsics only support the `rn` rounding
+mode, and do not take a rounding-mode operand.
+
+Not every combination of operand type, rounding mode and modifier maps to a
+PTX instruction. For the '`llvm.nvvm.fma{.ftz}{.sat|.relu}`' intrinsics, the
+supported combinations are:
+
+```{list-table}
+:widths: 25 25 25
+:header-rows: 1
+
+ * - Operand Type
+ - Rounding Modes
+ - Modifiers
+ * - `half`, `<2 x half>`
+ - `rn`
+ - `.ftz`, `.sat`, `.relu`
+ * - `bfloat`, `<2 x bfloat>`
+ - `rn`
+ - `.relu`
+ * - `float`
+ - `rn`, `rz`, `rp`, `rm`
+ - `.ftz`, `.sat`
+ * - `<2 x float>`
+ - `rn`, `rz`, `rp`, `rm`
+ - `.ftz`
+ * - `double`
+ - `rn`, `rz`, `rp`, `rm`
+ - None
+```
### Bit Manipulation Intrinsics
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 4a274fcc73b17b..0597b195a5a135 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1825,45 +1825,24 @@ let TargetPrefix = "nvvm" in {
//
// Fma
//
- foreach variant = ["", "_sat", "_relu"] in {
- foreach ftz = ["", "_ftz"] in {
- def int_nvvm_fma_rn # ftz # variant # _f16 :
- PureIntrinsic<[llvm_half_ty],
- [llvm_half_ty, llvm_half_ty, llvm_half_ty]>;
-
- def int_nvvm_fma_rn # ftz # variant # _f16x2 :
- PureIntrinsic<[llvm_v2f16_ty],
- [llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty]>;
- } // ftz
- } // variant
-
- foreach relu = ["", "_relu"] in {
- def int_nvvm_fma_rn # relu # _bf16 : NVVMBuiltin,
- PureIntrinsic<[llvm_bfloat_ty],
- [llvm_bfloat_ty, llvm_bfloat_ty, llvm_bfloat_ty]>;
-
- def int_nvvm_fma_rn # relu # _bf16x2 : NVVMBuiltin,
- PureIntrinsic<[llvm_v2bf16_ty],
- [llvm_v2bf16_ty, llvm_v2bf16_ty, llvm_v2bf16_ty]>;
-
- // oob (out-of-bounds) - clamps the result to 0 if either of the operand is
- // an OOB NaN value.
+ let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative,
+ IntrNoCreateUndefOrPoison, ImmArg<ArgIndex<3>>,
+ Range<ArgIndex<3>, 0, 4>,
+ ArgInfo<ArgIndex<3>,
+ [ArgName<"rnd">,
+ ImmArgPrinter<"printFPRoundingMode">]>] in
+ foreach ftz = ["", "_ftz"] in
+ foreach variant = ["", "_sat", "_relu"] in
+ def int_nvvm_fma # ftz # variant :
+ DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
+ LLVMMatchType<0>, llvm_i32_ty]>;
+
+ // oob (out-of-bounds) - clamps the result to 0 if either of the operand is
+ // an OOB NaN value.
+ foreach relu = ["", "_relu"] in
def int_nvvm_fma_rn_oob # relu : PureIntrinsic<[llvm_anyfloat_ty],
[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
- } // relu
-
- foreach rnd = ["_rn", "_rz", "_rm", "_rp"] in {
- foreach ftz = ["", "_ftz"] in {
- foreach sat = ["", "_sat"] in {
- def int_nvvm_fma # rnd # ftz # sat # _f : NVVMBuiltin,
- PureIntrinsic<[llvm_float_ty],
- [llvm_float_ty, llvm_float_ty, llvm_float_ty]>;
- } // sat
- } // ftz
- def int_nvvm_fma # rnd # _d : NVVMBuiltin,
- PureIntrinsic<[llvm_double_ty],
- [llvm_double_ty, llvm_double_ty, llvm_double_ty]>;
- } // rnd
//
// Rcp
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index c92fa1cf21f1bb..37c9c98cc26749 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -701,34 +701,72 @@ inline bool FPArithShouldFTZ(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_ftz:
case Intrinsic::nvvm_fadd_ftz_sat:
+ case Intrinsic::nvvm_fma_ftz:
+ case Intrinsic::nvvm_fma_ftz_sat:
+ case Intrinsic::nvvm_fma_ftz_relu:
case Intrinsic::nvvm_fmul_ftz:
case Intrinsic::nvvm_fmul_ftz_sat:
return true;
case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_sat:
+ case Intrinsic::nvvm_fma:
+ case Intrinsic::nvvm_fma_sat:
+ case Intrinsic::nvvm_fma_relu:
case Intrinsic::nvvm_fmul:
case Intrinsic::nvvm_fmul_sat:
return false;
}
- llvm_unreachable("Checking FTZ flag for invalid NVVM fadd/fmul intrinsic");
+ llvm_unreachable(
+ "Checking FTZ flag for invalid NVVM fadd/fmul/fma intrinsic");
}
inline bool FPArithIsSaturating(Intrinsic::ID IntrinsicID) {
switch (IntrinsicID) {
case Intrinsic::nvvm_fadd_sat:
case Intrinsic::nvvm_fadd_ftz_sat:
+ case Intrinsic::nvvm_fma_sat:
+ case Intrinsic::nvvm_fma_ftz_sat:
case Intrinsic::nvvm_fmul_sat:
case Intrinsic::nvvm_fmul_ftz_sat:
return true;
case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fma:
+ case Intrinsic::nvvm_fma_ftz:
+ case Intrinsic::nvvm_fma_relu:
+ case Intrinsic::nvvm_fma_ftz_relu:
case Intrinsic::nvvm_fmul:
case Intrinsic::nvvm_fmul_ftz:
return false;
}
- llvm_unreachable("Checking sat flag for invalid NVVM fadd/fmul intrinsic");
+ llvm_unreachable(
+ "Checking sat flag for invalid NVVM fadd/fmul/fma intrinsic");
+}
+
+inline bool FPArithIsRelu(Intrinsic::ID IntrinsicID) {
+ switch (IntrinsicID) {
+ case Intrinsic::nvvm_fma_relu:
+ case Intrinsic::nvvm_fma_ftz_relu:
+ return true;
+
+ case Intrinsic::nvvm_fadd:
+ case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fadd_sat:
+ case Intrinsic::nvvm_fadd_ftz_sat:
+ case Intrinsic::nvvm_fma:
+ case Intrinsic::nvvm_fma_ftz:
+ case Intrinsic::nvvm_fma_sat:
+ case Intrinsic::nvvm_fma_ftz_sat:
+ case Intrinsic::nvvm_fmul:
+ case Intrinsic::nvvm_fmul_ftz:
+ case Intrinsic::nvvm_fmul_sat:
+ case Intrinsic::nvvm_fmul_ftz_sat:
+ return false;
+ }
+ llvm_unreachable(
+ "Checking relu flag for invalid NVVM fadd/fmul/fma intrinsic");
}
inline bool FDivShouldFTZ(Intrinsic::ID IntrinsicID) {
@@ -774,49 +812,6 @@ inline APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID) {
llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM div");
}
-inline bool FMAShouldFTZ(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_fma_rm_ftz_f:
- case Intrinsic::nvvm_fma_rn_ftz_f:
- case Intrinsic::nvvm_fma_rp_ftz_f:
- case Intrinsic::nvvm_fma_rz_ftz_f:
- return true;
-
- case Intrinsic::nvvm_fma_rm_f:
- case Intrinsic::nvvm_fma_rn_f:
- case Intrinsic::nvvm_fma_rp_f:
- case Intrinsic::nvvm_fma_rz_f:
- case Intrinsic::nvvm_fma_rm_d:
- case Intrinsic::nvvm_fma_rn_d:
- case Intrinsic::nvvm_fma_rp_d:
- case Intrinsic::nvvm_fma_rz_d:
- return false;
- }
- llvm_unreachable("Checking FTZ flag for invalid NVVM fma intrinsic");
-}
-
-inline APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID) {
- switch (IntrinsicID) {
- case Intrinsic::nvvm_fma_rm_f:
- case Intrinsic::nvvm_fma_rm_d:
- case Intrinsic::nvvm_fma_rm_ftz_f:
- return APFloat::rmTowardNegative;
- case Intrinsic::nvvm_fma_rn_f:
- case Intrinsic::nvvm_fma_rn_d:
- case Intrinsic::nvvm_fma_rn_ftz_f:
- return APFloat::rmNearestTiesToEven;
- case Intrinsic::nvvm_fma_rp_f:
- case Intrinsic::nvvm_fma_rp_d:
- case Intrinsic::nvvm_fma_rp_ftz_f:
- return APFloat::rmTowardPositive;
- case Intrinsic::nvvm_fma_rz_f:
- case Intrinsic::nvvm_fma_rz_d:
- case Intrinsic::nvvm_fma_rz_ftz_f:
- return APFloat::rmTowardZero;
- }
- llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM fma");
-}
-
} // namespace nvvm
} // namespace llvm
#endif // LLVM_IR_NVVMINTRINSICUTILS_H
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 5169450936dc36..714d2329160994 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -2108,9 +2108,11 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
case Intrinsic::nvvm_sqrt_rn_ftz_f:
return !IsStrictFP;
- // NVVM fadd/fmul intrinsics with explicit rounding modes
+ // NVVM fadd/fmul/fma intrinsics with explicit rounding modes
case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fma:
+ case Intrinsic::nvvm_fma_ftz:
case Intrinsic::nvvm_fmul:
case Intrinsic::nvvm_fmul_ftz:
@@ -2128,20 +2130,6 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
case Intrinsic::nvvm_div_rp_ftz_f:
case Intrinsic::nvvm_div_rz_ftz_f:
- // NVVM fma intrinsics with explicit rounding modes
- case Intrinsic::nvvm_fma_rm_d:
- case Intrinsic::nvvm_fma_rn_d:
- case Intrinsic::nvvm_fma_rp_d:
- case Intrinsic::nvvm_fma_rz_d:
- case Intrinsic::nvvm_fma_rm_f:
- case Intrinsic::nvvm_fma_rn_f:
- case Intrinsic::nvvm_fma_rp_f:
- case Intrinsic::nvvm_fma_rz_f:
- case Intrinsic::nvvm_fma_rm_ftz_f:
- case Intrinsic::nvvm_fma_rn_ftz_f:
- case Intrinsic::nvvm_fma_rp_ftz_f:
- case Intrinsic::nvvm_fma_rz_ftz_f:
-
// Sign operations are actually bitwise operations, they do not raise
// exceptions even for SNANs.
case Intrinsic::fabs:
@@ -4297,37 +4285,6 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
return ConstantFP::get(Ty, V);
}
- case Intrinsic::nvvm_fma_rm_f:
- case Intrinsic::nvvm_fma_rn_f:
- case Intrinsic::nvvm_fma_rp_f:
- case Intrinsic::nvvm_fma_rz_f:
- case Intrinsic::nvvm_fma_rm_d:
- case Intrinsic::nvvm_fma_rn_d:
- case Intrinsic::nvvm_fma_rp_d:
- case Intrinsic::nvvm_fma_rz_d:
- case Intrinsic::nvvm_fma_rm_ftz_f:
- case Intrinsic::nvvm_fma_rn_ftz_f:
- case Intrinsic::nvvm_fma_rp_ftz_f:
- case Intrinsic::nvvm_fma_rz_ftz_f: {
- bool IsFTZ = nvvm::FMAShouldFTZ(IntrinsicID);
- APFloat A = IsFTZ ? FTZPreserveSign(C1) : C1;
- APFloat B = IsFTZ ? FTZPreserveSign(C2) : C2;
- APFloat C = IsFTZ ? FTZPreserveSign(C3) : C3;
-
- APFloat::roundingMode RoundMode =
- nvvm::GetFMARoundingMode(IntrinsicID);
-
- APFloat Res = A;
- APFloat::opStatus Status = Res.fusedMultiplyAdd(B, C, RoundMode);
-
- if (!Res.isNaN() &&
- (Status == APFloat::opOK || Status == APFloat::opInexact)) {
- Res = IsFTZ ? FTZPreserveSign(Res) : Res;
- return ConstantFP::get(Ty, Res);
- }
- return nullptr;
- }
-
case Intrinsic::amdgcn_cubeid:
case Intrinsic::amdgcn_cubema:
case Intrinsic::amdgcn_cubesc:
@@ -4437,6 +4394,45 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
return nullptr;
}
+static Constant *ConstantFoldScalarCall4(StringRef Name,
+ Intrinsic::ID IntrinsicID, Type *Ty,
+ ArrayRef<Constant *> Operands,
+ const TargetLibraryInfo *TLI = nullptr,
+ const CallBase *Call = nullptr) {
+ assert(Operands.size() == 4 && "Wrong number of operands.");
+
+ // TODO: Add constant folding for the _sat and _relu variants.
+ if (IntrinsicID == Intrinsic::nvvm_fma ||
+ IntrinsicID == Intrinsic::nvvm_fma_ftz) {
+ const auto *Op1 = dyn_cast<ConstantFP>(Operands[0]);
+ const auto *Op2 = dyn_cast<ConstantFP>(Operands[1]);
+ const auto *Op3 = dyn_cast<ConstantFP>(Operands[2]);
+ if (!Op1 || !Op2 || !Op3)
+ return nullptr;
+
+ bool IsFTZ = nvvm::FPArithShouldFTZ(IntrinsicID);
+ APFloat A =
+ IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
+ APFloat B =
+ IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
+ APFloat C =
+ IsFTZ ? FTZPreserveSign(Op3->getValueAPF()) : Op3->getValueAPF();
+
+ APFloat Res = A;
+ APFloat::opStatus Status = Res.fusedMultiplyAdd(
+ B, C, nvvm::GetRoundingModeFromImmArg(Operands[3]));
+
+ if (!Res.isNaN() &&
+ (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+ Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+ return ConstantFP::get(Ty, Res);
+ }
+ return nullptr;
+ }
+
+ return nullptr;
+}
+
static Constant *ConstantFoldScalarCall(StringRef Name,
Intrinsic::ID IntrinsicID, Type *Ty,
ArrayRef<Constant *> Operands,
@@ -4461,6 +4457,9 @@ static Constant *ConstantFoldScalarCall(StringRef Name,
if (Operands.size() == 3)
return ConstantFoldScalarCall3(Name, IntrinsicID, Ty, Operands, TLI, Call);
+ if (Operands.size() == 4)
+ return ConstantFoldScalarCall4(Name, IntrinsicID, Ty, Operands, TLI, Call);
+
return nullptr;
}
@@ -4655,6 +4654,8 @@ static Constant *ConstantFoldFixedVectorCall(
}
case Intrinsic::nvvm_fadd:
case Intrinsic::nvvm_fadd_ftz:
+ case Intrinsic::nvvm_fma:
+ case Intrinsic::nvvm_fma_ftz:
case Intrinsic::nvvm_fmul:
case Intrinsic::nvvm_fmul_ftz:
// The rounding mode operand is a scalar, so the lane-wise folding below
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index afb1cd1d48960e..fef13cb15d6fda 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1443,14 +1443,6 @@ shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
}
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
- if (Name.consume_front("fma.rn."))
- return StringSwitch<Intrinsic::ID>(Name)
- .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
- .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
- .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
- .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
- .Default(Intrinsic::not_intrinsic);
-
if (Name.consume_front("fmax."))
return StringSwitch<Intrinsic::ID>(Name)
.Case("bf16", Intrinsic::nvvm_fmax_bf16)
@@ -1562,6 +1554,36 @@ getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID (&IIDs)[2][2]) {
return std::make_pair(IIDs[IsFTZ][IsSat], *RoundingMode);
}
+static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
+getNVVMFMAUpgrade(StringRef Name) {
+ auto [Modifiers, Type] = Name.rsplit('.');
+ if (!is_contained({"f", "d", "f16", "f16x2", "bf16", "bf16x2"}, Type))
+ return std::nullopt;
+
+ std::optional<llvm::RoundingMode> RoundingMode =
+ StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
+ .Case("rn", llvm::RoundingMode::NearestTiesToEven)
+ .Case("rz", llvm::RoundingMode::TowardZero)
+ .Case("rm", llvm::RoundingMode::TowardNegative)
+ .Case("rp", llvm::RoundingMode::TowardPositive)
+ .Default(std::nullopt);
+ if (!RoundingMode)
+ return std::nullopt;
+
+ Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
+ .Case("", Intrinsic::nvvm_fma)
+ .Case(".ftz", Intrinsic::nvvm_fma_ftz)
+ .Case(".sat", Intrinsic::nvvm_fma_sat)
+ .Case(".ftz.sat", Intrinsic::nvvm_fma_ftz_sat)
+ .Case(".relu", Intrinsic::nvvm_fma_relu)
+ .Case(".ftz.relu", Intrinsic::nvvm_fma_ftz_relu)
+ .Default(Intrinsic::not_intrinsic);
+ if (IID == Intrinsic::not_intrinsic)
+ return std::nullopt;
+
+ return std::make_pair(IID, *RoundingMode);
+}
+
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
return Intrinsic::not_intrinsic;
@@ -2210,6 +2232,9 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
// nvvm.ex2.approx.{f,ftz.f,d,f16x2}
Expand =
Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
+ else if (Name.consume_front("fma."))
+ // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
+ Expand = getNVVMFMAUpgrade(Name).has_value();
else if (Name.consume_front("atomic.load."))
// nvvm.atomic.load.add.{f32,f64}.p
// nvvm.atomic.load.{inc,dec}.32.p
@@ -3310,6 +3335,33 @@ static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
: Intrinsic::nvvm_ex2_approx;
Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
+ } else if (Name.consume_front("fma.")) {
+ // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
+ auto FMA = getNVVMFMAUpgrade(Name);
+ assert(FMA && "unsupported nvvm.fma.* intrinsic");
+ auto [IID, RoundingMode] = *FMA;
+
+ // The legacy bf16 variants used integer storage types for the bf16
+ // operands; bitcast them to the floating-point types of the new
+ // overloaded intrinsic.
+ Type *Ty = CI->getType();
+ const bool IsLegacyBF16Storage = Ty->getScalarType()->isIntegerTy();
+ if (IsLegacyBF16Storage)
+ Ty = Name.ends_with("bf16x2")
+ ? FixedVectorType::get(Builder.getBFloatTy(), 2)
+ : static_cast<Type *>(Builder.getBFloatTy());
+
+ SmallVector<Value *, 4> Args;
+ for (unsigned I = 0; I != 3; ++I) {
+ Value *Arg = CI->getArgOperand(I);
+ Args.push_back(IsLegacyBF16Storage ? Builder.CreateBitCast(Arg, Ty)
+ : Arg);
+ }
+ Args.push_back(Builder.getInt32(static_cast<int>(RoundingMode)));
+
+ Rep = Builder.CreateIntrinsic(Ty, IID, Args);
+ if (IsLegacyBF16Storage)
+ Rep = Builder.CreateBitCast(Rep, CI->getType());
} else if (Name.starts_with("atomic.load.add.f32.p") ||
Name.starts_with("atomic.load.add.f64.p")) {
Value *Ptr = CI->getArgOperand(0);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 7d55ba810e1065..7453677065a588 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -7453,15 +7453,17 @@ static bool isSupportedFPArith(SDNode *N, const NVPTXSubtarget &STI,
const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
const bool IsFTZ = nvvm::FPArithShouldFTZ(IID);
const bool IsSat = nvvm::FPArithIsSaturating(IID);
+ const bool IsRelu = nvvm::FPArithIsRelu(IID);
switch (VT.getScalarType().getSimpleVT().SimpleTy) {
case MVT::f16:
return IsRN;
case MVT::bf16:
return IsRN && !IsSat && !IsFTZ && STI.hasNativeBF16Support(ISDOpcode);
case MVT::f32:
- return !VT.isVector() || (!IsSat && STI.hasF32x2Instructions());
+ return !IsRelu &&
+ (!VT.isVector() || (!IsSat && STI.hasF32x2Instructions()));
case MVT::f64:
- return !VT.isVector() && !IsSat && !IsFTZ;
+ return !VT.isVector() && !IsSat && !IsFTZ && !IsRelu;
default:
return false;
}
@@ -7502,6 +7504,18 @@ static SDValue combineIntrinsicWOChain(SDNode *N,
case Intrinsic::nvvm_spcompress:
case Intrinsic::nvvm_spdecompress:
return lowerSPIntrinsic(SDValue(N, 0), DCI.DAG);
+ case Intrinsic::nvvm_fma:
+ case Intrinsic::nvvm_fma_ftz:
+ case Intrinsic::nvvm_fma_sat:
+ case Intrinsic::nvvm_fma_ftz_sat:
+ case Intrinsic::nvvm_fma_relu:
+ case Intrinsic::nvvm_fma_ftz_relu: {
+ const auto RoundingMode = static_cast<APFloat::roundingMode>(
+ N->getConstantOperandAPInt(4).getSExtValue());
+ if (!isSupportedFPArith(N, STI, ISD::FMA, IID, RoundingMode))
+ return diagnoseUnsupportedFPArith(N, DCI.DAG, IID, RoundingMode);
+ break;
+ }
case Intrinsic::nvvm_fmul:
case Intrinsic::nvvm_fmul_ftz:
case Intrinsic::nvvm_fmul_sat:
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index c17cc827bdc82a..7000cca82f6498 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1975,6 +1975,26 @@ class SADInst<string OpcStr, NVPTXRegClass t_regclass,
[(set t_regclass:$dst, (IntOP s0_regclass:$src0, s1_regclass:$src1, s2_regclass:$src2))]>,
Requires<Preds>;
+multiclass F_MATH_3_RND_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP,
+ TImmLeaf rnd, list<Predicate> Preds = []> {
+ def NAME : BasicNVPTXInst<(outs t.RC:$dst),
+ (ins t.SrcOp:$src0, t.SrcOp:$src1, t.SrcOp:$src2), OpcStr>;
+ def : Pat<(t.Ty (IntOP t.Ty:$src0, t.Ty:$src1, t.Ty:$src2, rnd)),
+ (!cast<Instruction>(NAME) t.InOp:$src0, t.InOp:$src1, t.InOp:$src2)>,
+ Requires<Preds>;
+}
+
+multiclass F_MATH_3_RNDOP_TY<string OpcStr, RegTyInfo t, Intrinsic IntOP,
+ list<Predicate> Preds = []> {
+ def NAME : BasicFlagsNVPTXInst<(outs t.RC:$dst),
+ (ins t.SrcOp:$src0, t.SrcOp:$src1, t.SrcOp:$src2),
+ (ins FPRoundingMode:$rnd), OpcStr>;
+ def : Pat<(t.Ty (IntOP t.Ty:$src0, t.Ty:$src1, t.Ty:$src2, timm:$rnd)),
+ (!cast<Instruction>(NAME) t.InOp:$src0, t.InOp:$src1, t.InOp:$src2,
+ $rnd)>,
+ Requires<Preds>;
+}
+
defvar FPRoundingModes = ["rn", "rz", "rm", "rp"];
//
@@ -2321,72 +2341,61 @@ def : Pat<(int_nvvm_cos_approx_f f32:$a), (COS_APPROX_F32 reg_or_fpimm:$a, NoFTZ
// Fma
//
-class FMA_TUPLE<string V, Intrinsic I, RegTyInfo RT, list<Predicate> Preds = []> {
- string Variant = V;
- Intrinsic Intr = I;
- RegTyInfo RegType = RT;
- list<Predicate> Predicates = Preds;
+// f16/f16x2
+foreach t = [F16RT, F16X2RT] in {
+ foreach ftz = ["", "ftz"] in {
+ foreach sat = ["", "sat"] in
+ defm FMA_RN_ # !toupper(StrJoin<"_", [ftz, sat, t.PtxType]>.ret) :
+ F_MATH_3_RND_TY<StrJoin<".", ["fma.rn", ftz, sat, t.PtxType]>.ret, t,
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", ftz, sat]>.ret),
+ fp_rnd_rn_imm, [SM53]>;
+
+ defm FMA_RN_ # !toupper(StrJoin<"_", [ftz, "relu", t.PtxType]>.ret) :
+ F_MATH_3_RND_TY<StrJoin<".", ["fma.rn", ftz, "relu", t.PtxType]>.ret, t,
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", ftz, "relu"]>.ret),
+ fp_rnd_rn_imm, [SM80]>;
+ }
}
-multiclass FMA_INST {
- foreach P = [
- FMA_TUPLE<"_rn_f64", int_nvvm_fma_rn_d, F64RT>,
- FMA_TUPLE<"_rz_f64", int_nvvm_fma_rz_d, F64RT>,
- FMA_TUPLE<"_rm_f64", int_nvvm_fma_rm_d, F64RT>,
- FMA_TUPLE<"_rp_f64", int_nvvm_fma_rp_d, F64RT>,
-
- FMA_TUPLE<"_rn_ftz_f32", int_nvvm_fma_rn_ftz_f, F32RT>,
- FMA_TUPLE<"_rn_ftz_sat_f32", int_nvvm_fma_rn_ftz_sat_f, F32RT>,
- FMA_TUPLE<"_rn_f32", int_nvvm_fma_rn_f, F32RT>,
- FMA_TUPLE<"_rn_sat_f32", int_nvvm_fma_rn_sat_f, F32RT>,
- FMA_TUPLE<"_rz_ftz_f32", int_nvvm_fma_rz_ftz_f, F32RT>,
- FMA_TUPLE<"_rz_ftz_sat_f32", int_nvvm_fma_rz_ftz_sat_f, F32RT>,
- FMA_TUPLE<"_rz_f32", int_nvvm_fma_rz_f, F32RT>,
- FMA_TUPLE<"_rz_sat_f32", int_nvvm_fma_rz_sat_f, F32RT>,
- FMA_TUPLE<"_rm_f32", int_nvvm_fma_rm_f, F32RT>,
- FMA_TUPLE<"_rm_sat_f32", int_nvvm_fma_rm_sat_f, F32RT>,
- FMA_TUPLE<"_rm_ftz_f32", int_nvvm_fma_rm_ftz_f, F32RT>,
- FMA_TUPLE<"_rm_ftz_sat_f32", int_nvvm_fma_rm_ftz_sat_f, F32RT>,
- FMA_TUPLE<"_rp_f32", int_nvvm_fma_rp_f, F32RT>,
- FMA_TUPLE<"_rp_sat_f32", int_nvvm_fma_rp_sat_f, F32RT>,
- FMA_TUPLE<"_rp_ftz_f32", int_nvvm_fma_rp_ftz_f, F32RT>,
- FMA_TUPLE<"_rp_ftz_sat_f32", int_nvvm_fma_rp_ftz_sat_f, F32RT>,
-
- FMA_TUPLE<"_rn_f16", int_nvvm_fma_rn_f16, F16RT, [SM53]>,
- FMA_TUPLE<"_rn_ftz_f16", int_nvvm_fma_rn_ftz_f16, F16RT, [SM53]>,
- FMA_TUPLE<"_rn_sat_f16", int_nvvm_fma_rn_sat_f16, F16RT, [SM53]>,
- FMA_TUPLE<"_rn_ftz_sat_f16", int_nvvm_fma_rn_ftz_sat_f16, F16RT, [SM53]>,
- FMA_TUPLE<"_rn_relu_f16", int_nvvm_fma_rn_relu_f16, F16RT, [SM80]>,
- FMA_TUPLE<"_rn_ftz_relu_f16", int_nvvm_fma_rn_ftz_relu_f16, F16RT, [SM80]>,
-
- FMA_TUPLE<"_rn_bf16", int_nvvm_fma_rn_bf16, BF16RT, [SM80]>,
- FMA_TUPLE<"_rn_relu_bf16", int_nvvm_fma_rn_relu_bf16, BF16RT, [SM80]>,
-
- FMA_TUPLE<"_rn_f16x2", int_nvvm_fma_rn_f16x2, F16X2RT, [SM53]>,
- FMA_TUPLE<"_rn_ftz_f16x2", int_nvvm_fma_rn_ftz_f16x2, F16X2RT, [SM53]>,
- FMA_TUPLE<"_rn_sat_f16x2", int_nvvm_fma_rn_sat_f16x2, F16X2RT, [SM53]>,
- FMA_TUPLE<"_rn_ftz_sat_f16x2", int_nvvm_fma_rn_ftz_sat_f16x2, F16X2RT, [SM53]>,
- FMA_TUPLE<"_rn_relu_f16x2", int_nvvm_fma_rn_relu_f16x2, F16X2RT, [SM80]>,
- FMA_TUPLE<"_rn_ftz_relu_f16x2", int_nvvm_fma_rn_ftz_relu_f16x2, F16X2RT, [SM80]>,
-
- FMA_TUPLE<"_rn_bf16x2", int_nvvm_fma_rn_bf16x2, BF16X2RT, [SM80]>,
- FMA_TUPLE<"_rn_relu_bf16x2", int_nvvm_fma_rn_relu_bf16x2, BF16X2RT, [SM80]>,
- ] in {
- defm !toupper(P.Variant) : F_MATH_3<!strconcat("fma", !subst("_", ".", P.Variant)),
- P.RegType, P.Intr, P.Predicates>;
- }
+// bf16/bf16x2
+foreach t = [BF16RT, BF16X2RT] in
+ foreach relu = ["", "relu"] in
+ defm FMA_RN_ # !toupper(StrJoin<"_", [relu, t.PtxType]>.ret) :
+ F_MATH_3_RND_TY<StrJoin<".", ["fma.rn", relu, t.PtxType]>.ret, t,
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", relu]>.ret),
+ fp_rnd_rn_imm, [hasBF16Math, PTX70, SM80]>;
+
+// f32/f32x2/f64
+// Note: these are prefixed with FMA_RND to avoid clashing with the
+// fixed-rounding fma instructions for the generic fma node (FMA_F32 etc.).
+foreach ftz = ["", "ftz"] in {
+ foreach sat = ["", "sat"] in
+ defm StrJoin<"_", ["FMA_RND", !toupper(ftz), !toupper(sat), "F32"]>.ret :
+ F_MATH_3_RNDOP_TY<StrJoin<".", ["fma.${rnd}", ftz, sat, "f32"]>.ret, F32RT,
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", ftz, sat]>.ret)>;
+
+ defm StrJoin<"_", ["FMA_RND", !toupper(ftz), "F32X2"]>.ret :
+ F_MATH_3_RNDOP_TY<StrJoin<".", ["fma.${rnd}", ftz, "f32x2"]>.ret, F32X2RT,
+ !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", ftz]>.ret),
+ [hasF32x2Instructions]>;
}
-defm FMA : FMA_INST;
+defm FMA_RND_F64 :
+ F_MATH_3_RNDOP_TY<"fma.${rnd}.f64", F64RT, int_nvvm_fma>;
+
+// Mixed precision
+foreach rnd = FPRoundingModes in {
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd_" # rnd # "_imm");
-foreach rnd = FPRoundingModes in
foreach sat = ["", "sat"] in
foreach type = [f16, bf16] in
def FMA_ # !toupper(StrJoin<"_", [rnd, sat, "f32", type.LLVMName]>.ret) :
BasicNVPTXInst<(outs B32:$dst), (ins B16:$a, B16:$b, B32:$c),
StrJoin<".", ["fma", rnd, sat, "f32", type.LLVMName]>.ret, [(set f32:$dst,
- (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", rnd, sat, "f"]>.ret)
- (f32 (fpextend type:$a)), (f32 (fpextend type:$b)), f32:$c))]>, Requires<[SM100]>;
+ (!cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fma", sat]>.ret)
+ (f32 (fpextend type:$a)), (f32 (fpextend type:$b)), f32:$c, rnd_imm))]>,
+ Requires<[SM100]>;
+}
// Pattern for llvm.fma.f32 intrinsic when there is no FTZ flag
let Predicates = [SM100, doNoF32FTZ] in {
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 960cd7540aa907..309f11d439f24f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -18,6 +18,7 @@
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
+#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
@@ -156,24 +157,24 @@ static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
return {Intrinsic::floor, FTZ_MustBeOff};
case Intrinsic::nvvm_floor_ftz_f:
return {Intrinsic::floor, FTZ_MustBeOn};
- case Intrinsic::nvvm_fma_rn_d:
- return {Intrinsic::fma, FTZ_Any};
- case Intrinsic::nvvm_fma_rn_f:
- return {Intrinsic::fma, FTZ_MustBeOff};
- case Intrinsic::nvvm_fma_rn_ftz_f:
- return {Intrinsic::fma, FTZ_MustBeOn};
- case Intrinsic::nvvm_fma_rn_f16:
- return {Intrinsic::fma, FTZ_MustBeOff, true};
- case Intrinsic::nvvm_fma_rn_ftz_f16:
- return {Intrinsic::fma, FTZ_MustBeOn, true};
- case Intrinsic::nvvm_fma_rn_f16x2:
- return {Intrinsic::fma, FTZ_MustBeOff, true};
- case Intrinsic::nvvm_fma_rn_ftz_f16x2:
- return {Intrinsic::fma, FTZ_MustBeOn, true};
- case Intrinsic::nvvm_fma_rn_bf16:
- return {Intrinsic::fma, FTZ_MustBeOff, true};
- case Intrinsic::nvvm_fma_rn_bf16x2:
- return {Intrinsic::fma, FTZ_MustBeOff, true};
+ case Intrinsic::nvvm_fma:
+ case Intrinsic::nvvm_fma_ftz: {
+ // Only fma with the round-to-nearest-even rounding mode matches the
+ // semantics of llvm.fma.
+ if (nvvm::GetRoundingModeFromImmArg(II->getArgOperand(3)) !=
+ APFloat::rmNearestTiesToEven)
+ return {};
+ const bool IsFTZ = II->getIntrinsicID() == Intrinsic::nvvm_fma_ftz;
+ Type *EltTy = II->getType()->getScalarType();
+ if (EltTy->isDoubleTy())
+ return IsFTZ ? SimplifyAction{}
+ : SimplifyAction{Intrinsic::fma, FTZ_Any};
+ // ftz is not supported for bf16.
+ if (EltTy->isBFloatTy() && IsFTZ)
+ return {};
+ const bool IsHalfTy = EltTy->isHalfTy() || EltTy->isBFloatTy();
+ return {Intrinsic::fma, IsFTZ ? FTZ_MustBeOn : FTZ_MustBeOff, IsHalfTy};
+ }
case Intrinsic::nvvm_fmax_d:
return {Intrinsic::maximumnum, FTZ_Any};
case Intrinsic::nvvm_fmax_f:
@@ -317,6 +318,11 @@ static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
// Simplify to target-generic intrinsic.
if (Action.IID) {
SmallVector<Value *, 4> Args(II->args());
+ // The overloaded NVVM fma intrinsics have a trailing rounding-mode
+ // operand that llvm.fma does not take; it is known to be RN here.
+ if (II->getIntrinsicID() == Intrinsic::nvvm_fma ||
+ II->getIntrinsicID() == Intrinsic::nvvm_fma_ftz)
+ Args.pop_back();
// All the target-generic intrinsics currently of interest to us have one
// type argument, equal to that of the nvvm intrinsic's argument.
Type *Tys[] = {II->getArgOperand(0)->getType()};
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index a2174877ac0914..04180d10d9624c 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -836,6 +836,63 @@ define void @nvvm_mul(float %a, double %b, half %c, <2 x half> %d) {
ret void
}
+define void @nvvm_fma(float %a, double %b, half %c, <2 x half> %d,
+ bfloat %e, <2 x bfloat> %f) {
+; CHECK: call float @llvm.nvvm.fma.f32(float %a, float %a, float %a, /* rnd=rn */ i32 1)
+; CHECK: call float @llvm.nvvm.fma.ftz.f32(float %a, float %a, float %a, /* rnd=rz */ i32 0)
+; CHECK: call float @llvm.nvvm.fma.sat.f32(float %a, float %a, float %a, /* rnd=rm */ i32 3)
+; CHECK: call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %a, float %a, /* rnd=rp */ i32 2)
+; CHECK: call double @llvm.nvvm.fma.f64(double %b, double %b, double %b, /* rnd=rn */ i32 1)
+; CHECK: call double @llvm.nvvm.fma.f64(double %b, double %b, double %b, /* rnd=rz */ i32 0)
+; CHECK: call half @llvm.nvvm.fma.f16(half %c, half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fma.ftz.f16(half %c, half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fma.sat.f16(half %c, half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fma.ftz.sat.f16(half %c, half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fma.relu.f16(half %c, half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call half @llvm.nvvm.fma.ftz.relu.f16(half %c, half %c, half %c, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %d, <2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16(<2 x half> %d, <2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call <2 x half> @llvm.nvvm.fma.relu.v2f16(<2 x half> %d, <2 x half> %d, <2 x half> %d, /* rnd=rn */ i32 1)
+; CHECK: call bfloat @llvm.nvvm.fma.bf16(bfloat %e, bfloat %e, bfloat %e, /* rnd=rn */ i32 1)
+; CHECK: call bfloat @llvm.nvvm.fma.relu.bf16(bfloat %e, bfloat %e, bfloat %e, /* rnd=rn */ i32 1)
+; CHECK: call <2 x bfloat> @llvm.nvvm.fma.v2bf16(<2 x bfloat> %f, <2 x bfloat> %f, <2 x bfloat> %f, /* rnd=rn */ i32 1)
+; CHECK: call <2 x bfloat> @llvm.nvvm.fma.relu.v2bf16(<2 x bfloat> %f, <2 x bfloat> %f, <2 x bfloat> %f, /* rnd=rn */ i32 1)
+ %r1 = call float @llvm.nvvm.fma.rn.f(float %a, float %a, float %a)
+ %r2 = call float @llvm.nvvm.fma.rz.ftz.f(float %a, float %a, float %a)
+ %r3 = call float @llvm.nvvm.fma.rm.sat.f(float %a, float %a, float %a)
+ %r4 = call float @llvm.nvvm.fma.rp.ftz.sat.f(float %a, float %a, float %a)
+ %r5 = call double @llvm.nvvm.fma.rn.d(double %b, double %b, double %b)
+ %r6 = call double @llvm.nvvm.fma.rz.d(double %b, double %b, double %b)
+ %r7 = call half @llvm.nvvm.fma.rn.f16(half %c, half %c, half %c)
+ %r8 = call half @llvm.nvvm.fma.rn.ftz.f16(half %c, half %c, half %c)
+ %r9 = call half @llvm.nvvm.fma.rn.sat.f16(half %c, half %c, half %c)
+ %r10 = call half @llvm.nvvm.fma.rn.ftz.sat.f16(half %c, half %c, half %c)
+ %r11 = call half @llvm.nvvm.fma.rn.relu.f16(half %c, half %c, half %c)
+ %r12 = call half @llvm.nvvm.fma.rn.ftz.relu.f16(half %c, half %c, half %c)
+ %r13 = call <2 x half> @llvm.nvvm.fma.rn.f16x2(<2 x half> %d, <2 x half> %d, <2 x half> %d)
+ %r14 = call <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2(<2 x half> %d, <2 x half> %d, <2 x half> %d)
+ %r15 = call <2 x half> @llvm.nvvm.fma.rn.relu.f16x2(<2 x half> %d, <2 x half> %d, <2 x half> %d)
+ %r16 = call bfloat @llvm.nvvm.fma.rn.bf16(bfloat %e, bfloat %e, bfloat %e)
+ %r17 = call bfloat @llvm.nvvm.fma.rn.relu.bf16(bfloat %e, bfloat %e, bfloat %e)
+ %r18 = call <2 x bfloat> @llvm.nvvm.fma.rn.bf16x2(<2 x bfloat> %f, <2 x bfloat> %f, <2 x bfloat> %f)
+ %r19 = call <2 x bfloat> @llvm.nvvm.fma.rn.relu.bf16x2(<2 x bfloat> %f, <2 x bfloat> %f, <2 x bfloat> %f)
+ ret void
+}
+
+define void @nvvm_fma_legacy_bf16(i16 %a, i32 %b) {
+; CHECK-LABEL: @nvvm_fma_legacy_bf16
+; CHECK: [[R1:%.*]] = call bfloat @llvm.nvvm.fma.bf16(bfloat %{{.*}}, bfloat %{{.*}}, bfloat %{{.*}}, /* rnd=rn */ i32 1)
+; CHECK: bitcast bfloat [[R1]] to i16
+; CHECK: [[R2:%.*]] = call <2 x bfloat> @llvm.nvvm.fma.v2bf16(<2 x bfloat> %{{.*}}, <2 x bfloat> %{{.*}}, <2 x bfloat> %{{.*}}, /* rnd=rn */ i32 1)
+; CHECK: bitcast <2 x bfloat> [[R2]] to i32
+; CHECK: [[R3:%.*]] = call bfloat @llvm.nvvm.fma.relu.bf16(bfloat %{{.*}}, bfloat %{{.*}}, bfloat %{{.*}}, /* rnd=rn */ i32 1)
+; CHECK: bitcast bfloat [[R3]] to i16
+ %r1 = call i16 @llvm.nvvm.fma.rn.bf16(i16 %a, i16 %a, i16 %a)
+ %r2 = call i32 @llvm.nvvm.fma.rn.bf16x2(i32 %b, i32 %b, i32 %b)
+ %r3 = call i16 @llvm.nvvm.fma.rn.relu.bf16(i16 %a, i16 %a, i16 %a)
+ ret void
+}
+
declare void @llvm.nvvm.mbarrier.init(ptr, i32)
declare void @llvm.nvvm.mbarrier.init.shared(ptr addrspace(3), i32)
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index 8d1a00ae87eddc..b9aeed2b5d0ea6 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -132,17 +132,17 @@ define float @fma_sat_f32(float %a, float %b, float %c) {
; CHECK-NEXT: fma.rp.ftz.sat.f32 %r11, %r1, %r2, %r10;
; CHECK-NEXT: st.param.b32 [func_retval0], %r11;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.fma.rn.sat.f(float %a, float %b, float %c)
- %r2 = call float @llvm.nvvm.fma.rn.ftz.sat.f(float %a, float %b, float %r1)
+ %r1 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %c, i32 1)
+ %r2 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r1, i32 1)
- %r3 = call float @llvm.nvvm.fma.rz.sat.f(float %a, float %b, float %r2)
- %r4 = call float @llvm.nvvm.fma.rz.ftz.sat.f(float %a, float %b, float %r3)
+ %r3 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r2, i32 0)
+ %r4 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r3, i32 0)
- %r5 = call float @llvm.nvvm.fma.rm.sat.f(float %a, float %b, float %r4)
- %r6 = call float @llvm.nvvm.fma.rm.ftz.sat.f(float %a, float %b, float %r5)
+ %r5 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r4, i32 3)
+ %r6 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r5, i32 3)
- %r7 = call float @llvm.nvvm.fma.rp.sat.f(float %a, float %b, float %r6)
- %r8 = call float @llvm.nvvm.fma.rp.ftz.sat.f(float %a, float %b, float %r7)
+ %r7 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r6, i32 2)
+ %r8 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r7, i32 2)
ret float %r8
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll
new file mode 100644
index 00000000000000..cd933daeddc910
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 | FileCheck %s
+; RUN: %if ptxas-sm_100 && ptxas-isa-8.8 %{ llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 | %ptxas-verify -arch=sm_100 %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+define <2 x float> @fma_rn(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_rn(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [fma_rn_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [fma_rn_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [fma_rn_param_2];
+; CHECK-NEXT: fma.rn.f32x2 %rd4, %rd1, %rd2, %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NEXT: ret;
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @fma_rz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_rz(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [fma_rz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [fma_rz_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [fma_rz_param_2];
+; CHECK-NEXT: fma.rz.f32x2 %rd4, %rd1, %rd2, %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NEXT: ret;
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 0)
+ ret <2 x float> %r
+}
+
+define <2 x float> @fma_rm(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_rm(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [fma_rm_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [fma_rm_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [fma_rm_param_2];
+; CHECK-NEXT: fma.rm.f32x2 %rd4, %rd1, %rd2, %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NEXT: ret;
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 3)
+ ret <2 x float> %r
+}
+
+define <2 x float> @fma_rp(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_rp(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [fma_rp_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [fma_rp_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [fma_rp_param_2];
+; CHECK-NEXT: fma.rp.f32x2 %rd4, %rd1, %rd2, %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NEXT: ret;
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 2)
+ ret <2 x float> %r
+}
+
+define <2 x float> @fma_rn_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_rn_ftz(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [fma_rn_ftz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [fma_rn_ftz_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [fma_rn_ftz_param_2];
+; CHECK-NEXT: fma.rn.ftz.f32x2 %rd4, %rd1, %rd2, %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NEXT: ret;
+ %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @fma_rz_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_rz_ftz(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [fma_rz_ftz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [fma_rz_ftz_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [fma_rz_ftz_param_2];
+; CHECK-NEXT: fma.rz.ftz.f32x2 %rd4, %rd1, %rd2, %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NEXT: ret;
+ %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 0)
+ ret <2 x float> %r
+}
+
+define <2 x float> @fma_rm_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_rm_ftz(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [fma_rm_ftz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [fma_rm_ftz_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [fma_rm_ftz_param_2];
+; CHECK-NEXT: fma.rm.ftz.f32x2 %rd4, %rd1, %rd2, %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NEXT: ret;
+ %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 3)
+ ret <2 x float> %r
+}
+
+define <2 x float> @fma_rp_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: fma_rp_ftz(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [fma_rp_ftz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [fma_rp_ftz_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [fma_rp_ftz_param_2];
+; CHECK-NEXT: fma.rp.ftz.f32x2 %rd4, %rd1, %rd2, %rd3;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NEXT: ret;
+ %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 2)
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll b/llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll
new file mode 100644
index 00000000000000..bbf2521d635fea
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll
@@ -0,0 +1,65 @@
+; RUN: not llc < %s -mcpu=sm_100 -mattr=+ptx88 -march=nvptx64 2>&1 | FileCheck %s
+; RUN: not llc < %s -mcpu=sm_90 -mattr=+ptx78 -march=nvptx64 2>&1 | FileCheck %s --check-prefix=NOF32X2
+; RUN: not llc < %s -mcpu=sm_70 -mattr=+ptx70 -march=nvptx64 2>&1 | FileCheck %s --check-prefix=NOBF16
+
+target triple = "nvptx64-nvidia-cuda"
+
+; CHECK: error: {{.*}}llvm.nvvm.fma.sat with rounding mode rn and operand type v2f32 is not supported
+define <2 x float> @sat_f32x2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+ %r = call <2 x float> @llvm.nvvm.fma.sat.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 1)
+ ret <2 x float> %r
+}
+
+; NOF32X2: error: {{.*}}llvm.nvvm.fma with rounding mode rn and operand type v2f32 is not supported
+define <2 x float> @unsupported_f32x2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 1)
+ ret <2 x float> %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fma.ftz with rounding mode rn and operand type f64 is not supported
+define double @ftz_f64(double %a, double %b, double %c) {
+ %r = call double @llvm.nvvm.fma.ftz.f64(double %a, double %b, double %c, i32 1)
+ ret double %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fma.sat with rounding mode rz and operand type f16 is not supported
+define half @rz_f16(half %a, half %b, half %c) {
+ %r = call half @llvm.nvvm.fma.sat.f16(half %a, half %b, half %c, i32 0)
+ ret half %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fma.ftz with rounding mode rn and operand type bf16 is not supported
+define bfloat @ftz_bf16(bfloat %a, bfloat %b, bfloat %c) {
+ %r = call bfloat @llvm.nvvm.fma.ftz.bf16(bfloat %a, bfloat %b, bfloat %c, i32 1)
+ ret bfloat %r
+}
+
+; NOBF16: error: {{.*}}llvm.nvvm.fma with rounding mode rn and operand type bf16 is not supported
+define bfloat @unsupported_bf16(bfloat %a, bfloat %b, bfloat %c) {
+ %r = call bfloat @llvm.nvvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c, i32 1)
+ ret bfloat %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fma.relu with rounding mode rn and operand type f32 is not supported
+define float @relu_f32(float %a, float %b, float %c) {
+ %r = call float @llvm.nvvm.fma.relu.f32(float %a, float %b, float %c, i32 1)
+ ret float %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fma.relu with rounding mode rn and operand type f64 is not supported
+define double @relu_f64(double %a, double %b, double %c) {
+ %r = call double @llvm.nvvm.fma.relu.f64(double %a, double %b, double %c, i32 1)
+ ret double %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fma.relu with rounding mode rz and operand type f16 is not supported
+define half @relu_rz_f16(half %a, half %b, half %c) {
+ %r = call half @llvm.nvvm.fma.relu.f16(half %a, half %b, half %c, i32 0)
+ ret half %r
+}
+
+; CHECK: error: {{.*}}llvm.nvvm.fma with rounding mode rn and operand type v4f32 is not supported
+define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+ %r = call <4 x float> @llvm.nvvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c, i32 1)
+ ret <4 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins-sm53-ptx42.ll b/llvm/test/CodeGen/NVPTX/math-intrins-sm53-ptx42.ll
index ff0cf3eaafaccb..7936bc7c7d6b7a 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins-sm53-ptx42.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins-sm53-ptx42.ll
@@ -1,20 +1,20 @@
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | FileCheck %s
; RUN: %if ptxas-sm_53 && ptxas-isa-4.2 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_53 -mattr=+ptx42 | %ptxas-verify -arch=sm_53 %}
-declare half @llvm.nvvm.fma.rn.f16(half, half, half)
-declare half @llvm.nvvm.fma.rn.ftz.f16(half, half, half)
-declare half @llvm.nvvm.fma.rn.sat.f16(half, half, half)
-declare half @llvm.nvvm.fma.rn.ftz.sat.f16(half, half, half)
-declare <2 x half> @llvm.nvvm.fma.rn.f16x2(<2 x half>, <2 x half>, <2 x half>)
-declare <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2(<2 x half>, <2 x half>, <2 x half>)
-declare <2 x half> @llvm.nvvm.fma.rn.sat.f16x2(<2 x half>, <2 x half>, <2 x half>)
-declare <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2(<2 x half>, <2 x half>, <2 x half>)
+declare half @llvm.nvvm.fma.f16(half, half, half, i32)
+declare half @llvm.nvvm.fma.ftz.f16(half, half, half, i32)
+declare half @llvm.nvvm.fma.sat.f16(half, half, half, i32)
+declare half @llvm.nvvm.fma.ftz.sat.f16(half, half, half, i32)
+declare <2 x half> @llvm.nvvm.fma.v2f16(<2 x half>, <2 x half>, <2 x half>, i32)
+declare <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half>, <2 x half>, <2 x half>, i32)
+declare <2 x half> @llvm.nvvm.fma.sat.v2f16(<2 x half>, <2 x half>, <2 x half>, i32)
+declare <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16(<2 x half>, <2 x half>, <2 x half>, i32)
; CHECK-LABEL: fma_rn_f16
define half @fma_rn_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.f16
- %res = call half @llvm.nvvm.fma.rn.f16(half %0, half %1, half %2)
+ %res = call half @llvm.nvvm.fma.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
@@ -22,7 +22,7 @@ define half @fma_rn_f16(half %0, half %1, half %2) {
define half @fma_rn_ftz_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.f16
- %res = call half @llvm.nvvm.fma.rn.ftz.f16(half %0, half %1, half %2)
+ %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
@@ -30,7 +30,7 @@ define half @fma_rn_ftz_f16(half %0, half %1, half %2) {
define half @fma_rn_sat_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.sat.f16
- %res = call half @llvm.nvvm.fma.rn.sat.f16(half %0, half %1, half %2)
+ %res = call half @llvm.nvvm.fma.sat.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
@@ -38,7 +38,7 @@ define half @fma_rn_sat_f16(half %0, half %1, half %2) {
define half @fma_rn_ftz_sat_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.sat.f16
- %res = call half @llvm.nvvm.fma.rn.ftz.sat.f16(half %0, half %1, half %2)
+ %res = call half @llvm.nvvm.fma.ftz.sat.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
@@ -46,7 +46,7 @@ define half @fma_rn_ftz_sat_f16(half %0, half %1, half %2) {
define <2 x half> @fma_rn_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.rn.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ %res = call <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
@@ -54,7 +54,7 @@ define <2 x half> @fma_rn_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
define <2 x half> @fma_rn_ftz_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
@@ -62,7 +62,7 @@ define <2 x half> @fma_rn_ftz_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
define <2 x half> @fma_rn_sat_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.sat.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.rn.sat.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ %res = call <2 x half> @llvm.nvvm.fma.sat.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
@@ -70,6 +70,6 @@ define <2 x half> @fma_rn_sat_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
define <2 x half> @fma_rn_ftz_sat_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.sat.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-instcombine.ll b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-instcombine.ll
index e96fe5dfcba387..be6eadce562c00 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-instcombine.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-instcombine.ll
@@ -25,10 +25,10 @@ declare <2 x half> @llvm.nvvm.fmax.nan.f16x2(<2 x half>, <2 x half>)
declare <2 x half> @llvm.nvvm.fmax.ftz.nan.f16x2(<2 x half>, <2 x half>)
; f16 and f16x2 fma are available since ptx 4.2 and sm_53.
-declare half @llvm.nvvm.fma.rn.f16(half, half, half)
-declare half @llvm.nvvm.fma.rn.ftz.f16(half, half, half)
-declare <2 x half> @llvm.nvvm.fma.rn.f16x2(<2 x half>, <2 x half>, <2 x half>)
-declare <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2(<2 x half>, <2 x half>, <2 x half>)
+declare half @llvm.nvvm.fma.f16(half, half, half, i32)
+declare half @llvm.nvvm.fma.ftz.f16(half, half, half, i32)
+declare <2 x half> @llvm.nvvm.fma.v2f16(<2 x half>, <2 x half>, <2 x half>, i32)
+declare <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half>, <2 x half>, <2 x half>, i32)
; CHECK-LABEL: fmin_f16
define half @fmin_f16(half %0, half %1) {
@@ -272,49 +272,49 @@ define <2 x half> @fmax_ftz_nan_f16x2_no_attr(<2 x half> %0, <2 x half> %1) {
; CHECK-LABEL: fma_rn_f16
define half @fma_rn_f16(half %0, half %1, half %2) {
- ; CHECK-NOT: @llvm.nvvm.fma.rn.f16
+ ; CHECK-NOT: @llvm.nvvm.fma.f16
; CHECK: @llvm.fma.f16
- %res = call half @llvm.nvvm.fma.rn.f16(half %0, half %1, half %2)
+ %res = call half @llvm.nvvm.fma.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
; CHECK-LABEL: fma_rn_ftz_f16_no_attr
define half @fma_rn_ftz_f16_no_attr(half %0, half %1, half %2) {
; CHECK-NOT: @llvm.fma.f16
- ; CHECK: @llvm.nvvm.fma.rn.ftz.f16
- %res = call half @llvm.nvvm.fma.rn.ftz.f16(half %0, half %1, half %2)
+ ; CHECK: @llvm.nvvm.fma.ftz.f16
+ %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
; CHECK-LABEL: fma_rn_ftz_f16
define half @fma_rn_ftz_f16(half %0, half %1, half %2) #0 {
- ; CHECK-NOT: @llvm.nvvm.fma.rn.ftz.f16
+ ; CHECK-NOT: @llvm.nvvm.fma.ftz.f16
; CHECK: @llvm.fma.f16
- %res = call half @llvm.nvvm.fma.rn.ftz.f16(half %0, half %1, half %2)
+ %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
; CHECK-LABEL: fma_rn_f16x2
define <2 x half> @fma_rn_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
- ; CHECK-NOT: @llvm.nvvm.fma.rn.f16x2
+ ; CHECK-NOT: @llvm.nvvm.fma.v2f16
; CHECK: @llvm.fma.v2f16
- %res = call <2 x half> @llvm.nvvm.fma.rn.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ %res = call <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
; CHECK-LABEL: fma_rn_ftz_f16x2
define <2 x half> @fma_rn_ftz_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) #0 {
- ; CHECK-NOT: @llvm.nvvm.fma.rn.ftz.f16x2
+ ; CHECK-NOT: @llvm.nvvm.fma.ftz.v2f16
; CHECK: @llvm.fma.v2f16
- %res = call <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
; CHECK-LABEL: fma_rn_ftz_f16x2_no_attr
define <2 x half> @fma_rn_ftz_f16x2_no_attr(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: @llvm.fma.v2f16
- ; CHECK: @llvm.nvvm.fma.rn.ftz.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ ; CHECK: @llvm.nvvm.fma.ftz.v2f16
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70.ll b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70.ll
index fe2cb16a941302..807c5de49c9666 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70.ll
@@ -36,14 +36,14 @@ declare bfloat @llvm.nvvm.fmax.nan.bf16(bfloat, bfloat)
declare <2 x bfloat> @llvm.nvvm.fmax.bf16x2(<2 x bfloat>, <2 x bfloat>)
declare <2 x bfloat> @llvm.nvvm.fmax.nan.bf16x2(<2 x bfloat>, <2 x bfloat>)
-declare half @llvm.nvvm.fma.rn.relu.f16(half, half, half)
-declare half @llvm.nvvm.fma.rn.ftz.relu.f16(half, half, half)
-declare <2 x half> @llvm.nvvm.fma.rn.relu.f16x2(<2 x half>, <2 x half>, <2 x half>)
-declare <2 x half> @llvm.nvvm.fma.rn.ftz.relu.f16x2(<2 x half>, <2 x half>, <2 x half>)
-declare bfloat @llvm.nvvm.fma.rn.bf16(bfloat, bfloat, bfloat)
-declare bfloat @llvm.nvvm.fma.rn.relu.bf16(bfloat, bfloat, bfloat)
-declare <2 x bfloat> @llvm.nvvm.fma.rn.bf16x2(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
-declare <2 x bfloat> @llvm.nvvm.fma.rn.relu.bf16x2(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
+declare half @llvm.nvvm.fma.relu.f16(half, half, half, i32)
+declare half @llvm.nvvm.fma.ftz.relu.f16(half, half, half, i32)
+declare <2 x half> @llvm.nvvm.fma.relu.v2f16(<2 x half>, <2 x half>, <2 x half>, i32)
+declare <2 x half> @llvm.nvvm.fma.ftz.relu.v2f16(<2 x half>, <2 x half>, <2 x half>, i32)
+declare bfloat @llvm.nvvm.fma.bf16(bfloat, bfloat, bfloat, i32)
+declare bfloat @llvm.nvvm.fma.relu.bf16(bfloat, bfloat, bfloat, i32)
+declare <2 x bfloat> @llvm.nvvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>, i32)
+declare <2 x bfloat> @llvm.nvvm.fma.relu.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>, i32)
; CHECK-LABEL: abs_bf16
define bfloat @abs_bf16(bfloat %0) {
@@ -305,7 +305,7 @@ define <2 x bfloat> @fmax_nan_bf16x2(<2 x bfloat> %0, <2 x bfloat> %1) {
define half @fma_rn_relu_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.relu.f16
- %res = call half @llvm.nvvm.fma.rn.relu.f16(half %0, half %1, half %2)
+ %res = call half @llvm.nvvm.fma.relu.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
@@ -313,7 +313,7 @@ define half @fma_rn_relu_f16(half %0, half %1, half %2) {
define half @fma_rn_ftz_relu_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.relu.f16
- %res = call half @llvm.nvvm.fma.rn.ftz.relu.f16(half %0, half %1, half %2)
+ %res = call half @llvm.nvvm.fma.ftz.relu.f16(half %0, half %1, half %2, i32 1)
ret half %res
}
@@ -321,7 +321,7 @@ define half @fma_rn_ftz_relu_f16(half %0, half %1, half %2) {
define <2 x half> @fma_rn_relu_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.relu.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.rn.relu.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ %res = call <2 x half> @llvm.nvvm.fma.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
@@ -329,7 +329,7 @@ define <2 x half> @fma_rn_relu_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2
define <2 x half> @fma_rn_ftz_relu_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.relu.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.rn.ftz.relu.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
ret <2 x half> %res
}
@@ -337,7 +337,7 @@ define <2 x half> @fma_rn_ftz_relu_f16x2(<2 x half> %0, <2 x half> %1, <2 x half
define bfloat @fma_rn_bf16(bfloat %0, bfloat %1, bfloat %2) {
; CHECK-NOT: call
; CHECK: fma.rn.bf16
- %res = call bfloat @llvm.nvvm.fma.rn.bf16(bfloat %0, bfloat %1, bfloat %2)
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat %0, bfloat %1, bfloat %2, i32 1)
ret bfloat %res
}
@@ -345,7 +345,7 @@ define bfloat @fma_rn_bf16(bfloat %0, bfloat %1, bfloat %2) {
define bfloat @fma_rn_relu_bf16(bfloat %0, bfloat %1, bfloat %2) {
; CHECK-NOT: call
; CHECK: fma.rn.relu.bf16
- %res = call bfloat @llvm.nvvm.fma.rn.relu.bf16(bfloat %0, bfloat %1, bfloat %2)
+ %res = call bfloat @llvm.nvvm.fma.relu.bf16(bfloat %0, bfloat %1, bfloat %2, i32 1)
ret bfloat %res
}
@@ -353,7 +353,7 @@ define bfloat @fma_rn_relu_bf16(bfloat %0, bfloat %1, bfloat %2) {
define <2 x bfloat> @fma_rn_bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.bf16x2
- %res = call <2 x bfloat> @llvm.nvvm.fma.rn.bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2)
+ %res = call <2 x bfloat> @llvm.nvvm.fma.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2, i32 1)
ret <2 x bfloat> %res
}
@@ -361,6 +361,6 @@ define <2 x bfloat> @fma_rn_bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat
define <2 x bfloat> @fma_rn_relu_bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.relu.bf16x2
- %res = call <2 x bfloat> @llvm.nvvm.fma.rn.relu.bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2)
+ %res = call <2 x bfloat> @llvm.nvvm.fma.relu.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2, i32 1)
ret <2 x bfloat> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
index 9da6c9a39a34e7..28fb71a7f30f52 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
@@ -322,16 +322,16 @@ define float @test_fma_f32_f16_1(half %a, half %b, float %c) {
%r0 = fpext half %a to float
%r1 = fpext half %b to float
- %r2 = call float @llvm.nvvm.fma.rn.f(float %r0, float %r1, float %c)
- %r3 = call float @llvm.nvvm.fma.rz.f(float %r0, float %r1, float %r2)
- %r4 = call float @llvm.nvvm.fma.rm.f(float %r0, float %r1, float %r3)
- %r5 = call float @llvm.nvvm.fma.rp.f(float %r0, float %r1, float %r4)
+ %r2 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %c, i32 1)
+ %r3 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r2, i32 0)
+ %r4 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r3, i32 3)
+ %r5 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r4, i32 2)
; SAT
- %r6 = call float @llvm.nvvm.fma.rn.sat.f(float %r0, float %r1, float %r5)
- %r7 = call float @llvm.nvvm.fma.rz.sat.f(float %r0, float %r1, float %r6)
- %r8 = call float @llvm.nvvm.fma.rm.sat.f(float %r0, float %r1, float %r7)
- %r9 = call float @llvm.nvvm.fma.rp.sat.f(float %r0, float %r1, float %r8)
+ %r6 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r5, i32 1)
+ %r7 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r6, i32 0)
+ %r8 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r7, i32 3)
+ %r9 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r8, i32 2)
ret float %r8
}
@@ -393,16 +393,16 @@ define float @test_fma_f32_bf16_1(bfloat %a, bfloat %b, float %c) {
%r0 = fpext bfloat %a to float
%r1 = fpext bfloat %b to float
- %r2 = call float @llvm.nvvm.fma.rn.f(float %r0, float %r1, float %c)
- %r3 = call float @llvm.nvvm.fma.rz.f(float %r0, float %r1, float %r2)
- %r4 = call float @llvm.nvvm.fma.rm.f(float %r0, float %r1, float %r3)
- %r5 = call float @llvm.nvvm.fma.rp.f(float %r0, float %r1, float %r4)
+ %r2 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %c, i32 1)
+ %r3 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r2, i32 0)
+ %r4 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r3, i32 3)
+ %r5 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r4, i32 2)
; SAT
- %r6 = call float @llvm.nvvm.fma.rn.sat.f(float %r0, float %r1, float %r5)
- %r7 = call float @llvm.nvvm.fma.rz.sat.f(float %r0, float %r1, float %r6)
- %r8 = call float @llvm.nvvm.fma.rm.sat.f(float %r0, float %r1, float %r7)
- %r9 = call float @llvm.nvvm.fma.rp.sat.f(float %r0, float %r1, float %r8)
+ %r6 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r5, i32 1)
+ %r7 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r6, i32 0)
+ %r8 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r7, i32 3)
+ %r9 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r8, i32 2)
ret float %r8
}
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index ff3727e3bbb563..03d60f28d8fc90 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -84,21 +84,21 @@ define float @floor_float_ftz(float %a) #0 {
; CHECK-LABEL: @fma_double
define double @fma_double(double %a, double %b, double %c) #0 {
; CHECK: call double @llvm.fma.f64
- %ret = call double @llvm.nvvm.fma.rn.d(double %a, double %b, double %c)
+ %ret = call double @llvm.nvvm.fma.f64(double %a, double %b, double %c, i32 1)
ret double %ret
}
; CHECK-LABEL: @fma_float
define float @fma_float(float %a, float %b, float %c) #0 {
; NOFTZ: call float @llvm.fma.f32
-; FTZ: call float @llvm.nvvm.fma.rn.f
- %ret = call float @llvm.nvvm.fma.rn.f(float %a, float %b, float %c)
+; FTZ: call float @llvm.nvvm.fma.f32
+ %ret = call float @llvm.nvvm.fma.f32(float %a, float %b, float %c, i32 1)
ret float %ret
}
; CHECK-LABEL: @fma_float_ftz
define float @fma_float_ftz(float %a, float %b, float %c) #0 {
-; NOFTZ: call float @llvm.nvvm.fma.rn.ftz.f
+; NOFTZ: call float @llvm.nvvm.fma.ftz.f32
; FTZ: call float @llvm.fma.f32
- %ret = call float @llvm.nvvm.fma.rn.ftz.f(float %a, float %b, float %c)
+ %ret = call float @llvm.nvvm.fma.ftz.f32(float %a, float %b, float %c, i32 1)
ret float %ret
}
@@ -499,9 +499,9 @@ declare float @llvm.nvvm.fabs.ftz.f(float)
declare double @llvm.nvvm.floor.d(double)
declare float @llvm.nvvm.floor.f(float)
declare float @llvm.nvvm.floor.ftz.f(float)
-declare double @llvm.nvvm.fma.rn.d(double, double, double)
-declare float @llvm.nvvm.fma.rn.f(float, float, float)
-declare float @llvm.nvvm.fma.rn.ftz.f(float, float, float)
+declare double @llvm.nvvm.fma.f64(double, double, double, i32)
+declare float @llvm.nvvm.fma.f32(float, float, float, i32)
+declare float @llvm.nvvm.fma.ftz.f32(float, float, float, i32)
declare double @llvm.nvvm.fmax.d(double, double)
declare float @llvm.nvvm.fmax.f(float, float)
declare float @llvm.nvvm.fmax.ftz.f(float, float)
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-fma.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-fma.ll
index 54039fc13aa620..da7bb888e89913 100644
--- a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-fma.ll
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-fma.ll
@@ -12,7 +12,7 @@ define double @test_fma_2_3_4_rm_d() {
; CHECK-LABEL: define double @test_fma_2_3_4_rm_d() {
; CHECK-NEXT: ret double 1.000000e+01
;
- %res = call double @llvm.nvvm.fma.rm.d(double 2.0, double 3.0, double 4.0)
+ %res = call double @llvm.nvvm.fma.f64(double 2.0, double 3.0, double 4.0, /* rnd=rm */ i32 3)
ret double %res
}
@@ -20,7 +20,7 @@ define double @test_fma_2_3_4_rn_d() {
; CHECK-LABEL: define double @test_fma_2_3_4_rn_d() {
; CHECK-NEXT: ret double 1.000000e+01
;
- %res = call double @llvm.nvvm.fma.rn.d(double 2.0, double 3.0, double 4.0)
+ %res = call double @llvm.nvvm.fma.f64(double 2.0, double 3.0, double 4.0, /* rnd=rn */ i32 1)
ret double %res
}
@@ -28,7 +28,7 @@ define double @test_fma_2_3_4_rp_d() {
; CHECK-LABEL: define double @test_fma_2_3_4_rp_d() {
; CHECK-NEXT: ret double 1.000000e+01
;
- %res = call double @llvm.nvvm.fma.rp.d(double 2.0, double 3.0, double 4.0)
+ %res = call double @llvm.nvvm.fma.f64(double 2.0, double 3.0, double 4.0, /* rnd=rp */ i32 2)
ret double %res
}
@@ -36,7 +36,7 @@ define double @test_fma_2_3_4_rz_d() {
; CHECK-LABEL: define double @test_fma_2_3_4_rz_d() {
; CHECK-NEXT: ret double 1.000000e+01
;
- %res = call double @llvm.nvvm.fma.rz.d(double 2.0, double 3.0, double 4.0)
+ %res = call double @llvm.nvvm.fma.f64(double 2.0, double 3.0, double 4.0, /* rnd=rz */ i32 0)
ret double %res
}
@@ -44,7 +44,7 @@ define float @test_fma_2_3_4_rm_f() {
; CHECK-LABEL: define float @test_fma_2_3_4_rm_f() {
; CHECK-NEXT: ret float 1.000000e+01
;
- %res = call float @llvm.nvvm.fma.rm.f(float 2.0, float 3.0, float 4.0)
+ %res = call float @llvm.nvvm.fma.f32(float 2.0, float 3.0, float 4.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -52,7 +52,7 @@ define float @test_fma_2_3_4_rn_f() {
; CHECK-LABEL: define float @test_fma_2_3_4_rn_f() {
; CHECK-NEXT: ret float 1.000000e+01
;
- %res = call float @llvm.nvvm.fma.rn.f(float 2.0, float 3.0, float 4.0)
+ %res = call float @llvm.nvvm.fma.f32(float 2.0, float 3.0, float 4.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -60,7 +60,7 @@ define float @test_fma_2_3_4_rp_f() {
; CHECK-LABEL: define float @test_fma_2_3_4_rp_f() {
; CHECK-NEXT: ret float 1.000000e+01
;
- %res = call float @llvm.nvvm.fma.rp.f(float 2.0, float 3.0, float 4.0)
+ %res = call float @llvm.nvvm.fma.f32(float 2.0, float 3.0, float 4.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -68,7 +68,7 @@ define float @test_fma_2_3_4_rz_f() {
; CHECK-LABEL: define float @test_fma_2_3_4_rz_f() {
; CHECK-NEXT: ret float 1.000000e+01
;
- %res = call float @llvm.nvvm.fma.rz.f(float 2.0, float 3.0, float 4.0)
+ %res = call float @llvm.nvvm.fma.f32(float 2.0, float 3.0, float 4.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -76,7 +76,7 @@ define float @test_fma_2_3_4_rm_ftz_f() {
; CHECK-LABEL: define float @test_fma_2_3_4_rm_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+01
;
- %res = call float @llvm.nvvm.fma.rm.ftz.f(float 2.0, float 3.0, float 4.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 2.0, float 3.0, float 4.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -84,7 +84,7 @@ define float @test_fma_2_3_4_rn_ftz_f() {
; CHECK-LABEL: define float @test_fma_2_3_4_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+01
;
- %res = call float @llvm.nvvm.fma.rn.ftz.f(float 2.0, float 3.0, float 4.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 2.0, float 3.0, float 4.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -92,7 +92,7 @@ define float @test_fma_2_3_4_rp_ftz_f() {
; CHECK-LABEL: define float @test_fma_2_3_4_rp_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+01
;
- %res = call float @llvm.nvvm.fma.rp.ftz.f(float 2.0, float 3.0, float 4.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 2.0, float 3.0, float 4.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -100,7 +100,7 @@ define float @test_fma_2_3_4_rz_ftz_f() {
; CHECK-LABEL: define float @test_fma_2_3_4_rz_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+01
;
- %res = call float @llvm.nvvm.fma.rz.ftz.f(float 2.0, float 3.0, float 4.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 2.0, float 3.0, float 4.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -116,7 +116,7 @@ define double @test_fma_subnorm_to_norm_rm_d() {
; CHECK-LABEL: define double @test_fma_subnorm_to_norm_rm_d() {
; CHECK-NEXT: ret double f0x0010000000000000
;
- %res = call double @llvm.nvvm.fma.rm.d(double 0x0008000000000000, double 2.0, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double 0x0008000000000000, double 2.0, double 0.0, /* rnd=rm */ i32 3)
ret double %res
}
@@ -124,7 +124,7 @@ define double @test_fma_subnorm_to_norm_rn_d() {
; CHECK-LABEL: define double @test_fma_subnorm_to_norm_rn_d() {
; CHECK-NEXT: ret double f0x0010000000000000
;
- %res = call double @llvm.nvvm.fma.rn.d(double 0x0008000000000000, double 2.0, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double 0x0008000000000000, double 2.0, double 0.0, /* rnd=rn */ i32 1)
ret double %res
}
@@ -132,7 +132,7 @@ define double @test_fma_subnorm_to_norm_rp_d() {
; CHECK-LABEL: define double @test_fma_subnorm_to_norm_rp_d() {
; CHECK-NEXT: ret double f0x0010000000000000
;
- %res = call double @llvm.nvvm.fma.rp.d(double 0x0008000000000000, double 2.0, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double 0x0008000000000000, double 2.0, double 0.0, /* rnd=rp */ i32 2)
ret double %res
}
@@ -140,7 +140,7 @@ define double @test_fma_subnorm_to_norm_rz_d() {
; CHECK-LABEL: define double @test_fma_subnorm_to_norm_rz_d() {
; CHECK-NEXT: ret double f0x0010000000000000
;
- %res = call double @llvm.nvvm.fma.rz.d(double 0x0008000000000000, double 2.0, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double 0x0008000000000000, double 2.0, double 0.0, /* rnd=rz */ i32 0)
ret double %res
}
@@ -148,7 +148,7 @@ define float @test_fma_subnorm_to_norm_rm_f() {
; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rm_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fma.rm.f(float 0x3800000000000000, float 2.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x3800000000000000, float 2.0, float 0.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -156,7 +156,7 @@ define float @test_fma_subnorm_to_norm_rn_f() {
; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rn_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fma.rn.f(float 0x3800000000000000, float 2.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x3800000000000000, float 2.0, float 0.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -164,7 +164,7 @@ define float @test_fma_subnorm_to_norm_rp_f() {
; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rp_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fma.rp.f(float 0x3800000000000000, float 2.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x3800000000000000, float 2.0, float 0.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -172,7 +172,7 @@ define float @test_fma_subnorm_to_norm_rz_f() {
; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rz_f() {
; CHECK-NEXT: ret float f0x00800000
;
- %res = call float @llvm.nvvm.fma.rz.f(float 0x3800000000000000, float 2.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x3800000000000000, float 2.0, float 0.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -180,7 +180,7 @@ define float @test_fma_subnorm_to_norm_rm_ftz_f() {
; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rm_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fma.rm.ftz.f(float 0x3800000000000000, float 2.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x3800000000000000, float 2.0, float 0.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -188,7 +188,7 @@ define float @test_fma_subnorm_to_norm_rn_ftz_f() {
; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rn_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.ftz.f(float 0x3800000000000000, float 2.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x3800000000000000, float 2.0, float 0.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -196,7 +196,7 @@ define float @test_fma_subnorm_to_norm_rp_ftz_f() {
; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fma.rp.ftz.f(float 0x3800000000000000, float 2.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x3800000000000000, float 2.0, float 0.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -204,7 +204,7 @@ define float @test_fma_subnorm_to_norm_rz_ftz_f() {
; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fma.rz.ftz.f(float 0x3800000000000000, float 2.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x3800000000000000, float 2.0, float 0.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -220,7 +220,7 @@ define double @test_fma_norm_to_subnorm_rm_d() {
; CHECK-LABEL: define double @test_fma_norm_to_subnorm_rm_d() {
; CHECK-NEXT: ret double f0x0008000000000000
;
- %res = call double @llvm.nvvm.fma.rm.d(double 0x10000000000000, double 0.5, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double 0x10000000000000, double 0.5, double 0.0, /* rnd=rm */ i32 3)
ret double %res
}
@@ -228,7 +228,7 @@ define double @test_fma_norm_to_subnorm_rn_d() {
; CHECK-LABEL: define double @test_fma_norm_to_subnorm_rn_d() {
; CHECK-NEXT: ret double f0x0008000000000000
;
- %res = call double @llvm.nvvm.fma.rn.d(double 0x10000000000000, double 0.5, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double 0x10000000000000, double 0.5, double 0.0, /* rnd=rn */ i32 1)
ret double %res
}
@@ -236,7 +236,7 @@ define double @test_fma_norm_to_subnorm_rp_d() {
; CHECK-LABEL: define double @test_fma_norm_to_subnorm_rp_d() {
; CHECK-NEXT: ret double f0x0008000000000000
;
- %res = call double @llvm.nvvm.fma.rp.d(double 0x10000000000000, double 0.5, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double 0x10000000000000, double 0.5, double 0.0, /* rnd=rp */ i32 2)
ret double %res
}
@@ -244,7 +244,7 @@ define double @test_fma_norm_to_subnorm_rz_d() {
; CHECK-LABEL: define double @test_fma_norm_to_subnorm_rz_d() {
; CHECK-NEXT: ret double f0x0008000000000000
;
- %res = call double @llvm.nvvm.fma.rz.d(double 0x10000000000000, double 0.5, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double 0x10000000000000, double 0.5, double 0.0, /* rnd=rz */ i32 0)
ret double %res
}
@@ -252,7 +252,7 @@ define float @test_fma_norm_to_subnorm_rm_f() {
; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rm_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fma.rm.f(float 0x3810000000000000, float 0.5, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x3810000000000000, float 0.5, float 0.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -260,7 +260,7 @@ define float @test_fma_norm_to_subnorm_rn_f() {
; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rn_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fma.rn.f(float 0x3810000000000000, float 0.5, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x3810000000000000, float 0.5, float 0.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -268,7 +268,7 @@ define float @test_fma_norm_to_subnorm_rp_f() {
; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rp_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fma.rp.f(float 0x3810000000000000, float 0.5, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x3810000000000000, float 0.5, float 0.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -276,7 +276,7 @@ define float @test_fma_norm_to_subnorm_rz_f() {
; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rz_f() {
; CHECK-NEXT: ret float f0x00400000
;
- %res = call float @llvm.nvvm.fma.rz.f(float 0x3810000000000000, float 0.5, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x3810000000000000, float 0.5, float 0.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -284,7 +284,7 @@ define float @test_fma_norm_to_subnorm_rm_ftz_f() {
; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rm_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fma.rm.ftz.f(float 0x3810000000000000, float 0.5, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x3810000000000000, float 0.5, float 0.0, /* rnd=rm */ i32 3)
ret float %res
}
@@ -292,7 +292,7 @@ define float @test_fma_norm_to_subnorm_rn_ftz_f() {
; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rn_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.ftz.f(float 0x3810000000000000, float 0.5, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x3810000000000000, float 0.5, float 0.0, /* rnd=rn */ i32 1)
ret float %res
}
@@ -300,7 +300,7 @@ define float @test_fma_norm_to_subnorm_rp_ftz_f() {
; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rp_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fma.rp.ftz.f(float 0x3810000000000000, float 0.5, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x3810000000000000, float 0.5, float 0.0, /* rnd=rp */ i32 2)
ret float %res
}
@@ -308,7 +308,7 @@ define float @test_fma_norm_to_subnorm_rz_ftz_f() {
; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rz_ftz_f() {
; CHECK-NEXT: ret float 0.000000e+00
;
- %res = call float @llvm.nvvm.fma.rz.ftz.f(float 0x3810000000000000, float 0.5, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x3810000000000000, float 0.5, float 0.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -320,109 +320,109 @@ define float @test_fma_norm_to_subnorm_rz_ftz_f() {
define double @test_fma_nan_rm_d() {
; CHECK-LABEL: define double @test_fma_nan_rm_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fma.rm.d(double +qnan, double 1.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fma.f64(double +qnan, double 1.000000e+00, double 0.000000e+00, /* rnd=rm */ i32 3)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fma.rm.d(double +qnan, double 1.0, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double +qnan, double 1.0, double 0.0, /* rnd=rm */ i32 3)
ret double %res
}
define double @test_fma_nan_rn_d() {
; CHECK-LABEL: define double @test_fma_nan_rn_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fma.rn.d(double +qnan, double 1.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fma.f64(double +qnan, double 1.000000e+00, double 0.000000e+00, /* rnd=rn */ i32 1)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fma.rn.d(double +qnan, double 1.0, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double +qnan, double 1.0, double 0.0, /* rnd=rn */ i32 1)
ret double %res
}
define double @test_fma_nan_rp_d() {
; CHECK-LABEL: define double @test_fma_nan_rp_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fma.rp.d(double +qnan, double 1.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fma.f64(double +qnan, double 1.000000e+00, double 0.000000e+00, /* rnd=rp */ i32 2)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fma.rp.d(double +qnan, double 1.0, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double +qnan, double 1.0, double 0.0, /* rnd=rp */ i32 2)
ret double %res
}
define double @test_fma_nan_rz_d() {
; CHECK-LABEL: define double @test_fma_nan_rz_d() {
-; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fma.rz.d(double +qnan, double 1.000000e+00, double 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call double @llvm.nvvm.fma.f64(double +qnan, double 1.000000e+00, double 0.000000e+00, /* rnd=rz */ i32 0)
; CHECK-NEXT: ret double [[RES]]
;
- %res = call double @llvm.nvvm.fma.rz.d(double +qnan, double 1.0, double 0.0)
+ %res = call double @llvm.nvvm.fma.f64(double +qnan, double 1.0, double 0.0, /* rnd=rz */ i32 0)
ret double %res
}
define float @test_fma_nan_rm_f() {
; CHECK-LABEL: define float @test_fma_nan_rm_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.rm.f(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.f32(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00, /* rnd=rm */ i32 3)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fma.rm.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x7FFC000000000000, float 1.0, float 0.0, /* rnd=rm */ i32 3)
ret float %res
}
define float @test_fma_nan_rn_f() {
; CHECK-LABEL: define float @test_fma_nan_rn_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.rn.f(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.f32(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00, /* rnd=rn */ i32 1)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fma.rn.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x7FFC000000000000, float 1.0, float 0.0, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_fma_nan_rp_f() {
; CHECK-LABEL: define float @test_fma_nan_rp_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.rp.f(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.f32(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00, /* rnd=rp */ i32 2)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fma.rp.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x7FFC000000000000, float 1.0, float 0.0, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_fma_nan_rz_f() {
; CHECK-LABEL: define float @test_fma_nan_rz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.rz.f(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.f32(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00, /* rnd=rz */ i32 0)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fma.rz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.f32(float 0x7FFC000000000000, float 1.0, float 0.0, /* rnd=rz */ i32 0)
ret float %res
}
define float @test_fma_nan_rm_ftz_f() {
; CHECK-LABEL: define float @test_fma_nan_rm_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.rm.ftz.f(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.ftz.f32(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00, /* rnd=rm */ i32 3)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fma.rm.ftz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x7FFC000000000000, float 1.0, float 0.0, /* rnd=rm */ i32 3)
ret float %res
}
define float @test_fma_nan_rn_ftz_f() {
; CHECK-LABEL: define float @test_fma_nan_rn_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.rn.ftz.f(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.ftz.f32(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00, /* rnd=rn */ i32 1)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fma.rn.ftz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x7FFC000000000000, float 1.0, float 0.0, /* rnd=rn */ i32 1)
ret float %res
}
define float @test_fma_nan_rp_ftz_f() {
; CHECK-LABEL: define float @test_fma_nan_rp_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.rp.ftz.f(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.ftz.f32(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00, /* rnd=rp */ i32 2)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fma.rp.ftz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x7FFC000000000000, float 1.0, float 0.0, /* rnd=rp */ i32 2)
ret float %res
}
define float @test_fma_nan_rz_ftz_f() {
; CHECK-LABEL: define float @test_fma_nan_rz_ftz_f() {
-; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.rz.ftz.f(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT: [[RES:%.*]] = call float @llvm.nvvm.fma.ftz.f32(float +nan(0x200000), float 1.000000e+00, float 0.000000e+00, /* rnd=rz */ i32 0)
; CHECK-NEXT: ret float [[RES]]
;
- %res = call float @llvm.nvvm.fma.rz.ftz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 0x7FFC000000000000, float 1.0, float 0.0, /* rnd=rz */ i32 0)
ret float %res
}
@@ -437,7 +437,7 @@ define float @test_fma_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_fma_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rm.f(float 1.0, float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float 1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -445,7 +445,7 @@ define float @test_fma_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_fma_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.f(float 1.0, float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float 1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -453,7 +453,7 @@ define float @test_fma_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_fma_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.fma.rp.f(float 1.0, float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float 1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -461,7 +461,7 @@ define float @test_fma_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_fma_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float 1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -469,7 +469,7 @@ define float @test_fma_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_fma_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rm.ftz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float 1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -477,7 +477,7 @@ define float @test_fma_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_fma_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.ftz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float 1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -485,7 +485,7 @@ define float @test_fma_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_fma_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0x3F800001
;
- %res = call float @llvm.nvvm.fma.rp.ftz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float 1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -493,7 +493,7 @@ define float @test_fma_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_fma_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rz.ftz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float 1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -509,7 +509,7 @@ define double @test_fma_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_fma_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rm.d(double 1.0, double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double 1.0, double 0x3C90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -517,7 +517,7 @@ define double @test_fma_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_fma_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rn.d(double 1.0, double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double 1.0, double 0x3C90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -525,7 +525,7 @@ define double @test_fma_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_fma_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0x3FF0000000000001
;
- %res = call double @llvm.nvvm.fma.rp.d(double 1.0, double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double 1.0, double 0x3C90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -533,7 +533,7 @@ define double @test_fma_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_fma_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rz.d(double 1.0, double 1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double 1.0, double 0x3C90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -549,7 +549,7 @@ define float @test_fma_neg_1_plus_ulp_rm_f() {
; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rm_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rm.f(float 1.0, float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float -1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -557,7 +557,7 @@ define float @test_fma_neg_1_plus_ulp_rn_f() {
; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.f(float 1.0, float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float -1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -565,7 +565,7 @@ define float @test_fma_neg_1_plus_ulp_rp_f() {
; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rp_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fma.rp.f(float 1.0, float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float -1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -573,7 +573,7 @@ define float @test_fma_neg_1_plus_ulp_rz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fma.rz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float -1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -581,7 +581,7 @@ define float @test_fma_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rm.ftz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float -1.0, float 0x3E60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -589,7 +589,7 @@ define float @test_fma_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.ftz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float -1.0, float 0x3E60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -597,7 +597,7 @@ define float @test_fma_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fma.rp.ftz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float -1.0, float 0x3E60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -605,7 +605,7 @@ define float @test_fma_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0xBF7FFFFF
;
- %res = call float @llvm.nvvm.fma.rz.ftz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float -1.0, float 0x3E60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -621,7 +621,7 @@ define double @test_fma_neg_1_plus_ulp_rm_d() {
; CHECK-LABEL: define double @test_fma_neg_1_plus_ulp_rm_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rm.d(double 1.0, double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double -1.0, double 0x3C90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -629,7 +629,7 @@ define double @test_fma_neg_1_plus_ulp_rn_d() {
; CHECK-LABEL: define double @test_fma_neg_1_plus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rn.d(double 1.0, double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double -1.0, double 0x3C90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -637,7 +637,7 @@ define double @test_fma_neg_1_plus_ulp_rp_d() {
; CHECK-LABEL: define double @test_fma_neg_1_plus_ulp_rp_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fma.rp.d(double 1.0, double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double -1.0, double 0x3C90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -645,7 +645,7 @@ define double @test_fma_neg_1_plus_ulp_rz_d() {
; CHECK-LABEL: define double @test_fma_neg_1_plus_ulp_rz_d() {
; CHECK-NEXT: ret double f0xBFEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fma.rz.d(double 1.0, double -1.0, double 0x3C90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double -1.0, double 0x3C90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -661,7 +661,7 @@ define float @test_fma_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_fma_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fma.rm.f(float 1.0, float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float 1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -669,7 +669,7 @@ define float @test_fma_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_fma_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.f(float 1.0, float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float 1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -677,7 +677,7 @@ define float @test_fma_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_fma_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rp.f(float 1.0, float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float 1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -685,7 +685,7 @@ define float @test_fma_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_fma_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fma.rz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float 1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -693,7 +693,7 @@ define float @test_fma_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_fma_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fma.rm.ftz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float 1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -701,7 +701,7 @@ define float @test_fma_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_fma_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.ftz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float 1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -709,7 +709,7 @@ define float @test_fma_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_fma_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float 1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rp.ftz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float 1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -717,7 +717,7 @@ define float @test_fma_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_fma_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float f0x3F7FFFFF
;
- %res = call float @llvm.nvvm.fma.rz.ftz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float 1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -733,7 +733,7 @@ define double @test_fma_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_fma_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fma.rm.d(double 1.0, double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double 1.0, double 0xBC90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -741,7 +741,7 @@ define double @test_fma_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_fma_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rn.d(double 1.0, double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double 1.0, double 0xBC90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -749,7 +749,7 @@ define double @test_fma_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_fma_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double 1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rp.d(double 1.0, double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double 1.0, double 0xBC90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -757,7 +757,7 @@ define double @test_fma_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_fma_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double f0x3FEFFFFFFFFFFFFF
;
- %res = call double @llvm.nvvm.fma.rz.d(double 1.0, double 1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double 1.0, double 0xBC90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
@@ -773,7 +773,7 @@ define float @test_fma_neg_1_minus_ulp_rm_f() {
; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rm_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.fma.rm.f(float 1.0, float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float -1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -781,7 +781,7 @@ define float @test_fma_neg_1_minus_ulp_rn_f() {
; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rn_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.f(float 1.0, float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float -1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -789,7 +789,7 @@ define float @test_fma_neg_1_minus_ulp_rp_f() {
; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rp_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rp.f(float 1.0, float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float -1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -797,7 +797,7 @@ define float @test_fma_neg_1_minus_ulp_rz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.f32(float 1.0, float -1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -805,7 +805,7 @@ define float @test_fma_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rm_ftz_f() {
; CHECK-NEXT: ret float f0xBF800001
;
- %res = call float @llvm.nvvm.fma.rm.ftz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float -1.0, float 0xBE60000000000000, /* rnd=rm */ i32 3)
ret float %res
}
@@ -813,7 +813,7 @@ define float @test_fma_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rn_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rn.ftz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float -1.0, float 0xBE60000000000000, /* rnd=rn */ i32 1)
ret float %res
}
@@ -821,7 +821,7 @@ define float @test_fma_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rp_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rp.ftz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float -1.0, float 0xBE60000000000000, /* rnd=rp */ i32 2)
ret float %res
}
@@ -829,7 +829,7 @@ define float @test_fma_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rz_ftz_f() {
; CHECK-NEXT: ret float -1.000000e+00
;
- %res = call float @llvm.nvvm.fma.rz.ftz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+ %res = call float @llvm.nvvm.fma.ftz.f32(float 1.0, float -1.0, float 0xBE60000000000000, /* rnd=rz */ i32 0)
ret float %res
}
@@ -845,7 +845,7 @@ define double @test_fma_neg_1_minus_ulp_rm_d() {
; CHECK-LABEL: define double @test_fma_neg_1_minus_ulp_rm_d() {
; CHECK-NEXT: ret double f0xBFF0000000000001
;
- %res = call double @llvm.nvvm.fma.rm.d(double 1.0, double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double -1.0, double 0xBC90000000000000, /* rnd=rm */ i32 3)
ret double %res
}
@@ -853,7 +853,7 @@ define double @test_fma_neg_1_minus_ulp_rn_d() {
; CHECK-LABEL: define double @test_fma_neg_1_minus_ulp_rn_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rn.d(double 1.0, double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double -1.0, double 0xBC90000000000000, /* rnd=rn */ i32 1)
ret double %res
}
@@ -861,7 +861,7 @@ define double @test_fma_neg_1_minus_ulp_rp_d() {
; CHECK-LABEL: define double @test_fma_neg_1_minus_ulp_rp_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rp.d(double 1.0, double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double -1.0, double 0xBC90000000000000, /* rnd=rp */ i32 2)
ret double %res
}
@@ -869,6 +869,278 @@ define double @test_fma_neg_1_minus_ulp_rz_d() {
; CHECK-LABEL: define double @test_fma_neg_1_minus_ulp_rz_d() {
; CHECK-NEXT: ret double -1.000000e+00
;
- %res = call double @llvm.nvvm.fma.rz.d(double 1.0, double -1.0, double 0xBC90000000000000)
+ %res = call double @llvm.nvvm.fma.f64(double 1.0, double -1.0, double 0xBC90000000000000, /* rnd=rz */ i32 0)
ret double %res
}
+
+;###############################################################
+;# FMA(2.0, 3.0, 4.0) - f16/bf16 #
+;###############################################################
+; Tests FMA with regular numbers that produce a precise result
+
+define half @test_fma_2_3_4_rm_f16() {
+; CHECK-LABEL: define half @test_fma_2_3_4_rm_f16() {
+; CHECK-NEXT: ret half 1.000000e+01
+;
+ %res = call half @llvm.nvvm.fma.f16(half 2.0, half 3.0, half 4.0, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_fma_2_3_4_rn_f16() {
+; CHECK-LABEL: define half @test_fma_2_3_4_rn_f16() {
+; CHECK-NEXT: ret half 1.000000e+01
+;
+ %res = call half @llvm.nvvm.fma.f16(half 2.0, half 3.0, half 4.0, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_fma_2_3_4_rp_f16() {
+; CHECK-LABEL: define half @test_fma_2_3_4_rp_f16() {
+; CHECK-NEXT: ret half 1.000000e+01
+;
+ %res = call half @llvm.nvvm.fma.f16(half 2.0, half 3.0, half 4.0, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_fma_2_3_4_rz_f16() {
+; CHECK-LABEL: define half @test_fma_2_3_4_rz_f16() {
+; CHECK-NEXT: ret half 1.000000e+01
+;
+ %res = call half @llvm.nvvm.fma.f16(half 2.0, half 3.0, half 4.0, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+define half @test_fma_2_3_4_rn_ftz_f16() {
+; CHECK-LABEL: define half @test_fma_2_3_4_rn_ftz_f16() {
+; CHECK-NEXT: ret half 1.000000e+01
+;
+ %res = call half @llvm.nvvm.fma.ftz.f16(half 2.0, half 3.0, half 4.0, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define bfloat @test_fma_2_3_4_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_2_3_4_rm_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+01
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 2.0, bfloat 3.0, bfloat 4.0, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_2_3_4_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_2_3_4_rn_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+01
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 2.0, bfloat 3.0, bfloat 4.0, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_2_3_4_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_2_3_4_rp_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+01
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 2.0, bfloat 3.0, bfloat 4.0, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_2_3_4_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_2_3_4_rz_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+01
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 2.0, bfloat 3.0, bfloat 4.0, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
+;###############################################################
+;# FMA(NaN, 1.0, 0.0) - f16/bf16 #
+;###############################################################
+; Tests FMA with NaN input to verify that the instruction is preserved
+
+define half @test_fma_nan_rn_f16() {
+; CHECK-LABEL: define half @test_fma_nan_rn_f16() {
+; CHECK-NEXT: [[RES:%.*]] = call half @llvm.nvvm.fma.f16(half +qnan, half 1.000000e+00, half 0.000000e+00, /* rnd=rn */ i32 1)
+; CHECK-NEXT: ret half [[RES]]
+;
+ %res = call half @llvm.nvvm.fma.f16(half 0x7FF8000000000000, half 1.0, half 0.0, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define bfloat @test_fma_nan_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_nan_rn_bf16() {
+; CHECK-NEXT: [[RES:%.*]] = call bfloat @llvm.nvvm.fma.bf16(bfloat +qnan, bfloat 1.000000e+00, bfloat 0.000000e+00, /* rnd=rn */ i32 1)
+; CHECK-NEXT: ret bfloat [[RES]]
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 0x7FF8000000000000, bfloat 1.0, bfloat 0.0, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+;###############################################################
+;# FMA(1.0, 1.0, 2^(-12)) - f16 #
+;###############################################################
+; Tests f16 FMA with 1.0 and 2^(-12) where different rounding modes produce
+; different results.
+
+define half @test_fma_1_plus_ulp_rm_f16() {
+; CHECK-LABEL: define half @test_fma_1_plus_ulp_rm_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 1.0, half 1.0, half 0xH1000, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_fma_1_plus_ulp_rn_f16() {
+; CHECK-LABEL: define half @test_fma_1_plus_ulp_rn_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 1.0, half 1.0, half 0xH1000, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_fma_1_plus_ulp_rp_f16() {
+; CHECK-LABEL: define half @test_fma_1_plus_ulp_rp_f16() {
+; CHECK-NEXT: ret half 1.000980e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 1.0, half 1.0, half 0xH1000, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_fma_1_plus_ulp_rz_f16() {
+; CHECK-LABEL: define half @test_fma_1_plus_ulp_rz_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 1.0, half 1.0, half 0xH1000, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+;###############################################################
+;# FMA(1.0, -1.0, -2^(-12)) - f16 #
+;###############################################################
+; Tests f16 FMA producing a negative result just below -1.0.
+
+define half @test_fma_neg_1_minus_ulp_rm_f16() {
+; CHECK-LABEL: define half @test_fma_neg_1_minus_ulp_rm_f16() {
+; CHECK-NEXT: ret half -1.000980e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 1.0, half -1.0, half 0xH9000, /* rnd=rm */ i32 3)
+ ret half %res
+}
+
+define half @test_fma_neg_1_minus_ulp_rn_f16() {
+; CHECK-LABEL: define half @test_fma_neg_1_minus_ulp_rn_f16() {
+; CHECK-NEXT: ret half -1.000000e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 1.0, half -1.0, half 0xH9000, /* rnd=rn */ i32 1)
+ ret half %res
+}
+
+define half @test_fma_neg_1_minus_ulp_rp_f16() {
+; CHECK-LABEL: define half @test_fma_neg_1_minus_ulp_rp_f16() {
+; CHECK-NEXT: ret half -1.000000e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 1.0, half -1.0, half 0xH9000, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_fma_neg_1_minus_ulp_rz_f16() {
+; CHECK-LABEL: define half @test_fma_neg_1_minus_ulp_rz_f16() {
+; CHECK-NEXT: ret half -1.000000e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 1.0, half -1.0, half 0xH9000, /* rnd=rz */ i32 0)
+ ret half %res
+}
+
+;###############################################################
+;# FMA(1.0, 1.0, 2^(-9)) - bf16 #
+;###############################################################
+; Tests bf16 FMA with 1.0 and 2^(-9) where different rounding modes produce
+; different results.
+
+define bfloat @test_fma_1_plus_ulp_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_1_plus_ulp_rm_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 1.0, bfloat 1.0, bfloat 0xR3B00, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_1_plus_ulp_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_1_plus_ulp_rn_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 1.0, bfloat 1.0, bfloat 0xR3B00, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_1_plus_ulp_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_1_plus_ulp_rp_bf16() {
+; CHECK-NEXT: ret bfloat 1.007810e+00
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 1.0, bfloat 1.0, bfloat 0xR3B00, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_1_plus_ulp_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_1_plus_ulp_rz_bf16() {
+; CHECK-NEXT: ret bfloat 1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 1.0, bfloat 1.0, bfloat 0xR3B00, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
+;###############################################################
+;# FMA(1.0, -1.0, -2^(-9)) - bf16 #
+;###############################################################
+; Tests bf16 FMA producing a negative result just below -1.0.
+
+define bfloat @test_fma_neg_1_minus_ulp_rm_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_neg_1_minus_ulp_rm_bf16() {
+; CHECK-NEXT: ret bfloat -1.007810e+00
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 1.0, bfloat -1.0, bfloat 0xRBB00, /* rnd=rm */ i32 3)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_neg_1_minus_ulp_rn_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_neg_1_minus_ulp_rn_bf16() {
+; CHECK-NEXT: ret bfloat -1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 1.0, bfloat -1.0, bfloat 0xRBB00, /* rnd=rn */ i32 1)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_neg_1_minus_ulp_rp_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_neg_1_minus_ulp_rp_bf16() {
+; CHECK-NEXT: ret bfloat -1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 1.0, bfloat -1.0, bfloat 0xRBB00, /* rnd=rp */ i32 2)
+ ret bfloat %res
+}
+
+define bfloat @test_fma_neg_1_minus_ulp_rz_bf16() {
+; CHECK-LABEL: define bfloat @test_fma_neg_1_minus_ulp_rz_bf16() {
+; CHECK-NEXT: ret bfloat -1.000000e+00
+;
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat 1.0, bfloat -1.0, bfloat 0xRBB00, /* rnd=rz */ i32 0)
+ ret bfloat %res
+}
+
+;###############################################################
+;# FMA(Subnormal, 2.0, Normal) - f16 FTZ #
+;###############################################################
+; Tests that FTZ flushes a subnormal f16 input to zero before the operation.
+; 0xH0200 is a subnormal f16 value (2^-19).
+
+define half @test_fma_subnorm_input_rp_f16() {
+; CHECK-LABEL: define half @test_fma_subnorm_input_rp_f16() {
+; CHECK-NEXT: ret half 1.000980e+00
+;
+ %res = call half @llvm.nvvm.fma.f16(half 0xH0200, half 2.0, half 1.0, /* rnd=rp */ i32 2)
+ ret half %res
+}
+
+define half @test_fma_subnorm_input_rp_ftz_f16() {
+; CHECK-LABEL: define half @test_fma_subnorm_input_rp_ftz_f16() {
+; CHECK-NEXT: ret half 1.000000e+00
+;
+ %res = call half @llvm.nvvm.fma.ftz.f16(half 0xH0200, half 2.0, half 1.0, /* rnd=rp */ i32 2)
+ ret half %res
+}
diff --git a/llvm/test/Verifier/NVPTX/fma.ll b/llvm/test/Verifier/NVPTX/fma.ll
new file mode 100644
index 00000000000000..4c700526db3b98
--- /dev/null
+++ b/llvm/test/Verifier/NVPTX/fma.ll
@@ -0,0 +1,16 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+declare float @llvm.nvvm.fma.f32(float, float, float, i32 immarg)
+
+define void @test_fma_rounding_mode(float %a) {
+ ; CHECK: immarg value 4 for arg 3 out of range [0,4)
+ call float @llvm.nvvm.fma.f32(float %a, float %a, float %a, i32 4)
+
+ ; CHECK: immarg value 7 for arg 3 out of range [0,4)
+ call float @llvm.nvvm.fma.f32(float %a, float %a, float %a, i32 7)
+
+ ; CHECK: immarg value -1 for arg 3 out of range [0,4)
+ call float @llvm.nvvm.fma.f32(float %a, float %a, float %a, i32 -1)
+
+ ret void
+}
diff --git a/llvm/test/Verifier/invalid_intrinsic_signatures.ll b/llvm/test/Verifier/invalid_intrinsic_signatures.ll
index 0b46c24c7f84d5..5782a8bc2290b8 100644
--- a/llvm/test/Verifier/invalid_intrinsic_signatures.ll
+++ b/llvm/test/Verifier/invalid_intrinsic_signatures.ll
@@ -541,8 +541,8 @@ declare i32 @llvm.call.preallocated.setup(i32)
declare double @llvm.fptrunc.round.f64.f64(double, i16)
; CHECK: intrinsic argument 0 type expected half, but got i16
-; CHECK-NEXT: declare half @llvm.nvvm.fma.rn.f16(i16, half, half)
-declare half @llvm.nvvm.fma.rn.f16(i16, half, half)
+; CHECK-NEXT: declare half @llvm.nvvm.fmax.f16(i16, half)
+declare half @llvm.nvvm.fmax.f16(i16, half)
; CHECK: intrinsic return type expected bfloat, but got half
; CHECK-NEXT: declare half @llvm.arm.neon.vcvtbfp2bf(float)
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index f7f2503eee4284..c554a7768c4dba 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -522,116 +522,53 @@ void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
llvm::IRBuilderBase &builder) {
auto thisOp = cast<NVVM::FmaOp>(op);
mlir::NVVM::FPRoundingMode rndMode = thisOp.getRnd();
- unsigned rndIndex = static_cast<unsigned>(rndMode) - 1; // 1-4 mapped to 0-3
- mlir::NVVM::SaturationMode satMode = thisOp.getSat();
bool isFTZ = thisOp.getFtz();
bool isRelu = thisOp.getRelu();
- bool isSat = satMode == NVVM::SaturationMode::SAT;
+ bool isSat = thisOp.getSat() == NVVM::SaturationMode::SAT;
bool isOOB = thisOp.getOob();
mlir::Type opType = thisOp.getRes().getType();
llvm::Type *opTypeLLVM = mt.convertType(opType);
- bool isVectorFma = opTypeLLVM->isVectorTy();
llvm::Value *argA = mt.lookupValue(thisOp.getA());
llvm::Value *argB = mt.lookupValue(thisOp.getB());
llvm::Value *argC = mt.lookupValue(thisOp.getC());
- static constexpr llvm::Intrinsic::ID f16IDs[] = {
- llvm::Intrinsic::nvvm_fma_rn_f16,
- llvm::Intrinsic::nvvm_fma_rn_f16x2,
- llvm::Intrinsic::nvvm_fma_rn_ftz_f16,
- llvm::Intrinsic::nvvm_fma_rn_ftz_f16x2,
- llvm::Intrinsic::nvvm_fma_rn_sat_f16,
- llvm::Intrinsic::nvvm_fma_rn_sat_f16x2,
- llvm::Intrinsic::nvvm_fma_rn_ftz_sat_f16,
- llvm::Intrinsic::nvvm_fma_rn_ftz_sat_f16x2,
- llvm::Intrinsic::nvvm_fma_rn_relu_f16,
- llvm::Intrinsic::nvvm_fma_rn_relu_f16x2,
- llvm::Intrinsic::nvvm_fma_rn_ftz_relu_f16,
- llvm::Intrinsic::nvvm_fma_rn_ftz_relu_f16x2};
-
- static constexpr llvm::Intrinsic::ID bf16IDs[] = {
- llvm::Intrinsic::nvvm_fma_rn_bf16, llvm::Intrinsic::nvvm_fma_rn_bf16x2,
- llvm::Intrinsic::nvvm_fma_rn_relu_bf16,
- llvm::Intrinsic::nvvm_fma_rn_relu_bf16x2};
-
- static constexpr llvm::Intrinsic::ID f32IDs[] = {
- llvm::Intrinsic::nvvm_fma_rn_f,
- llvm::Intrinsic::nvvm_fma_rm_f,
- llvm::Intrinsic::nvvm_fma_rp_f,
- llvm::Intrinsic::nvvm_fma_rz_f,
- llvm::Intrinsic::nvvm_fma_rn_sat_f,
- llvm::Intrinsic::nvvm_fma_rm_sat_f,
- llvm::Intrinsic::nvvm_fma_rp_sat_f,
- llvm::Intrinsic::nvvm_fma_rz_sat_f,
- llvm::Intrinsic::nvvm_fma_rn_ftz_f,
- llvm::Intrinsic::nvvm_fma_rm_ftz_f,
- llvm::Intrinsic::nvvm_fma_rp_ftz_f,
- llvm::Intrinsic::nvvm_fma_rz_ftz_f,
- llvm::Intrinsic::nvvm_fma_rn_ftz_sat_f,
- llvm::Intrinsic::nvvm_fma_rm_ftz_sat_f,
- llvm::Intrinsic::nvvm_fma_rp_ftz_sat_f,
- llvm::Intrinsic::nvvm_fma_rz_ftz_sat_f,
- };
-
- static constexpr llvm::Intrinsic::ID f64IDs[] = {
- llvm::Intrinsic::nvvm_fma_rn_d, llvm::Intrinsic::nvvm_fma_rm_d,
- llvm::Intrinsic::nvvm_fma_rp_d, llvm::Intrinsic::nvvm_fma_rz_d};
-
- auto fmaIntrinsic = [&](llvm::Intrinsic::ID IID,
- llvm::Type *retType) -> llvm::Value * {
- return createScalarizedIntrinsicCall(
- builder, IID, opTypeLLVM, {argA, argB, argC}, /*retType=*/retType);
- };
-
- // f16 + f16 -> f16 / vector<2xf16> + vector<2xf16> -> vector<2xf16>
- if (opTypeLLVM->getScalarType()->isHalfTy()) {
- llvm::Value *result;
- if (isOOB) {
- result = fmaIntrinsic(isRelu ? llvm::Intrinsic::nvvm_fma_rn_oob_relu
- : llvm::Intrinsic::nvvm_fma_rn_oob,
- opTypeLLVM);
- } else {
- unsigned index =
- (isRelu << 3) | (isSat << 2) | (isFTZ << 1) |
- isVectorFma; // Op verifier ensures that this index is valid
- result = fmaIntrinsic(f16IDs[index], opTypeLLVM);
- }
- mt.mapValue(thisOp.getRes(), result);
+ // The out-of-bounds variants have their own overloaded intrinsics, which
+ // only support the RN rounding mode and take no rounding-mode operand.
+ if (isOOB) {
+ llvm::Intrinsic::ID id = isRelu ? llvm::Intrinsic::nvvm_fma_rn_oob_relu
+ : llvm::Intrinsic::nvvm_fma_rn_oob;
+ mt.mapValue(thisOp.getRes(), createIntrinsicCall(builder, id, opTypeLLVM,
+ {argA, argB, argC}));
return;
}
- // bf16 + bf16 -> bf16 / vector<2xbf16> + vector<2xbf16> -> vector<2xbf16>
- if (opTypeLLVM->getScalarType()->isBFloatTy()) {
- llvm::Value *result;
- if (isOOB) {
- result = fmaIntrinsic(isRelu ? llvm::Intrinsic::nvvm_fma_rn_oob_relu
- : llvm::Intrinsic::nvvm_fma_rn_oob,
- opTypeLLVM);
- } else {
- unsigned index = (isRelu << 1) | isVectorFma;
- result = fmaIntrinsic(bf16IDs[index], opTypeLLVM);
- }
- mt.mapValue(thisOp.getRes(), result);
- return;
- }
+ // Overloaded fma intrinsic IDs, indexed by [`.ftz`][`.sat`/`.relu`].
+ static constexpr llvm::Intrinsic::ID fmaIDs[2][3] = {
+ {llvm::Intrinsic::nvvm_fma, llvm::Intrinsic::nvvm_fma_sat,
+ llvm::Intrinsic::nvvm_fma_relu},
+ {llvm::Intrinsic::nvvm_fma_ftz, llvm::Intrinsic::nvvm_fma_ftz_sat,
+ llvm::Intrinsic::nvvm_fma_ftz_relu}};
- // f64 + f64 -> f64 / vector<2xf64> + vector<2xf64> -> vector<2xf64>
- if (opTypeLLVM->getScalarType()->isDoubleTy()) {
- mt.mapValue(thisOp.getRes(),
- fmaIntrinsic(f64IDs[rndIndex], opTypeLLVM->getScalarType()));
- return;
- }
+ // Op verifier ensures that sat and relu are mutually exclusive.
+ llvm::Intrinsic::ID id = fmaIDs[isFTZ][isSat ? 1 : (isRelu ? 2 : 0)];
+ llvm::Value *rnd = builder.getInt32(
+ static_cast<int>(getLLVMRoundingModeForFPArith(rndMode)));
- // f32 + f32 -> f32 / vector<2xf32> + vector<2xf32> -> vector<2xf32>
- const unsigned numRndModes = 4; // RN, RM, RP, RZ
- if (opTypeLLVM->getScalarType()->isFloatTy()) {
- unsigned index = ((isFTZ << 1) | isSat) * numRndModes + rndIndex;
- mt.mapValue(thisOp.getRes(),
- fmaIntrinsic(f32IDs[index], opTypeLLVM->getScalarType()));
+ // For f64 vector fma, and f32 vector fma with saturation, we need to
+ // scalarize the intrinsic call.
+ llvm::Type *scalarTypeLLVM = opTypeLLVM->getScalarType();
+ if (opTypeLLVM->isVectorTy() && (scalarTypeLLVM->isDoubleTy() ||
+ (isSat && scalarTypeLLVM->isFloatTy()))) {
+ mt.mapValue(thisOp.getRes(), createScalarizedIntrinsicCall(
+ builder, id, opTypeLLVM,
+ {argA, argB, argC, rnd}, scalarTypeLLVM));
return;
}
+
+ mt.mapValue(thisOp.getRes(), createIntrinsicCall(builder, id, opTypeLLVM,
+ {argA, argB, argC, rnd}));
}
namespace {
diff --git a/mlir/test/Target/LLVMIR/nvvm/fma/fma.mlir b/mlir/test/Target/LLVMIR/nvvm/fma/fma.mlir
index 268da3fce6bd80..82a65a8b4b0a48 100644
--- a/mlir/test/Target/LLVMIR/nvvm/fma/fma.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/fma/fma.mlir
@@ -2,12 +2,12 @@
llvm.func @fma_f16(%a: f16, %b: f16, %c: f16) -> f16 {
// CHECK-LABEL: define half @fma_f16(half %0, half %1, half %2) {
- // CHECK-NEXT: %4 = call half @llvm.nvvm.fma.rn.f16(half %0, half %1, half %2)
- // CHECK-NEXT: %5 = call half @llvm.nvvm.fma.rn.ftz.f16(half %0, half %1, half %4)
- // CHECK-NEXT: %6 = call half @llvm.nvvm.fma.rn.sat.f16(half %0, half %1, half %5)
- // CHECK-NEXT: %7 = call half @llvm.nvvm.fma.rn.ftz.sat.f16(half %0, half %1, half %6)
- // CHECK-NEXT: %8 = call half @llvm.nvvm.fma.rn.relu.f16(half %0, half %1, half %7)
- // CHECK-NEXT: %9 = call half @llvm.nvvm.fma.rn.ftz.relu.f16(half %0, half %1, half %8)
+ // CHECK-NEXT: %4 = call half @llvm.nvvm.fma.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call half @llvm.nvvm.fma.sat.f16(half %0, half %1, half %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call half @llvm.nvvm.fma.ftz.sat.f16(half %0, half %1, half %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %8 = call half @llvm.nvvm.fma.relu.f16(half %0, half %1, half %7, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %9 = call half @llvm.nvvm.fma.ftz.relu.f16(half %0, half %1, half %8, /* rnd=rn */ i32 1)
// CHECK-NEXT: %10 = call half @llvm.nvvm.fma.rn.oob.f16(half %0, half %1, half %9)
// CHECK-NEXT: %11 = call half @llvm.nvvm.fma.rn.oob.relu.f16(half %0, half %1, half %10)
// CHECK-NEXT: ret half %11
@@ -25,8 +25,8 @@ llvm.func @fma_f16(%a: f16, %b: f16, %c: f16) -> f16 {
llvm.func @fma_bf16(%a: bf16, %b: bf16, %c: bf16) -> bf16 {
// CHECK-LABEL: define bfloat @fma_bf16(bfloat %0, bfloat %1, bfloat %2) {
- // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fma.rn.bf16(bfloat %0, bfloat %1, bfloat %2)
- // CHECK-NEXT: %5 = call bfloat @llvm.nvvm.fma.rn.relu.bf16(bfloat %0, bfloat %1, bfloat %4)
+ // CHECK-NEXT: %4 = call bfloat @llvm.nvvm.fma.bf16(bfloat %0, bfloat %1, bfloat %2, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call bfloat @llvm.nvvm.fma.relu.bf16(bfloat %0, bfloat %1, bfloat %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call bfloat @llvm.nvvm.fma.rn.oob.bf16(bfloat %0, bfloat %1, bfloat %5)
// CHECK-NEXT: %7 = call bfloat @llvm.nvvm.fma.rn.oob.relu.bf16(bfloat %0, bfloat %1, bfloat %6)
// CHECK-NEXT: ret bfloat %7
@@ -40,10 +40,10 @@ llvm.func @fma_bf16(%a: bf16, %b: bf16, %c: bf16) -> bf16 {
llvm.func @fma_f32_rn(%a: f32, %b: f32, %c: f32) -> f32 {
// CHECK-LABEL: define float @fma_f32_rn(float %0, float %1, float %2) {
- // CHECK-NEXT: %4 = call float @llvm.nvvm.fma.rn.f(float %0, float %1, float %2)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.fma.rn.ftz.f(float %0, float %1, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fma.rn.sat.f(float %0, float %1, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.rn.ftz.sat.f(float %0, float %1, float %6)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fma.f32(float %0, float %1, float %2, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fma.ftz.f32(float %0, float %1, float %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fma.sat.f32(float %0, float %1, float %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.ftz.sat.f32(float %0, float %1, float %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: ret float %7
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rn> : f32
@@ -55,10 +55,10 @@ llvm.func @fma_f32_rn(%a: f32, %b: f32, %c: f32) -> f32 {
llvm.func @fma_f32_rm(%a: f32, %b: f32, %c: f32) -> f32 {
// CHECK-LABEL: define float @fma_f32_rm(float %0, float %1, float %2) {
- // CHECK-NEXT: %4 = call float @llvm.nvvm.fma.rm.f(float %0, float %1, float %2)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.fma.rm.ftz.f(float %0, float %1, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fma.rm.sat.f(float %0, float %1, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.rm.ftz.sat.f(float %0, float %1, float %6)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fma.f32(float %0, float %1, float %2, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fma.ftz.f32(float %0, float %1, float %4, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fma.sat.f32(float %0, float %1, float %5, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.ftz.sat.f32(float %0, float %1, float %6, /* rnd=rm */ i32 3)
// CHECK-NEXT: ret float %7
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rm> : f32
@@ -70,10 +70,10 @@ llvm.func @fma_f32_rm(%a: f32, %b: f32, %c: f32) -> f32 {
llvm.func @fma_f32_rp(%a: f32, %b: f32, %c: f32) -> f32 {
// CHECK-LABEL: define float @fma_f32_rp(float %0, float %1, float %2) {
- // CHECK-NEXT: %4 = call float @llvm.nvvm.fma.rp.f(float %0, float %1, float %2)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.fma.rp.ftz.f(float %0, float %1, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fma.rp.sat.f(float %0, float %1, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.rp.ftz.sat.f(float %0, float %1, float %6)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fma.f32(float %0, float %1, float %2, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fma.ftz.f32(float %0, float %1, float %4, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fma.sat.f32(float %0, float %1, float %5, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.ftz.sat.f32(float %0, float %1, float %6, /* rnd=rp */ i32 2)
// CHECK-NEXT: ret float %7
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rp> : f32
@@ -85,10 +85,10 @@ llvm.func @fma_f32_rp(%a: f32, %b: f32, %c: f32) -> f32 {
llvm.func @fma_f32_rz(%a: f32, %b: f32, %c: f32) -> f32 {
// CHECK-LABEL: define float @fma_f32_rz(float %0, float %1, float %2) {
- // CHECK-NEXT: %4 = call float @llvm.nvvm.fma.rz.f(float %0, float %1, float %2)
- // CHECK-NEXT: %5 = call float @llvm.nvvm.fma.rz.ftz.f(float %0, float %1, float %4)
- // CHECK-NEXT: %6 = call float @llvm.nvvm.fma.rz.sat.f(float %0, float %1, float %5)
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.rz.ftz.sat.f(float %0, float %1, float %6)
+ // CHECK-NEXT: %4 = call float @llvm.nvvm.fma.f32(float %0, float %1, float %2, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %5 = call float @llvm.nvvm.fma.ftz.f32(float %0, float %1, float %4, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %6 = call float @llvm.nvvm.fma.sat.f32(float %0, float %1, float %5, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.ftz.sat.f32(float %0, float %1, float %6, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret float %7
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rz> : f32
@@ -100,10 +100,10 @@ llvm.func @fma_f32_rz(%a: f32, %b: f32, %c: f32) -> f32 {
llvm.func @fma_f64(%a: f64, %b: f64, %c: f64) -> f64 {
// CHECK-LABEL: define double @fma_f64(double %0, double %1, double %2) {
- // CHECK-NEXT: %4 = call double @llvm.nvvm.fma.rn.d(double %0, double %1, double %2)
- // CHECK-NEXT: %5 = call double @llvm.nvvm.fma.rm.d(double %0, double %1, double %4)
- // CHECK-NEXT: %6 = call double @llvm.nvvm.fma.rp.d(double %0, double %1, double %5)
- // CHECK-NEXT: %7 = call double @llvm.nvvm.fma.rz.d(double %0, double %1, double %6)
+ // CHECK-NEXT: %4 = call double @llvm.nvvm.fma.f64(double %0, double %1, double %2, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call double @llvm.nvvm.fma.f64(double %0, double %1, double %4, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %6 = call double @llvm.nvvm.fma.f64(double %0, double %1, double %5, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %7 = call double @llvm.nvvm.fma.f64(double %0, double %1, double %6, /* rnd=rz */ i32 0)
// CHECK-NEXT: ret double %7
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rn> : f64
diff --git a/mlir/test/Target/LLVMIR/nvvm/fma/fma_vector.mlir b/mlir/test/Target/LLVMIR/nvvm/fma/fma_vector.mlir
index 211f0b0d9f153b..9868d876fccc4a 100644
--- a/mlir/test/Target/LLVMIR/nvvm/fma/fma_vector.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/fma/fma_vector.mlir
@@ -2,12 +2,12 @@
llvm.func @fma_f16(%a: vector<2xf16>, %b: vector<2xf16>, %c: vector<2xf16>) -> vector<2xf16> {
// CHECK-LABEL: define <2 x half> @fma_f16(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
- // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fma.rn.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
- // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fma.rn.ftz.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %4)
- // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fma.rn.sat.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %5)
- // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fma.rn.ftz.sat.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %6)
- // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fma.rn.relu.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %7)
- // CHECK-NEXT: %9 = call <2 x half> @llvm.nvvm.fma.rn.ftz.relu.f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %8)
+ // CHECK-NEXT: %4 = call <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = call <2 x half> @llvm.nvvm.fma.sat.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %5, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %7 = call <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %6, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %8 = call <2 x half> @llvm.nvvm.fma.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %7, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %9 = call <2 x half> @llvm.nvvm.fma.ftz.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %8, /* rnd=rn */ i32 1)
// CHECK-NEXT: %10 = call <2 x half> @llvm.nvvm.fma.rn.oob.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %9)
// CHECK-NEXT: %11 = call <2 x half> @llvm.nvvm.fma.rn.oob.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %10)
// CHECK-NEXT: ret <2 x half> %11
@@ -25,8 +25,8 @@ llvm.func @fma_f16(%a: vector<2xf16>, %b: vector<2xf16>, %c: vector<2xf16>) -> v
llvm.func @fma_bf16(%a: vector<2xbf16>, %b: vector<2xbf16>, %c: vector<2xbf16>) -> vector<2xbf16> {
// CHECK-LABEL: define <2 x bfloat> @fma_bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2) {
- // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fma.rn.bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2)
- // CHECK-NEXT: %5 = call <2 x bfloat> @llvm.nvvm.fma.rn.relu.bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %4)
+ // CHECK-NEXT: %4 = call <2 x bfloat> @llvm.nvvm.fma.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call <2 x bfloat> @llvm.nvvm.fma.relu.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %4, /* rnd=rn */ i32 1)
// CHECK-NEXT: %6 = call <2 x bfloat> @llvm.nvvm.fma.rn.oob.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %5)
// CHECK-NEXT: %7 = call <2 x bfloat> @llvm.nvvm.fma.rn.oob.relu.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %6)
// CHECK-NEXT: ret <2 x bfloat> %7
@@ -40,47 +40,29 @@ llvm.func @fma_bf16(%a: vector<2xbf16>, %b: vector<2xbf16>, %c: vector<2xbf16>)
llvm.func @fma_f32_rn(%a: vector<2xf32>, %b: vector<2xf32>, %c: vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @fma_f32_rn(<2 x float> %0, <2 x float> %1, <2 x float> %2) {
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %6 = extractelement <2 x float> %2, i32 0
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.rn.f(float %4, float %5, float %6)
- // CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
- // CHECK-NEXT: %9 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %10 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %2, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fma.rn.f(float %9, float %10, float %11)
- // CHECK-NEXT: %13 = insertelement <2 x float> %8, float %12, i32 1
- // CHECK-NEXT: %14 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %16 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %17 = call float @llvm.nvvm.fma.rn.ftz.f(float %14, float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> poison, float %17, i32 0
- // CHECK-NEXT: %19 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %20 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %21 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %22 = call float @llvm.nvvm.fma.rn.ftz.f(float %19, float %20, float %21)
- // CHECK-NEXT: %23 = insertelement <2 x float> %18, float %22, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %25 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %23, i32 0
- // CHECK-NEXT: %27 = call float @llvm.nvvm.fma.rn.sat.f(float %24, float %25, float %26)
- // CHECK-NEXT: %28 = insertelement <2 x float> poison, float %27, i32 0
- // CHECK-NEXT: %29 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %30 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %31 = extractelement <2 x float> %23, i32 1
- // CHECK-NEXT: %32 = call float @llvm.nvvm.fma.rn.sat.f(float %29, float %30, float %31)
- // CHECK-NEXT: %33 = insertelement <2 x float> %28, float %32, i32 1
- // CHECK-NEXT: %34 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %36 = extractelement <2 x float> %33, i32 0
- // CHECK-NEXT: %37 = call float @llvm.nvvm.fma.rn.ftz.sat.f(float %34, float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> poison, float %37, i32 0
- // CHECK-NEXT: %39 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %40 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %41 = extractelement <2 x float> %33, i32 1
- // CHECK-NEXT: %42 = call float @llvm.nvvm.fma.rn.ftz.sat.f(float %39, float %40, float %41)
- // CHECK-NEXT: %43 = insertelement <2 x float> %38, float %42, i32 1
- // CHECK-NEXT: ret <2 x float> %43
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %5 = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %4, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %6 = extractelement <2 x float> %0, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %1, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fma.sat.f32(float %6, float %7, float %8, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %10 = insertelement <2 x float> poison, float %9, i32 0
+ // CHECK-NEXT: %11 = extractelement <2 x float> %0, i32 1
+ // CHECK-NEXT: %12 = extractelement <2 x float> %1, i32 1
+ // CHECK-NEXT: %13 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fma.sat.f32(float %11, float %12, float %13, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %15 = insertelement <2 x float> %10, float %14, i32 1
+ // CHECK-NEXT: %16 = extractelement <2 x float> %0, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %1, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fma.ftz.sat.f32(float %16, float %17, float %18, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %0, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %1, i32 1
+ // CHECK-NEXT: %23 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fma.ftz.sat.f32(float %21, float %22, float %23, /* rnd=rn */ i32 1)
+ // CHECK-NEXT: %25 = insertelement <2 x float> %20, float %24, i32 1
+ // CHECK-NEXT: ret <2 x float> %25
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rn> : vector<2xf32>
%f1 = nvvm.fma %a, %b, %f0, rnd = <rn> ftz = true : vector<2xf32>
@@ -91,47 +73,29 @@ llvm.func @fma_f32_rn(%a: vector<2xf32>, %b: vector<2xf32>, %c: vector<2xf32>) -
llvm.func @fma_f32_rm(%a: vector<2xf32>, %b: vector<2xf32>, %c: vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @fma_f32_rm(<2 x float> %0, <2 x float> %1, <2 x float> %2) {
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %6 = extractelement <2 x float> %2, i32 0
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.rm.f(float %4, float %5, float %6)
- // CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
- // CHECK-NEXT: %9 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %10 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %2, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fma.rm.f(float %9, float %10, float %11)
- // CHECK-NEXT: %13 = insertelement <2 x float> %8, float %12, i32 1
- // CHECK-NEXT: %14 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %16 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %17 = call float @llvm.nvvm.fma.rm.ftz.f(float %14, float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> poison, float %17, i32 0
- // CHECK-NEXT: %19 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %20 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %21 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %22 = call float @llvm.nvvm.fma.rm.ftz.f(float %19, float %20, float %21)
- // CHECK-NEXT: %23 = insertelement <2 x float> %18, float %22, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %25 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %23, i32 0
- // CHECK-NEXT: %27 = call float @llvm.nvvm.fma.rm.sat.f(float %24, float %25, float %26)
- // CHECK-NEXT: %28 = insertelement <2 x float> poison, float %27, i32 0
- // CHECK-NEXT: %29 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %30 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %31 = extractelement <2 x float> %23, i32 1
- // CHECK-NEXT: %32 = call float @llvm.nvvm.fma.rm.sat.f(float %29, float %30, float %31)
- // CHECK-NEXT: %33 = insertelement <2 x float> %28, float %32, i32 1
- // CHECK-NEXT: %34 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %36 = extractelement <2 x float> %33, i32 0
- // CHECK-NEXT: %37 = call float @llvm.nvvm.fma.rm.ftz.sat.f(float %34, float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> poison, float %37, i32 0
- // CHECK-NEXT: %39 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %40 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %41 = extractelement <2 x float> %33, i32 1
- // CHECK-NEXT: %42 = call float @llvm.nvvm.fma.rm.ftz.sat.f(float %39, float %40, float %41)
- // CHECK-NEXT: %43 = insertelement <2 x float> %38, float %42, i32 1
- // CHECK-NEXT: ret <2 x float> %43
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %5 = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %4, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %6 = extractelement <2 x float> %0, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %1, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fma.sat.f32(float %6, float %7, float %8, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %10 = insertelement <2 x float> poison, float %9, i32 0
+ // CHECK-NEXT: %11 = extractelement <2 x float> %0, i32 1
+ // CHECK-NEXT: %12 = extractelement <2 x float> %1, i32 1
+ // CHECK-NEXT: %13 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fma.sat.f32(float %11, float %12, float %13, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %15 = insertelement <2 x float> %10, float %14, i32 1
+ // CHECK-NEXT: %16 = extractelement <2 x float> %0, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %1, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fma.ftz.sat.f32(float %16, float %17, float %18, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %0, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %1, i32 1
+ // CHECK-NEXT: %23 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fma.ftz.sat.f32(float %21, float %22, float %23, /* rnd=rm */ i32 3)
+ // CHECK-NEXT: %25 = insertelement <2 x float> %20, float %24, i32 1
+ // CHECK-NEXT: ret <2 x float> %25
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rm> : vector<2xf32>
%f1 = nvvm.fma %a, %b, %f0, rnd = <rm> ftz = true : vector<2xf32>
@@ -142,47 +106,29 @@ llvm.func @fma_f32_rm(%a: vector<2xf32>, %b: vector<2xf32>, %c: vector<2xf32>) -
llvm.func @fma_f32_rp(%a: vector<2xf32>, %b: vector<2xf32>, %c: vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @fma_f32_rp(<2 x float> %0, <2 x float> %1, <2 x float> %2) {
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %6 = extractelement <2 x float> %2, i32 0
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.rp.f(float %4, float %5, float %6)
- // CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
- // CHECK-NEXT: %9 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %10 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %2, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fma.rp.f(float %9, float %10, float %11)
- // CHECK-NEXT: %13 = insertelement <2 x float> %8, float %12, i32 1
- // CHECK-NEXT: %14 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %16 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %17 = call float @llvm.nvvm.fma.rp.ftz.f(float %14, float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> poison, float %17, i32 0
- // CHECK-NEXT: %19 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %20 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %21 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %22 = call float @llvm.nvvm.fma.rp.ftz.f(float %19, float %20, float %21)
- // CHECK-NEXT: %23 = insertelement <2 x float> %18, float %22, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %25 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %23, i32 0
- // CHECK-NEXT: %27 = call float @llvm.nvvm.fma.rp.sat.f(float %24, float %25, float %26)
- // CHECK-NEXT: %28 = insertelement <2 x float> poison, float %27, i32 0
- // CHECK-NEXT: %29 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %30 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %31 = extractelement <2 x float> %23, i32 1
- // CHECK-NEXT: %32 = call float @llvm.nvvm.fma.rp.sat.f(float %29, float %30, float %31)
- // CHECK-NEXT: %33 = insertelement <2 x float> %28, float %32, i32 1
- // CHECK-NEXT: %34 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %36 = extractelement <2 x float> %33, i32 0
- // CHECK-NEXT: %37 = call float @llvm.nvvm.fma.rp.ftz.sat.f(float %34, float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> poison, float %37, i32 0
- // CHECK-NEXT: %39 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %40 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %41 = extractelement <2 x float> %33, i32 1
- // CHECK-NEXT: %42 = call float @llvm.nvvm.fma.rp.ftz.sat.f(float %39, float %40, float %41)
- // CHECK-NEXT: %43 = insertelement <2 x float> %38, float %42, i32 1
- // CHECK-NEXT: ret <2 x float> %43
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %5 = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %4, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %6 = extractelement <2 x float> %0, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %1, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fma.sat.f32(float %6, float %7, float %8, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %10 = insertelement <2 x float> poison, float %9, i32 0
+ // CHECK-NEXT: %11 = extractelement <2 x float> %0, i32 1
+ // CHECK-NEXT: %12 = extractelement <2 x float> %1, i32 1
+ // CHECK-NEXT: %13 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fma.sat.f32(float %11, float %12, float %13, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %15 = insertelement <2 x float> %10, float %14, i32 1
+ // CHECK-NEXT: %16 = extractelement <2 x float> %0, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %1, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fma.ftz.sat.f32(float %16, float %17, float %18, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %0, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %1, i32 1
+ // CHECK-NEXT: %23 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fma.ftz.sat.f32(float %21, float %22, float %23, /* rnd=rp */ i32 2)
+ // CHECK-NEXT: %25 = insertelement <2 x float> %20, float %24, i32 1
+ // CHECK-NEXT: ret <2 x float> %25
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rp> : vector<2xf32>
%f1 = nvvm.fma %a, %b, %f0, rnd = <rp> ftz = true : vector<2xf32>
@@ -193,47 +139,29 @@ llvm.func @fma_f32_rp(%a: vector<2xf32>, %b: vector<2xf32>, %c: vector<2xf32>) -
llvm.func @fma_f32_rz(%a: vector<2xf32>, %b: vector<2xf32>, %c: vector<2xf32>) -> vector<2xf32> {
// CHECK-LABEL: define <2 x float> @fma_f32_rz(<2 x float> %0, <2 x float> %1, <2 x float> %2) {
- // CHECK-NEXT: %4 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %5 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %6 = extractelement <2 x float> %2, i32 0
- // CHECK-NEXT: %7 = call float @llvm.nvvm.fma.rz.f(float %4, float %5, float %6)
- // CHECK-NEXT: %8 = insertelement <2 x float> poison, float %7, i32 0
- // CHECK-NEXT: %9 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %10 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %11 = extractelement <2 x float> %2, i32 1
- // CHECK-NEXT: %12 = call float @llvm.nvvm.fma.rz.f(float %9, float %10, float %11)
- // CHECK-NEXT: %13 = insertelement <2 x float> %8, float %12, i32 1
- // CHECK-NEXT: %14 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %15 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %16 = extractelement <2 x float> %13, i32 0
- // CHECK-NEXT: %17 = call float @llvm.nvvm.fma.rz.ftz.f(float %14, float %15, float %16)
- // CHECK-NEXT: %18 = insertelement <2 x float> poison, float %17, i32 0
- // CHECK-NEXT: %19 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %20 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %21 = extractelement <2 x float> %13, i32 1
- // CHECK-NEXT: %22 = call float @llvm.nvvm.fma.rz.ftz.f(float %19, float %20, float %21)
- // CHECK-NEXT: %23 = insertelement <2 x float> %18, float %22, i32 1
- // CHECK-NEXT: %24 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %25 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %26 = extractelement <2 x float> %23, i32 0
- // CHECK-NEXT: %27 = call float @llvm.nvvm.fma.rz.sat.f(float %24, float %25, float %26)
- // CHECK-NEXT: %28 = insertelement <2 x float> poison, float %27, i32 0
- // CHECK-NEXT: %29 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %30 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %31 = extractelement <2 x float> %23, i32 1
- // CHECK-NEXT: %32 = call float @llvm.nvvm.fma.rz.sat.f(float %29, float %30, float %31)
- // CHECK-NEXT: %33 = insertelement <2 x float> %28, float %32, i32 1
- // CHECK-NEXT: %34 = extractelement <2 x float> %0, i32 0
- // CHECK-NEXT: %35 = extractelement <2 x float> %1, i32 0
- // CHECK-NEXT: %36 = extractelement <2 x float> %33, i32 0
- // CHECK-NEXT: %37 = call float @llvm.nvvm.fma.rz.ftz.sat.f(float %34, float %35, float %36)
- // CHECK-NEXT: %38 = insertelement <2 x float> poison, float %37, i32 0
- // CHECK-NEXT: %39 = extractelement <2 x float> %0, i32 1
- // CHECK-NEXT: %40 = extractelement <2 x float> %1, i32 1
- // CHECK-NEXT: %41 = extractelement <2 x float> %33, i32 1
- // CHECK-NEXT: %42 = call float @llvm.nvvm.fma.rz.ftz.sat.f(float %39, float %40, float %41)
- // CHECK-NEXT: %43 = insertelement <2 x float> %38, float %42, i32 1
- // CHECK-NEXT: ret <2 x float> %43
+ // CHECK-NEXT: %4 = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %2, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %5 = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %0, <2 x float> %1, <2 x float> %4, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %6 = extractelement <2 x float> %0, i32 0
+ // CHECK-NEXT: %7 = extractelement <2 x float> %1, i32 0
+ // CHECK-NEXT: %8 = extractelement <2 x float> %5, i32 0
+ // CHECK-NEXT: %9 = call float @llvm.nvvm.fma.sat.f32(float %6, float %7, float %8, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %10 = insertelement <2 x float> poison, float %9, i32 0
+ // CHECK-NEXT: %11 = extractelement <2 x float> %0, i32 1
+ // CHECK-NEXT: %12 = extractelement <2 x float> %1, i32 1
+ // CHECK-NEXT: %13 = extractelement <2 x float> %5, i32 1
+ // CHECK-NEXT: %14 = call float @llvm.nvvm.fma.sat.f32(float %11, float %12, float %13, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %15 = insertelement <2 x float> %10, float %14, i32 1
+ // CHECK-NEXT: %16 = extractelement <2 x float> %0, i32 0
+ // CHECK-NEXT: %17 = extractelement <2 x float> %1, i32 0
+ // CHECK-NEXT: %18 = extractelement <2 x float> %15, i32 0
+ // CHECK-NEXT: %19 = call float @llvm.nvvm.fma.ftz.sat.f32(float %16, float %17, float %18, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %20 = insertelement <2 x float> poison, float %19, i32 0
+ // CHECK-NEXT: %21 = extractelement <2 x float> %0, i32 1
+ // CHECK-NEXT: %22 = extractelement <2 x float> %1, i32 1
+ // CHECK-NEXT: %23 = extractelement <2 x float> %15, i32 1
+ // CHECK-NEXT: %24 = call float @llvm.nvvm.fma.ftz.sat.f32(float %21, float %22, float %23, /* rnd=rz */ i32 0)
+ // CHECK-NEXT: %25 = insertelement <2 x float> %20, float %24, i32 1
+ // CHECK-NEXT: ret <2 x float> %25
// CHECK-NEXT: }
%f0 = nvvm.fma %a, %b, %c, rnd = <rz> : vector<2xf32>
%f1 = nvvm.fma %a, %b, %f0, rnd = <rz> ftz = true : vector<2xf32>
@@ -247,42 +175,42 @@ llvm.func @fma_f64(%a: vector<2xf64>, %b: vector<2xf64>, %c: vector<2xf64>) -> v
// CHECK-NEXT: %4 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %5 = extractelement <2 x double> %1, i32 0
// CHECK-NEXT: %6 = extractelement <2 x double> %2, i32 0
- // CHECK-NEXT: %7 = call double @llvm.nvvm.fma.rn.d(double %4, double %5, double %6)
+ // CHECK-NEXT: %7 = call double @llvm.nvvm.fma.f64(double %4, double %5, double %6, /* rnd=rn */ i32 1)
// CHECK-NEXT: %8 = insertelement <2 x double> poison, double %7, i32 0
// CHECK-NEXT: %9 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %10 = extractelement <2 x double> %1, i32 1
// CHECK-NEXT: %11 = extractelement <2 x double> %2, i32 1
- // CHECK-NEXT: %12 = call double @llvm.nvvm.fma.rn.d(double %9, double %10, double %11)
+ // CHECK-NEXT: %12 = call double @llvm.nvvm.fma.f64(double %9, double %10, double %11, /* rnd=rn */ i32 1)
// CHECK-NEXT: %13 = insertelement <2 x double> %8, double %12, i32 1
// CHECK-NEXT: %14 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %15 = extractelement <2 x double> %1, i32 0
// CHECK-NEXT: %16 = extractelement <2 x double> %13, i32 0
- // CHECK-NEXT: %17 = call double @llvm.nvvm.fma.rm.d(double %14, double %15, double %16)
+ // CHECK-NEXT: %17 = call double @llvm.nvvm.fma.f64(double %14, double %15, double %16, /* rnd=rm */ i32 3)
// CHECK-NEXT: %18 = insertelement <2 x double> poison, double %17, i32 0
// CHECK-NEXT: %19 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %20 = extractelement <2 x double> %1, i32 1
// CHECK-NEXT: %21 = extractelement <2 x double> %13, i32 1
- // CHECK-NEXT: %22 = call double @llvm.nvvm.fma.rm.d(double %19, double %20, double %21)
+ // CHECK-NEXT: %22 = call double @llvm.nvvm.fma.f64(double %19, double %20, double %21, /* rnd=rm */ i32 3)
// CHECK-NEXT: %23 = insertelement <2 x double> %18, double %22, i32 1
// CHECK-NEXT: %24 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %25 = extractelement <2 x double> %1, i32 0
// CHECK-NEXT: %26 = extractelement <2 x double> %23, i32 0
- // CHECK-NEXT: %27 = call double @llvm.nvvm.fma.rp.d(double %24, double %25, double %26)
+ // CHECK-NEXT: %27 = call double @llvm.nvvm.fma.f64(double %24, double %25, double %26, /* rnd=rp */ i32 2)
// CHECK-NEXT: %28 = insertelement <2 x double> poison, double %27, i32 0
// CHECK-NEXT: %29 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %30 = extractelement <2 x double> %1, i32 1
// CHECK-NEXT: %31 = extractelement <2 x double> %23, i32 1
- // CHECK-NEXT: %32 = call double @llvm.nvvm.fma.rp.d(double %29, double %30, double %31)
+ // CHECK-NEXT: %32 = call double @llvm.nvvm.fma.f64(double %29, double %30, double %31, /* rnd=rp */ i32 2)
// CHECK-NEXT: %33 = insertelement <2 x double> %28, double %32, i32 1
// CHECK-NEXT: %34 = extractelement <2 x double> %0, i32 0
// CHECK-NEXT: %35 = extractelement <2 x double> %1, i32 0
// CHECK-NEXT: %36 = extractelement <2 x double> %33, i32 0
- // CHECK-NEXT: %37 = call double @llvm.nvvm.fma.rz.d(double %34, double %35, double %36)
+ // CHECK-NEXT: %37 = call double @llvm.nvvm.fma.f64(double %34, double %35, double %36, /* rnd=rz */ i32 0)
// CHECK-NEXT: %38 = insertelement <2 x double> poison, double %37, i32 0
// CHECK-NEXT: %39 = extractelement <2 x double> %0, i32 1
// CHECK-NEXT: %40 = extractelement <2 x double> %1, i32 1
// CHECK-NEXT: %41 = extractelement <2 x double> %33, i32 1
- // CHECK-NEXT: %42 = call double @llvm.nvvm.fma.rz.d(double %39, double %40, double %41)
+ // CHECK-NEXT: %42 = call double @llvm.nvvm.fma.f64(double %39, double %40, double %41, /* rnd=rz */ i32 0)
// CHECK-NEXT: %43 = insertelement <2 x double> %38, double %42, i32 1
// CHECK-NEXT: ret <2 x double> %43
// CHECK-NEXT: }
>From edcab85fd79ab60f52c0f90252d7ecfee2be823f Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 16 Oct 2026 04:12:55 +0000
Subject: [PATCH 2/4] clean-up
---
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 2 +-
llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp | 7 ++-----
.../Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp | 5 +----
3 files changed, 4 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 7000cca82f6498..a1e911b87256aa 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2383,7 +2383,7 @@ foreach ftz = ["", "ftz"] in {
defm FMA_RND_F64 :
F_MATH_3_RNDOP_TY<"fma.${rnd}.f64", F64RT, int_nvvm_fma>;
-// Mixed precision
+// Mixed Precision
foreach rnd = FPRoundingModes in {
defvar rnd_imm = !cast<TImmLeaf>("fp_rnd_" # rnd # "_imm");
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 309f11d439f24f..a0e40da887a498 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -159,8 +159,7 @@ static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
return {Intrinsic::floor, FTZ_MustBeOn};
case Intrinsic::nvvm_fma:
case Intrinsic::nvvm_fma_ftz: {
- // Only fma with the round-to-nearest-even rounding mode matches the
- // semantics of llvm.fma.
+ // Only replace if the rounding mode is RN.
if (nvvm::GetRoundingModeFromImmArg(II->getArgOperand(3)) !=
APFloat::rmNearestTiesToEven)
return {};
@@ -169,7 +168,6 @@ static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
if (EltTy->isDoubleTy())
return IsFTZ ? SimplifyAction{}
: SimplifyAction{Intrinsic::fma, FTZ_Any};
- // ftz is not supported for bf16.
if (EltTy->isBFloatTy() && IsFTZ)
return {};
const bool IsHalfTy = EltTy->isHalfTy() || EltTy->isBFloatTy();
@@ -318,8 +316,7 @@ static Instruction *convertNvvmIntrinsicToLlvm(InstCombiner &IC,
// Simplify to target-generic intrinsic.
if (Action.IID) {
SmallVector<Value *, 4> Args(II->args());
- // The overloaded NVVM fma intrinsics have a trailing rounding-mode
- // operand that llvm.fma does not take; it is known to be RN here.
+ // llvm.fma does not take the trailing rounding-mode operand.
if (II->getIntrinsicID() == Intrinsic::nvvm_fma ||
II->getIntrinsicID() == Intrinsic::nvvm_fma_ftz)
Args.pop_back();
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index c554a7768c4dba..b34872f807c1d2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -534,8 +534,7 @@ void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
llvm::Value *argB = mt.lookupValue(thisOp.getB());
llvm::Value *argC = mt.lookupValue(thisOp.getC());
- // The out-of-bounds variants have their own overloaded intrinsics, which
- // only support the RN rounding mode and take no rounding-mode operand.
+ // The oob variants only support RN and take no rounding-mode operand.
if (isOOB) {
llvm::Intrinsic::ID id = isRelu ? llvm::Intrinsic::nvvm_fma_rn_oob_relu
: llvm::Intrinsic::nvvm_fma_rn_oob;
@@ -544,14 +543,12 @@ void NVVM::FmaOp::lowerFmaToLLVMIR(Operation &op, LLVM::ModuleTranslation &mt,
return;
}
- // Overloaded fma intrinsic IDs, indexed by [`.ftz`][`.sat`/`.relu`].
static constexpr llvm::Intrinsic::ID fmaIDs[2][3] = {
{llvm::Intrinsic::nvvm_fma, llvm::Intrinsic::nvvm_fma_sat,
llvm::Intrinsic::nvvm_fma_relu},
{llvm::Intrinsic::nvvm_fma_ftz, llvm::Intrinsic::nvvm_fma_ftz_sat,
llvm::Intrinsic::nvvm_fma_ftz_relu}};
- // Op verifier ensures that sat and relu are mutually exclusive.
llvm::Intrinsic::ID id = fmaIDs[isFTZ][isSat ? 1 : (isRelu ? 2 : 0)];
llvm::Value *rnd = builder.getInt32(
static_cast<int>(getLLVMRoundingModeForFPArith(rndMode)));
>From 501891d7e13bf1ca41055555236b7f537c2ebd80 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 16 Oct 2026 09:26:50 +0000
Subject: [PATCH 3/4] change DefaultAttrsIntrinsic to PureIntrinsic
---
llvm/include/llvm/IR/IntrinsicsNVVM.td | 25 +++++++++++++------------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 0597b195a5a135..fa24e69664800d 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1825,18 +1825,19 @@ let TargetPrefix = "nvvm" in {
//
// Fma
//
- let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative,
- IntrNoCreateUndefOrPoison, ImmArg<ArgIndex<3>>,
- Range<ArgIndex<3>, 0, 4>,
- ArgInfo<ArgIndex<3>,
- [ArgName<"rnd">,
- ImmArgPrinter<"printFPRoundingMode">]>] in
- foreach ftz = ["", "_ftz"] in
- foreach variant = ["", "_sat", "_relu"] in
- def int_nvvm_fma # ftz # variant :
- DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
- [LLVMMatchType<0>, LLVMMatchType<0>,
- LLVMMatchType<0>, llvm_i32_ty]>;
+ // TODO: Add tests for commutativity along with missing mixed precision
+ // patterns.
+ foreach ftz = ["", "_ftz"] in
+ foreach variant = ["", "_sat", "_relu"] in
+ def int_nvvm_fma # ftz # variant :
+ PureIntrinsic<[llvm_anyfloat_ty],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
+ LLVMMatchType<0>, llvm_i32_ty],
+ [Commutative, ImmArg<ArgIndex<3>>,
+ Range<ArgIndex<3>, 0, 4>,
+ ArgInfo<ArgIndex<3>,
+ [ArgName<"rnd">,
+ ImmArgPrinter<"printFPRoundingMode">]>]>;
// oob (out-of-bounds) - clamps the result to 0 if either of the operand is
// an OOB NaN value.
>From 193da7bb6a8b19db5db5e15772416a8dcd861057 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Fri, 16 Oct 2026 10:11:41 +0000
Subject: [PATCH 4/4] regenerate test files with pretty printing
---
llvm/test/CodeGen/NVPTX/fp-arith-sat.ll | 64 ++++++-------
llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll | 16 ++--
llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll | 20 ++--
.../CodeGen/NVPTX/math-intrins-sm53-ptx42.ll | 16 ++--
.../math-intrins-sm80-ptx70-instcombine.ll | 12 +--
.../CodeGen/NVPTX/math-intrins-sm80-ptx70.ll | 16 ++--
llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll | 96 +++++++++----------
.../InstCombine/NVPTX/nvvm-intrins.ll | 6 +-
8 files changed, 123 insertions(+), 123 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
index b9aeed2b5d0ea6..b7e3365bb2a72c 100644
--- a/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-arith-sat.ll
@@ -20,17 +20,17 @@ define float @add_sat_f32(float %a, float %b) {
; CHECK-NEXT: add.rp.ftz.sat.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, i32 1)
- %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, i32 1)
+ %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %b, /* rnd=rn */ i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r1, /* rnd=rn */ i32 1)
- %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, i32 0)
+ %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r2, /* rnd=rz */ i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r3, /* rnd=rz */ i32 0)
- %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, i32 3)
- %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, i32 3)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r4, /* rnd=rm */ i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r5, /* rnd=rm */ i32 3)
- %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, i32 2)
- %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, i32 2)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %r6, /* rnd=rp */ i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %r7, /* rnd=rp */ i32 2)
ret float %r8
}
@@ -54,28 +54,28 @@ define float @sub_sat_f32(float %a, float %b) {
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, i32 1)
+ %r1 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f0, /* rnd=rn */ i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, i32 1)
+ %r2 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f1, /* rnd=rn */ i32 1)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, i32 0)
+ %r3 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f2, /* rnd=rz */ i32 0)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, i32 0)
+ %r4 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f3, /* rnd=rz */ i32 0)
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, i32 3)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f4, /* rnd=rm */ i32 3)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, i32 3)
+ %r6 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f5, /* rnd=rm */ i32 3)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, i32 2)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %a, float %f6, /* rnd=rp */ i32 2)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.ftz.sat.f32(float %a, float %f7, /* rnd=rp */ i32 2)
ret float %r8
}
@@ -98,17 +98,17 @@ define float @mul_sat_f32(float %a, float %b) {
; CHECK-NEXT: mul.rp.ftz.sat.f32 %r10, %r1, %r9;
; CHECK-NEXT: st.param.b32 [func_retval0], %r10;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.fmul.sat.f32(float %a, float %b, i32 1)
- %r2 = call float @llvm.nvvm.fmul.ftz.sat.f32(float %a, float %r1, i32 1)
+ %r1 = call float @llvm.nvvm.fmul.sat.f32(float %a, float %b, /* rnd=rn */ i32 1)
+ %r2 = call float @llvm.nvvm.fmul.ftz.sat.f32(float %a, float %r1, /* rnd=rn */ i32 1)
- %r3 = call float @llvm.nvvm.fmul.sat.f32(float %a, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fmul.ftz.sat.f32(float %a, float %r3, i32 0)
+ %r3 = call float @llvm.nvvm.fmul.sat.f32(float %a, float %r2, /* rnd=rz */ i32 0)
+ %r4 = call float @llvm.nvvm.fmul.ftz.sat.f32(float %a, float %r3, /* rnd=rz */ i32 0)
- %r5 = call float @llvm.nvvm.fmul.sat.f32(float %a, float %r4, i32 3)
- %r6 = call float @llvm.nvvm.fmul.ftz.sat.f32(float %a, float %r5, i32 3)
+ %r5 = call float @llvm.nvvm.fmul.sat.f32(float %a, float %r4, /* rnd=rm */ i32 3)
+ %r6 = call float @llvm.nvvm.fmul.ftz.sat.f32(float %a, float %r5, /* rnd=rm */ i32 3)
- %r7 = call float @llvm.nvvm.fmul.sat.f32(float %a, float %r6, i32 2)
- %r8 = call float @llvm.nvvm.fmul.ftz.sat.f32(float %a, float %r7, i32 2)
+ %r7 = call float @llvm.nvvm.fmul.sat.f32(float %a, float %r6, /* rnd=rp */ i32 2)
+ %r8 = call float @llvm.nvvm.fmul.ftz.sat.f32(float %a, float %r7, /* rnd=rp */ i32 2)
ret float %r8
}
@@ -132,17 +132,17 @@ define float @fma_sat_f32(float %a, float %b, float %c) {
; CHECK-NEXT: fma.rp.ftz.sat.f32 %r11, %r1, %r2, %r10;
; CHECK-NEXT: st.param.b32 [func_retval0], %r11;
; CHECK-NEXT: ret;
- %r1 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %c, i32 1)
- %r2 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r1, i32 1)
+ %r1 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %c, /* rnd=rn */ i32 1)
+ %r2 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r1, /* rnd=rn */ i32 1)
- %r3 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r3, i32 0)
+ %r3 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r2, /* rnd=rz */ i32 0)
+ %r4 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r3, /* rnd=rz */ i32 0)
- %r5 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r4, i32 3)
- %r6 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r5, i32 3)
+ %r5 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r4, /* rnd=rm */ i32 3)
+ %r6 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r5, /* rnd=rm */ i32 3)
- %r7 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r6, i32 2)
- %r8 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r7, i32 2)
+ %r7 = call float @llvm.nvvm.fma.sat.f32(float %a, float %b, float %r6, /* rnd=rp */ i32 2)
+ %r8 = call float @llvm.nvvm.fma.ftz.sat.f32(float %a, float %b, float %r7, /* rnd=rp */ i32 2)
ret float %r8
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll b/llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll
index cd933daeddc910..6648932c653ad7 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fma-f32x2.ll
@@ -16,7 +16,7 @@ define <2 x float> @fma_rn(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
; CHECK-NEXT: fma.rn.f32x2 %rd4, %rd1, %rd2, %rd3;
; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
- %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 1)
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rn */ i32 1)
ret <2 x float> %r
}
@@ -32,7 +32,7 @@ define <2 x float> @fma_rz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
; CHECK-NEXT: fma.rz.f32x2 %rd4, %rd1, %rd2, %rd3;
; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
- %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 0)
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rz */ i32 0)
ret <2 x float> %r
}
@@ -48,7 +48,7 @@ define <2 x float> @fma_rm(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
; CHECK-NEXT: fma.rm.f32x2 %rd4, %rd1, %rd2, %rd3;
; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
- %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 3)
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rm */ i32 3)
ret <2 x float> %r
}
@@ -64,7 +64,7 @@ define <2 x float> @fma_rp(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
; CHECK-NEXT: fma.rp.f32x2 %rd4, %rd1, %rd2, %rd3;
; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
- %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 2)
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rp */ i32 2)
ret <2 x float> %r
}
@@ -80,7 +80,7 @@ define <2 x float> @fma_rn_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
; CHECK-NEXT: fma.rn.ftz.f32x2 %rd4, %rd1, %rd2, %rd3;
; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
- %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 1)
+ %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rn */ i32 1)
ret <2 x float> %r
}
@@ -96,7 +96,7 @@ define <2 x float> @fma_rz_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
; CHECK-NEXT: fma.rz.ftz.f32x2 %rd4, %rd1, %rd2, %rd3;
; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
- %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 0)
+ %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rz */ i32 0)
ret <2 x float> %r
}
@@ -112,7 +112,7 @@ define <2 x float> @fma_rm_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
; CHECK-NEXT: fma.rm.ftz.f32x2 %rd4, %rd1, %rd2, %rd3;
; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
- %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 3)
+ %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rm */ i32 3)
ret <2 x float> %r
}
@@ -128,6 +128,6 @@ define <2 x float> @fma_rp_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
; CHECK-NEXT: fma.rp.ftz.f32x2 %rd4, %rd1, %rd2, %rd3;
; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd4;
; CHECK-NEXT: ret;
- %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 2)
+ %r = call <2 x float> @llvm.nvvm.fma.ftz.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rp */ i32 2)
ret <2 x float> %r
}
diff --git a/llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll b/llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll
index bbf2521d635fea..054f87b9333292 100644
--- a/llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll
+++ b/llvm/test/CodeGen/NVPTX/fp-fma-invalid.ll
@@ -6,60 +6,60 @@ target triple = "nvptx64-nvidia-cuda"
; CHECK: error: {{.*}}llvm.nvvm.fma.sat with rounding mode rn and operand type v2f32 is not supported
define <2 x float> @sat_f32x2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
- %r = call <2 x float> @llvm.nvvm.fma.sat.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 1)
+ %r = call <2 x float> @llvm.nvvm.fma.sat.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rn */ i32 1)
ret <2 x float> %r
}
; NOF32X2: error: {{.*}}llvm.nvvm.fma with rounding mode rn and operand type v2f32 is not supported
define <2 x float> @unsupported_f32x2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
- %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, i32 1)
+ %r = call <2 x float> @llvm.nvvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c, /* rnd=rn */ i32 1)
ret <2 x float> %r
}
; CHECK: error: {{.*}}llvm.nvvm.fma.ftz with rounding mode rn and operand type f64 is not supported
define double @ftz_f64(double %a, double %b, double %c) {
- %r = call double @llvm.nvvm.fma.ftz.f64(double %a, double %b, double %c, i32 1)
+ %r = call double @llvm.nvvm.fma.ftz.f64(double %a, double %b, double %c, /* rnd=rn */ i32 1)
ret double %r
}
; CHECK: error: {{.*}}llvm.nvvm.fma.sat with rounding mode rz and operand type f16 is not supported
define half @rz_f16(half %a, half %b, half %c) {
- %r = call half @llvm.nvvm.fma.sat.f16(half %a, half %b, half %c, i32 0)
+ %r = call half @llvm.nvvm.fma.sat.f16(half %a, half %b, half %c, /* rnd=rz */ i32 0)
ret half %r
}
; CHECK: error: {{.*}}llvm.nvvm.fma.ftz with rounding mode rn and operand type bf16 is not supported
define bfloat @ftz_bf16(bfloat %a, bfloat %b, bfloat %c) {
- %r = call bfloat @llvm.nvvm.fma.ftz.bf16(bfloat %a, bfloat %b, bfloat %c, i32 1)
+ %r = call bfloat @llvm.nvvm.fma.ftz.bf16(bfloat %a, bfloat %b, bfloat %c, /* rnd=rn */ i32 1)
ret bfloat %r
}
; NOBF16: error: {{.*}}llvm.nvvm.fma with rounding mode rn and operand type bf16 is not supported
define bfloat @unsupported_bf16(bfloat %a, bfloat %b, bfloat %c) {
- %r = call bfloat @llvm.nvvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c, i32 1)
+ %r = call bfloat @llvm.nvvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c, /* rnd=rn */ i32 1)
ret bfloat %r
}
; CHECK: error: {{.*}}llvm.nvvm.fma.relu with rounding mode rn and operand type f32 is not supported
define float @relu_f32(float %a, float %b, float %c) {
- %r = call float @llvm.nvvm.fma.relu.f32(float %a, float %b, float %c, i32 1)
+ %r = call float @llvm.nvvm.fma.relu.f32(float %a, float %b, float %c, /* rnd=rn */ i32 1)
ret float %r
}
; CHECK: error: {{.*}}llvm.nvvm.fma.relu with rounding mode rn and operand type f64 is not supported
define double @relu_f64(double %a, double %b, double %c) {
- %r = call double @llvm.nvvm.fma.relu.f64(double %a, double %b, double %c, i32 1)
+ %r = call double @llvm.nvvm.fma.relu.f64(double %a, double %b, double %c, /* rnd=rn */ i32 1)
ret double %r
}
; CHECK: error: {{.*}}llvm.nvvm.fma.relu with rounding mode rz and operand type f16 is not supported
define half @relu_rz_f16(half %a, half %b, half %c) {
- %r = call half @llvm.nvvm.fma.relu.f16(half %a, half %b, half %c, i32 0)
+ %r = call half @llvm.nvvm.fma.relu.f16(half %a, half %b, half %c, /* rnd=rz */ i32 0)
ret half %r
}
; CHECK: error: {{.*}}llvm.nvvm.fma with rounding mode rn and operand type v4f32 is not supported
define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
- %r = call <4 x float> @llvm.nvvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c, i32 1)
+ %r = call <4 x float> @llvm.nvvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c, /* rnd=rn */ i32 1)
ret <4 x float> %r
}
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins-sm53-ptx42.ll b/llvm/test/CodeGen/NVPTX/math-intrins-sm53-ptx42.ll
index 7936bc7c7d6b7a..03499b861b635d 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins-sm53-ptx42.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins-sm53-ptx42.ll
@@ -14,7 +14,7 @@ declare <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16(<2 x half>, <2 x half>, <2 x hal
define half @fma_rn_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.f16
- %res = call half @llvm.nvvm.fma.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -22,7 +22,7 @@ define half @fma_rn_f16(half %0, half %1, half %2) {
define half @fma_rn_ftz_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.f16
- %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -30,7 +30,7 @@ define half @fma_rn_ftz_f16(half %0, half %1, half %2) {
define half @fma_rn_sat_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.sat.f16
- %res = call half @llvm.nvvm.fma.sat.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.sat.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -38,7 +38,7 @@ define half @fma_rn_sat_f16(half %0, half %1, half %2) {
define half @fma_rn_ftz_sat_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.sat.f16
- %res = call half @llvm.nvvm.fma.ftz.sat.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.ftz.sat.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -46,7 +46,7 @@ define half @fma_rn_ftz_sat_f16(half %0, half %1, half %2) {
define <2 x half> @fma_rn_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
@@ -54,7 +54,7 @@ define <2 x half> @fma_rn_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
define <2 x half> @fma_rn_ftz_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
@@ -62,7 +62,7 @@ define <2 x half> @fma_rn_ftz_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
define <2 x half> @fma_rn_sat_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.sat.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.sat.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.sat.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
@@ -70,6 +70,6 @@ define <2 x half> @fma_rn_sat_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
define <2 x half> @fma_rn_ftz_sat_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.sat.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.sat.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-instcombine.ll b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-instcombine.ll
index be6eadce562c00..4997a2114824d1 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-instcombine.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70-instcombine.ll
@@ -274,7 +274,7 @@ define <2 x half> @fmax_ftz_nan_f16x2_no_attr(<2 x half> %0, <2 x half> %1) {
define half @fma_rn_f16(half %0, half %1, half %2) {
; CHECK-NOT: @llvm.nvvm.fma.f16
; CHECK: @llvm.fma.f16
- %res = call half @llvm.nvvm.fma.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -282,7 +282,7 @@ define half @fma_rn_f16(half %0, half %1, half %2) {
define half @fma_rn_ftz_f16_no_attr(half %0, half %1, half %2) {
; CHECK-NOT: @llvm.fma.f16
; CHECK: @llvm.nvvm.fma.ftz.f16
- %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -290,7 +290,7 @@ define half @fma_rn_ftz_f16_no_attr(half %0, half %1, half %2) {
define half @fma_rn_ftz_f16(half %0, half %1, half %2) #0 {
; CHECK-NOT: @llvm.nvvm.fma.ftz.f16
; CHECK: @llvm.fma.f16
- %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.ftz.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -298,7 +298,7 @@ define half @fma_rn_ftz_f16(half %0, half %1, half %2) #0 {
define <2 x half> @fma_rn_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: @llvm.nvvm.fma.v2f16
; CHECK: @llvm.fma.v2f16
- %res = call <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
@@ -306,7 +306,7 @@ define <2 x half> @fma_rn_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
define <2 x half> @fma_rn_ftz_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) #0 {
; CHECK-NOT: @llvm.nvvm.fma.ftz.v2f16
; CHECK: @llvm.fma.v2f16
- %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
@@ -314,7 +314,7 @@ define <2 x half> @fma_rn_ftz_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2)
define <2 x half> @fma_rn_ftz_f16x2_no_attr(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: @llvm.fma.v2f16
; CHECK: @llvm.nvvm.fma.ftz.v2f16
- %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70.ll b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70.ll
index 807c5de49c9666..2b4bacc138f1c8 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins-sm80-ptx70.ll
@@ -305,7 +305,7 @@ define <2 x bfloat> @fmax_nan_bf16x2(<2 x bfloat> %0, <2 x bfloat> %1) {
define half @fma_rn_relu_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.relu.f16
- %res = call half @llvm.nvvm.fma.relu.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.relu.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -313,7 +313,7 @@ define half @fma_rn_relu_f16(half %0, half %1, half %2) {
define half @fma_rn_ftz_relu_f16(half %0, half %1, half %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.relu.f16
- %res = call half @llvm.nvvm.fma.ftz.relu.f16(half %0, half %1, half %2, i32 1)
+ %res = call half @llvm.nvvm.fma.ftz.relu.f16(half %0, half %1, half %2, /* rnd=rn */ i32 1)
ret half %res
}
@@ -321,7 +321,7 @@ define half @fma_rn_ftz_relu_f16(half %0, half %1, half %2) {
define <2 x half> @fma_rn_relu_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.relu.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
@@ -329,7 +329,7 @@ define <2 x half> @fma_rn_relu_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2
define <2 x half> @fma_rn_ftz_relu_f16x2(<2 x half> %0, <2 x half> %1, <2 x half> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.ftz.relu.f16x2
- %res = call <2 x half> @llvm.nvvm.fma.ftz.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, i32 1)
+ %res = call <2 x half> @llvm.nvvm.fma.ftz.relu.v2f16(<2 x half> %0, <2 x half> %1, <2 x half> %2, /* rnd=rn */ i32 1)
ret <2 x half> %res
}
@@ -337,7 +337,7 @@ define <2 x half> @fma_rn_ftz_relu_f16x2(<2 x half> %0, <2 x half> %1, <2 x half
define bfloat @fma_rn_bf16(bfloat %0, bfloat %1, bfloat %2) {
; CHECK-NOT: call
; CHECK: fma.rn.bf16
- %res = call bfloat @llvm.nvvm.fma.bf16(bfloat %0, bfloat %1, bfloat %2, i32 1)
+ %res = call bfloat @llvm.nvvm.fma.bf16(bfloat %0, bfloat %1, bfloat %2, /* rnd=rn */ i32 1)
ret bfloat %res
}
@@ -345,7 +345,7 @@ define bfloat @fma_rn_bf16(bfloat %0, bfloat %1, bfloat %2) {
define bfloat @fma_rn_relu_bf16(bfloat %0, bfloat %1, bfloat %2) {
; CHECK-NOT: call
; CHECK: fma.rn.relu.bf16
- %res = call bfloat @llvm.nvvm.fma.relu.bf16(bfloat %0, bfloat %1, bfloat %2, i32 1)
+ %res = call bfloat @llvm.nvvm.fma.relu.bf16(bfloat %0, bfloat %1, bfloat %2, /* rnd=rn */ i32 1)
ret bfloat %res
}
@@ -353,7 +353,7 @@ define bfloat @fma_rn_relu_bf16(bfloat %0, bfloat %1, bfloat %2) {
define <2 x bfloat> @fma_rn_bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.bf16x2
- %res = call <2 x bfloat> @llvm.nvvm.fma.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2, i32 1)
+ %res = call <2 x bfloat> @llvm.nvvm.fma.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2, /* rnd=rn */ i32 1)
ret <2 x bfloat> %res
}
@@ -361,6 +361,6 @@ define <2 x bfloat> @fma_rn_bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat
define <2 x bfloat> @fma_rn_relu_bf16x2(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2) {
; CHECK-NOT: call
; CHECK: fma.rn.relu.bf16x2
- %res = call <2 x bfloat> @llvm.nvvm.fma.relu.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2, i32 1)
+ %res = call <2 x bfloat> @llvm.nvvm.fma.relu.v2bf16(<2 x bfloat> %0, <2 x bfloat> %1, <2 x bfloat> %2, /* rnd=rn */ i32 1)
ret <2 x bfloat> %res
}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
index 28fb71a7f30f52..b1eeeef512c58d 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-fp.ll
@@ -27,16 +27,16 @@ define float @test_add_f32_f16_1(half %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext half %a to float
- %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, i32 1)
- %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, i32 0)
- %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, i32 3)
- %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, i32 2)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, /* rnd=rn */ i32 1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, /* rnd=rz */ i32 0)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, /* rnd=rm */ i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, /* rnd=rp */ i32 2)
; SAT
- %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, i32 1)
- %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, i32 0)
- %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, i32 3)
- %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, i32 2)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, /* rnd=rn */ i32 1)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, /* rnd=rz */ i32 0)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, /* rnd=rm */ i32 3)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, /* rnd=rp */ i32 2)
ret float %r8
}
@@ -93,16 +93,16 @@ define float @test_add_f32_bf16_1(bfloat %a, float %b) {
; CHECK-NEXT: ret;
%r0 = fpext bfloat %a to float
- %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, i32 1)
- %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, i32 0)
- %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, i32 3)
- %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, i32 2)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %b, /* rnd=rn */ i32 1)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %r1, /* rnd=rz */ i32 0)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %r2, /* rnd=rm */ i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %r3, /* rnd=rp */ i32 2)
; SAT
- %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, i32 1)
- %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, i32 0)
- %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, i32 3)
- %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, i32 2)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r4, /* rnd=rn */ i32 1)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r5, /* rnd=rz */ i32 0)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r6, /* rnd=rm */ i32 3)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %r7, /* rnd=rp */ i32 2)
ret float %r8
}
@@ -160,29 +160,29 @@ define float @test_sub_f32_f16_1(half %a, float %b) {
%r0 = fpext half %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, i32 1)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, /* rnd=rn */ i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, i32 0)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, /* rnd=rz */ i32 0)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, i32 3)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, /* rnd=rm */ i32 3)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, i32 3)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, /* rnd=rm */ i32 3)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, i32 1)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, /* rnd=rn */ i32 1)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, i32 0)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, /* rnd=rz */ i32 0)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, i32 3)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, /* rnd=rm */ i32 3)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, /* rnd=rp */ i32 2)
ret float %r7
}
@@ -240,29 +240,29 @@ define float @test_sub_f32_bf16_1(bfloat %a, float %b) {
%r0 = fpext bfloat %a to float
%f0 = fneg float %b
- %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, i32 1)
+ %r1 = call float @llvm.nvvm.fadd.f32(float %r0, float %f0, /* rnd=rn */ i32 1)
%f1 = fneg float %r1
- %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, i32 0)
+ %r2 = call float @llvm.nvvm.fadd.f32(float %r0, float %f1, /* rnd=rz */ i32 0)
%f2 = fneg float %r2
- %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, i32 3)
+ %r3 = call float @llvm.nvvm.fadd.f32(float %r0, float %f2, /* rnd=rm */ i32 3)
%f3 = fneg float %r3
- %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, i32 2)
+ %r4 = call float @llvm.nvvm.fadd.f32(float %r0, float %f3, /* rnd=rp */ i32 2)
; SAT
%f4 = fneg float %r4
- %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, i32 1)
+ %r5 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f4, /* rnd=rn */ i32 1)
%f5 = fneg float %r5
- %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, i32 0)
+ %r6 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f5, /* rnd=rz */ i32 0)
%f6 = fneg float %r6
- %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, i32 3)
+ %r7 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f6, /* rnd=rm */ i32 3)
%f7 = fneg float %r7
- %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, i32 2)
+ %r8 = call float @llvm.nvvm.fadd.sat.f32(float %r0, float %f7, /* rnd=rp */ i32 2)
ret float %r8
}
@@ -322,16 +322,16 @@ define float @test_fma_f32_f16_1(half %a, half %b, float %c) {
%r0 = fpext half %a to float
%r1 = fpext half %b to float
- %r2 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %c, i32 1)
- %r3 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r3, i32 3)
- %r5 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r4, i32 2)
+ %r2 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %c, /* rnd=rn */ i32 1)
+ %r3 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r2, /* rnd=rz */ i32 0)
+ %r4 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r3, /* rnd=rm */ i32 3)
+ %r5 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r4, /* rnd=rp */ i32 2)
; SAT
- %r6 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r5, i32 1)
- %r7 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r6, i32 0)
- %r8 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r7, i32 3)
- %r9 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r8, i32 2)
+ %r6 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r5, /* rnd=rn */ i32 1)
+ %r7 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r6, /* rnd=rz */ i32 0)
+ %r8 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r7, /* rnd=rm */ i32 3)
+ %r9 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r8, /* rnd=rp */ i32 2)
ret float %r8
}
@@ -393,16 +393,16 @@ define float @test_fma_f32_bf16_1(bfloat %a, bfloat %b, float %c) {
%r0 = fpext bfloat %a to float
%r1 = fpext bfloat %b to float
- %r2 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %c, i32 1)
- %r3 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r2, i32 0)
- %r4 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r3, i32 3)
- %r5 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r4, i32 2)
+ %r2 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %c, /* rnd=rn */ i32 1)
+ %r3 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r2, /* rnd=rz */ i32 0)
+ %r4 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r3, /* rnd=rm */ i32 3)
+ %r5 = call float @llvm.nvvm.fma.f32(float %r0, float %r1, float %r4, /* rnd=rp */ i32 2)
; SAT
- %r6 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r5, i32 1)
- %r7 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r6, i32 0)
- %r8 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r7, i32 3)
- %r9 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r8, i32 2)
+ %r6 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r5, /* rnd=rn */ i32 1)
+ %r7 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r6, /* rnd=rz */ i32 0)
+ %r8 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r7, /* rnd=rm */ i32 3)
+ %r9 = call float @llvm.nvvm.fma.sat.f32(float %r0, float %r1, float %r8, /* rnd=rp */ i32 2)
ret float %r8
}
diff --git a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
index 03d60f28d8fc90..45cb8b0f2c36d1 100644
--- a/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
+++ b/llvm/test/Transforms/InstCombine/NVPTX/nvvm-intrins.ll
@@ -84,21 +84,21 @@ define float @floor_float_ftz(float %a) #0 {
; CHECK-LABEL: @fma_double
define double @fma_double(double %a, double %b, double %c) #0 {
; CHECK: call double @llvm.fma.f64
- %ret = call double @llvm.nvvm.fma.f64(double %a, double %b, double %c, i32 1)
+ %ret = call double @llvm.nvvm.fma.f64(double %a, double %b, double %c, /* rnd=rn */ i32 1)
ret double %ret
}
; CHECK-LABEL: @fma_float
define float @fma_float(float %a, float %b, float %c) #0 {
; NOFTZ: call float @llvm.fma.f32
; FTZ: call float @llvm.nvvm.fma.f32
- %ret = call float @llvm.nvvm.fma.f32(float %a, float %b, float %c, i32 1)
+ %ret = call float @llvm.nvvm.fma.f32(float %a, float %b, float %c, /* rnd=rn */ i32 1)
ret float %ret
}
; CHECK-LABEL: @fma_float_ftz
define float @fma_float_ftz(float %a, float %b, float %c) #0 {
; NOFTZ: call float @llvm.nvvm.fma.ftz.f32
; FTZ: call float @llvm.fma.f32
- %ret = call float @llvm.nvvm.fma.ftz.f32(float %a, float %b, float %c, i32 1)
+ %ret = call float @llvm.nvvm.fma.ftz.f32(float %a, float %b, float %c, /* rnd=rn */ i32 1)
ret float %ret
}
More information about the Mlir-commits
mailing list