Author: Rajat Bajpai
Date: 2024-07-16T12:45:21+05:30
New Revision: 3a0e01543212c1e1c80f5665aee0e12289e06a29
URL: https://github.com/llvm/llvm-project/commit/3a0e01543212c1e1c80f5665aee0e12289e06a29
DIFF: https://github.com/llvm/llvm-project/commit/3a0e01543212c1e1c80f5665aee0e12289e06a29.diff
LOG: [NVPTX] Lower -1/x to neg.f64(rcp.rn.f64) instead of fdiv (#98343)
The NVPTX backend lowers 1/x to rcp.rn.f64 instruction instead of slower
fdiv instruction. However, in the case of -1/x, it uses the slower fdiv
instruction. After this change, -1/x will be lowered into neg.f64
(rcp.rn.f64).
Added:
llvm/test/CodeGen/NVPTX/rcp-opt.ll
Modified:
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index f37822f764bed..7f1ac8688007e 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -1150,6 +1150,18 @@ def DoubleConst1 : PatLeaf<(fpimm), [{
return &N->getValueAPF().getSemantics() == &llvm::APFloat::IEEEdouble() &&
N->getValueAPF().convertToDouble() == 1.0;
}]>;
+// Constant -1.0 (double)
+def DoubleConstNeg1 : PatLeaf<(fpimm), [{
+ return &N->getValueAPF().getSemantics() == &llvm::APFloat::IEEEdouble() &&
+ N->getValueAPF().convertToDouble() == -1.0;
+}]>;
+
+
+// Constant -X -> X (double)
+def NegDoubleConst : SDNodeXForm<fpimm, [{
+ return CurDAG->getTargetConstantFP(-(N->getValueAPF()),
+ SDLoc(N), MVT::f64);
+}]>;
// Loads FP16 constant into a register.
//
@@ -1225,6 +1237,11 @@ def FDIV64ri :
"div.rn.f64 \t$dst, $a, $b;",
[(set Float64Regs:$dst, (fdiv Float64Regs:$a, fpimm:$b))]>;
+// fdiv will be converted to rcp
+// fneg (fdiv 1.0, X) => fneg (rcp.rn X)
+def : Pat<(fdiv DoubleConstNeg1:$a, Float64Regs:$b),
+ (FNEGf64 (FDIV641r (NegDoubleConst node:$a), Float64Regs:$b))>;
+
//
// F32 Approximate reciprocal
//
diff --git a/llvm/test/CodeGen/NVPTX/rcp-opt.ll b/llvm/test/CodeGen/NVPTX/rcp-opt.ll
new file mode 100644
index 0000000000000..e2443c27e8490
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/rcp-opt.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 | %ptxas-verify %}
+
+target triple = "nvptx64-nvidia-cuda"
+
+;; Check if fneg (fdiv 1, X) lowers to fneg (rcp.rn X).
+
+define double @test1(double %in) {
+; CHECK-LABEL: test1(
+; CHECK: {
+; CHECK-NEXT: .reg .f64 %fd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f64 %fd1, [test1_param_0];
+; CHECK-NEXT: rcp.rn.f64 %fd2, %fd1;
+; CHECK-NEXT: neg.f64 %fd3, %fd2;
+; CHECK-NEXT: st.param.f64 [func_retval0+0], %fd3;
+; CHECK-NEXT: ret;
+ %div = fdiv double 1.000000e+00, %in
+ %neg = fsub double -0.000000e+00, %div
+ ret double %neg
+}
+
+;; Check if fdiv -1, X lowers to fneg (rcp.rn X).
+
+define double @test2(double %in) {
+; CHECK-LABEL: test2(
+; CHECK: {
+; CHECK-NEXT: .reg .f64 %fd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f64 %fd1, [test2_param_0];
+; CHECK-NEXT: rcp.rn.f64 %fd2, %fd1;
+; CHECK-NEXT: neg.f64 %fd3, %fd2;
+; CHECK-NEXT: st.param.f64 [func_retval0+0], %fd3;
+; CHECK-NEXT: ret;
+ %div = fdiv double -1.000000e+00, %in
+ ret double %div
+}
+
+;; Check if fdiv 1, (fneg X) lowers to fneg (rcp.rn X).
+
+define double @test3(double %in) {
+; CHECK-LABEL: test3(
+; CHECK: {
+; CHECK-NEXT: .reg .f64 %fd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.f64 %fd1, [test3_param_0];
+; CHECK-NEXT: rcp.rn.f64 %fd2, %fd1;
+; CHECK-NEXT: neg.f64 %fd3, %fd2;
+; CHECK-NEXT: st.param.f64 [func_retval0+0], %fd3;
+; CHECK-NEXT: ret;
+ %neg = fsub double -0.000000e+00, %in
+ %div = fdiv double 1.000000e+00, %neg
+ ret double %div
+}