[flang-commits] [flang] 1b9d0de - [Flang] Change complex divide lowering
Kiran Chandramohan via flang-commits
flang-commits at lists.llvm.org
Fri May 5 06:08:24 PDT 2023
Author: Kiran Chandramohan
Date: 2023-05-05T13:04:21Z
New Revision: 1b9d0deb6d53db760a91bcf63c9b2509f95af2bb
URL: https://github.com/llvm/llvm-project/commit/1b9d0deb6d53db760a91bcf63c9b2509f95af2bb
DIFF: https://github.com/llvm/llvm-project/commit/1b9d0deb6d53db760a91bcf63c9b2509f95af2bb.diff
LOG: [Flang] Change complex divide lowering
Currently complex division is lowered to a fir.divc operation and the
fir.divc is later converted to a sequence of llvm operations to perform
complex division, however this causes issues for extreme values when
the calculations overflow.
This patch changes the lowering of complex division to use the Intrinsic
Call functionality to lower into library calls (for single, double,
extended and quad precisions) or an MLIR complex dialect division operation
(for half and bfloat precisions).
Note 1: If the Complex To Standard conversion of division operation
matures then we can use it for all precisions. Currently it has the
same issues as the conversion of fir.divc.
Note 2: A previous patch (D145808) did the same but during conversion of
the fir.divc operation. But using function calls at that stage leads to
ABI issues since the conversion to LLVM is not aware of the complex target
rewrite.
Note 3: If the patch is accepted, fir.divc can be removed from FIR.
Reviewed By: vzakhari, PeteSteinfeld, DavidTruby
Differential Revision: https://reviews.llvm.org/D149546
Added:
Modified:
flang/include/flang/Optimizer/Builder/IntrinsicCall.h
flang/lib/Lower/ConvertExpr.cpp
flang/lib/Optimizer/Builder/IntrinsicCall.cpp
flang/test/Lower/assignment.f90
flang/test/Lower/complex-operations.f90
Removed:
################################################################################
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 2cc797f9b3f2a..0d32976b6f606 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -98,6 +98,11 @@ mlir::Value genMax(fir::FirOpBuilder &, mlir::Location,
mlir::Value genMin(fir::FirOpBuilder &, mlir::Location,
llvm::ArrayRef<mlir::Value> args);
+/// Generate Complex divide with the given expected
+/// result type.
+mlir::Value genDivC(fir::FirOpBuilder &builder, mlir::Location loc,
+ mlir::Type resultType, mlir::Value x, mlir::Value y);
+
/// Generate power function x**y with the given expected
/// result type.
mlir::Value genPow(fir::FirOpBuilder &, mlir::Location, mlir::Type resultType,
diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp
index 7f89fcf88f87d..ba084d1efb276 100644
--- a/flang/lib/Lower/ConvertExpr.cpp
+++ b/flang/lib/Lower/ConvertExpr.cpp
@@ -1075,7 +1075,16 @@ class ScalarExprLowering {
GENBIN(Multiply, Complex, fir::MulcOp)
GENBIN(Divide, Integer, mlir::arith::DivSIOp)
GENBIN(Divide, Real, mlir::arith::DivFOp)
- GENBIN(Divide, Complex, fir::DivcOp)
+
+ template <int KIND>
+ ExtValue genval(const Fortran::evaluate::Divide<Fortran::evaluate::Type<
+ Fortran::common::TypeCategory::Complex, KIND>> &op) {
+ mlir::Type ty =
+ converter.genType(Fortran::common::TypeCategory::Complex, KIND);
+ mlir::Value lhs = genunbox(op.left());
+ mlir::Value rhs = genunbox(op.right());
+ return fir::genDivC(builder, getLoc(), ty, lhs, rhs);
+ }
template <Fortran::common::TypeCategory TC, int KIND>
ExtValue genval(
@@ -5077,7 +5086,21 @@ class ArrayExprLowering {
GENBIN(Multiply, Complex, fir::MulcOp)
GENBIN(Divide, Integer, mlir::arith::DivSIOp)
GENBIN(Divide, Real, mlir::arith::DivFOp)
- GENBIN(Divide, Complex, fir::DivcOp)
+
+ template <int KIND>
+ CC genarr(const Fortran::evaluate::Divide<Fortran::evaluate::Type<
+ Fortran::common::TypeCategory::Complex, KIND>> &x) {
+ mlir::Location loc = getLoc();
+ mlir::Type ty =
+ converter.genType(Fortran::common::TypeCategory::Complex, KIND);
+ auto lf = genarr(x.left());
+ auto rf = genarr(x.right());
+ return [=](IterSpace iters) -> ExtValue {
+ mlir::Value lhs = fir::getBase(lf(iters));
+ mlir::Value rhs = fir::getBase(rf(iters));
+ return fir::genDivC(builder, loc, ty, lhs, rhs);
+ };
+ }
template <Fortran::common::TypeCategory TC, int KIND>
CC genarr(
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index d877be0cbc0ae..ca3214d67dee6 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -1345,6 +1345,18 @@ static constexpr MathOperation mathOperations[] = {
{"cosh", "cosh", genF64F64FuncType, genLibCall},
{"cosh", "ccoshf", genComplexComplexFuncType<4>, genLibCall},
{"cosh", "ccosh", genComplexComplexFuncType<8>, genLibCall},
+ {"divc",
+ {},
+ genComplexComplexComplexFuncType<2>,
+ genComplexMathOp<mlir::complex::DivOp>},
+ {"divc",
+ {},
+ genComplexComplexComplexFuncType<3>,
+ genComplexMathOp<mlir::complex::DivOp>},
+ {"divc", "__divsc3", genComplexComplexComplexFuncType<4>, genLibCall},
+ {"divc", "__divdc3", genComplexComplexComplexFuncType<8>, genLibCall},
+ {"divc", "__divxc3", genComplexComplexComplexFuncType<10>, genLibCall},
+ {"divc", "__divtc3", genComplexComplexComplexFuncType<16>, genLibCall},
{"erf", "erff", genF32F32FuncType, genMathOp<mlir::math::ErfOp>},
{"erf", "erf", genF64F64FuncType, genMathOp<mlir::math::ErfOp>},
{"erfc", "erfcf", genF32F32FuncType, genLibCall},
@@ -5661,6 +5673,11 @@ mlir::Value fir::genMin(fir::FirOpBuilder &builder, mlir::Location loc,
args);
}
+mlir::Value fir::genDivC(fir::FirOpBuilder &builder, mlir::Location loc,
+ mlir::Type type, mlir::Value x, mlir::Value y) {
+ return IntrinsicLibrary{builder, loc}.genRuntimeCall("divc", type, {x, y});
+}
+
mlir::Value fir::genPow(fir::FirOpBuilder &builder, mlir::Location loc,
mlir::Type type, mlir::Value x, mlir::Value y) {
// TODO: since there is no libm version of pow with integer exponent,
diff --git a/flang/test/Lower/assignment.f90 b/flang/test/Lower/assignment.f90
index 8e836ce30cdb3..69c77fdcfb7cc 100644
--- a/flang/test/Lower/assignment.f90
+++ b/flang/test/Lower/assignment.f90
@@ -251,7 +251,7 @@ real function divf(a, b)
! CHECK: %[[FCTRES:.*]] = fir.alloca !fir.complex<4>
! CHECK: %[[A_VAL:.*]] = fir.load %[[A]] : !fir.ref<!fir.complex<4>>
! CHECK: %[[B_VAL:.*]] = fir.load %[[B]] : !fir.ref<!fir.complex<4>>
-! CHECK: %[[DIV:.*]] = fir.divc %[[A_VAL]], %[[B_VAL]] : !fir.complex<4>
+! CHECK: %[[DIV:.*]] = fir.call @__divsc3(%[[A_VAL]], %[[B_VAL]]) fastmath<contract> : (!fir.complex<4>, !fir.complex<4>) -> !fir.complex<4>
! CHECK: fir.store %[[DIV]] to %[[FCTRES]] : !fir.ref<!fir.complex<4>>
! CHECK: %[[RET:.*]] = fir.load %[[FCTRES]] : !fir.ref<!fir.complex<4>>
! CHECK: return %[[RET]] : !fir.complex<4>
diff --git a/flang/test/Lower/complex-operations.f90 b/flang/test/Lower/complex-operations.f90
index b11ce6ee7f215..ee4b6c1bcb21a 100644
--- a/flang/test/Lower/complex-operations.f90
+++ b/flang/test/Lower/complex-operations.f90
@@ -27,11 +27,74 @@ subroutine mul_test(a,b,c)
a = b * c
end subroutine mul_test
-! CHECK-LABEL: @_QPdiv_test
-subroutine div_test(a,b,c)
- complex :: a, b, c
- ! CHECK-NOT: fir.extract_value
- ! CHECK-NOT: fir.insert_value
- ! CHECK: fir.divc {{.*}}: !fir.complex
+! CHECK-LABEL: @_QPdiv_test_half
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<2>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<2>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<2>>
+! CHECK: %[[BVAL_CVT:.*]] = fir.convert %[[BVAL]] : (!fir.complex<2>) -> complex<f16>
+! CHECK: %[[CVAL_CVT:.*]] = fir.convert %[[CVAL]] : (!fir.complex<2>) -> complex<f16>
+! CHECK: %[[AVAL_CVT:.*]] = complex.div %[[BVAL_CVT]], %[[CVAL_CVT]] : complex<f16>
+! CHECK: %[[AVAL:.*]] = fir.convert %[[AVAL_CVT]] : (complex<f16>) -> !fir.complex<2>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<2>>
+subroutine div_test_half(a,b,c)
+ complex(kind=2) :: a, b, c
+ a = b / c
+end subroutine div_test_half
+
+! CHECK-LABEL: @_QPdiv_test_bfloat
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<3>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<3>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<3>>
+! CHECK: %[[BVAL_CVT:.*]] = fir.convert %[[BVAL]] : (!fir.complex<3>) -> complex<bf16>
+! CHECK: %[[CVAL_CVT:.*]] = fir.convert %[[CVAL]] : (!fir.complex<3>) -> complex<bf16>
+! CHECK: %[[AVAL_CVT:.*]] = complex.div %[[BVAL_CVT]], %[[CVAL_CVT]] : complex<bf16>
+! CHECK: %[[AVAL:.*]] = fir.convert %[[AVAL_CVT]] : (complex<bf16>) -> !fir.complex<3>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<3>>
+subroutine div_test_bfloat(a,b,c)
+ complex(kind=3) :: a, b, c
+ a = b / c
+end subroutine div_test_bfloat
+
+! CHECK-LABEL: @_QPdiv_test_single
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<4>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<4>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<4>>
+! CHECK: %[[AVAL:.*]] = fir.call @__divsc3(%[[BVAL]], %[[CVAL]]) fastmath<contract> : (!fir.complex<4>, !fir.complex<4>) -> !fir.complex<4>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<4>>
+subroutine div_test_single(a,b,c)
+ complex(kind=4) :: a, b, c
+ a = b / c
+end subroutine div_test_single
+
+! CHECK-LABEL: @_QPdiv_test_double
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<8>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<8>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<8>>
+! CHECK: %[[AVAL:.*]] = fir.call @__divdc3(%[[BVAL]], %[[CVAL]]) fastmath<contract> : (!fir.complex<8>, !fir.complex<8>) -> !fir.complex<8>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<8>>
+subroutine div_test_double(a,b,c)
+ complex(kind=8) :: a, b, c
+ a = b / c
+end subroutine div_test_double
+
+! CHECK-LABEL: @_QPdiv_test_extended
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<10>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<10>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<10>>
+! CHECK: %[[AVAL:.*]] = fir.call @__divxc3(%[[BVAL]], %[[CVAL]]) fastmath<contract> : (!fir.complex<10>, !fir.complex<10>) -> !fir.complex<10>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<10>>
+subroutine div_test_extended(a,b,c)
+ complex(kind=10) :: a, b, c
+ a = b / c
+end subroutine div_test_extended
+
+! CHECK-LABEL: @_QPdiv_test_quad
+! CHECK-SAME: %[[AREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}}, %[[BREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}}, %[[CREF:.*]]: !fir.ref<!fir.complex<16>> {{.*}})
+! CHECK: %[[BVAL:.*]] = fir.load %[[BREF]] : !fir.ref<!fir.complex<16>>
+! CHECK: %[[CVAL:.*]] = fir.load %[[CREF]] : !fir.ref<!fir.complex<16>>
+! CHECK: %[[AVAL:.*]] = fir.call @__divtc3(%[[BVAL]], %[[CVAL]]) fastmath<contract> : (!fir.complex<16>, !fir.complex<16>) -> !fir.complex<16>
+! CHECK: fir.store %[[AVAL]] to %[[AREF]] : !fir.ref<!fir.complex<16>>
+subroutine div_test_quad(a,b,c)
+ complex(kind=16) :: a, b, c
a = b / c
-end subroutine div_test
+end subroutine div_test_quad
More information about the flang-commits
mailing list