[Libclc-dev] [PATCH 1/1] math/fma: Add fp32 software implementation
Jan Vesely via Libclc-dev
libclc-dev at lists.llvm.org
Wed Jun 6 18:25:16 PDT 2018
On Tue, 2018-05-29 at 16:09 -0400, Jan Vesely wrote:
> On Sat, 2018-05-19 at 19:46 -0400, Jan Vesely wrote:
> > Passes CTS on carrizo (when forced to use sw fma) and turks.
> > Signed-off-by: Jan Vesely <jan.vesely at rutgers.edu>
> > ---
> > generic/include/clc/math/fma.h | 11 +-
> > generic/include/clc/math/ternary_decl.inc | 1 +
> > generic/include/math/clc_fma.h | 11 ++
> > generic/lib/SOURCES | 2 +
> > generic/lib/math/clc_fma.cl | 158 ++++++++++++++++++++++
> > generic/lib/math/fma.cl | 7 +
> > generic/lib/math/fma.inc | 7 +
> > 7 files changed, 192 insertions(+), 5 deletions(-)
> > create mode 100644 generic/include/clc/math/ternary_decl.inc
> > create mode 100644 generic/include/math/clc_fma.h
> > create mode 100644 generic/lib/math/clc_fma.cl
> > create mode 100644 generic/lib/math/fma.cl
> > create mode 100644 generic/lib/math/fma.inc
> >
> > diff --git a/generic/include/clc/math/fma.h b/generic/include/clc/math/fma.h
> > index 02d39f6..c7c23eb 100644
> > --- a/generic/include/clc/math/fma.h
> > +++ b/generic/include/clc/math/fma.h
> > @@ -1,6 +1,7 @@
> > -#undef fma
> > -#define fma __clc_fma
> > +#define __CLC_BODY <clc/math/ternary_decl.inc>
> > +#define __CLC_FUNCTION fma
> >
> > -#define __CLC_FUNCTION __clc_fma
> > -#define __CLC_INTRINSIC "llvm.fma"
> > -#include <clc/math/ternary_intrin.inc>
> > +#include <clc/math/gentype.inc>
> > +
> > +#undef __CLC_BODY
> > +#undef __CLC_FUNCTION
> > diff --git a/generic/include/clc/math/ternary_decl.inc b/generic/include/clc/math/ternary_decl.inc
> > new file mode 100644
> > index 0000000..0598684
> > --- /dev/null
> > +++ b/generic/include/clc/math/ternary_decl.inc
> > @@ -0,0 +1 @@
> > +_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b, __CLC_GENTYPE c);
> > diff --git a/generic/include/math/clc_fma.h b/generic/include/math/clc_fma.h
> > new file mode 100644
> > index 0000000..3ae6a74
> > --- /dev/null
> > +++ b/generic/include/math/clc_fma.h
> > @@ -0,0 +1,11 @@
> > +#define __CLC_FUNCTION __clc_fma
> > +#define __CLC_INTRINSIC "llvm.fma"
> > +#include <clc/math/ternary_intrin.inc>
> > +
> > +#define __FLOAT_ONLY
> > +#define __CLC_FUNCTION __clc_sw_fma
> > +#define __CLC_BODY <clc/math/ternary_decl.inc>
> > +#include <clc/math/gentype.inc>
> > +#undef __CLC_BODY
> > +#undef __CLC_FUNCTION
> > +#undef __FLOAT_ONLY
> > diff --git a/generic/lib/SOURCES b/generic/lib/SOURCES
> > index 1f1a427..565750e 100644
> > --- a/generic/lib/SOURCES
> > +++ b/generic/lib/SOURCES
> > @@ -101,6 +101,8 @@ math/exp2.cl
> > math/clc_exp10.cl
> > math/exp10.cl
> > math/fdim.cl
> > +math/clc_fma.cl
> > +math/fma.cl
> > math/fmax.cl
> > math/fmin.cl
> > math/clc_fmod.cl
> > diff --git a/generic/lib/math/clc_fma.cl b/generic/lib/math/clc_fma.cl
> > new file mode 100644
> > index 0000000..dee90e9
> > --- /dev/null
> > +++ b/generic/lib/math/clc_fma.cl
> > @@ -0,0 +1,158 @@
> > +/*
> > + * Copyright (c) 2014 Advanced Micro Devices, Inc.
> > + *
> > + * Permission is hereby granted, free of charge, to any person obtaining a copy
> > + * of this software and associated documentation files (the "Software"), to deal
> > + * in the Software without restriction, including without limitation the rights
> > + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
> > + * copies of the Software, and to permit persons to whom the Software is
> > + * furnished to do so, subject to the following conditions:
> > + *
> > + * The above copyright notice and this permission notice shall be included in
> > + * all copies or substantial portions of the Software.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> > + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> > + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
> > + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> > + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
> > + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
> > + * THE SOFTWARE.
> > + */
> > +
> > +#include <clc/clc.h>
> > +
> > +#include "config.h"
> > +#include "math.h"
> > +#include "../clcmacro.h"
> > +
> > +struct fp {
> > + ulong mantissa;
> > + int exponent;
> > + uint sign;
> > +};
> > +
> > +_CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c)
> > +{
> > + /* special cases */
> > + if (isnan(a) || isnan(b) || isnan(c) || isinf(a) || isinf(b))
> > + return mad(a, b, c);
> > +
> > + /* If only c is inf, and both a,b are regular numbers, the result is c*/
> > + if (isinf(c))
> > + return c;
> > +
> > + a = __clc_flush_denormal_if_not_supported(a);
> > + b = __clc_flush_denormal_if_not_supported(b);
> > + c = __clc_flush_denormal_if_not_supported(c);
> > +
> > + if (c == 0)
> > + return a * b;
> > +
> > + struct fp st_a, st_b, st_c;
> > +
> > + st_a.exponent = a == .0f ? 0 : ((as_uint(a) & 0x7f800000) >> 23) - 127;
> > + st_b.exponent = b == .0f ? 0 : ((as_uint(b) & 0x7f800000) >> 23) - 127;
> > + st_c.exponent = c == .0f ? 0 : ((as_uint(c) & 0x7f800000) >> 23) - 127;
> > +
> > + st_a.mantissa = a == .0f ? 0 : (as_uint(a) & 0x7fffff) | 0x800000;
> > + st_b.mantissa = b == .0f ? 0 : (as_uint(b) & 0x7fffff) | 0x800000;
> > + st_c.mantissa = c == .0f ? 0 : (as_uint(c) & 0x7fffff) | 0x800000;
> > +
> > + st_a.sign = as_uint(a) & 0x80000000;
> > + st_b.sign = as_uint(b) & 0x80000000;
> > + st_c.sign = as_uint(c) & 0x80000000;
> > +
> > + // Multiplication.
> > + // Move the product to the highest bits to maximize precision
> > + // mantissa is 24 bits => product is 48 bits, 2bits non-fraction.
> > + // Add one bit for future addition overflow,
> > + // add another bit to detect subtraction underflow
> > + struct fp st_mul;
> > + st_mul.sign = st_a.sign ^ st_b.sign;
> > + st_mul.mantissa = (st_a.mantissa * st_b.mantissa) << 14ul;
> > + st_mul.exponent = st_mul.mantissa ? st_a.exponent + st_b.exponent : 0;
> > +
> > + // FIXME: Detecting a == 0 || b == 0 above crashed GCN isel
> > + if (st_mul.exponent == 0 && st_mul.mantissa == 0)
> > + return c;
> > +
> > +// Mantissa is 23 fractional bits, shift it the same way as product mantissa
> > +#define C_ADJUST 37ul
> > +
> > + // both exponents are bias adjusted
> > + int exp_diff = st_mul.exponent - st_c.exponent;
> > +
> > + st_c.mantissa <<= C_ADJUST;
> > + ulong cutoff_bits = 0;
> > + ulong cutoff_mask = (1ul << abs(exp_diff)) - 1ul;
> > + if (exp_diff > 0) {
> > + cutoff_bits = exp_diff >= 64 ? st_c.mantissa : (st_c.mantissa & cutoff_mask);
> > + st_c.mantissa = exp_diff >= 64 ? 0 : (st_c.mantissa >> exp_diff);
> > + } else {
> > + cutoff_bits = -exp_diff >= 64 ? st_mul.mantissa : (st_mul.mantissa & cutoff_mask);
> > + st_mul.mantissa = -exp_diff >= 64 ? 0 : (st_mul.mantissa >> -exp_diff);
> > + }
> > +
> > + struct fp st_fma;
> > + st_fma.sign = st_mul.sign;
> > + st_fma.exponent = max(st_mul.exponent, st_c.exponent);
> > + if (st_c.sign == st_mul.sign) {
> > + st_fma.mantissa = st_mul.mantissa + st_c.mantissa;
> > + } else {
> > + // cutoff bits borrow one
> > + st_fma.mantissa = st_mul.mantissa - st_c.mantissa - (cutoff_bits && (st_mul.exponent > st_c.exponent) ? 1 : 0);
> > + }
> > +
> > + // underflow: st_c.sign != st_mul.sign, and magnitude switches the sign
> > + if (st_fma.mantissa > LONG_MAX) {
> > + st_fma.mantissa = 0 - st_fma.mantissa;
> > + st_fma.sign = st_mul.sign ^ 0x80000000;
> > + }
> > +
> > + // detect overflow/underflow
> > + int overflow_bits = 3 - clz(st_fma.mantissa);
> > +
> > + // adjust exponent
> > + st_fma.exponent += overflow_bits;
> > +
> > + // handle underflow
> > + if (overflow_bits < 0) {
> > + st_fma.mantissa <<= -overflow_bits;
> > + overflow_bits = 0;
> > + }
> > +
> > + // rounding
> > + ulong trunc_mask = (1ul << (C_ADJUST + overflow_bits)) - 1;
> > + ulong trunc_bits = (st_fma.mantissa & trunc_mask) | (cutoff_bits != 0);
> > + ulong last_bit = st_fma.mantissa & (1ul << (C_ADJUST + overflow_bits));
> > + ulong grs_bits = (0x4ul << (C_ADJUST - 3 + overflow_bits));
> > +
> > + // round to nearest even
> > + if ((trunc_bits > grs_bits) ||
> > + (trunc_bits == grs_bits && last_bit != 0))
> > + st_fma.mantissa += (1ul << (C_ADJUST + overflow_bits));
> > +
> > + // Shift mantissa back to bit 23
> > + st_fma.mantissa = (st_fma.mantissa >> (C_ADJUST + overflow_bits));
> > +
> > + // Detect rounding overflow
> > + if (st_fma.mantissa > 0xffffff) {
> > + ++st_fma.exponent;
> > + st_fma.mantissa >>= 1;
> > + }
> > +
> > + if (st_fma.mantissa == 0)
> > + return .0f;
> > +
> > + // Flating point range limit
> > + if (st_fma.exponent > 127)
> > + return as_float(as_uint(INFINITY) | st_fma.sign);
> > +
> > + // Flush denormals
> > + if (st_fma.exponent <= -127)
> > + return as_float(st_fma.sign);
> > +
> > + return as_float(st_fma.sign | ((st_fma.exponent + 127) << 23) | ((uint)st_fma.mantissa & 0x7fffff));
> > +}
> > +_CLC_TERNARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_sw_fma, float, float, float)
> > diff --git a/generic/lib/math/fma.cl b/generic/lib/math/fma.cl
> > new file mode 100644
> > index 0000000..9ad81be
> > --- /dev/null
> > +++ b/generic/lib/math/fma.cl
> > @@ -0,0 +1,7 @@
> > +#include <clc/clc.h>
> > +
> > +#include "math.h"
> > +#include "math/clc_fma.h"
> > +
> > +#define __CLC_BODY <fma.inc>
> > +#include <clc/math/gentype.inc>
> > diff --git a/generic/lib/math/fma.inc b/generic/lib/math/fma.inc
> > new file mode 100644
> > index 0000000..654208f
> > --- /dev/null
> > +++ b/generic/lib/math/fma.inc
> > @@ -0,0 +1,7 @@
> > +_CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE fma(__CLC_GENTYPE a, __CLC_GENTYPE b, __CLC_GENTYPE c) {
> > +#if __CLC_FPSIZE == 32 && HAVE_HW_FMA32() == 0
> > + return __clc_sw_fma(a, b, c);
> > +#else
> > + return __clc_fma(a, b, c);
> > +#endif
> > +}
>
> ping.
ping2.
Jan
>
> Jan
--
Jan Vesely <jan.vesely at rutgers.edu>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: This is a digitally signed message part
URL: <http://lists.llvm.org/pipermail/libclc-dev/attachments/20180606/8538e878/attachment.sig>
More information about the Libclc-dev
mailing list