[libclc] [libclc] Move mad_sat to CLC; optimize for vector types (PR #125517)
Fraser Cormack via cfe-commits
cfe-commits at lists.llvm.org
Mon Feb 3 07:39:12 PST 2025
https://github.com/frasercrmck updated https://github.com/llvm/llvm-project/pull/125517
>From 1404e1c7b4f133458751048b3f73800b90f98584 Mon Sep 17 00:00:00 2001
From: Fraser Cormack <fraser at codeplay.com>
Date: Wed, 29 Jan 2025 13:46:26 +0000
Subject: [PATCH] [libclc] Move mad_sat to CLC; optimize for vector
This commit moves the mad_sat builtin to the CLC library.
It also optimizes it for vector types by avoiding scalarization. To help do
this it transforms the previous control-flow code into vector select
code. This has also been done for the scalar versions for simplicity.
---
libclc/clc/include/clc/integer/clc_mad_sat.h | 12 ++
libclc/clc/include/clc/integer/definitions.h | 4 +
libclc/clc/lib/clspv/SOURCES | 1 +
libclc/clc/lib/generic/SOURCES | 1 +
libclc/clc/lib/generic/integer/clc_mad_sat.cl | 119 ++++++++++++++++++
libclc/clc/lib/spirv/SOURCES | 1 +
libclc/generic/lib/integer/mad_sat.cl | 73 +----------
7 files changed, 142 insertions(+), 69 deletions(-)
create mode 100644 libclc/clc/include/clc/integer/clc_mad_sat.h
create mode 100644 libclc/clc/lib/generic/integer/clc_mad_sat.cl
diff --git a/libclc/clc/include/clc/integer/clc_mad_sat.h b/libclc/clc/include/clc/integer/clc_mad_sat.h
new file mode 100644
index 00000000000000..c474d067b0f628
--- /dev/null
+++ b/libclc/clc/include/clc/integer/clc_mad_sat.h
@@ -0,0 +1,12 @@
+#ifndef __CLC_INTEGER_CLC_MAD_SAT_H__
+#define __CLC_INTEGER_CLC_MAD_SAT_H__
+
+#define __CLC_FUNCTION __clc_mad_sat
+#define __CLC_BODY <clc/shared/ternary_decl.inc>
+
+#include <clc/integer/gentype.inc>
+
+#undef __CLC_BODY
+#undef __CLC_FUNCTION
+
+#endif // __CLC_INTEGER_CLC_MAD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/definitions.h b/libclc/clc/include/clc/integer/definitions.h
index 18a9e54dec75c6..1b51c11a93f801 100644
--- a/libclc/clc/include/clc/integer/definitions.h
+++ b/libclc/clc/include/clc/integer/definitions.h
@@ -13,8 +13,12 @@
#define SHRT_MAX 32767
#define SHRT_MIN (-32767 - 1)
#define UCHAR_MAX 255
+#define UCHAR_MIN 0
#define USHRT_MAX 65535
+#define USHRT_MIN 0
#define UINT_MAX 0xffffffff
+#define UINT_MIN 0
#define ULONG_MAX 0xffffffffffffffffUL
+#define ULONG_MIN 0UL
#endif // __CLC_INTEGER_DEFINITIONS_H__
diff --git a/libclc/clc/lib/clspv/SOURCES b/libclc/clc/lib/clspv/SOURCES
index 2581abe64f1443..c96a6fc15bf677 100644
--- a/libclc/clc/lib/clspv/SOURCES
+++ b/libclc/clc/lib/clspv/SOURCES
@@ -2,6 +2,7 @@
../generic/integer/clc_clz.cl
../generic/integer/clc_hadd.cl
../generic/integer/clc_mad24.cl
+../generic/integer/clc_mad_sat.cl
../generic/integer/clc_mul24.cl
../generic/integer/clc_mul_hi.cl
../generic/integer/clc_popcount.cl
diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES
index 2f4df168f70745..5fd882eb1bb256 100644
--- a/libclc/clc/lib/generic/SOURCES
+++ b/libclc/clc/lib/generic/SOURCES
@@ -8,6 +8,7 @@ integer/clc_add_sat.cl
integer/clc_clz.cl
integer/clc_hadd.cl
integer/clc_mad24.cl
+integer/clc_mad_sat.cl
integer/clc_mul24.cl
integer/clc_mul_hi.cl
integer/clc_popcount.cl
diff --git a/libclc/clc/lib/generic/integer/clc_mad_sat.cl b/libclc/clc/lib/generic/integer/clc_mad_sat.cl
new file mode 100644
index 00000000000000..51bd5fc01ab4b2
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_mad_sat.cl
@@ -0,0 +1,119 @@
+#include <clc/clcmacro.h>
+#include <clc/integer/clc_add_sat.h>
+#include <clc/integer/clc_mad24.h>
+#include <clc/integer/clc_mul_hi.h>
+#include <clc/integer/clc_upsample.h>
+#include <clc/integer/definitions.h>
+#include <clc/internal/clc.h>
+#include <clc/relational/clc_select.h>
+#include <clc/shared/clc_clamp.h>
+
+#define __CLC_CONVERT_TY(X, TY) __builtin_convertvector(X, TY)
+
+// Macro for defining mad_sat variants for char/uchar/short/ushort
+// FIXME: Once using __clc_convert_ty, can easily unify scalar and vector defs
+#define _CLC_DEFINE_SIMPLE_MAD_SAT(TYPE, UP_TYPE, LIT_PREFIX) \
+ _CLC_OVERLOAD _CLC_DEF TYPE __clc_mad_sat(TYPE x, TYPE y, TYPE z) { \
+ return __clc_clamp( \
+ (UP_TYPE)__clc_mad24((UP_TYPE)x, (UP_TYPE)y, (UP_TYPE)z), \
+ (UP_TYPE)LIT_PREFIX##_MIN, (UP_TYPE)LIT_PREFIX##_MAX); \
+ }
+
+#define _CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE, UP_TYPE, LIT_PREFIX) \
+ _CLC_OVERLOAD _CLC_DEF TYPE __clc_mad_sat(TYPE x, TYPE y, TYPE z) { \
+ UP_TYPE upscaled_mad = __clc_mad24(__CLC_CONVERT_TY(x, UP_TYPE), \
+ __CLC_CONVERT_TY(y, UP_TYPE), \
+ __CLC_CONVERT_TY(z, UP_TYPE)); \
+ UP_TYPE clamped_mad = __clc_clamp(upscaled_mad, (UP_TYPE)LIT_PREFIX##_MIN, \
+ (UP_TYPE)LIT_PREFIX##_MAX); \
+ return __CLC_CONVERT_TY(clamped_mad, TYPE); \
+ }
+
+#define _CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(TYPE, UP_TYPE, LIT_PREFIX) \
+ _CLC_DEFINE_SIMPLE_MAD_SAT(TYPE, UP_TYPE, LIT_PREFIX) \
+ _CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##2, UP_TYPE##2, LIT_PREFIX) \
+ _CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##3, UP_TYPE##3, LIT_PREFIX) \
+ _CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##4, UP_TYPE##4, LIT_PREFIX) \
+ _CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##8, UP_TYPE##8, LIT_PREFIX) \
+ _CLC_DEFINE_SIMPLE_MAD_SAT_VEC(TYPE##16, UP_TYPE##16, LIT_PREFIX)
+
+_CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(char, int, CHAR)
+_CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(uchar, uint, UCHAR)
+_CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(short, int, SHRT)
+_CLC_DEFINE_SIMPLE_MAD_SAT_ALL_TYS(ushort, uint, USHRT)
+
+// Macro for defining mad_sat variants for uint/ulong
+#define __CLC_DEFINE_UINTLONG_MAD_SAT(UTYPE, STYPE, ULIT_PREFIX) \
+ _CLC_OVERLOAD _CLC_DEF UTYPE __clc_mad_sat(UTYPE x, UTYPE y, UTYPE z) { \
+ STYPE has_mul_hi = __clc_mul_hi(x, y) != (UTYPE)0; \
+ return __clc_select(__clc_add_sat(x * y, z), (UTYPE)ULIT_PREFIX##_MAX, \
+ has_mul_hi); \
+ }
+
+#define __CLC_DEFINE_UINTLONG_MAD_SAT_ALL_TYS(UTY, STY, ULIT_PREFIX) \
+ __CLC_DEFINE_UINTLONG_MAD_SAT(UTY, STY, ULIT_PREFIX) \
+ __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##2, STY##2, ULIT_PREFIX) \
+ __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##3, STY##3, ULIT_PREFIX) \
+ __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##4, STY##4, ULIT_PREFIX) \
+ __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##8, STY##8, ULIT_PREFIX) \
+ __CLC_DEFINE_UINTLONG_MAD_SAT(UTY##16, STY##16, ULIT_PREFIX)
+
+__CLC_DEFINE_UINTLONG_MAD_SAT_ALL_TYS(uint, int, UINT)
+__CLC_DEFINE_UINTLONG_MAD_SAT_ALL_TYS(ulong, long, ULONG)
+
+// Macro for defining mad_sat variants for int
+#define __CLC_DEFINE_SINT_MAD_SAT(INTTY, UINTTY, SLONGTY) \
+ _CLC_OVERLOAD _CLC_DEF INTTY __clc_mad_sat(INTTY x, INTTY y, INTTY z) { \
+ INTTY mhi = __clc_mul_hi(x, y); \
+ UINTTY mlo = __clc_as_##UINTTY(x * y); \
+ SLONGTY m = __clc_upsample(mhi, mlo); \
+ m += __CLC_CONVERT_TY(z, SLONGTY); \
+ m = __clc_clamp(m, (SLONGTY)INT_MIN, (SLONGTY)INT_MAX); \
+ return __CLC_CONVERT_TY(m, INTTY); \
+ }
+
+// FIXME: Once using __clc_convert_ty, can easily unify scalar and vector defs
+#define __CLC_DEFINE_SINT_MAD_SAT_ALL_TYS(INTTY, UINTTY, SLONGTY) \
+ _CLC_OVERLOAD _CLC_DEF INTTY __clc_mad_sat(INTTY x, INTTY y, INTTY z) { \
+ INTTY mhi = __clc_mul_hi(x, y); \
+ UINTTY mlo = __clc_as_##UINTTY(x * y); \
+ SLONGTY m = __clc_upsample(mhi, mlo); \
+ m += z; \
+ return __clc_clamp(m, (SLONGTY)INT_MIN, (SLONGTY)INT_MAX); \
+ } \
+ __CLC_DEFINE_SINT_MAD_SAT(INTTY##2, UINTTY##2, SLONGTY##2) \
+ __CLC_DEFINE_SINT_MAD_SAT(INTTY##3, UINTTY##3, SLONGTY##3) \
+ __CLC_DEFINE_SINT_MAD_SAT(INTTY##4, UINTTY##4, SLONGTY##4) \
+ __CLC_DEFINE_SINT_MAD_SAT(INTTY##8, UINTTY##8, SLONGTY##8) \
+ __CLC_DEFINE_SINT_MAD_SAT(INTTY##16, UINTTY##16, SLONGTY##16)
+
+__CLC_DEFINE_SINT_MAD_SAT_ALL_TYS(int, uint, long)
+
+// Macro for defining mad_sat variants for long
+#define __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY, ULONGTY) \
+ _CLC_OVERLOAD _CLC_DEF SLONGTY __clc_mad_sat(SLONGTY x, SLONGTY y, \
+ SLONGTY z) { \
+ SLONGTY hi = __clc_mul_hi(x, y); \
+ ULONGTY ulo = __clc_as_##ULONGTY(x * y); \
+ SLONGTY max1 = (x < 0) == (y < 0) && hi != 0; \
+ SLONGTY max2 = hi == 0 && ulo >= LONG_MAX && \
+ (z > 0 || (ulo + __clc_as_##ULONGTY(z)) > LONG_MAX); \
+ SLONGTY min1 = (((x < 0) != (y < 0)) && hi != -1); \
+ SLONGTY min2 = \
+ hi == -1 && ulo <= ((ULONGTY)LONG_MAX + 1UL) && \
+ (z < 0 || __clc_as_##ULONGTY(z) < ((ULONGTY)LONG_MAX - ulo)); \
+ SLONGTY ret = __clc_as_##SLONGTY(ulo + __clc_as_##ULONGTY(z)); \
+ ret = __clc_select(ret, (SLONGTY)LONG_MAX, (SLONGTY)(max1 || max2)); \
+ ret = __clc_select(ret, (SLONGTY)LONG_MIN, (SLONGTY)(min1 || min2)); \
+ return ret; \
+ }
+
+#define __CLC_DEFINE_SLONG_MAD_SAT_ALL_TYS(SLONGTY, ULONGTY) \
+ __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY, ULONGTY) \
+ __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##2, ULONGTY##2) \
+ __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##3, ULONGTY##3) \
+ __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##4, ULONGTY##4) \
+ __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##8, ULONGTY##8) \
+ __CLC_DEFINE_SLONG_MAD_SAT(SLONGTY##16, ULONGTY##16)
+
+__CLC_DEFINE_SLONG_MAD_SAT_ALL_TYS(long, ulong)
diff --git a/libclc/clc/lib/spirv/SOURCES b/libclc/clc/lib/spirv/SOURCES
index ddc9e4c49d8626..c3cc4068225d81 100644
--- a/libclc/clc/lib/spirv/SOURCES
+++ b/libclc/clc/lib/spirv/SOURCES
@@ -6,6 +6,7 @@
../generic/integer/clc_clz.cl
../generic/integer/clc_hadd.cl
../generic/integer/clc_mad24.cl
+../generic/integer/clc_mad_sat.cl
../generic/integer/clc_mul24.cl
../generic/integer/clc_mul_hi.cl
../generic/integer/clc_popcount.cl
diff --git a/libclc/generic/lib/integer/mad_sat.cl b/libclc/generic/lib/integer/mad_sat.cl
index 2372eaacd6da82..e9c44364deee0c 100644
--- a/libclc/generic/lib/integer/mad_sat.cl
+++ b/libclc/generic/lib/integer/mad_sat.cl
@@ -1,72 +1,7 @@
#include <clc/clc.h>
-#include <clc/clcmacro.h>
+#include <clc/integer/clc_mad_sat.h>
-_CLC_OVERLOAD _CLC_DEF char mad_sat(char x, char y, char z) {
- return clamp((short)mad24((short)x, (short)y, (short)z), (short)CHAR_MIN, (short) CHAR_MAX);
-}
+#define FUNCTION mad_sat
+#define __CLC_BODY <clc/shared/ternary_def.inc>
-_CLC_OVERLOAD _CLC_DEF uchar mad_sat(uchar x, uchar y, uchar z) {
- return clamp((ushort)mad24((ushort)x, (ushort)y, (ushort)z), (ushort)0, (ushort) UCHAR_MAX);
-}
-
-_CLC_OVERLOAD _CLC_DEF short mad_sat(short x, short y, short z) {
- return clamp((int)mad24((int)x, (int)y, (int)z), (int)SHRT_MIN, (int) SHRT_MAX);
-}
-
-_CLC_OVERLOAD _CLC_DEF ushort mad_sat(ushort x, ushort y, ushort z) {
- return clamp((uint)mad24((uint)x, (uint)y, (uint)z), (uint)0, (uint) USHRT_MAX);
-}
-
-_CLC_OVERLOAD _CLC_DEF int mad_sat(int x, int y, int z) {
- int mhi = mul_hi(x, y);
- uint mlo = x * y;
- long m = upsample(mhi, mlo);
- m += z;
- if (m > INT_MAX)
- return INT_MAX;
- if (m < INT_MIN)
- return INT_MIN;
- return m;
-}
-
-_CLC_OVERLOAD _CLC_DEF uint mad_sat(uint x, uint y, uint z) {
- if (mul_hi(x, y) != 0)
- return UINT_MAX;
- return add_sat(x * y, z);
-}
-
-_CLC_OVERLOAD _CLC_DEF long mad_sat(long x, long y, long z) {
- long hi = mul_hi(x, y);
- ulong ulo = x * y;
- long slo = x * y;
- /* Big overflow of more than 2 bits, add can't fix this */
- if (((x < 0) == (y < 0)) && hi != 0)
- return LONG_MAX;
- /* Low overflow in mul and z not neg enough to correct it */
- if (hi == 0 && ulo >= LONG_MAX && (z > 0 || (ulo + z) > LONG_MAX))
- return LONG_MAX;
- /* Big overflow of more than 2 bits, add can't fix this */
- if (((x < 0) != (y < 0)) && hi != -1)
- return LONG_MIN;
- /* Low overflow in mul and z not pos enough to correct it */
- if (hi == -1 && ulo <= ((ulong)LONG_MAX + 1UL) && (z < 0 || z < (LONG_MAX - ulo)))
- return LONG_MIN;
- /* We have checked all conditions, any overflow in addition returns
- * the correct value */
- return ulo + z;
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong mad_sat(ulong x, ulong y, ulong z) {
- if (mul_hi(x, y) != 0)
- return ULONG_MAX;
- return add_sat(x * y, z);
-}
-
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, mad_sat, char, char, char)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, mad_sat, uchar, uchar, uchar)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, mad_sat, short, short, short)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, mad_sat, ushort, ushort, ushort)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, mad_sat, int, int, int)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, mad_sat, uint, uint, uint)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, mad_sat, long, long, long)
-_CLC_TERNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, mad_sat, ulong, ulong, ulong)
+#include <clc/integer/gentype.inc>
More information about the cfe-commits
mailing list