[clang] bd07c22 - [Clang] Add support for scalable vectors in __builtin_reduce_* functions (#87750)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Apr 29 07:45:37 PDT 2024
Author: Lawrence Benson
Date: 2024-04-29T16:45:33+02:00
New Revision: bd07c22e5372789c3eb47b9009029d5e99e0ef9f
URL: https://github.com/llvm/llvm-project/commit/bd07c22e5372789c3eb47b9009029d5e99e0ef9f
DIFF: https://github.com/llvm/llvm-project/commit/bd07c22e5372789c3eb47b9009029d5e99e0ef9f.diff
LOG: [Clang] Add support for scalable vectors in __builtin_reduce_* functions (#87750)
Currently, a lot of `__builtin_reduce_*` function do not support
scalable vectors, i.e., ARM SVE and RISCV V. This PR adds support for
them. The main code change is to use a different path to extract the
type from the vectors, the rest is the same and LLVM supports the reduce
functions for `vscale` vectors.
This PR adds scalable vector support for:
- `__builtin_reduce_add`
- `__builtin_reduce_mul`
- `__builtin_reduce_xor`
- `__builtin_reduce_or`
- `__builtin_reduce_and`
- `__builtin_reduce_min`
- `__builtin_reduce_max`
Note: For all except `min/max`, the element type must still be an
integer value. Adding floating point support for `add` and `mul` is
still an open TODO.
Added:
Modified:
clang/docs/LanguageExtensions.rst
clang/docs/ReleaseNotes.rst
clang/include/clang/AST/Type.h
clang/lib/AST/Type.cpp
clang/lib/CodeGen/CGBuiltin.cpp
clang/lib/Sema/SemaChecking.cpp
clang/test/CodeGen/builtins-reduction-math.c
Removed:
################################################################################
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 127d1b6dd48252..87cb743856b07e 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -711,6 +711,8 @@ even-odd element pair with indices ``i * 2`` and ``i * 2 + 1`` with
power of 2, the vector is widened with neutral elements for the reduction
at the end to the next power of 2.
+These reductions support both fixed-sized and scalable vector types.
+
Example:
.. code-block:: c++
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 347c81253d39e6..4cb2462ae64956 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -222,6 +222,7 @@ Non-comprehensive list of changes in this release
- ``__typeof_unqual__`` is available in all C modes as an extension, which behaves
like ``typeof_unqual`` from C23, similar to ``__typeof__`` and ``typeof``.
+- ``__builtin_reduce_{add|mul|xor|or|and|min|max}`` builtins now support scalable vectors.
* Shared libraries linked with either the ``-ffast-math``, ``-Ofast``, or
``-funsafe-math-optimizations`` flags will no longer enable flush-to-zero
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index dff02d4861b3db..fa2b47ed3d2350 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -2378,6 +2378,10 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
/// 'riscv_rvv_vector_bits' type attribute as VectorType.
QualType getRVVEltType(const ASTContext &Ctx) const;
+ /// Returns the representative type for the element of a sizeless vector
+ /// builtin type.
+ QualType getSizelessVectorEltType(const ASTContext &Ctx) const;
+
/// Types are partitioned into 3 broad categories (C99 6.2.5p1):
/// object types, function types, and incomplete types.
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 8aaa6801d85b8b..68e81f45b4c28e 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -2510,6 +2510,18 @@ bool Type::isSveVLSBuiltinType() const {
return false;
}
+QualType Type::getSizelessVectorEltType(const ASTContext &Ctx) const {
+ assert(isSizelessVectorType() && "Must be sizeless vector type");
+ // Currently supports SVE and RVV
+ if (isSVESizelessBuiltinType())
+ return getSveEltType(Ctx);
+
+ if (isRVVSizelessBuiltinType())
+ return getRVVEltType(Ctx);
+
+ llvm_unreachable("Unhandled type");
+}
+
QualType Type::getSveEltType(const ASTContext &Ctx) const {
assert(isSveVLSBuiltinType() && "unsupported type!");
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index d08ab539148914..a370734e00d3e1 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -3885,9 +3885,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
}
case Builtin::BI__builtin_reduce_max: {
- auto GetIntrinsicID = [](QualType QT) {
+ auto GetIntrinsicID = [this](QualType QT) {
if (auto *VecTy = QT->getAs<VectorType>())
QT = VecTy->getElementType();
+ else if (QT->isSizelessVectorType())
+ QT = QT->getSizelessVectorEltType(CGM.getContext());
+
if (QT->isSignedIntegerType())
return llvm::Intrinsic::vector_reduce_smax;
if (QT->isUnsignedIntegerType())
@@ -3900,9 +3903,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
}
case Builtin::BI__builtin_reduce_min: {
- auto GetIntrinsicID = [](QualType QT) {
+ auto GetIntrinsicID = [this](QualType QT) {
if (auto *VecTy = QT->getAs<VectorType>())
QT = VecTy->getElementType();
+ else if (QT->isSizelessVectorType())
+ QT = QT->getSizelessVectorEltType(CGM.getContext());
+
if (QT->isSignedIntegerType())
return llvm::Intrinsic::vector_reduce_smin;
if (QT->isUnsignedIntegerType())
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index e33113ab9c4c1d..e26cf20d10363b 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -3164,13 +3164,20 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
const Expr *Arg = TheCall->getArg(0);
const auto *TyA = Arg->getType()->getAs<VectorType>();
- if (!TyA) {
+
+ QualType ElTy;
+ if (TyA)
+ ElTy = TyA->getElementType();
+ else if (Arg->getType()->isSizelessVectorType())
+ ElTy = Arg->getType()->getSizelessVectorEltType(Context);
+
+ if (ElTy.isNull()) {
Diag(Arg->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* vector ty*/ 4 << Arg->getType();
return ExprError();
}
- TheCall->setType(TyA->getElementType());
+ TheCall->setType(ElTy);
break;
}
@@ -3186,12 +3193,20 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
const Expr *Arg = TheCall->getArg(0);
const auto *TyA = Arg->getType()->getAs<VectorType>();
- if (!TyA || !TyA->getElementType()->isIntegerType()) {
+
+ QualType ElTy;
+ if (TyA)
+ ElTy = TyA->getElementType();
+ else if (Arg->getType()->isSizelessVectorType())
+ ElTy = Arg->getType()->getSizelessVectorEltType(Context);
+
+ if (ElTy.isNull() || !ElTy->isIntegerType()) {
Diag(Arg->getBeginLoc(), diag::err_builtin_invalid_arg_type)
<< 1 << /* vector of integers */ 6 << Arg->getType();
return ExprError();
}
- TheCall->setType(TyA->getElementType());
+
+ TheCall->setType(ElTy);
break;
}
diff --git a/clang/test/CodeGen/builtins-reduction-math.c b/clang/test/CodeGen/builtins-reduction-math.c
index 34f39cea5265ea..acafe9222d59fd 100644
--- a/clang/test/CodeGen/builtins-reduction-math.c
+++ b/clang/test/CodeGen/builtins-reduction-math.c
@@ -1,5 +1,8 @@
// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -emit-llvm -disable-llvm-passes -o - | FileCheck %s
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -O1 -triple aarch64 -target-feature +sve %s -emit-llvm -disable-llvm-passes -o - | FileCheck --check-prefixes=SVE %s
+
typedef float float4 __attribute__((ext_vector_type(4)));
typedef short int si8 __attribute__((ext_vector_type(8)));
typedef unsigned int u4 __attribute__((ext_vector_type(4)));
@@ -134,3 +137,53 @@ void test_builtin_reduce_and(si8 vi1, u4 vu1) {
// CHECK-NEXT: call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[VU1]])
unsigned r3 = __builtin_reduce_and(vu1);
}
+
+#if defined(__ARM_FEATURE_SVE)
+#include <arm_sve.h>
+
+void test_builtin_reduce_SVE(int a, unsigned long long b, short c, float d) {
+ // SVE-LABEL: void @test_builtin_reduce_SVE(
+
+ svint32_t vec_a = svdup_s32(a);
+ svuint64_t vec_b = svdup_u64(b);
+ svint16_t vec_c1 = svdup_s16(c);
+ svuint16_t vec_c2 = svdup_u16(c);
+ svfloat32_t vec_d = svdup_f32(d);
+
+ // SVE: [[VF1:%.+]] = load <vscale x 4 x i32>, ptr %vec_a
+ // SVE-NEXT: call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[VF1]])
+ int r1 = __builtin_reduce_add(vec_a);
+
+ // SVE: [[VF2:%.+]] = load <vscale x 4 x i32>, ptr %vec_a
+ // SVE-NEXT: call i32 @llvm.vector.reduce.mul.nxv4i32(<vscale x 4 x i32> [[VF2]])
+ int r2 = __builtin_reduce_mul(vec_a);
+
+ // SVE: [[VF3:%.+]] = load <vscale x 2 x i64>, ptr %vec_b
+ // SVE-NEXT: call i64 @llvm.vector.reduce.xor.nxv2i64(<vscale x 2 x i64> [[VF3]])
+ long long r3 = __builtin_reduce_xor(vec_b);
+
+ // SVE: [[VF4:%.+]] = load <vscale x 2 x i64>, ptr %vec_b
+ // SVE-NEXT: call i64 @llvm.vector.reduce.or.nxv2i64(<vscale x 2 x i64> [[VF4]])
+ long long r4 = __builtin_reduce_or(vec_b);
+
+ // SVE: [[VF5:%.+]] = load <vscale x 2 x i64>, ptr %vec_b
+ // SVE-NEXT: call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> [[VF5]])
+ long long r5 = __builtin_reduce_and(vec_b);
+
+ // SVE: [[VF6:%.+]] = load <vscale x 8 x i16>, ptr %vec_c1
+ // SVE-NEXT: call i16 @llvm.vector.reduce.smax.nxv8i16(<vscale x 8 x i16> [[VF6]])
+ short r6 = __builtin_reduce_max(vec_c1);
+
+ // SVE: [[VF7:%.+]] = load <vscale x 8 x i16>, ptr %vec_c2
+ // SVE-NEXT: call i16 @llvm.vector.reduce.umin.nxv8i16(<vscale x 8 x i16> [[VF7]])
+ unsigned short r7 = __builtin_reduce_min(vec_c2);
+
+ // SVE: [[VF8:%.+]] = load <vscale x 4 x float>, ptr %vec_d
+ // SVE-NEXT: call float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> [[VF8]])
+ float r8 = __builtin_reduce_max(vec_d);
+
+ // SVE: [[VF9:%.+]] = load <vscale x 4 x float>, ptr %vec_d
+ // SVE-NEXT: call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[VF9]])
+ float r9 = __builtin_reduce_min(vec_d);
+}
+#endif
More information about the cfe-commits
mailing list