[clang] [Clang][AArch64]Refactor typespec handling in SveEmitter.cpp (PR #117717)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Dec 2 05:12:24 PST 2024
https://github.com/SpencerAbson updated https://github.com/llvm/llvm-project/pull/117717
>From 850b7c0173f47a382093ff345d9bf35ee9e1643e Mon Sep 17 00:00:00 2001
From: Spencer Abson <Spencer.Abson at arm.com>
Date: Tue, 26 Nov 2024 13:49:12 +0000
Subject: [PATCH 1/9] Refactor parts of SveEmitter.cpp
---
clang/include/clang/Basic/arm_sve.td | 28 +-
...#12752a66d88e6d5bc8de5376bca6898e3e71f901# | 1874 +++++++++++++++++
clang/utils/TableGen/SveEmitter.cpp | 369 ++--
3 files changed, 2048 insertions(+), 223 deletions(-)
create mode 100644 clang/utils/TableGen/#12752a66d88e6d5bc8de5376bca6898e3e71f901#
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index b36e592042da0b..e551d6e46b8f33 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -762,14 +762,14 @@ def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone,
////////////////////////////////////////////////////////////////////////////////
// While comparisons
-def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
////////////////////////////////////////////////////////////////////////////////
// Counting bit
@@ -1365,10 +1365,10 @@ def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNon
def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
-def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
+def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
}
let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in {
@@ -2326,7 +2326,7 @@ let SVETargetGuard = "sve2p1,bf16", SMETargetGuard = "sme2p1,bf16" in {
// Multi-vector convert to/from floating-point.
//
let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in {
- def SVCVT_F16_X2 : SInst<"svcvt_f16[_f32_x2]", "e2", "f", MergeNone, "aarch64_sve_fcvt_x2", [IsStreaming],[]>;
+ def SVCVT_F16_X2 : SInst<"svcvt_f16[_f32_x2]", "h2", "f", MergeNone, "aarch64_sve_fcvt_x2", [IsStreaming],[]>;
def SVCVT_BF16_X2 : SInst<"svcvt_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvt_x2", [IsOverloadNone, IsStreaming],[]>;
def SVCVT_F32_U32_X2 : SInst<"svcvt_{d}[_u32_x2]", "2.d2.u", "f", MergeNone, "aarch64_sve_ucvtf_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>;
@@ -2348,7 +2348,7 @@ let SVETargetGuard = InvalidMode, SMETargetGuard = "sme-f16f16" in {
// Multi-vector floating-point convert from single-precision to interleaved half-precision/BFloat16
//
let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in {
- def SVCVTN_F16_X2 : SInst<"svcvtn_f16[_f32_x2]", "e2", "f", MergeNone, "aarch64_sve_fcvtn_x2", [IsStreaming],[]>;
+ def SVCVTN_F16_X2 : SInst<"svcvtn_f16[_f32_x2]", "h2", "f", MergeNone, "aarch64_sve_fcvtn_x2", [IsStreaming],[]>;
def SVCVTN_BF16_X2 : SInst<"svcvtn_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvtn_x2", [IsOverloadNone, IsStreaming],[]>;
}
diff --git a/clang/utils/TableGen/#12752a66d88e6d5bc8de5376bca6898e3e71f901# b/clang/utils/TableGen/#12752a66d88e6d5bc8de5376bca6898e3e71f901#
new file mode 100644
index 00000000000000..40a798928f4d69
--- /dev/null
+++ b/clang/utils/TableGen/#12752a66d88e6d5bc8de5376bca6898e3e71f901#
@@ -0,0 +1,1874 @@
+//===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This tablegen backend is responsible for emitting arm_sve.h, which includes
+// a declaration and definition of each function specified by the ARM C/C++
+// Language Extensions (ACLE).
+//
+// For details, visit:
+// https://developer.arm.com/architectures/system-architectures/software-standards/acle
+//
+// Each SVE instruction is implemented in terms of 1 or more functions which
+// are suffixed with the element type of the input vectors. Functions may be
+// implemented in terms of generic vector operations such as +, *, -, etc. or
+// by calling a __builtin_-prefixed function which will be handled by clang's
+// CodeGen library.
+//
+// See also the documentation in include/clang/Basic/arm_sve.td.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include "ImmCheck.h"
+#include <array>
+#include <cctype>
+#include <set>
+#include <sstream>
+#include <string>
+#include <tuple>
+
+using namespace llvm;
+
+enum ClassKind {
+ ClassNone,
+ ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
+ ClassG, // Overloaded name without type suffix
+};
+
+enum class ACLEKind { SVE, SME };
+
+using TypeSpec = std::string;
+
+namespace {
+class SVEType {
+ bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
+ bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
+ Svcount;
+ unsigned Bitwidth, ElementBitwidth, NumVectors;
+
+public:
+ SVEType() : SVEType("", 'v') {}
+
+ SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
+ : Float(false), Signed(true), Immediate(false), Void(false),
+ Constant(false), Pointer(false), BFloat(false), DefaultType(false),
+ IsScalable(true), Predicate(false), PredicatePattern(false),
+ PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
+ NumVectors(NumVectors) {
+ if (!TS.empty())
+ applyTypespec(TS);
+ applyModifier(CharMod);
+ }
+
+ SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
+ NumVectors = NumV;
+ }
+
+ bool isPointer() const { return Pointer; }
+ bool isVoidPointer() const { return Pointer && Void; }
+ bool isSigned() const { return Signed; }
+ bool isImmediate() const { return Immediate; }
+ bool isScalar() const { return NumVectors == 0; }
+ bool isVector() const { return NumVectors > 0; }
+ bool isScalableVector() const { return isVector() && IsScalable; }
+ bool isFixedLengthVector() const { return isVector() && !IsScalable; }
+ bool isChar() const { return ElementBitwidth == 8; }
+ bool isVoid() const { return Void && !Pointer; }
+ bool isDefault() const { return DefaultType; }
+ bool isFloat() const { return Float && !BFloat; }
+ bool isBFloat() const { return BFloat && !Float; }
+ bool isFloatingPoint() const { return Float || BFloat; }
+ bool isInteger() const {
+ return !isFloatingPoint() && !Predicate && !Svcount;
+ }
+ bool isScalarPredicate() const {
+ return !isFloatingPoint() && Predicate && NumVectors == 0;
+ }
+ bool isPredicateVector() const { return Predicate; }
+ bool isPredicatePattern() const { return PredicatePattern; }
+ bool isPrefetchOp() const { return PrefetchOp; }
+ bool isSvcount() const { return Svcount; }
+ bool isConstant() const { return Constant; }
+ unsigned getElementSizeInBits() const { return ElementBitwidth; }
+ unsigned getNumVectors() const { return NumVectors; }
+
+ unsigned getNumElements() const {
+ assert(ElementBitwidth != ~0U);
+ return Bitwidth / ElementBitwidth;
+ }
+ unsigned getSizeInBits() const {
+ return Bitwidth;
+ }
+
+ /// Return the string representation of a type, which is an encoded
+ /// string for passing to the BUILTIN() macro in Builtins.def.
+ std::string builtin_str() const;
+
+ /// Return the C/C++ string representation of a type for use in the
+ /// arm_sve.h header file.
+ std::string str() const;
+
+private:
+ /// Creates the type based on the typespec string in TS.
+ void applyTypespec(StringRef TS);
+
+ /// Applies a prototype modifier to the type.
+ void applyModifier(char Mod);
+};
+
+class SVEEmitter;
+
+/// The main grunt class. This represents an instantiation of an intrinsic with
+/// a particular typespec and prototype.
+class Intrinsic {
+ /// The unmangled name.
+ std::string Name;
+
+ /// The name of the corresponding LLVM IR intrinsic.
+ std::string LLVMName;
+
+ /// Intrinsic prototype.
+ std::string Proto;
+
+ /// The base type spec for this intrinsic.
+ TypeSpec BaseTypeSpec;
+
+ /// The base class kind. Most intrinsics use ClassS, which has full type
+ /// info for integers (_s32/_u32), or ClassG which is used for overloaded
+ /// intrinsics.
+ ClassKind Class;
+
+ /// The architectural #ifdef guard.
+ std::string SVEGuard, SMEGuard;
+
+ // The merge suffix such as _m, _x or _z.
+ std::string MergeSuffix;
+
+ /// The types of return value [0] and parameters [1..].
+ std::vector<SVEType> Types;
+
+ /// The "base type", which is VarType('d', BaseTypeSpec).
+ SVEType BaseType;
+
+ uint64_t Flags;
+
+ SmallVector<ImmCheck, 2> ImmChecks;
+
+public:
+ Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
+ StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
+ uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
+ ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
+ StringRef SMEGuard);
+
+ ~Intrinsic()=default;
+
+ std::string getName() const { return Name; }
+ std::string getLLVMName() const { return LLVMName; }
+ std::string getProto() const { return Proto; }
+ TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
+ SVEType getBaseType() const { return BaseType; }
+
+ StringRef getSVEGuard() const { return SVEGuard; }
+ StringRef getSMEGuard() const { return SMEGuard; }
+ void printGuard(raw_ostream &OS) const {
+ if (!SVEGuard.empty() && SMEGuard.empty())
+ OS << SVEGuard;
+ else if (SVEGuard.empty() && !SMEGuard.empty())
+ OS << SMEGuard;
+ else {
+ if (SVEGuard.find(",") != std::string::npos ||
+ SVEGuard.find("|") != std::string::npos)
+ OS << "(" << SVEGuard << ")";
+ else
+ OS << SVEGuard;
+ OS << "|";
+ if (SMEGuard.find(",") != std::string::npos ||
+ SMEGuard.find("|") != std::string::npos)
+ OS << "(" << SMEGuard << ")";
+ else
+ OS << SMEGuard;
+ }
+ }
+ ClassKind getClassKind() const { return Class; }
+
+ SVEType getReturnType() const { return Types[0]; }
+ ArrayRef<SVEType> getTypes() const { return Types; }
+ SVEType getParamType(unsigned I) const { return Types[I + 1]; }
+ unsigned getNumParams() const {
+ return Proto.size() - (2 * llvm::count(Proto, '.')) - 1;
+ }
+
+ uint64_t getFlags() const { return Flags; }
+ bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
+
+ ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
+
+ /// Return the type string for a BUILTIN() macro in Builtins.def.
+ std::string getBuiltinTypeStr();
+
+ /// Return the name, mangled with type information. The name is mangled for
+ /// ClassS, so will add type suffixes such as _u32/_s32.
+ std::string getMangledName() const { return mangleName(ClassS); }
+
+ /// As above, but mangles the LLVM name instead.
+ std::string getMangledLLVMName() const { return mangleLLVMName(); }
+
+ /// Returns true if the intrinsic is overloaded, in that it should also generate
+ /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
+ /// 'svld1_u32(..)'.
+ static bool isOverloadedIntrinsic(StringRef Name) {
+ auto BrOpen = Name.find('[');
+ auto BrClose = Name.find(']');
+ return BrOpen != std::string::npos && BrClose != std::string::npos;
+ }
+
+ /// Return true if the intrinsic takes a splat operand.
+ bool hasSplat() const {
+ // These prototype modifiers are described in arm_sve.td.
+ return Proto.find_first_of("ajfrKLR@") != std::string::npos;
+ }
+
+ /// Return the parameter index of the splat operand.
+ unsigned getSplatIdx() const {
+ unsigned I = 1, Param = 0;
+ for (; I < Proto.size(); ++I, ++Param) {
+ if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
+ Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
+ Proto[I] == 'R' || Proto[I] == '@')
+ break;
+
+ // Multivector modifier can be skipped
+ if (Proto[I] == '.')
+ I += 2;
+ }
+ assert(I != Proto.size() && "Prototype has no splat operand");
+ return Param;
+ }
+
+ /// Emits the intrinsic declaration to the ostream.
+ void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
+
+private:
+ std::string getMergeSuffix() const { return MergeSuffix; }
+ std::string mangleName(ClassKind LocalCK) const;
+ std::string mangleLLVMName() const;
+ std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
+ std::string Proto) const;
+};
+
+class SVEEmitter {
+private:
+ // The reinterpret builtins are generated separately because they
+ // need the cross product of all types (121 functions in total),
+ // which is inconvenient to specify in the arm_sve.td file or
+ // generate in CGBuiltin.cpp.
+ struct ReinterpretTypeInfo {
+ SVEType BaseType;
+ const char *Suffix;
+ };
+
+ static const std::array<ReinterpretTypeInfo, 12> Reinterprets;
+
+ RecordKeeper &Records;
+ llvm::StringMap<uint64_t> EltTypes;
+ llvm::StringMap<uint64_t> MemEltTypes;
+ llvm::StringMap<uint64_t> FlagTypes;
+ llvm::StringMap<uint64_t> MergeTypes;
+ llvm::StringMap<uint64_t> ImmCheckTypes;
+
+public:
+ SVEEmitter(RecordKeeper &R) : Records(R) {
+ for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
+ EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
+ for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
+ MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
+ for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
+ FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
+ for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
+ MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
+ for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
+ ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
+ }
+
+ /// Returns the enum value for the immcheck type
+ unsigned getEnumValueForImmCheck(StringRef C) const {
+ auto It = ImmCheckTypes.find(C);
+ if (It != ImmCheckTypes.end())
+ return It->getValue();
+ llvm_unreachable("Unsupported imm check");
+ }
+
+ /// Returns the enum value for the flag type
+ uint64_t getEnumValueForFlag(StringRef C) const {
+ auto Res = FlagTypes.find(C);
+ if (Res != FlagTypes.end())
+ return Res->getValue();
+ llvm_unreachable("Unsupported flag");
+ }
+
+ // Returns the SVETypeFlags for a given value and mask.
+ uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
+ auto It = FlagTypes.find(MaskName);
+ if (It != FlagTypes.end()) {
+ uint64_t Mask = It->getValue();
+ unsigned Shift = llvm::countr_zero(Mask);
+ assert(Shift < 64 && "Mask value produced an invalid shift value");
+ return (V << Shift) & Mask;
+ }
+ llvm_unreachable("Unsupported flag");
+ }
+
+ // Returns the SVETypeFlags for the given element type.
+ uint64_t encodeEltType(StringRef EltName) {
+ auto It = EltTypes.find(EltName);
+ if (It != EltTypes.end())
+ return encodeFlag(It->getValue(), "EltTypeMask");
+ llvm_unreachable("Unsupported EltType");
+ }
+
+ // Returns the SVETypeFlags for the given memory element type.
+ uint64_t encodeMemoryElementType(uint64_t MT) {
+ return encodeFlag(MT, "MemEltTypeMask");
+ }
+
+ // Returns the SVETypeFlags for the given merge type.
+ uint64_t encodeMergeType(uint64_t MT) {
+ return encodeFlag(MT, "MergeTypeMask");
+ }
+
+ // Returns the SVETypeFlags for the given splat operand.
+ unsigned encodeSplatOperand(unsigned SplatIdx) {
+ assert(SplatIdx < 7 && "SplatIdx out of encodable range");
+ return encodeFlag(SplatIdx + 1, "SplatOperandMask");
+ }
+
+ // Returns the SVETypeFlags value for the given SVEType.
+ uint64_t encodeTypeFlags(const SVEType &T);
+
+ /// Emit arm_sve.h.
+ void createHeader(raw_ostream &o);
+
+ // Emits core intrinsics in both arm_sme.h and arm_sve.h
+ void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
+ ACLEKind Kind);
+
+ /// Emit all the __builtin prototypes and code needed by Sema.
+ void createBuiltins(raw_ostream &o);
+
+ /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
+ void createCodeGenMap(raw_ostream &o);
+
+ /// Emit all the range checks for the immediates.
+ void createRangeChecks(raw_ostream &o);
+
+ /// Create the SVETypeFlags used in CGBuiltins
+ void createTypeFlags(raw_ostream &o);
+
+ /// Emit arm_sme.h.
+ void createSMEHeader(raw_ostream &o);
+
+ /// Emit all the SME __builtin prototypes and code needed by Sema.
+ void createSMEBuiltins(raw_ostream &o);
+
+ /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
+ void createSMECodeGenMap(raw_ostream &o);
+
+ /// Create a table for a builtin's requirement for PSTATE.SM.
+ void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
+
+ /// Emit all the range checks for the immediates.
+ void createSMERangeChecks(raw_ostream &o);
+
+ /// Create a table for a builtin's requirement for PSTATE.ZA.
+ void createBuiltinZAState(raw_ostream &OS);
+
+ /// Create intrinsic and add it to \p Out
+ void createIntrinsic(Record *R,
+ SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
+};
+
+const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
+ {{{SVEType("c", 'd'), "s8"},
+ {SVEType("Uc", 'd'), "u8"},
+ {SVEType("s", 'd'), "s16"},
+ {SVEType("Us", 'd'), "u16"},
+ {SVEType("i", 'd'), "s32"},
+ {SVEType("Ui", 'd'), "u32"},
+ {SVEType("l", 'd'), "s64"},
+ {SVEType("Ul", 'd'), "u64"},
+ {SVEType("h", 'd'), "f16"},
+ {SVEType("b", 'd'), "bf16"},
+ {SVEType("f", 'd'), "f32"},
+ {SVEType("d", 'd'), "f64"}}};
+
+} // end anonymous namespace
+
+
+//===----------------------------------------------------------------------===//
+// Type implementation
+//===----------------------------------------------------------------------===//
+
+std::string SVEType::builtin_str() const {
+ std::string S;
+ if (isVoid())
+ return "v";
+
+ if (isScalarPredicate())
+ return "b";
+
+ if (isSvcount())
+ return "Qa";
+
+ if (isVoidPointer())
+ S += "v";
+ else if (!isFloatingPoint())
+ switch (ElementBitwidth) {
+ case 1: S += "b"; break;
+ case 8: S += "c"; break;
+ case 16: S += "s"; break;
+ case 32: S += "i"; break;
+ case 64: S += "Wi"; break;
+ case 128: S += "LLLi"; break;
+ default: llvm_unreachable("Unhandled case!");
+ }
+ else if (isFloat())
+ switch (ElementBitwidth) {
+ case 16: S += "h"; break;
+ case 32: S += "f"; break;
+ case 64: S += "d"; break;
+ default: llvm_unreachable("Unhandled case!");
+ }
+ else if (isBFloat()) {
+ assert(ElementBitwidth == 16 && "Not a valid BFloat.");
+ S += "y";
+ }
+
+ if (!isFloatingPoint()) {
+ if ((isChar() || isPointer()) && !isVoidPointer()) {
+ // Make chars and typed pointers explicitly signed.
+ if (Signed)
+ S = "S" + S;
+ else if (!Signed)
+ S = "U" + S;
+ } else if (!isVoidPointer() && !Signed) {
+ S = "U" + S;
+ }
+ }
+
+ // Constant indices are "int", but have the "constant expression" modifier.
+ if (isImmediate()) {
+ assert(!isFloat() && "fp immediates are not supported");
+ S = "I" + S;
+ }
+
+ if (isScalar()) {
+ if (Constant) S += "C";
+ if (Pointer) S += "*";
+ return S;
+ }
+
+ if (isFixedLengthVector())
+ return "V" + utostr(getNumElements() * NumVectors) + S;
+ return "q" + utostr(getNumElements() * NumVectors) + S;
+}
+
+std::string SVEType::str() const {
+ if (isPredicatePattern())
+ return "enum svpattern";
+
+ if (isPrefetchOp())
+ return "enum svprfop";
+
+ std::string S;
+ if (Void)
+ S += "void";
+ else {
+ if (isScalableVector() || isSvcount())
+ S += "sv";
+ if (!Signed && !isFloatingPoint())
+ S += "u";
+
+ if (Float)
+ S += "float";
+ else if (isSvcount())
+ S += "count";
+ else if (isScalarPredicate() || isPredicateVector())
+ S += "bool";
+ else if (isBFloat())
+ S += "bfloat";
+ else
+ S += "int";
+
+ if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
+ S += utostr(ElementBitwidth);
+ if (isFixedLengthVector())
+ S += "x" + utostr(getNumElements());
+ if (NumVectors > 1)
+ S += "x" + utostr(NumVectors);
+ if (!isScalarPredicate())
+ S += "_t";
+ }
+
+ if (Constant)
+ S += " const";
+ if (Pointer)
+ S += " *";
+
+ return S;
+}
+
+void SVEType::applyTypespec(StringRef TS) {
+ for (char I : TS) {
+ switch (I) {
+ case 'Q':
+ Svcount = true;
+ break;
+ case 'P':
+ Predicate = true;
+ break;
+ case 'U':
+ Signed = false;
+ break;
+ case 'c':
+ ElementBitwidth = 8;
+ break;
+ case 's':
+ ElementBitwidth = 16;
+ break;
+ case 'i':
+ ElementBitwidth = 32;
+ break;
+ case 'l':
+ ElementBitwidth = 64;
+ break;
+ case 'q':
+ ElementBitwidth = 128;
+ break;
+ case 'h':
+ Float = true;
+ ElementBitwidth = 16;
+ break;
+ case 'f':
+ Float = true;
+ ElementBitwidth = 32;
+ break;
+ case 'd':
+ Float = true;
+ ElementBitwidth = 64;
+ break;
+ case 'b':
+ BFloat = true;
+ Float = false;
+ ElementBitwidth = 16;
+ break;
+ default:
+ llvm_unreachable("Unhandled type code!");
+ }
+ }
+ assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
+}
+
+void SVEType::applyModifier(char Mod) {
+ switch (Mod) {
+ case 'v':
+ Void = true;
+ break;
+ case 'd':
+ DefaultType = true;
+ break;
+ case 'c':
+ Constant = true;
+ [[fallthrough]];
+ case 'p':
+ Pointer = true;
+ Bitwidth = ElementBitwidth;
+ NumVectors = 0;
+ break;
+ case 'e':
+ Signed = false;
+ ElementBitwidth /= 2;
+ break;
+ case 'h':
+ ElementBitwidth /= 2;
+ break;
+ case 'q':
+ ElementBitwidth /= 4;
+ break;
+ case 'b':
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth /= 4;
+ break;
+ case 'o':
+ ElementBitwidth *= 4;
+ break;
+ case 'P':
+ Signed = true;
+ Float = false;
+ BFloat = false;
+ Predicate = true;
+ Svcount = false;
+ Bitwidth = 16;
+ ElementBitwidth = 1;
+ break;
+ case '{':
+ IsScalable = false;
+ Bitwidth = 128;
+ NumVectors = 1;
+ break;
+ case 's':
+ case 'a':
+ Bitwidth = ElementBitwidth;
+ NumVectors = 0;
+ break;
+ case 'R':
+ ElementBitwidth /= 2;
+ NumVectors = 0;
+ break;
+ case 'r':
+ ElementBitwidth /= 4;
+ NumVectors = 0;
+ break;
+ case '@':
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth /= 4;
+ NumVectors = 0;
+ break;
+ case 'K':
+ Signed = true;
+ Float = false;
+ BFloat = false;
+ Bitwidth = ElementBitwidth;
+ NumVectors = 0;
+ break;
+ case 'L':
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ Bitwidth = ElementBitwidth;
+ NumVectors = 0;
+ break;
+ case 'u':
+ Predicate = false;
+ Svcount = false;
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ break;
+ case 'x':
+ Predicate = false;
+ Svcount = false;
+ Signed = true;
+ Float = false;
+ BFloat = false;
+ break;
+ case 'i':
+ Predicate = false;
+ Svcount = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = Bitwidth = 64;
+ NumVectors = 0;
+ Signed = false;
+ Immediate = true;
+ break;
+ case 'I':
+ Predicate = false;
+ Svcount = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = Bitwidth = 32;
+ NumVectors = 0;
+ Signed = true;
+ Immediate = true;
+ PredicatePattern = true;
+ break;
+ case 'J':
+ Predicate = false;
+ Svcount = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = Bitwidth = 32;
+ NumVectors = 0;
+ Signed = true;
+ Immediate = true;
+ PrefetchOp = true;
+ break;
+ case 'k':
+ Predicate = false;
+ Svcount = false;
+ Signed = true;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = Bitwidth = 32;
+ NumVectors = 0;
+ break;
+ case 'l':
+ Predicate = false;
+ Svcount = false;
+ Signed = true;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = Bitwidth = 64;
+ NumVectors = 0;
+ break;
+ case 'm':
+ Predicate = false;
+ Svcount = false;
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = Bitwidth = 32;
+ NumVectors = 0;
+ break;
+ case 'n':
+ Predicate = false;
+ Svcount = false;
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = Bitwidth = 64;
+ NumVectors = 0;
+ break;
+ case 'w':
+ ElementBitwidth = 64;
+ break;
+ case 'j':
+ ElementBitwidth = Bitwidth = 64;
+ NumVectors = 0;
+ break;
+ case 'f':
+ Signed = false;
+ ElementBitwidth = Bitwidth = 64;
+ NumVectors = 0;
+ break;
+ case 'g':
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = 64;
+ break;
+ case '[':
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = 8;
+ break;
+ case 't':
+ Signed = true;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = 32;
+ break;
+ case 'z':
+ Signed = false;
+ Float = false;
+ BFloat = false;
+ ElementBitwidth = 32;
+ break;
+ case 'O':
+ Predicate = false;
+ Svcount = false;
+ Float = true;
+ ElementBitwidth = 16;
+ break;
+ case 'M':
+ Predicate = false;
+ Svcount = false;
+ Float = true;
+ BFloat = false;
+ ElementBitwidth = 32;
+ break;
+ case 'N':
+ Predicate = false;
+ Svcount = false;
+ Float = true;
+ ElementBitwidth = 64;
+ break;
+ case 'Q':
+ Constant = true;
+ Pointer = true;
+ Void = true;
+ NumVectors = 0;
+ break;
+ case 'S':
+ Constant = true;
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 8;
+ NumVectors = 0;
+ Signed = true;
+ break;
+ case 'W':
+ Constant = true;
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 8;
+ NumVectors = 0;
+ Signed = false;
+ break;
+ case 'T':
+ Constant = true;
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 16;
+ NumVectors = 0;
+ Signed = true;
+ break;
+ case 'X':
+ Constant = true;
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 16;
+ NumVectors = 0;
+ Signed = false;
+ break;
+ case 'Y':
+ Constant = true;
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 32;
+ NumVectors = 0;
+ Signed = false;
+ break;
+ case 'U':
+ Constant = true;
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 32;
+ NumVectors = 0;
+ Signed = true;
+ break;
+ case '%':
+ Pointer = true;
+ Void = true;
+ NumVectors = 0;
+ break;
+ case 'A':
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 8;
+ NumVectors = 0;
+ Signed = true;
+ break;
+ case 'B':
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 16;
+ NumVectors = 0;
+ Signed = true;
+ break;
+ case 'C':
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 32;
+ NumVectors = 0;
+ Signed = true;
+ break;
+ case 'D':
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 64;
+ NumVectors = 0;
+ Signed = true;
+ break;
+ case 'E':
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 8;
+ NumVectors = 0;
+ Signed = false;
+ break;
+ case 'F':
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 16;
+ NumVectors = 0;
+ Signed = false;
+ break;
+ case 'G':
+ Pointer = true;
+ ElementBitwidth = Bitwidth = 32;
+ NumVectors = 0;
+ Signed = false;
+ break;
+ case '$':
+ Predicate = false;
+ Svcount = false;
+ Float = false;
+ BFloat = true;
+ ElementBitwidth = 16;
+ break;
+ case '}':
+ Predicate = false;
+ Signed = true;
+ Svcount = true;
+ NumVectors = 0;
+ Float = false;
+ BFloat = false;
+ break;
+ case '.':
+ llvm_unreachable(". is never a type in itself");
+ break;
+ default:
+ llvm_unreachable("Unhandled character!");
+ }
+}
+
+/// Returns the modifier and number of vectors for the given operand \p Op.
+std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
+ for (unsigned P = 0; !Proto.empty(); ++P) {
+ unsigned NumVectors = 1;
+ unsigned CharsToSkip = 1;
+ char Mod = Proto[0];
+ if (Mod == '2' || Mod == '3' || Mod == '4') {
+ NumVectors = Mod - '0';
+ Mod = 'd';
+ if (Proto.size() > 1 && Proto[1] == '.') {
+ Mod = Proto[2];
+ CharsToSkip = 3;
+ }
+ }
+
+ if (P == Op)
+ return {Mod, NumVectors};
+
+ Proto = Proto.drop_front(CharsToSkip);
+ }
+ llvm_unreachable("Unexpected Op");
+}
+
+//===----------------------------------------------------------------------===//
+// Intrinsic implementation
+//===----------------------------------------------------------------------===//
+
+Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
+ StringRef MergeSuffix, uint64_t MemoryElementTy,
+ StringRef LLVMName, uint64_t Flags,
+ ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
+ SVEEmitter &Emitter, StringRef SVEGuard,
+ StringRef SMEGuard)
+ : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
+ BaseTypeSpec(BT), Class(Class), SVEGuard(SVEGuard.str()),
+ SMEGuard(SMEGuard.str()), MergeSuffix(MergeSuffix.str()),
+ BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks.begin(), Checks.end()) {
+ // Types[0] is the return value.
+ for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
+ char Mod;
+ unsigned NumVectors;
+ std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
+ SVEType T(BaseTypeSpec, Mod, NumVectors);
+ Types.push_back(T);
+
+ // Add range checks for immediates
+ if (I > 0) {
+ if (T.isPredicatePattern())
+ ImmChecks.emplace_back(
+ I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
+ else if (T.isPrefetchOp())
+ ImmChecks.emplace_back(
+ I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
+ }
+ }
+
+ // Set flags based on properties
+ this->Flags |= Emitter.encodeTypeFlags(BaseType);
+ this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
+ this->Flags |= Emitter.encodeMergeType(MergeTy);
+ if (hasSplat())
+ this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
+}
+
+std::string Intrinsic::getBuiltinTypeStr() {
+ std::string S = getReturnType().builtin_str();
+ for (unsigned I = 0; I < getNumParams(); ++I)
+ S += getParamType(I).builtin_str();
+
+ return S;
+}
+
+std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
+ std::string Proto) const {
+ std::string Ret = Name;
+ while (Ret.find('{') != std::string::npos) {
+ size_t Pos = Ret.find('{');
+ size_t End = Ret.find('}');
+ unsigned NumChars = End - Pos + 1;
+ assert(NumChars == 3 && "Unexpected template argument");
+
+ SVEType T;
+ char C = Ret[Pos+1];
+ switch(C) {
+ default:
+ llvm_unreachable("Unknown predication specifier");
+ case 'd':
+ T = SVEType(TS, 'd');
+ break;
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ T = SVEType(TS, Proto[C - '0']);
+ break;
+ }
+
+ // Replace templated arg with the right suffix (e.g. u32)
+ std::string TypeCode;
+ if (T.isInteger())
+ TypeCode = T.isSigned() ? 's' : 'u';
+ else if (T.isSvcount())
+ TypeCode = 'c';
+ else if (T.isPredicateVector())
+ TypeCode = 'b';
+ else if (T.isBFloat())
+ TypeCode = "bf";
+ else
+ TypeCode = 'f';
+ Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
+ }
+
+ return Ret;
+}
+
+std::string Intrinsic::mangleLLVMName() const {
+ std::string S = getLLVMName();
+
+ // Replace all {d} like expressions with e.g. 'u32'
+ return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
+}
+
+std::string Intrinsic::mangleName(ClassKind LocalCK) const {
+ std::string S = getName();
+
+ if (LocalCK == ClassG) {
+ // Remove the square brackets and everything in between.
+ while (S.find('[') != std::string::npos) {
+ auto Start = S.find('[');
+ auto End = S.find(']');
+ S.erase(Start, (End-Start)+1);
+ }
+ } else {
+ // Remove the square brackets.
+ while (S.find('[') != std::string::npos) {
+ auto BrPos = S.find('[');
+ if (BrPos != std::string::npos)
+ S.erase(BrPos, 1);
+ BrPos = S.find(']');
+ if (BrPos != std::string::npos)
+ S.erase(BrPos, 1);
+ }
+ }
+
+ // Replace all {d} like expressions with e.g. 'u32'
+ return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
+ getMergeSuffix();
+}
+
+void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
+ ACLEKind Kind) const {
+ bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
+
+ std::string FullName = mangleName(ClassS);
+ std::string ProtoName = mangleName(getClassKind());
+ OS << (IsOverloaded ? "__aio " : "__ai ")
+ << "__attribute__((__clang_arm_builtin_alias(";
+
+ switch (Kind) {
+ case ACLEKind::SME:
+ OS << "__builtin_sme_" << FullName << ")";
+ break;
+ case ACLEKind::SVE:
+ OS << "__builtin_sve_" << FullName << ")";
+ break;
+ }
+
+ OS << "))\n";
+
+ OS << getTypes()[0].str() << " " << ProtoName << "(";
+ for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
+ if (I != 0)
+ OS << ", ";
+ OS << getTypes()[I + 1].str();
+ }
+ OS << ");\n";
+}
+
+//===----------------------------------------------------------------------===//
+// SVEEmitter implementation
+//===----------------------------------------------------------------------===//
+uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
+ if (T.isFloat()) {
+ switch (T.getElementSizeInBits()) {
+ case 16:
+ return encodeEltType("EltTyFloat16");
+ case 32:
+ return encodeEltType("EltTyFloat32");
+ case 64:
+ return encodeEltType("EltTyFloat64");
+ default:
+ llvm_unreachable("Unhandled float element bitwidth!");
+ }
+ }
+
+ if (T.isBFloat()) {
+ assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
+ return encodeEltType("EltTyBFloat16");
+ }
+
+ if (T.isPredicateVector() || T.isSvcount()) {
+ switch (T.getElementSizeInBits()) {
+ case 8:
+ return encodeEltType("EltTyBool8");
+ case 16:
+ return encodeEltType("EltTyBool16");
+ case 32:
+ return encodeEltType("EltTyBool32");
+ case 64:
+ return encodeEltType("EltTyBool64");
+ default:
+ llvm_unreachable("Unhandled predicate element bitwidth!");
+ }
+ }
+
+ switch (T.getElementSizeInBits()) {
+ case 8:
+ return encodeEltType("EltTyInt8");
+ case 16:
+ return encodeEltType("EltTyInt16");
+ case 32:
+ return encodeEltType("EltTyInt32");
+ case 64:
+ return encodeEltType("EltTyInt64");
+ case 128:
+ return encodeEltType("EltTyInt128");
+ default:
+ llvm_unreachable("Unhandled integer element bitwidth!");
+ }
+}
+
+void SVEEmitter::createIntrinsic(
+ Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
+ StringRef Name = R->getValueAsString("Name");
+ StringRef Proto = R->getValueAsString("Prototype");
+ StringRef Types = R->getValueAsString("Types");
+ StringRef SVEGuard = R->getValueAsString("SVETargetGuard");
+ StringRef SMEGuard = R->getValueAsString("SMETargetGuard");
+ StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
+ uint64_t Merge = R->getValueAsInt("Merge");
+ StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
+ uint64_t MemEltType = R->getValueAsInt("MemEltType");
+ std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
+ std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
+
+ int64_t Flags = 0;
+ for (auto FlagRec : FlagsList)
+ Flags |= FlagRec->getValueAsInt("Value");
+
+ // Create a dummy TypeSpec for non-overloaded builtins.
+ if (Types.empty()) {
+ assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
+ "Expect TypeSpec for overloaded builtin!");
+ Types = "i";
+ }
+
+ // Extract type specs from string
+ SmallVector<TypeSpec, 8> TypeSpecs;
+ TypeSpec Acc;
+ for (char I : Types) {
+ Acc.push_back(I);
+ if (islower(I)) {
+ TypeSpecs.push_back(TypeSpec(Acc));
+ Acc.clear();
+ }
+ }
+
+ // Remove duplicate type specs.
+ llvm::sort(TypeSpecs);
+ TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
+ TypeSpecs.end());
+
+ // Create an Intrinsic for each type spec.
+ for (auto TS : TypeSpecs) {
+ // Collate a list of range/option checks for the immediates.
+ SmallVector<ImmCheck, 2> ImmChecks;
+ for (auto *R : ImmCheckList) {
+ int64_t Arg = R->getValueAsInt("Arg");
+ int64_t EltSizeArg = R->getValueAsInt("EltSizeArg");
+ int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
+ assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");
+
+ unsigned ElementSizeInBits = 0;
+ char Mod;
+ unsigned NumVectors;
+ std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1);
+ if (EltSizeArg >= 0)
+ ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
+ ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
+ }
+
+ Out.push_back(std::make_unique<Intrinsic>(
+ Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
+ TS, ClassS, *this, SVEGuard, SMEGuard));
+
+ // Also generate the short-form (e.g. svadd_m) for the given type-spec.
+ if (Intrinsic::isOverloadedIntrinsic(Name))
+ Out.push_back(std::make_unique<Intrinsic>(
+ Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
+ ImmChecks, TS, ClassG, *this, SVEGuard, SMEGuard));
+ }
+}
+
+void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
+ SVEEmitter &Emitter,
+ ACLEKind Kind) {
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ for (auto *R : RV)
+ createIntrinsic(R, Defs);
+
+ // Sort intrinsics in header file by following order/priority:
+ // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
+ // - Class (is intrinsic overloaded or not)
+ // - Intrinsic name
+ std::stable_sort(Defs.begin(), Defs.end(),
+ [](const std::unique_ptr<Intrinsic> &A,
+ const std::unique_ptr<Intrinsic> &B) {
+ auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
+ return std::make_tuple(
+ I->getSVEGuard().str() + I->getSMEGuard().str(),
+ (unsigned)I->getClassKind(), I->getName());
+ };
+ return ToTuple(A) < ToTuple(B);
+ });
+
+ // Actually emit the intrinsic declarations.
+ for (auto &I : Defs)
+ I->emitIntrinsic(OS, Emitter, Kind);
+}
+
+void SVEEmitter::createHeader(raw_ostream &OS) {
+ OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
+ "-----------------------------------===\n"
+ " *\n"
+ " *\n"
+ " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
+ "Exceptions.\n"
+ " * See https://llvm.org/LICENSE.txt for license information.\n"
+ " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
+ " *\n"
+ " *===-----------------------------------------------------------------"
+ "------===\n"
+ " */\n\n";
+
+ OS << "#ifndef __ARM_SVE_H\n";
+ OS << "#define __ARM_SVE_H\n\n";
+
+ OS << "#if !defined(__LITTLE_ENDIAN__)\n";
+ OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
+ OS << "#endif\n";
+
+ OS << "#include <stdint.h>\n\n";
+ OS << "#ifdef __cplusplus\n";
+ OS << "extern \"C\" {\n";
+ OS << "#else\n";
+ OS << "#include <stdbool.h>\n";
+ OS << "#endif\n\n";
+
+ OS << "typedef __fp16 float16_t;\n";
+ OS << "typedef float float32_t;\n";
+ OS << "typedef double float64_t;\n";
+
+ OS << "typedef __SVInt8_t svint8_t;\n";
+ OS << "typedef __SVInt16_t svint16_t;\n";
+ OS << "typedef __SVInt32_t svint32_t;\n";
+ OS << "typedef __SVInt64_t svint64_t;\n";
+ OS << "typedef __SVUint8_t svuint8_t;\n";
+ OS << "typedef __SVUint16_t svuint16_t;\n";
+ OS << "typedef __SVUint32_t svuint32_t;\n";
+ OS << "typedef __SVUint64_t svuint64_t;\n";
+ OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
+
+ OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
+
+ OS << "#include <arm_bf16.h>\n";
+ OS << "#include <arm_vector_types.h>\n";
+
+ OS << "typedef __SVFloat32_t svfloat32_t;\n";
+ OS << "typedef __SVFloat64_t svfloat64_t;\n";
+ OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
+ OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
+ OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
+ OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
+ OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
+ OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
+ OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
+ OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
+ OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
+ OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
+ OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
+ OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
+ OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
+ OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
+ OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
+ OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
+ OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
+ OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
+ OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
+ OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
+ OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
+ OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
+ OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
+ OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
+ OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
+ OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
+ OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
+ OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
+ OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
+ OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
+ OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
+ OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
+ OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
+ OS << "typedef __SVBool_t svbool_t;\n";
+ OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
+ OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
+
+ OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
+ OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
+ OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
+
+ OS << "typedef __SVCount_t svcount_t;\n\n";
+
+ OS << "enum svpattern\n";
+ OS << "{\n";
+ OS << " SV_POW2 = 0,\n";
+ OS << " SV_VL1 = 1,\n";
+ OS << " SV_VL2 = 2,\n";
+ OS << " SV_VL3 = 3,\n";
+ OS << " SV_VL4 = 4,\n";
+ OS << " SV_VL5 = 5,\n";
+ OS << " SV_VL6 = 6,\n";
+ OS << " SV_VL7 = 7,\n";
+ OS << " SV_VL8 = 8,\n";
+ OS << " SV_VL16 = 9,\n";
+ OS << " SV_VL32 = 10,\n";
+ OS << " SV_VL64 = 11,\n";
+ OS << " SV_VL128 = 12,\n";
+ OS << " SV_VL256 = 13,\n";
+ OS << " SV_MUL4 = 29,\n";
+ OS << " SV_MUL3 = 30,\n";
+ OS << " SV_ALL = 31\n";
+ OS << "};\n\n";
+
+ OS << "enum svprfop\n";
+ OS << "{\n";
+ OS << " SV_PLDL1KEEP = 0,\n";
+ OS << " SV_PLDL1STRM = 1,\n";
+ OS << " SV_PLDL2KEEP = 2,\n";
+ OS << " SV_PLDL2STRM = 3,\n";
+ OS << " SV_PLDL3KEEP = 4,\n";
+ OS << " SV_PLDL3STRM = 5,\n";
+ OS << " SV_PSTL1KEEP = 8,\n";
+ OS << " SV_PSTL1STRM = 9,\n";
+ OS << " SV_PSTL2KEEP = 10,\n";
+ OS << " SV_PSTL2STRM = 11,\n";
+ OS << " SV_PSTL3KEEP = 12,\n";
+ OS << " SV_PSTL3STRM = 13\n";
+ OS << "};\n\n";
+
+ OS << "/* Function attributes */\n";
+ OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
+ "__nodebug__))\n\n";
+ OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
+ "__nodebug__, __overloadable__))\n\n";
+
+ // Add reinterpret functions.
+ for (auto [N, Suffix] :
+ std::initializer_list<std::pair<unsigned, const char *>>{
+ {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
+ for (auto ShortForm : {false, true})
+ for (const ReinterpretTypeInfo &To : Reinterprets) {
+ SVEType ToV(To.BaseType, N);
+ for (const ReinterpretTypeInfo &From : Reinterprets) {
+ SVEType FromV(From.BaseType, N);
+ OS << "__aio "
+ "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
+ "reinterpret_"
+ << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
+ << ToV.str() << " svreinterpret_" << To.Suffix;
+ if (!ShortForm)
+ OS << "_" << From.Suffix << Suffix;
+ OS << "(" << FromV.str() << " op);\n";
+ }
+ }
+ }
+
+ createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);
+
+ OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
+ OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
+
+ OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
+ OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
+ OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
+ OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
+
+ OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
+ OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
+
+ OS << "#ifdef __cplusplus\n";
+ OS << "} // extern \"C\"\n";
+ OS << "#endif\n\n";
+ OS << "#undef __ai\n\n";
+ OS << "#undef __aio\n\n";
+ OS << "#endif /* __ARM_SVE_H */\n";
+}
+
+void SVEEmitter::createBuiltins(raw_ostream &OS) {
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ for (auto *R : RV)
+ createIntrinsic(R, Defs);
+
+ // The mappings must be sorted based on BuiltinID.
+ llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
+ const std::unique_ptr<Intrinsic> &B) {
+ return A->getMangledName() < B->getMangledName();
+ });
+
+ OS << "#ifdef GET_SVE_BUILTINS\n";
+ for (auto &Def : Defs) {
+ // Only create BUILTINs for non-overloaded intrinsics, as overloaded
+ // declarations only live in the header file.
+ if (Def->getClassKind() != ClassG) {
+ OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
+ << Def->getBuiltinTypeStr() << "\", \"n\", \"";
+ Def->printGuard(OS);
+ OS << "\")\n";
+ }
+ }
+
+ // Add reinterpret functions.
+ for (auto [N, Suffix] :
+ std::initializer_list<std::pair<unsigned, const char *>>{
+ {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
+ for (const ReinterpretTypeInfo &To : Reinterprets) {
+ SVEType ToV(To.BaseType, N);
+ for (const ReinterpretTypeInfo &From : Reinterprets) {
+ SVEType FromV(From.BaseType, N);
+ OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_"
+ << From.Suffix << Suffix << +", \"" << ToV.builtin_str()
+ << FromV.builtin_str() << "\", \"n\", \"sme|sve\")\n";
+ }
+ }
+ }
+
+ OS << "#endif\n\n";
+}
+
+void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ for (auto *R : RV)
+ createIntrinsic(R, Defs);
+
+ // The mappings must be sorted based on BuiltinID.
+ llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
+ const std::unique_ptr<Intrinsic> &B) {
+ return A->getMangledName() < B->getMangledName();
+ });
+
+ OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
+ for (auto &Def : Defs) {
+ // Builtins only exist for non-overloaded intrinsics, overloaded
+ // declarations only live in the header file.
+ if (Def->getClassKind() == ClassG)
+ continue;
+
+ uint64_t Flags = Def->getFlags();
+ auto FlagString = std::to_string(Flags);
+
+ std::string LLVMName = Def->getMangledLLVMName();
+ std::string Builtin = Def->getMangledName();
+ if (!LLVMName.empty())
+ OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
+ << "),\n";
+ else
+ OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
+ }
+ OS << "#endif\n\n";
+}
+
+void SVEEmitter::createRangeChecks(raw_ostream &OS) {
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ for (auto *R : RV)
+ createIntrinsic(R, Defs);
+
+ // The mappings must be sorted based on BuiltinID.
+ llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
+ const std::unique_ptr<Intrinsic> &B) {
+ return A->getMangledName() < B->getMangledName();
+ });
+
+
+ OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
+
+ // Ensure these are only emitted once.
+ std::set<std::string> Emitted;
+
+ for (auto &Def : Defs) {
+ if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
+ Def->getImmChecks().empty())
+ continue;
+
+ OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
+ for (auto &Check : Def->getImmChecks())
+ OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
+ << Check.getKind() << ", " << Check.getElementSizeInBits() << ", "
+ << Check.getBitWidth() << "));\n";
+ OS << " break;\n";
+
+ Emitted.insert(Def->getMangledName());
+ }
+
+ OS << "#endif\n\n";
+}
+
+/// Create the SVETypeFlags used in CGBuiltins
+void SVEEmitter::createTypeFlags(raw_ostream &OS) {
+ OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
+ for (auto &KV : FlagTypes)
+ OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
+ OS << "#endif\n\n";
+
+ OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
+ for (auto &KV : EltTypes)
+ OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
+ OS << "#endif\n\n";
+
+ OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
+ for (auto &KV : MemEltTypes)
+ OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
+ OS << "#endif\n\n";
+
+ OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
+ for (auto &KV : MergeTypes)
+ OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
+ OS << "#endif\n\n";
+
+ OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";
+ for (auto &KV : ImmCheckTypes)
+ OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
+ OS << "#endif\n\n";
+}
+
+void SVEEmitter::createSMEHeader(raw_ostream &OS) {
+ OS << "/*===---- arm_sme.h - ARM SME intrinsics "
+ "------===\n"
+ " *\n"
+ " *\n"
+ " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
+ "Exceptions.\n"
+ " * See https://llvm.org/LICENSE.txt for license information.\n"
+ " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
+ " *\n"
+ " *===-----------------------------------------------------------------"
+ "------===\n"
+ " */\n\n";
+
+ OS << "#ifndef __ARM_SME_H\n";
+ OS << "#define __ARM_SME_H\n\n";
+
+ OS << "#if !defined(__LITTLE_ENDIAN__)\n";
+ OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
+ OS << "#endif\n";
+
+ OS << "#include <arm_sve.h>\n\n";
+ OS << "#include <stddef.h>\n\n";
+
+ OS << "/* Function attributes */\n";
+ OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
+ "__nodebug__))\n\n";
+ OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
+ "__nodebug__, __overloadable__))\n\n";
+
+ OS << "#ifdef __cplusplus\n";
+ OS << "extern \"C\" {\n";
+ OS << "#endif\n\n";
+
+ OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
+
+ OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
+ OS << " uint64_t x0, x1;\n";
+ OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
+ OS << " return x0 & (1ULL << 63);\n";
+ OS << "}\n\n";
+
+ OS << "__ai bool __arm_in_streaming_mode(void) __arm_streaming_compatible "
+ "{\n";
+ OS << " uint64_t x0, x1;\n";
+ OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
+ OS << " return x0 & 1;\n";
+ OS << "}\n\n";
+
+ OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
+ OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
+ OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
+ OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
+
+ OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
+ "__arm_streaming_compatible __arm_out(\"za\") "
+ "{ }\n\n";
+
+ createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);
+
+ OS << "#ifdef __cplusplus\n";
+ OS << "} // extern \"C\"\n";
+ OS << "#endif\n\n";
+ OS << "#undef __ai\n\n";
+ OS << "#endif /* __ARM_SME_H */\n";
+}
+
+void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ for (auto *R : RV) {
+ createIntrinsic(R, Defs);
+ }
+
+ // The mappings must be sorted based on BuiltinID.
+ llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
+ const std::unique_ptr<Intrinsic> &B) {
+ return A->getMangledName() < B->getMangledName();
+ });
+
+ OS << "#ifdef GET_SME_BUILTINS\n";
+ for (auto &Def : Defs) {
+ // Only create BUILTINs for non-overloaded intrinsics, as overloaded
+ // declarations only live in the header file.
+ if (Def->getClassKind() != ClassG) {
+ OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
+ << Def->getBuiltinTypeStr() << "\", \"n\", \"";
+ Def->printGuard(OS);
+ OS << "\")\n";
+ }
+ }
+
+ OS << "#endif\n\n";
+}
+
+void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ for (auto *R : RV) {
+ createIntrinsic(R, Defs);
+ }
+
+ // The mappings must be sorted based on BuiltinID.
+ llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
+ const std::unique_ptr<Intrinsic> &B) {
+ return A->getMangledName() < B->getMangledName();
+ });
+
+ OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
+ for (auto &Def : Defs) {
+ // Builtins only exist for non-overloaded intrinsics, overloaded
+ // declarations only live in the header file.
+ if (Def->getClassKind() == ClassG)
+ continue;
+
+ uint64_t Flags = Def->getFlags();
+ auto FlagString = std::to_string(Flags);
+
+ std::string LLVMName = Def->getLLVMName();
+ std::string Builtin = Def->getMangledName();
+ if (!LLVMName.empty())
+ OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
+ << "),\n";
+ else
+ OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
+ }
+ OS << "#endif\n\n";
+}
+
+void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ for (auto *R : RV) {
+ createIntrinsic(R, Defs);
+ }
+
+ // The mappings must be sorted based on BuiltinID.
+ llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
+ const std::unique_ptr<Intrinsic> &B) {
+ return A->getMangledName() < B->getMangledName();
+ });
+
+
+ OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
+
+ // Ensure these are only emitted once.
+ std::set<std::string> Emitted;
+
+ for (auto &Def : Defs) {
+ if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
+ Def->getImmChecks().empty())
+ continue;
+
+ OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
+ for (auto &Check : Def->getImmChecks())
+ OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
+ << Check.getKind() << ", " << Check.getElementSizeInBits() << ", "
+ << Check.getBitWidth() << "));\n";
+ OS << " break;\n";
+
+ Emitted.insert(Def->getMangledName());
+ }
+
+ OS << "#endif\n\n";
+}
+
+void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ for (auto *R : RV)
+ createIntrinsic(R, Defs);
+
+ std::map<std::string, std::set<std::string>> IntrinsicsPerState;
+ for (auto &Def : Defs) {
+ std::string Key;
+ auto AddToKey = [&Key](const std::string &S) -> void {
+ Key = Key.empty() ? S : (Key + " | " + S);
+ };
+
+ if (Def->isFlagSet(getEnumValueForFlag("IsInZA")))
+ AddToKey("ArmInZA");
+ else if (Def->isFlagSet(getEnumValueForFlag("IsOutZA")))
+ AddToKey("ArmOutZA");
+ else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))
+ AddToKey("ArmInOutZA");
+
+ if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")))
+ AddToKey("ArmInZT0");
+ else if (Def->isFlagSet(getEnumValueForFlag("IsOutZT0")))
+ AddToKey("ArmOutZT0");
+ else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))
+ AddToKey("ArmInOutZT0");
+
+ if (!Key.empty())
+ IntrinsicsPerState[Key].insert(Def->getMangledName());
+ }
+
+ OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
+ for (auto &KV : IntrinsicsPerState) {
+ for (StringRef Name : KV.second)
+ OS << "case SME::BI__builtin_sme_" << Name << ":\n";
+ OS << " return " << KV.first << ";\n";
+ }
+ OS << "#endif\n\n";
+}
+
+void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
+ std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
+ SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
+ for (auto *R : RV)
+ createIntrinsic(R, Defs);
+
+ StringRef ExtensionKind;
+ switch (Kind) {
+ case ACLEKind::SME:
+ ExtensionKind = "SME";
+ break;
+ case ACLEKind::SVE:
+ ExtensionKind = "SVE";
+ break;
+ }
+
+ OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
+
+ llvm::StringMap<std::set<std::string>> StreamingMap;
+
+ uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");
+ uint64_t VerifyRuntimeMode = getEnumValueForFlag("VerifyRuntimeMode");
+ uint64_t IsStreamingCompatibleFlag =
+ getEnumValueForFlag("IsStreamingCompatible");
+
+ for (auto &Def : Defs) {
+ if (!Def->isFlagSet(VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
+ !Def->getSMEGuard().empty())
+ report_fatal_error("Missing VerifyRuntimeMode flag");
+
+ if (Def->isFlagSet(IsStreamingFlag))
+ StreamingMap["ArmStreaming"].insert(Def->getMangledName());
+ else if (Def->isFlagSet(VerifyRuntimeMode))
+ StreamingMap["VerifyRuntimeMode"].insert(Def->getMangledName());
+ else if (Def->isFlagSet(IsStreamingCompatibleFlag))
+ StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());
+ else
+ StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());
+ }
+
+ for (auto BuiltinType : StreamingMap.keys()) {
+ for (auto Name : StreamingMap[BuiltinType]) {
+ OS << "case " << ExtensionKind << "::BI__builtin_"
+ << ExtensionKind.lower() << "_";
+ OS << Name << ":\n";
+ }
+ OS << " BuiltinType = " << BuiltinType << ";\n";
+ OS << " break;\n";
+ }
+
+ OS << "#endif\n\n";
+}
+
+namespace clang {
+void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createHeader(OS);
+}
+
+void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createBuiltins(OS);
+}
+
+void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createCodeGenMap(OS);
+}
+
+void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createRangeChecks(OS);
+}
+
+void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createTypeFlags(OS);
+}
+
+void EmitSveStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);
+}
+
+void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createSMEHeader(OS);
+}
+
+void EmitSmeBuiltins(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createSMEBuiltins(OS);
+}
+
+void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createSMECodeGenMap(OS);
+}
+
+void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createSMERangeChecks(OS);
+}
+
+void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);
+}
+
+void EmitSmeBuiltinZAState(RecordKeeper &Records, raw_ostream &OS) {
+ SVEEmitter(Records).createBuiltinZAState(OS);
+}
+} // End namespace clang
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index e9fa01ea98dced..60754327014c47 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -50,20 +50,31 @@ using TypeSpec = std::string;
namespace {
class SVEType {
- bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat, MFloat;
- bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
- Svcount, Fpm;
+
+ enum TypeKind {
+ Void,
+ Float,
+ SInt,
+ UInt,
+ BFloat16,
+ MFloat8,
+ Svcount,
+ PrefetchOp,
+ PredicatePattern,
+ Predicate,
+ Fpm
+ };
+ TypeKind Kind;
+ bool Immediate, Constant, Pointer, DefaultType, IsScalable;
unsigned Bitwidth, ElementBitwidth, NumVectors;
public:
SVEType() : SVEType("", 'v') {}
SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
- : Float(false), Signed(true), Immediate(false), Void(false),
- Constant(false), Pointer(false), BFloat(false), MFloat(false),
- DefaultType(false), IsScalable(true), Predicate(false),
- PredicatePattern(false), PrefetchOp(false), Svcount(false), Fpm(false),
- Bitwidth(128), ElementBitwidth(~0U), NumVectors(NumVectors) {
+ : Kind(SInt), Immediate(false), Constant(false), Pointer(false),
+ DefaultType(false), IsScalable(true), Bitwidth(128),
+ ElementBitwidth(~0U), NumVectors(NumVectors) {
if (!TS.empty())
applyTypespec(TS);
applyModifier(CharMod);
@@ -74,34 +85,32 @@ class SVEType {
}
bool isPointer() const { return Pointer; }
- bool isVoidPointer() const { return Pointer && Void; }
- bool isSigned() const { return Signed; }
+ bool isConstant() const { return Constant; }
bool isImmediate() const { return Immediate; }
+ bool isSigned() const { return Kind != UInt; }
bool isScalar() const { return NumVectors == 0; }
bool isVector() const { return NumVectors > 0; }
bool isScalableVector() const { return isVector() && IsScalable; }
bool isFixedLengthVector() const { return isVector() && !IsScalable; }
- bool isChar() const { return ElementBitwidth == 8 && !MFloat; }
- bool isVoid() const { return Void && !Pointer; }
+ bool isChar() const { return ElementBitwidth == 8 && isInteger(); }
+ bool isVoid() const { return Kind == Void; }
bool isDefault() const { return DefaultType; }
- bool isFloat() const { return Float && !BFloat && !MFloat; }
- bool isBFloat() const { return BFloat && !Float && !MFloat; }
- bool isMFloat() const {
- return MFloat && !BFloat && !Float;
- }
- bool isFloatingPoint() const { return Float || BFloat; }
- bool isInteger() const {
- return !isFloatingPoint() && !Predicate && !Svcount;
+ bool isFloat() const { return Kind == Float; }
+ bool isBFloat() const { return Kind == BFloat16; }
+ bool isMFloat() const { return Kind == MFloat8; }
+ bool isTypedPointer() const { return Pointer && Kind != Void; }
+ bool isFloatingPoint() const {
+ return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
}
+ bool isInteger() const { return Kind == SInt || Kind == UInt; }
bool isScalarPredicate() const {
- return !isFloatingPoint() && Predicate && NumVectors == 0;
+ return Kind == Predicate && NumVectors == 0;
}
- bool isPredicateVector() const { return Predicate; }
- bool isPredicatePattern() const { return PredicatePattern; }
- bool isPrefetchOp() const { return PrefetchOp; }
- bool isSvcount() const { return Svcount; }
- bool isConstant() const { return Constant; }
- bool isFpm() const { return Fpm; }
+ bool isPredicate() const { return Kind == Predicate; }
+ bool isPredicatePattern() const { return Kind == PredicatePattern; }
+ bool isPrefetchOp() const { return Kind == PrefetchOp; }
+ bool isSvcount() const { return Kind == Svcount; }
+ bool isFpm() const { return Kind == Fpm; }
unsigned getElementSizeInBits() const { return ElementBitwidth; }
unsigned getNumVectors() const { return NumVectors; }
@@ -425,9 +434,7 @@ const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
//===----------------------------------------------------------------------===//
std::string SVEType::builtin_str() const {
- std::string S;
- if (isVoid())
- return "v";
+ std::string OutStr;
if (isScalarPredicate())
return "b";
@@ -435,62 +442,81 @@ std::string SVEType::builtin_str() const {
if (isSvcount())
return "Qa";
- if (isVoidPointer())
- S += "v";
- else if (!isFloatingPoint())
- switch (ElementBitwidth) {
- case 1: S += "b"; break;
- case 8: S += "c"; break;
- case 16: S += "s"; break;
- case 32: S += "i"; break;
- case 64: S += "Wi"; break;
- case 128: S += "LLLi"; break;
- default: llvm_unreachable("Unhandled case!");
- }
- else if (isFloat())
+ if (isVoid()) {
+ OutStr += "v";
+ if (!isPointer())
+ return OutStr;
+ } else if (isFloat()) {
switch (ElementBitwidth) {
- case 16: S += "h"; break;
- case 32: S += "f"; break;
- case 64: S += "d"; break;
- default: llvm_unreachable("Unhandled case!");
+ case 16:
+ OutStr += "h";
+ break;
+ case 32:
+ OutStr += "f";
+ break;
+ case 64:
+ OutStr += "d";
+ break;
+ default:
+ llvm_unreachable("Unhandled float type!");
}
- else if (isBFloat()) {
+ } else if (isBFloat()) {
assert(ElementBitwidth == 16 && "Not a valid BFloat.");
- S += "y";
+ OutStr += "y";
} else if (isMFloat()) {
assert(ElementBitwidth == 8 && "Not a valid MFloat.");
- S += "m";
+ OutStr += "m";
+ } else {
+ switch (ElementBitwidth) {
+ case 1:
+ OutStr += "b";
+ break;
+ case 8:
+ OutStr += "c";
+ break;
+ case 16:
+ OutStr += "s";
+ break;
+ case 32:
+ OutStr += "i";
+ break;
+ case 64:
+ OutStr += "Wi";
+ break;
+ case 128:
+ OutStr += "LLLi";
+ break;
+ default:
+ llvm_unreachable("Unhandled bitwidth!");
+ }
}
- if (!isFloatingPoint()) {
- if ((isChar() || isPointer()) && !isVoidPointer()) {
- // Make chars and typed pointers explicitly signed.
- if (Signed)
- S = "S" + S;
- else if (!Signed)
- S = "U" + S;
- } else if (!isVoidPointer() && !Signed) {
- S = "U" + S;
- }
+ // Make chars and typed pointers explicitly signed.
+ if (!isFloatingPoint() && !isVoid()) {
+ if ((ElementBitwidth == 8 || isPointer()) && isSigned())
+ OutStr = "S" + OutStr;
+ if (!isSigned())
+ OutStr = "U" + OutStr;
}
// Constant indices are "int", but have the "constant expression" modifier.
if (isImmediate()) {
- assert(!isFloat() && "fp immediates are not supported");
- S = "I" + S;
+ assert(!isFloatingPoint() && "fp immediates are not supported");
+ OutStr = "I" + OutStr;
}
if (isScalar()) {
- if (Constant) S += "C";
- if (Pointer) S += "*";
- return S;
+ if (Constant)
+ OutStr += "C";
+ if (Pointer)
+ OutStr += "*";
+ return OutStr;
}
if (isFixedLengthVector())
- return "V" + utostr(getNumElements() * NumVectors) + S;
- return "q" + utostr(getNumElements() * NumVectors) + S;
+ return "V" + utostr(getNumElements() * NumVectors) + OutStr;
+ return "q" + utostr(getNumElements() * NumVectors) + OutStr;
}
-
std::string SVEType::str() const {
if (isPredicatePattern())
return "enum svpattern";
@@ -502,28 +528,30 @@ std::string SVEType::str() const {
return "fpm_t";
std::string S;
- if (Void)
+ if (isVoid())
S += "void";
else {
if (isScalableVector() || isSvcount())
S += "sv";
- if (!Signed && !isFloatingPoint())
- S += "u";
- if (Float)
+ if (isFloat())
S += "float";
else if (isSvcount())
S += "count";
- else if (isScalarPredicate() || isPredicateVector())
+ else if (isPredicate())
S += "bool";
else if (isBFloat())
S += "bfloat";
else if (isMFloat())
S += "mfloat";
- else
- S += "int";
+ else {
+ if (isSigned())
+ S += "int";
+ else
+ S += "uint";
+ };
- if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
+ if (!isPredicate() && !isSvcount())
S += utostr(ElementBitwidth);
if (isFixedLengthVector())
S += "x" + utostr(getNumElements());
@@ -545,13 +573,13 @@ void SVEType::applyTypespec(StringRef TS) {
for (char I : TS) {
switch (I) {
case 'Q':
- Svcount = true;
+ Kind = Svcount;
break;
case 'P':
- Predicate = true;
+ Kind = Predicate;
break;
case 'U':
- Signed = false;
+ Kind = UInt;
break;
case 'c':
ElementBitwidth = 8;
@@ -569,28 +597,23 @@ void SVEType::applyTypespec(StringRef TS) {
ElementBitwidth = 128;
break;
case 'h':
- Float = true;
+ Kind = Float;
ElementBitwidth = 16;
break;
case 'f':
- Float = true;
+ Kind = Float;
ElementBitwidth = 32;
break;
case 'd':
- Float = true;
+ Kind = Float;
ElementBitwidth = 64;
break;
case 'b':
- BFloat = true;
- Float = false;
- MFloat = false;
+ Kind = BFloat16;
ElementBitwidth = 16;
break;
case 'm':
- Signed = false;
- MFloat = true;
- Float = false;
- BFloat = false;
+ Kind = MFloat8;
ElementBitwidth = 8;
break;
default:
@@ -603,7 +626,7 @@ void SVEType::applyTypespec(StringRef TS) {
void SVEType::applyModifier(char Mod) {
switch (Mod) {
case 'v':
- Void = true;
+ Kind = Void;
break;
case 'd':
DefaultType = true;
@@ -617,7 +640,7 @@ void SVEType::applyModifier(char Mod) {
NumVectors = 0;
break;
case 'e':
- Signed = false;
+ Kind = UInt;
ElementBitwidth /= 2;
break;
case 'h':
@@ -627,20 +650,14 @@ void SVEType::applyModifier(char Mod) {
ElementBitwidth /= 4;
break;
case 'b':
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
ElementBitwidth /= 4;
break;
case 'o':
ElementBitwidth *= 4;
break;
case 'P':
- Signed = true;
- Float = false;
- BFloat = false;
- Predicate = true;
- Svcount = false;
+ Kind = Predicate;
Bitwidth = 16;
ElementBitwidth = 1;
break;
@@ -663,108 +680,66 @@ void SVEType::applyModifier(char Mod) {
NumVectors = 0;
break;
case '@':
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
ElementBitwidth /= 4;
NumVectors = 0;
break;
case 'K':
- Signed = true;
- Float = false;
- BFloat = false;
+ Kind = SInt;
Bitwidth = ElementBitwidth;
NumVectors = 0;
break;
case 'L':
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
Bitwidth = ElementBitwidth;
NumVectors = 0;
break;
case 'u':
- Predicate = false;
- Svcount = false;
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
break;
case 'x':
- Predicate = false;
- Svcount = false;
- Signed = true;
- Float = false;
- BFloat = false;
+ Kind = SInt;
break;
case 'i':
- Predicate = false;
- Svcount = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
ElementBitwidth = Bitwidth = 64;
NumVectors = 0;
- Signed = false;
Immediate = true;
break;
case 'I':
- Predicate = false;
- Svcount = false;
- Float = false;
- BFloat = false;
+ Kind = PredicatePattern;
ElementBitwidth = Bitwidth = 32;
NumVectors = 0;
- Signed = true;
Immediate = true;
- PredicatePattern = true;
break;
case 'J':
- Predicate = false;
- Svcount = false;
- Float = false;
- BFloat = false;
+ Kind = PrefetchOp;
ElementBitwidth = Bitwidth = 32;
NumVectors = 0;
- Signed = true;
Immediate = true;
- PrefetchOp = true;
break;
case 'k':
- Predicate = false;
- Svcount = false;
- Signed = true;
- Float = false;
- BFloat = false;
+ Kind = SInt;
ElementBitwidth = Bitwidth = 32;
NumVectors = 0;
break;
case 'l':
- Predicate = false;
- Svcount = false;
- Signed = true;
- Float = false;
- BFloat = false;
+ Kind = SInt;
ElementBitwidth = Bitwidth = 64;
NumVectors = 0;
break;
case 'm':
- Predicate = false;
- Svcount = false;
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
ElementBitwidth = Bitwidth = 32;
NumVectors = 0;
break;
case '>':
- Fpm = true;
- [[fallthrough]];
+ Kind = Fpm;
+ ElementBitwidth = Bitwidth = 64;
+ NumVectors = 0;
+ break;
case 'n':
- Predicate = false;
- Svcount = false;
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
ElementBitwidth = Bitwidth = 64;
NumVectors = 0;
break;
@@ -776,167 +751,143 @@ void SVEType::applyModifier(char Mod) {
NumVectors = 0;
break;
case 'f':
- Signed = false;
+ Kind = UInt;
ElementBitwidth = Bitwidth = 64;
NumVectors = 0;
break;
case 'g':
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
ElementBitwidth = 64;
break;
case '[':
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
ElementBitwidth = 8;
break;
case 't':
- Signed = true;
- Float = false;
- BFloat = false;
+ Kind = SInt;
ElementBitwidth = 32;
break;
case 'z':
- Signed = false;
- Float = false;
- BFloat = false;
+ Kind = UInt;
ElementBitwidth = 32;
break;
case 'O':
- Predicate = false;
- Svcount = false;
- Float = true;
+ Kind = Float;
ElementBitwidth = 16;
break;
case 'M':
- Predicate = false;
- Svcount = false;
- Float = true;
- BFloat = false;
+ Kind = Float;
ElementBitwidth = 32;
break;
case 'N':
- Predicate = false;
- Svcount = false;
- Float = true;
+ Kind = Float;
ElementBitwidth = 64;
break;
case 'Q':
+ Kind = Void;
Constant = true;
Pointer = true;
- Void = true;
NumVectors = 0;
break;
case 'S':
+ Kind = SInt;
Constant = true;
Pointer = true;
ElementBitwidth = Bitwidth = 8;
NumVectors = 0;
- Signed = true;
break;
case 'W':
+ Kind = UInt;
Constant = true;
Pointer = true;
ElementBitwidth = Bitwidth = 8;
NumVectors = 0;
- Signed = false;
break;
case 'T':
+ Kind = SInt;
Constant = true;
Pointer = true;
ElementBitwidth = Bitwidth = 16;
NumVectors = 0;
- Signed = true;
break;
case 'X':
+ Kind = UInt;
Constant = true;
Pointer = true;
ElementBitwidth = Bitwidth = 16;
NumVectors = 0;
- Signed = false;
break;
case 'Y':
+ Kind = UInt;
Constant = true;
Pointer = true;
ElementBitwidth = Bitwidth = 32;
NumVectors = 0;
- Signed = false;
break;
case 'U':
+ Kind = SInt;
Constant = true;
Pointer = true;
ElementBitwidth = Bitwidth = 32;
NumVectors = 0;
- Signed = true;
break;
case '%':
+ Kind = Void;
Pointer = true;
- Void = true;
NumVectors = 0;
break;
case 'A':
+ Kind = SInt;
Pointer = true;
ElementBitwidth = Bitwidth = 8;
NumVectors = 0;
- Signed = true;
break;
case 'B':
+ Kind = SInt;
Pointer = true;
ElementBitwidth = Bitwidth = 16;
NumVectors = 0;
- Signed = true;
break;
case 'C':
+ Kind = SInt;
Pointer = true;
ElementBitwidth = Bitwidth = 32;
NumVectors = 0;
- Signed = true;
break;
case 'D':
+ Kind = SInt;
Pointer = true;
ElementBitwidth = Bitwidth = 64;
NumVectors = 0;
- Signed = true;
break;
case 'E':
+ Kind = UInt;
Pointer = true;
ElementBitwidth = Bitwidth = 8;
NumVectors = 0;
- Signed = false;
break;
case 'F':
+ Kind = UInt;
Pointer = true;
ElementBitwidth = Bitwidth = 16;
NumVectors = 0;
- Signed = false;
break;
case 'G':
+ Kind = UInt;
Pointer = true;
ElementBitwidth = Bitwidth = 32;
NumVectors = 0;
- Signed = false;
break;
case '$':
- Predicate = false;
- Svcount = false;
- Float = false;
- BFloat = true;
+ Kind = BFloat16;
ElementBitwidth = 16;
break;
case '}':
- Predicate = false;
- Signed = true;
- Svcount = true;
+ Kind = Svcount;
NumVectors = 0;
- Float = false;
- BFloat = false;
break;
case '~':
- Float = false;
- BFloat = false;
- MFloat = true;
+ Kind = MFloat8;
ElementBitwidth = 8;
break;
case '.':
@@ -1061,7 +1012,7 @@ std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
TypeCode = T.isSigned() ? 's' : 'u';
else if (T.isSvcount())
TypeCode = 'c';
- else if (T.isPredicateVector())
+ else if (T.isPredicate())
TypeCode = 'b';
else if (T.isBFloat())
TypeCode = "bf";
@@ -1165,7 +1116,7 @@ uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
return encodeEltType("EltTyMFloat8");
}
- if (T.isPredicateVector() || T.isSvcount()) {
+ if (T.isPredicate() || T.isSvcount()) {
switch (T.getElementSizeInBits()) {
case 8:
return encodeEltType("EltTyBool8");
>From 20e5fc18074d3955716e014d87b32488c70a37ae Mon Sep 17 00:00:00 2001
From: Spencer Abson <Spencer.Abson at arm.com>
Date: Tue, 26 Nov 2024 16:35:51 +0000
Subject: [PATCH 2/9] Replace isSigned()
---
clang/utils/TableGen/SveEmitter.cpp | 33 ++++++++++++++---------------
1 file changed, 16 insertions(+), 17 deletions(-)
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 60754327014c47..01238654f8ce65 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -87,7 +87,6 @@ class SVEType {
bool isPointer() const { return Pointer; }
bool isConstant() const { return Constant; }
bool isImmediate() const { return Immediate; }
- bool isSigned() const { return Kind != UInt; }
bool isScalar() const { return NumVectors == 0; }
bool isVector() const { return NumVectors > 0; }
bool isScalableVector() const { return isVector() && IsScalable; }
@@ -98,11 +97,12 @@ class SVEType {
bool isFloat() const { return Kind == Float; }
bool isBFloat() const { return Kind == BFloat16; }
bool isMFloat() const { return Kind == MFloat8; }
- bool isTypedPointer() const { return Pointer && Kind != Void; }
bool isFloatingPoint() const {
return Kind == Float || Kind == BFloat16 || Kind == MFloat8;
}
bool isInteger() const { return Kind == SInt || Kind == UInt; }
+ bool isSignedInteger() const { return Kind == SInt; }
+ bool isUnsignedInteger() const { return Kind == UInt; }
bool isScalarPredicate() const {
return Kind == Predicate && NumVectors == 0;
}
@@ -491,13 +491,11 @@ std::string SVEType::builtin_str() const {
}
}
- // Make chars and typed pointers explicitly signed.
- if (!isFloatingPoint() && !isVoid()) {
- if ((ElementBitwidth == 8 || isPointer()) && isSigned())
- OutStr = "S" + OutStr;
- if (!isSigned())
- OutStr = "U" + OutStr;
- }
+ // Make chars and integer pointers explicitly signed.
+ if((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
+ OutStr = "S" + OutStr;
+ else if(isUnsignedInteger())
+ OutStr = "U" + OutStr;
// Constant indices are "int", but have the "constant expression" modifier.
if (isImmediate()) {
@@ -544,12 +542,10 @@ std::string SVEType::str() const {
S += "bfloat";
else if (isMFloat())
S += "mfloat";
- else {
- if (isSigned())
- S += "int";
- else
- S += "uint";
- };
+ else if (isSignedInteger())
+ S += "int";
+ else if (isUnsignedInteger())
+ S += "uint";
if (!isPredicate() && !isSvcount())
S += utostr(ElementBitwidth);
@@ -1008,8 +1004,11 @@ std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
// Replace templated arg with the right suffix (e.g. u32)
std::string TypeCode;
- if (T.isInteger())
- TypeCode = T.isSigned() ? 's' : 'u';
+
+ if(T.isSignedInteger())
+ TypeCode = 's';
+ else if (T.isUnsignedInteger())
+ TypeCode = 'u';
else if (T.isSvcount())
TypeCode = 'c';
else if (T.isPredicate())
>From ad46796c38ddaab3291d1640a04426007a8f826a Mon Sep 17 00:00:00 2001
From: Spencer Abson <Spencer.Abson at arm.com>
Date: Tue, 26 Nov 2024 16:43:41 +0000
Subject: [PATCH 3/9] [NFC] Fix format
---
clang/utils/TableGen/SveEmitter.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 01238654f8ce65..2f91f30b4f69eb 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -492,9 +492,9 @@ std::string SVEType::builtin_str() const {
}
// Make chars and integer pointers explicitly signed.
- if((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
+ if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
OutStr = "S" + OutStr;
- else if(isUnsignedInteger())
+ else if (isUnsignedInteger())
OutStr = "U" + OutStr;
// Constant indices are "int", but have the "constant expression" modifier.
@@ -1005,7 +1005,7 @@ std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
// Replace templated arg with the right suffix (e.g. u32)
std::string TypeCode;
- if(T.isSignedInteger())
+ if (T.isSignedInteger())
TypeCode = 's';
else if (T.isUnsignedInteger())
TypeCode = 'u';
>From 3146eaaf7e942cc867dcef35165ee4ebcf6ef84d Mon Sep 17 00:00:00 2001
From: Spencer Abson <Spencer.Abson at arm.com>
Date: Wed, 27 Nov 2024 13:32:38 +0000
Subject: [PATCH 4/9] Refactor builtin_str()
---
clang/utils/TableGen/SveEmitter.cpp | 112 ++++++++++++++--------------
1 file changed, 55 insertions(+), 57 deletions(-)
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 2f91f30b4f69eb..7f167c717d1e4b 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -136,6 +136,9 @@ class SVEType {
/// Applies a prototype modifier to the type.
void applyModifier(char Mod);
+
+ /// Get the builtin base for this SVEType, e.g, 'Wi' for svint64_t.
+ std::string builtinBaseType() const;
};
class SVEEmitter;
@@ -433,88 +436,82 @@ const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
// Type implementation
//===----------------------------------------------------------------------===//
-std::string SVEType::builtin_str() const {
- std::string OutStr;
-
- if (isScalarPredicate())
- return "b";
-
- if (isSvcount())
+std::string SVEType::builtinBaseType() const {
+ switch (Kind) {
+ case TypeKind::Void:
+ return "v";
+ case TypeKind::Svcount:
return "Qa";
-
- if (isVoid()) {
- OutStr += "v";
- if (!isPointer())
- return OutStr;
- } else if (isFloat()) {
+ case TypeKind::BFloat16:
+ assert(ElementBitwidth == 16 && "Invalid BFloat16!");
+ return "y";
+ case TypeKind::MFloat8:
+ assert(ElementBitwidth == 8 && "Invalid MFloat8!");
+ return "c";
+ case TypeKind::Float:
switch (ElementBitwidth) {
case 16:
- OutStr += "h";
- break;
+ return "h";
case 32:
- OutStr += "f";
- break;
+ return "f";
case 64:
- OutStr += "d";
- break;
+ return "d";
default:
- llvm_unreachable("Unhandled float type!");
+ llvm_unreachable("Unhandled float width!");
}
- } else if (isBFloat()) {
- assert(ElementBitwidth == 16 && "Not a valid BFloat.");
- OutStr += "y";
- } else if (isMFloat()) {
- assert(ElementBitwidth == 8 && "Not a valid MFloat.");
- OutStr += "m";
- } else {
+ case TypeKind::Predicate:
+ if (isScalar())
+ return "b";
+ [[fallthrough]];
+ // SInt/UInt, PredicatePattern, PrefetchOp.
+ default:
switch (ElementBitwidth) {
case 1:
- OutStr += "b";
- break;
+ return "b";
case 8:
- OutStr += "c";
- break;
+ return "c";
case 16:
- OutStr += "s";
- break;
+ return "s";
case 32:
- OutStr += "i";
- break;
+ return "i";
case 64:
- OutStr += "Wi";
- break;
+ return "Wi";
case 128:
- OutStr += "LLLi";
- break;
+ return "LLLi";
default:
llvm_unreachable("Unhandled bitwidth!");
}
}
+}
- // Make chars and integer pointers explicitly signed.
- if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
- OutStr = "S" + OutStr;
- else if (isUnsignedInteger())
- OutStr = "U" + OutStr;
+std::string SVEType::builtin_str() const {
+
+ std::string Prefix;
- // Constant indices are "int", but have the "constant expression" modifier.
- if (isImmediate()) {
+ if (isScalableVector())
+ Prefix = "q" + llvm::utostr(getNumElements() * NumVectors);
+ else if (isFixedLengthVector())
+ Prefix = "V" + llvm::utostr(getNumElements() * NumVectors);
+ else if (isImmediate()) {
assert(!isFloatingPoint() && "fp immediates are not supported");
- OutStr = "I" + OutStr;
+ Prefix = "I";
}
- if (isScalar()) {
- if (Constant)
- OutStr += "C";
- if (Pointer)
- OutStr += "*";
- return OutStr;
- }
+ // Make chars and integer pointers explicitly signed.
+ if ((ElementBitwidth == 8 || isPointer()) && isSignedInteger())
+ Prefix += "S";
+ else if (isUnsignedInteger())
+ Prefix += "U";
- if (isFixedLengthVector())
- return "V" + utostr(getNumElements() * NumVectors) + OutStr;
- return "q" + utostr(getNumElements() * NumVectors) + OutStr;
+ std::string BuiltinStr = Prefix + builtinBaseType();
+ if (isConstant())
+ BuiltinStr += "C";
+ if (isPointer())
+ BuiltinStr += "*";
+
+ return BuiltinStr;
}
+
std::string SVEType::str() const {
if (isPredicatePattern())
return "enum svpattern";
@@ -623,6 +620,7 @@ void SVEType::applyModifier(char Mod) {
switch (Mod) {
case 'v':
Kind = Void;
+ NumVectors = 0;
break;
case 'd':
DefaultType = true;
>From a9f0e6fb4a225b7ee92057ac7bfc1923cc118b9c Mon Sep 17 00:00:00 2001
From: Spencer Abson <Spencer.Abson at arm.com>
Date: Wed, 27 Nov 2024 14:00:10 +0000
Subject: [PATCH 5/9] Refactor SVEType::str()
---
clang/utils/TableGen/SveEmitter.cpp | 87 +++++++++++++++--------------
1 file changed, 45 insertions(+), 42 deletions(-)
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 7f167c717d1e4b..2309830b367a79 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -513,53 +513,56 @@ std::string SVEType::builtin_str() const {
}
std::string SVEType::str() const {
- if (isPredicatePattern())
- return "enum svpattern";
+ std::string TypeStr;
- if (isPrefetchOp())
+ switch (Kind) {
+ case TypeKind::PrefetchOp:
return "enum svprfop";
-
- if (isFpm())
- return "fpm_t";
-
- std::string S;
- if (isVoid())
- S += "void";
- else {
- if (isScalableVector() || isSvcount())
- S += "sv";
-
- if (isFloat())
- S += "float";
- else if (isSvcount())
- S += "count";
- else if (isPredicate())
- S += "bool";
- else if (isBFloat())
- S += "bfloat";
- else if (isMFloat())
- S += "mfloat";
- else if (isSignedInteger())
- S += "int";
- else if (isUnsignedInteger())
- S += "uint";
-
- if (!isPredicate() && !isSvcount())
- S += utostr(ElementBitwidth);
- if (isFixedLengthVector())
- S += "x" + utostr(getNumElements());
- if (NumVectors > 1)
- S += "x" + utostr(NumVectors);
- if (!isScalarPredicate())
- S += "_t";
+ case TypeKind::PredicatePattern:
+ return "enum svpattern";
+ case TypeKind::Fpm:
+ TypeStr += "fpm";
+ break;
+ case TypeKind::Void:
+ TypeStr += "void";
+ break;
+ case TypeKind::Float:
+ TypeStr += "float" + llvm::utostr(ElementBitwidth);
+ break;
+ case TypeKind::Svcount:
+ TypeStr += "svcount";
+ break;
+ case TypeKind::Predicate:
+ TypeStr += "bool";
+ break;
+ case TypeKind::BFloat16:
+ TypeStr += "bfloat16";
+ break;
+ case TypeKind::MFloat8:
+ TypeStr += "mfloat8";
+ break;
+ case TypeKind::SInt:
+ TypeStr += "int" + llvm::utostr(ElementBitwidth);
+ break;
+ case TypeKind::UInt:
+ TypeStr += "uint" + llvm::utostr(ElementBitwidth);
}
- if (Constant)
- S += " const";
- if (Pointer)
- S += " *";
+ if (isFixedLengthVector())
+ TypeStr += "x" + llvm::utostr(getNumElements());
+ else if (isScalableVector())
+ TypeStr = "sv" + TypeStr;
- return S;
+ if (NumVectors > 1)
+ TypeStr += "x" + llvm::utostr(NumVectors);
+ if (!isScalarPredicate() && !isVoid())
+ TypeStr += "_t";
+ if (isConstant())
+ TypeStr += " const";
+ if (isPointer())
+ TypeStr += " *";
+
+ return TypeStr;
}
void SVEType::applyTypespec(StringRef TS) {
>From 6288a81491bdeb8a33fe039b206b7ec38bdbca30 Mon Sep 17 00:00:00 2001
From: Spencer Abson <Spencer.Abson at arm.com>
Date: Thu, 28 Nov 2024 14:41:48 +0000
Subject: [PATCH 6/9] Fix improper predicate builtin strings
---
clang/utils/TableGen/SveEmitter.cpp | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 2309830b367a79..c8d94f69708d4b 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -116,7 +116,7 @@ class SVEType {
unsigned getNumElements() const {
assert(ElementBitwidth != ~0U);
- return Bitwidth / ElementBitwidth;
+ return isPredicate() ? 16 : (Bitwidth / ElementBitwidth);
}
unsigned getSizeInBits() const {
return Bitwidth;
@@ -137,7 +137,7 @@ class SVEType {
/// Applies a prototype modifier to the type.
void applyModifier(char Mod);
- /// Get the builtin base for this SVEType, e.g, 'Wi' for svint64_t.
+ /// Get the builtin base for this SVEType, e.g. 'Wi' for svint64_t.
std::string builtinBaseType() const;
};
@@ -442,6 +442,11 @@ std::string SVEType::builtinBaseType() const {
return "v";
case TypeKind::Svcount:
return "Qa";
+ case TypeKind::PrefetchOp:
+ case TypeKind::PredicatePattern:
+ return "i";
+ case TypeKind::Predicate:
+ return "b";
case TypeKind::BFloat16:
assert(ElementBitwidth == 16 && "Invalid BFloat16!");
return "y";
@@ -459,11 +464,8 @@ std::string SVEType::builtinBaseType() const {
default:
llvm_unreachable("Unhandled float width!");
}
- case TypeKind::Predicate:
- if (isScalar())
- return "b";
- [[fallthrough]];
- // SInt/UInt, PredicatePattern, PrefetchOp.
+ case TypeKind::SInt:
+ case TypeKind::UInt:
default:
switch (ElementBitwidth) {
case 1:
@@ -482,10 +484,10 @@ std::string SVEType::builtinBaseType() const {
llvm_unreachable("Unhandled bitwidth!");
}
}
+ llvm_unreachable("Unhandled TypeKind!");
}
std::string SVEType::builtin_str() const {
-
std::string Prefix;
if (isScalableVector())
>From e9f10f9e144103349e4adb1ee8d7ced2f208e486 Mon Sep 17 00:00:00 2001
From: Spencer Abson <Spencer.Abson at arm.com>
Date: Thu, 28 Nov 2024 14:57:26 +0000
Subject: [PATCH 7/9] Remove default from typekind switch
---
clang/utils/TableGen/SveEmitter.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index c8d94f69708d4b..333cfe29434e4e 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -466,7 +466,6 @@ std::string SVEType::builtinBaseType() const {
}
case TypeKind::SInt:
case TypeKind::UInt:
- default:
switch (ElementBitwidth) {
case 1:
return "b";
>From 643a12ab5f862f0deeb9484de1f4c98371c9c81d Mon Sep 17 00:00:00 2001
From: Spencer Abson <Spencer.Abson at arm.com>
Date: Fri, 29 Nov 2024 11:49:18 +0000
Subject: [PATCH 8/9] Rebase and add 'Invalid' TypeKind
---
clang/utils/TableGen/SveEmitter.cpp | 19 ++++++++++++++++++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index 333cfe29434e4e..8e0af6bc732895 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -52,6 +52,7 @@ namespace {
class SVEType {
enum TypeKind {
+ Invalid,
Void,
Float,
SInt,
@@ -72,7 +73,7 @@ class SVEType {
SVEType() : SVEType("", 'v') {}
SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
- : Kind(SInt), Immediate(false), Constant(false), Pointer(false),
+ : Kind(Invalid), Immediate(false), Constant(false), Pointer(false),
DefaultType(false), IsScalable(true), Bitwidth(128),
ElementBitwidth(~0U), NumVectors(NumVectors) {
if (!TS.empty())
@@ -111,6 +112,7 @@ class SVEType {
bool isPrefetchOp() const { return Kind == PrefetchOp; }
bool isSvcount() const { return Kind == Svcount; }
bool isFpm() const { return Kind == Fpm; }
+ bool isInvalid() const { return Kind == Invalid; }
unsigned getElementSizeInBits() const { return ElementBitwidth; }
unsigned getNumVectors() const { return NumVectors; }
@@ -445,6 +447,8 @@ std::string SVEType::builtinBaseType() const {
case TypeKind::PrefetchOp:
case TypeKind::PredicatePattern:
return "i";
+ case TypeKind::Fpm:
+ return "Wi";
case TypeKind::Predicate:
return "b";
case TypeKind::BFloat16:
@@ -482,6 +486,8 @@ std::string SVEType::builtinBaseType() const {
default:
llvm_unreachable("Unhandled bitwidth!");
}
+ case TypeKind::Invalid:
+ llvm_unreachable("Attempting to resolve builtin string from Invalid type!");
}
llvm_unreachable("Unhandled TypeKind!");
}
@@ -547,6 +553,9 @@ std::string SVEType::str() const {
break;
case TypeKind::UInt:
TypeStr += "uint" + llvm::utostr(ElementBitwidth);
+ break;
+ case TypeKind::Invalid:
+ llvm_unreachable("Attempting to resolve type name from Invalid type!");
}
if (isFixedLengthVector())
@@ -570,27 +579,35 @@ void SVEType::applyTypespec(StringRef TS) {
for (char I : TS) {
switch (I) {
case 'Q':
+ assert(Kind == Invalid && "Invalid use of modifer!");
Kind = Svcount;
break;
case 'P':
+ assert(Kind == Invalid && "Invalid use of modifer!");
Kind = Predicate;
break;
case 'U':
+ assert(Kind == Invalid && "Invalid use of modifer!");
Kind = UInt;
break;
case 'c':
+ Kind = isInvalid() ? SInt : Kind;
ElementBitwidth = 8;
break;
case 's':
+ Kind = isInvalid() ? SInt : Kind;
ElementBitwidth = 16;
break;
case 'i':
+ Kind = isInvalid() ? SInt : Kind;
ElementBitwidth = 32;
break;
case 'l':
+ Kind = isInvalid() ? SInt : Kind;
ElementBitwidth = 64;
break;
case 'q':
+ Kind = SInt;
ElementBitwidth = 128;
break;
case 'h':
>From 88a49cf59cf6bcfacfabb35172a4c8aca666ca26 Mon Sep 17 00:00:00 2001
From: SpencerAbson <Spencer.Abson at arm.com>
Date: Mon, 2 Dec 2024 13:12:10 +0000
Subject: [PATCH 9/9] Delete rouge rebase file
---
...#12752a66d88e6d5bc8de5376bca6898e3e71f901# | 1874 -----------------
1 file changed, 1874 deletions(-)
delete mode 100644 clang/utils/TableGen/#12752a66d88e6d5bc8de5376bca6898e3e71f901#
diff --git a/clang/utils/TableGen/#12752a66d88e6d5bc8de5376bca6898e3e71f901# b/clang/utils/TableGen/#12752a66d88e6d5bc8de5376bca6898e3e71f901#
deleted file mode 100644
index 40a798928f4d69..00000000000000
--- a/clang/utils/TableGen/#12752a66d88e6d5bc8de5376bca6898e3e71f901#
+++ /dev/null
@@ -1,1874 +0,0 @@
-//===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This tablegen backend is responsible for emitting arm_sve.h, which includes
-// a declaration and definition of each function specified by the ARM C/C++
-// Language Extensions (ACLE).
-//
-// For details, visit:
-// https://developer.arm.com/architectures/system-architectures/software-standards/acle
-//
-// Each SVE instruction is implemented in terms of 1 or more functions which
-// are suffixed with the element type of the input vectors. Functions may be
-// implemented in terms of generic vector operations such as +, *, -, etc. or
-// by calling a __builtin_-prefixed function which will be handled by clang's
-// CodeGen library.
-//
-// See also the documentation in include/clang/Basic/arm_sve.td.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/TableGen/Error.h"
-#include "llvm/TableGen/Record.h"
-#include "ImmCheck.h"
-#include <array>
-#include <cctype>
-#include <set>
-#include <sstream>
-#include <string>
-#include <tuple>
-
-using namespace llvm;
-
-enum ClassKind {
- ClassNone,
- ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix
- ClassG, // Overloaded name without type suffix
-};
-
-enum class ACLEKind { SVE, SME };
-
-using TypeSpec = std::string;
-
-namespace {
-class SVEType {
- bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
- bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
- Svcount;
- unsigned Bitwidth, ElementBitwidth, NumVectors;
-
-public:
- SVEType() : SVEType("", 'v') {}
-
- SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
- : Float(false), Signed(true), Immediate(false), Void(false),
- Constant(false), Pointer(false), BFloat(false), DefaultType(false),
- IsScalable(true), Predicate(false), PredicatePattern(false),
- PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
- NumVectors(NumVectors) {
- if (!TS.empty())
- applyTypespec(TS);
- applyModifier(CharMod);
- }
-
- SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
- NumVectors = NumV;
- }
-
- bool isPointer() const { return Pointer; }
- bool isVoidPointer() const { return Pointer && Void; }
- bool isSigned() const { return Signed; }
- bool isImmediate() const { return Immediate; }
- bool isScalar() const { return NumVectors == 0; }
- bool isVector() const { return NumVectors > 0; }
- bool isScalableVector() const { return isVector() && IsScalable; }
- bool isFixedLengthVector() const { return isVector() && !IsScalable; }
- bool isChar() const { return ElementBitwidth == 8; }
- bool isVoid() const { return Void && !Pointer; }
- bool isDefault() const { return DefaultType; }
- bool isFloat() const { return Float && !BFloat; }
- bool isBFloat() const { return BFloat && !Float; }
- bool isFloatingPoint() const { return Float || BFloat; }
- bool isInteger() const {
- return !isFloatingPoint() && !Predicate && !Svcount;
- }
- bool isScalarPredicate() const {
- return !isFloatingPoint() && Predicate && NumVectors == 0;
- }
- bool isPredicateVector() const { return Predicate; }
- bool isPredicatePattern() const { return PredicatePattern; }
- bool isPrefetchOp() const { return PrefetchOp; }
- bool isSvcount() const { return Svcount; }
- bool isConstant() const { return Constant; }
- unsigned getElementSizeInBits() const { return ElementBitwidth; }
- unsigned getNumVectors() const { return NumVectors; }
-
- unsigned getNumElements() const {
- assert(ElementBitwidth != ~0U);
- return Bitwidth / ElementBitwidth;
- }
- unsigned getSizeInBits() const {
- return Bitwidth;
- }
-
- /// Return the string representation of a type, which is an encoded
- /// string for passing to the BUILTIN() macro in Builtins.def.
- std::string builtin_str() const;
-
- /// Return the C/C++ string representation of a type for use in the
- /// arm_sve.h header file.
- std::string str() const;
-
-private:
- /// Creates the type based on the typespec string in TS.
- void applyTypespec(StringRef TS);
-
- /// Applies a prototype modifier to the type.
- void applyModifier(char Mod);
-};
-
-class SVEEmitter;
-
-/// The main grunt class. This represents an instantiation of an intrinsic with
-/// a particular typespec and prototype.
-class Intrinsic {
- /// The unmangled name.
- std::string Name;
-
- /// The name of the corresponding LLVM IR intrinsic.
- std::string LLVMName;
-
- /// Intrinsic prototype.
- std::string Proto;
-
- /// The base type spec for this intrinsic.
- TypeSpec BaseTypeSpec;
-
- /// The base class kind. Most intrinsics use ClassS, which has full type
- /// info for integers (_s32/_u32), or ClassG which is used for overloaded
- /// intrinsics.
- ClassKind Class;
-
- /// The architectural #ifdef guard.
- std::string SVEGuard, SMEGuard;
-
- // The merge suffix such as _m, _x or _z.
- std::string MergeSuffix;
-
- /// The types of return value [0] and parameters [1..].
- std::vector<SVEType> Types;
-
- /// The "base type", which is VarType('d', BaseTypeSpec).
- SVEType BaseType;
-
- uint64_t Flags;
-
- SmallVector<ImmCheck, 2> ImmChecks;
-
-public:
- Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
- StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
- uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
- ClassKind Class, SVEEmitter &Emitter, StringRef SVEGuard,
- StringRef SMEGuard);
-
- ~Intrinsic()=default;
-
- std::string getName() const { return Name; }
- std::string getLLVMName() const { return LLVMName; }
- std::string getProto() const { return Proto; }
- TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
- SVEType getBaseType() const { return BaseType; }
-
- StringRef getSVEGuard() const { return SVEGuard; }
- StringRef getSMEGuard() const { return SMEGuard; }
- void printGuard(raw_ostream &OS) const {
- if (!SVEGuard.empty() && SMEGuard.empty())
- OS << SVEGuard;
- else if (SVEGuard.empty() && !SMEGuard.empty())
- OS << SMEGuard;
- else {
- if (SVEGuard.find(",") != std::string::npos ||
- SVEGuard.find("|") != std::string::npos)
- OS << "(" << SVEGuard << ")";
- else
- OS << SVEGuard;
- OS << "|";
- if (SMEGuard.find(",") != std::string::npos ||
- SMEGuard.find("|") != std::string::npos)
- OS << "(" << SMEGuard << ")";
- else
- OS << SMEGuard;
- }
- }
- ClassKind getClassKind() const { return Class; }
-
- SVEType getReturnType() const { return Types[0]; }
- ArrayRef<SVEType> getTypes() const { return Types; }
- SVEType getParamType(unsigned I) const { return Types[I + 1]; }
- unsigned getNumParams() const {
- return Proto.size() - (2 * llvm::count(Proto, '.')) - 1;
- }
-
- uint64_t getFlags() const { return Flags; }
- bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}
-
- ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }
-
- /// Return the type string for a BUILTIN() macro in Builtins.def.
- std::string getBuiltinTypeStr();
-
- /// Return the name, mangled with type information. The name is mangled for
- /// ClassS, so will add type suffixes such as _u32/_s32.
- std::string getMangledName() const { return mangleName(ClassS); }
-
- /// As above, but mangles the LLVM name instead.
- std::string getMangledLLVMName() const { return mangleLLVMName(); }
-
- /// Returns true if the intrinsic is overloaded, in that it should also generate
- /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
- /// 'svld1_u32(..)'.
- static bool isOverloadedIntrinsic(StringRef Name) {
- auto BrOpen = Name.find('[');
- auto BrClose = Name.find(']');
- return BrOpen != std::string::npos && BrClose != std::string::npos;
- }
-
- /// Return true if the intrinsic takes a splat operand.
- bool hasSplat() const {
- // These prototype modifiers are described in arm_sve.td.
- return Proto.find_first_of("ajfrKLR@") != std::string::npos;
- }
-
- /// Return the parameter index of the splat operand.
- unsigned getSplatIdx() const {
- unsigned I = 1, Param = 0;
- for (; I < Proto.size(); ++I, ++Param) {
- if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
- Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
- Proto[I] == 'R' || Proto[I] == '@')
- break;
-
- // Multivector modifier can be skipped
- if (Proto[I] == '.')
- I += 2;
- }
- assert(I != Proto.size() && "Prototype has no splat operand");
- return Param;
- }
-
- /// Emits the intrinsic declaration to the ostream.
- void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;
-
-private:
- std::string getMergeSuffix() const { return MergeSuffix; }
- std::string mangleName(ClassKind LocalCK) const;
- std::string mangleLLVMName() const;
- std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
- std::string Proto) const;
-};
-
-class SVEEmitter {
-private:
- // The reinterpret builtins are generated separately because they
- // need the cross product of all types (121 functions in total),
- // which is inconvenient to specify in the arm_sve.td file or
- // generate in CGBuiltin.cpp.
- struct ReinterpretTypeInfo {
- SVEType BaseType;
- const char *Suffix;
- };
-
- static const std::array<ReinterpretTypeInfo, 12> Reinterprets;
-
- RecordKeeper &Records;
- llvm::StringMap<uint64_t> EltTypes;
- llvm::StringMap<uint64_t> MemEltTypes;
- llvm::StringMap<uint64_t> FlagTypes;
- llvm::StringMap<uint64_t> MergeTypes;
- llvm::StringMap<uint64_t> ImmCheckTypes;
-
-public:
- SVEEmitter(RecordKeeper &R) : Records(R) {
- for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
- EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
- for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
- MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
- for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
- FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
- for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
- MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
- for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
- ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
- }
-
- /// Returns the enum value for the immcheck type
- unsigned getEnumValueForImmCheck(StringRef C) const {
- auto It = ImmCheckTypes.find(C);
- if (It != ImmCheckTypes.end())
- return It->getValue();
- llvm_unreachable("Unsupported imm check");
- }
-
- /// Returns the enum value for the flag type
- uint64_t getEnumValueForFlag(StringRef C) const {
- auto Res = FlagTypes.find(C);
- if (Res != FlagTypes.end())
- return Res->getValue();
- llvm_unreachable("Unsupported flag");
- }
-
- // Returns the SVETypeFlags for a given value and mask.
- uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
- auto It = FlagTypes.find(MaskName);
- if (It != FlagTypes.end()) {
- uint64_t Mask = It->getValue();
- unsigned Shift = llvm::countr_zero(Mask);
- assert(Shift < 64 && "Mask value produced an invalid shift value");
- return (V << Shift) & Mask;
- }
- llvm_unreachable("Unsupported flag");
- }
-
- // Returns the SVETypeFlags for the given element type.
- uint64_t encodeEltType(StringRef EltName) {
- auto It = EltTypes.find(EltName);
- if (It != EltTypes.end())
- return encodeFlag(It->getValue(), "EltTypeMask");
- llvm_unreachable("Unsupported EltType");
- }
-
- // Returns the SVETypeFlags for the given memory element type.
- uint64_t encodeMemoryElementType(uint64_t MT) {
- return encodeFlag(MT, "MemEltTypeMask");
- }
-
- // Returns the SVETypeFlags for the given merge type.
- uint64_t encodeMergeType(uint64_t MT) {
- return encodeFlag(MT, "MergeTypeMask");
- }
-
- // Returns the SVETypeFlags for the given splat operand.
- unsigned encodeSplatOperand(unsigned SplatIdx) {
- assert(SplatIdx < 7 && "SplatIdx out of encodable range");
- return encodeFlag(SplatIdx + 1, "SplatOperandMask");
- }
-
- // Returns the SVETypeFlags value for the given SVEType.
- uint64_t encodeTypeFlags(const SVEType &T);
-
- /// Emit arm_sve.h.
- void createHeader(raw_ostream &o);
-
- // Emits core intrinsics in both arm_sme.h and arm_sve.h
- void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
- ACLEKind Kind);
-
- /// Emit all the __builtin prototypes and code needed by Sema.
- void createBuiltins(raw_ostream &o);
-
- /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
- void createCodeGenMap(raw_ostream &o);
-
- /// Emit all the range checks for the immediates.
- void createRangeChecks(raw_ostream &o);
-
- /// Create the SVETypeFlags used in CGBuiltins
- void createTypeFlags(raw_ostream &o);
-
- /// Emit arm_sme.h.
- void createSMEHeader(raw_ostream &o);
-
- /// Emit all the SME __builtin prototypes and code needed by Sema.
- void createSMEBuiltins(raw_ostream &o);
-
- /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
- void createSMECodeGenMap(raw_ostream &o);
-
- /// Create a table for a builtin's requirement for PSTATE.SM.
- void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);
-
- /// Emit all the range checks for the immediates.
- void createSMERangeChecks(raw_ostream &o);
-
- /// Create a table for a builtin's requirement for PSTATE.ZA.
- void createBuiltinZAState(raw_ostream &OS);
-
- /// Create intrinsic and add it to \p Out
- void createIntrinsic(Record *R,
- SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
-};
-
-const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
- {{{SVEType("c", 'd'), "s8"},
- {SVEType("Uc", 'd'), "u8"},
- {SVEType("s", 'd'), "s16"},
- {SVEType("Us", 'd'), "u16"},
- {SVEType("i", 'd'), "s32"},
- {SVEType("Ui", 'd'), "u32"},
- {SVEType("l", 'd'), "s64"},
- {SVEType("Ul", 'd'), "u64"},
- {SVEType("h", 'd'), "f16"},
- {SVEType("b", 'd'), "bf16"},
- {SVEType("f", 'd'), "f32"},
- {SVEType("d", 'd'), "f64"}}};
-
-} // end anonymous namespace
-
-
-//===----------------------------------------------------------------------===//
-// Type implementation
-//===----------------------------------------------------------------------===//
-
-std::string SVEType::builtin_str() const {
- std::string S;
- if (isVoid())
- return "v";
-
- if (isScalarPredicate())
- return "b";
-
- if (isSvcount())
- return "Qa";
-
- if (isVoidPointer())
- S += "v";
- else if (!isFloatingPoint())
- switch (ElementBitwidth) {
- case 1: S += "b"; break;
- case 8: S += "c"; break;
- case 16: S += "s"; break;
- case 32: S += "i"; break;
- case 64: S += "Wi"; break;
- case 128: S += "LLLi"; break;
- default: llvm_unreachable("Unhandled case!");
- }
- else if (isFloat())
- switch (ElementBitwidth) {
- case 16: S += "h"; break;
- case 32: S += "f"; break;
- case 64: S += "d"; break;
- default: llvm_unreachable("Unhandled case!");
- }
- else if (isBFloat()) {
- assert(ElementBitwidth == 16 && "Not a valid BFloat.");
- S += "y";
- }
-
- if (!isFloatingPoint()) {
- if ((isChar() || isPointer()) && !isVoidPointer()) {
- // Make chars and typed pointers explicitly signed.
- if (Signed)
- S = "S" + S;
- else if (!Signed)
- S = "U" + S;
- } else if (!isVoidPointer() && !Signed) {
- S = "U" + S;
- }
- }
-
- // Constant indices are "int", but have the "constant expression" modifier.
- if (isImmediate()) {
- assert(!isFloat() && "fp immediates are not supported");
- S = "I" + S;
- }
-
- if (isScalar()) {
- if (Constant) S += "C";
- if (Pointer) S += "*";
- return S;
- }
-
- if (isFixedLengthVector())
- return "V" + utostr(getNumElements() * NumVectors) + S;
- return "q" + utostr(getNumElements() * NumVectors) + S;
-}
-
-std::string SVEType::str() const {
- if (isPredicatePattern())
- return "enum svpattern";
-
- if (isPrefetchOp())
- return "enum svprfop";
-
- std::string S;
- if (Void)
- S += "void";
- else {
- if (isScalableVector() || isSvcount())
- S += "sv";
- if (!Signed && !isFloatingPoint())
- S += "u";
-
- if (Float)
- S += "float";
- else if (isSvcount())
- S += "count";
- else if (isScalarPredicate() || isPredicateVector())
- S += "bool";
- else if (isBFloat())
- S += "bfloat";
- else
- S += "int";
-
- if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
- S += utostr(ElementBitwidth);
- if (isFixedLengthVector())
- S += "x" + utostr(getNumElements());
- if (NumVectors > 1)
- S += "x" + utostr(NumVectors);
- if (!isScalarPredicate())
- S += "_t";
- }
-
- if (Constant)
- S += " const";
- if (Pointer)
- S += " *";
-
- return S;
-}
-
-void SVEType::applyTypespec(StringRef TS) {
- for (char I : TS) {
- switch (I) {
- case 'Q':
- Svcount = true;
- break;
- case 'P':
- Predicate = true;
- break;
- case 'U':
- Signed = false;
- break;
- case 'c':
- ElementBitwidth = 8;
- break;
- case 's':
- ElementBitwidth = 16;
- break;
- case 'i':
- ElementBitwidth = 32;
- break;
- case 'l':
- ElementBitwidth = 64;
- break;
- case 'q':
- ElementBitwidth = 128;
- break;
- case 'h':
- Float = true;
- ElementBitwidth = 16;
- break;
- case 'f':
- Float = true;
- ElementBitwidth = 32;
- break;
- case 'd':
- Float = true;
- ElementBitwidth = 64;
- break;
- case 'b':
- BFloat = true;
- Float = false;
- ElementBitwidth = 16;
- break;
- default:
- llvm_unreachable("Unhandled type code!");
- }
- }
- assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
-}
-
-void SVEType::applyModifier(char Mod) {
- switch (Mod) {
- case 'v':
- Void = true;
- break;
- case 'd':
- DefaultType = true;
- break;
- case 'c':
- Constant = true;
- [[fallthrough]];
- case 'p':
- Pointer = true;
- Bitwidth = ElementBitwidth;
- NumVectors = 0;
- break;
- case 'e':
- Signed = false;
- ElementBitwidth /= 2;
- break;
- case 'h':
- ElementBitwidth /= 2;
- break;
- case 'q':
- ElementBitwidth /= 4;
- break;
- case 'b':
- Signed = false;
- Float = false;
- BFloat = false;
- ElementBitwidth /= 4;
- break;
- case 'o':
- ElementBitwidth *= 4;
- break;
- case 'P':
- Signed = true;
- Float = false;
- BFloat = false;
- Predicate = true;
- Svcount = false;
- Bitwidth = 16;
- ElementBitwidth = 1;
- break;
- case '{':
- IsScalable = false;
- Bitwidth = 128;
- NumVectors = 1;
- break;
- case 's':
- case 'a':
- Bitwidth = ElementBitwidth;
- NumVectors = 0;
- break;
- case 'R':
- ElementBitwidth /= 2;
- NumVectors = 0;
- break;
- case 'r':
- ElementBitwidth /= 4;
- NumVectors = 0;
- break;
- case '@':
- Signed = false;
- Float = false;
- BFloat = false;
- ElementBitwidth /= 4;
- NumVectors = 0;
- break;
- case 'K':
- Signed = true;
- Float = false;
- BFloat = false;
- Bitwidth = ElementBitwidth;
- NumVectors = 0;
- break;
- case 'L':
- Signed = false;
- Float = false;
- BFloat = false;
- Bitwidth = ElementBitwidth;
- NumVectors = 0;
- break;
- case 'u':
- Predicate = false;
- Svcount = false;
- Signed = false;
- Float = false;
- BFloat = false;
- break;
- case 'x':
- Predicate = false;
- Svcount = false;
- Signed = true;
- Float = false;
- BFloat = false;
- break;
- case 'i':
- Predicate = false;
- Svcount = false;
- Float = false;
- BFloat = false;
- ElementBitwidth = Bitwidth = 64;
- NumVectors = 0;
- Signed = false;
- Immediate = true;
- break;
- case 'I':
- Predicate = false;
- Svcount = false;
- Float = false;
- BFloat = false;
- ElementBitwidth = Bitwidth = 32;
- NumVectors = 0;
- Signed = true;
- Immediate = true;
- PredicatePattern = true;
- break;
- case 'J':
- Predicate = false;
- Svcount = false;
- Float = false;
- BFloat = false;
- ElementBitwidth = Bitwidth = 32;
- NumVectors = 0;
- Signed = true;
- Immediate = true;
- PrefetchOp = true;
- break;
- case 'k':
- Predicate = false;
- Svcount = false;
- Signed = true;
- Float = false;
- BFloat = false;
- ElementBitwidth = Bitwidth = 32;
- NumVectors = 0;
- break;
- case 'l':
- Predicate = false;
- Svcount = false;
- Signed = true;
- Float = false;
- BFloat = false;
- ElementBitwidth = Bitwidth = 64;
- NumVectors = 0;
- break;
- case 'm':
- Predicate = false;
- Svcount = false;
- Signed = false;
- Float = false;
- BFloat = false;
- ElementBitwidth = Bitwidth = 32;
- NumVectors = 0;
- break;
- case 'n':
- Predicate = false;
- Svcount = false;
- Signed = false;
- Float = false;
- BFloat = false;
- ElementBitwidth = Bitwidth = 64;
- NumVectors = 0;
- break;
- case 'w':
- ElementBitwidth = 64;
- break;
- case 'j':
- ElementBitwidth = Bitwidth = 64;
- NumVectors = 0;
- break;
- case 'f':
- Signed = false;
- ElementBitwidth = Bitwidth = 64;
- NumVectors = 0;
- break;
- case 'g':
- Signed = false;
- Float = false;
- BFloat = false;
- ElementBitwidth = 64;
- break;
- case '[':
- Signed = false;
- Float = false;
- BFloat = false;
- ElementBitwidth = 8;
- break;
- case 't':
- Signed = true;
- Float = false;
- BFloat = false;
- ElementBitwidth = 32;
- break;
- case 'z':
- Signed = false;
- Float = false;
- BFloat = false;
- ElementBitwidth = 32;
- break;
- case 'O':
- Predicate = false;
- Svcount = false;
- Float = true;
- ElementBitwidth = 16;
- break;
- case 'M':
- Predicate = false;
- Svcount = false;
- Float = true;
- BFloat = false;
- ElementBitwidth = 32;
- break;
- case 'N':
- Predicate = false;
- Svcount = false;
- Float = true;
- ElementBitwidth = 64;
- break;
- case 'Q':
- Constant = true;
- Pointer = true;
- Void = true;
- NumVectors = 0;
- break;
- case 'S':
- Constant = true;
- Pointer = true;
- ElementBitwidth = Bitwidth = 8;
- NumVectors = 0;
- Signed = true;
- break;
- case 'W':
- Constant = true;
- Pointer = true;
- ElementBitwidth = Bitwidth = 8;
- NumVectors = 0;
- Signed = false;
- break;
- case 'T':
- Constant = true;
- Pointer = true;
- ElementBitwidth = Bitwidth = 16;
- NumVectors = 0;
- Signed = true;
- break;
- case 'X':
- Constant = true;
- Pointer = true;
- ElementBitwidth = Bitwidth = 16;
- NumVectors = 0;
- Signed = false;
- break;
- case 'Y':
- Constant = true;
- Pointer = true;
- ElementBitwidth = Bitwidth = 32;
- NumVectors = 0;
- Signed = false;
- break;
- case 'U':
- Constant = true;
- Pointer = true;
- ElementBitwidth = Bitwidth = 32;
- NumVectors = 0;
- Signed = true;
- break;
- case '%':
- Pointer = true;
- Void = true;
- NumVectors = 0;
- break;
- case 'A':
- Pointer = true;
- ElementBitwidth = Bitwidth = 8;
- NumVectors = 0;
- Signed = true;
- break;
- case 'B':
- Pointer = true;
- ElementBitwidth = Bitwidth = 16;
- NumVectors = 0;
- Signed = true;
- break;
- case 'C':
- Pointer = true;
- ElementBitwidth = Bitwidth = 32;
- NumVectors = 0;
- Signed = true;
- break;
- case 'D':
- Pointer = true;
- ElementBitwidth = Bitwidth = 64;
- NumVectors = 0;
- Signed = true;
- break;
- case 'E':
- Pointer = true;
- ElementBitwidth = Bitwidth = 8;
- NumVectors = 0;
- Signed = false;
- break;
- case 'F':
- Pointer = true;
- ElementBitwidth = Bitwidth = 16;
- NumVectors = 0;
- Signed = false;
- break;
- case 'G':
- Pointer = true;
- ElementBitwidth = Bitwidth = 32;
- NumVectors = 0;
- Signed = false;
- break;
- case '$':
- Predicate = false;
- Svcount = false;
- Float = false;
- BFloat = true;
- ElementBitwidth = 16;
- break;
- case '}':
- Predicate = false;
- Signed = true;
- Svcount = true;
- NumVectors = 0;
- Float = false;
- BFloat = false;
- break;
- case '.':
- llvm_unreachable(". is never a type in itself");
- break;
- default:
- llvm_unreachable("Unhandled character!");
- }
-}
-
-/// Returns the modifier and number of vectors for the given operand \p Op.
-std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
- for (unsigned P = 0; !Proto.empty(); ++P) {
- unsigned NumVectors = 1;
- unsigned CharsToSkip = 1;
- char Mod = Proto[0];
- if (Mod == '2' || Mod == '3' || Mod == '4') {
- NumVectors = Mod - '0';
- Mod = 'd';
- if (Proto.size() > 1 && Proto[1] == '.') {
- Mod = Proto[2];
- CharsToSkip = 3;
- }
- }
-
- if (P == Op)
- return {Mod, NumVectors};
-
- Proto = Proto.drop_front(CharsToSkip);
- }
- llvm_unreachable("Unexpected Op");
-}
-
-//===----------------------------------------------------------------------===//
-// Intrinsic implementation
-//===----------------------------------------------------------------------===//
-
-Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
- StringRef MergeSuffix, uint64_t MemoryElementTy,
- StringRef LLVMName, uint64_t Flags,
- ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
- SVEEmitter &Emitter, StringRef SVEGuard,
- StringRef SMEGuard)
- : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
- BaseTypeSpec(BT), Class(Class), SVEGuard(SVEGuard.str()),
- SMEGuard(SMEGuard.str()), MergeSuffix(MergeSuffix.str()),
- BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks.begin(), Checks.end()) {
- // Types[0] is the return value.
- for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
- char Mod;
- unsigned NumVectors;
- std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
- SVEType T(BaseTypeSpec, Mod, NumVectors);
- Types.push_back(T);
-
- // Add range checks for immediates
- if (I > 0) {
- if (T.isPredicatePattern())
- ImmChecks.emplace_back(
- I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
- else if (T.isPrefetchOp())
- ImmChecks.emplace_back(
- I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
- }
- }
-
- // Set flags based on properties
- this->Flags |= Emitter.encodeTypeFlags(BaseType);
- this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
- this->Flags |= Emitter.encodeMergeType(MergeTy);
- if (hasSplat())
- this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
-}
-
-std::string Intrinsic::getBuiltinTypeStr() {
- std::string S = getReturnType().builtin_str();
- for (unsigned I = 0; I < getNumParams(); ++I)
- S += getParamType(I).builtin_str();
-
- return S;
-}
-
-std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
- std::string Proto) const {
- std::string Ret = Name;
- while (Ret.find('{') != std::string::npos) {
- size_t Pos = Ret.find('{');
- size_t End = Ret.find('}');
- unsigned NumChars = End - Pos + 1;
- assert(NumChars == 3 && "Unexpected template argument");
-
- SVEType T;
- char C = Ret[Pos+1];
- switch(C) {
- default:
- llvm_unreachable("Unknown predication specifier");
- case 'd':
- T = SVEType(TS, 'd');
- break;
- case '0':
- case '1':
- case '2':
- case '3':
- T = SVEType(TS, Proto[C - '0']);
- break;
- }
-
- // Replace templated arg with the right suffix (e.g. u32)
- std::string TypeCode;
- if (T.isInteger())
- TypeCode = T.isSigned() ? 's' : 'u';
- else if (T.isSvcount())
- TypeCode = 'c';
- else if (T.isPredicateVector())
- TypeCode = 'b';
- else if (T.isBFloat())
- TypeCode = "bf";
- else
- TypeCode = 'f';
- Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
- }
-
- return Ret;
-}
-
-std::string Intrinsic::mangleLLVMName() const {
- std::string S = getLLVMName();
-
- // Replace all {d} like expressions with e.g. 'u32'
- return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
-}
-
-std::string Intrinsic::mangleName(ClassKind LocalCK) const {
- std::string S = getName();
-
- if (LocalCK == ClassG) {
- // Remove the square brackets and everything in between.
- while (S.find('[') != std::string::npos) {
- auto Start = S.find('[');
- auto End = S.find(']');
- S.erase(Start, (End-Start)+1);
- }
- } else {
- // Remove the square brackets.
- while (S.find('[') != std::string::npos) {
- auto BrPos = S.find('[');
- if (BrPos != std::string::npos)
- S.erase(BrPos, 1);
- BrPos = S.find(']');
- if (BrPos != std::string::npos)
- S.erase(BrPos, 1);
- }
- }
-
- // Replace all {d} like expressions with e.g. 'u32'
- return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
- getMergeSuffix();
-}
-
-void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
- ACLEKind Kind) const {
- bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;
-
- std::string FullName = mangleName(ClassS);
- std::string ProtoName = mangleName(getClassKind());
- OS << (IsOverloaded ? "__aio " : "__ai ")
- << "__attribute__((__clang_arm_builtin_alias(";
-
- switch (Kind) {
- case ACLEKind::SME:
- OS << "__builtin_sme_" << FullName << ")";
- break;
- case ACLEKind::SVE:
- OS << "__builtin_sve_" << FullName << ")";
- break;
- }
-
- OS << "))\n";
-
- OS << getTypes()[0].str() << " " << ProtoName << "(";
- for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
- if (I != 0)
- OS << ", ";
- OS << getTypes()[I + 1].str();
- }
- OS << ");\n";
-}
-
-//===----------------------------------------------------------------------===//
-// SVEEmitter implementation
-//===----------------------------------------------------------------------===//
-uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
- if (T.isFloat()) {
- switch (T.getElementSizeInBits()) {
- case 16:
- return encodeEltType("EltTyFloat16");
- case 32:
- return encodeEltType("EltTyFloat32");
- case 64:
- return encodeEltType("EltTyFloat64");
- default:
- llvm_unreachable("Unhandled float element bitwidth!");
- }
- }
-
- if (T.isBFloat()) {
- assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
- return encodeEltType("EltTyBFloat16");
- }
-
- if (T.isPredicateVector() || T.isSvcount()) {
- switch (T.getElementSizeInBits()) {
- case 8:
- return encodeEltType("EltTyBool8");
- case 16:
- return encodeEltType("EltTyBool16");
- case 32:
- return encodeEltType("EltTyBool32");
- case 64:
- return encodeEltType("EltTyBool64");
- default:
- llvm_unreachable("Unhandled predicate element bitwidth!");
- }
- }
-
- switch (T.getElementSizeInBits()) {
- case 8:
- return encodeEltType("EltTyInt8");
- case 16:
- return encodeEltType("EltTyInt16");
- case 32:
- return encodeEltType("EltTyInt32");
- case 64:
- return encodeEltType("EltTyInt64");
- case 128:
- return encodeEltType("EltTyInt128");
- default:
- llvm_unreachable("Unhandled integer element bitwidth!");
- }
-}
-
-void SVEEmitter::createIntrinsic(
- Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
- StringRef Name = R->getValueAsString("Name");
- StringRef Proto = R->getValueAsString("Prototype");
- StringRef Types = R->getValueAsString("Types");
- StringRef SVEGuard = R->getValueAsString("SVETargetGuard");
- StringRef SMEGuard = R->getValueAsString("SMETargetGuard");
- StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
- uint64_t Merge = R->getValueAsInt("Merge");
- StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
- uint64_t MemEltType = R->getValueAsInt("MemEltType");
- std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
- std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");
-
- int64_t Flags = 0;
- for (auto FlagRec : FlagsList)
- Flags |= FlagRec->getValueAsInt("Value");
-
- // Create a dummy TypeSpec for non-overloaded builtins.
- if (Types.empty()) {
- assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
- "Expect TypeSpec for overloaded builtin!");
- Types = "i";
- }
-
- // Extract type specs from string
- SmallVector<TypeSpec, 8> TypeSpecs;
- TypeSpec Acc;
- for (char I : Types) {
- Acc.push_back(I);
- if (islower(I)) {
- TypeSpecs.push_back(TypeSpec(Acc));
- Acc.clear();
- }
- }
-
- // Remove duplicate type specs.
- llvm::sort(TypeSpecs);
- TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
- TypeSpecs.end());
-
- // Create an Intrinsic for each type spec.
- for (auto TS : TypeSpecs) {
- // Collate a list of range/option checks for the immediates.
- SmallVector<ImmCheck, 2> ImmChecks;
- for (auto *R : ImmCheckList) {
- int64_t Arg = R->getValueAsInt("Arg");
- int64_t EltSizeArg = R->getValueAsInt("EltSizeArg");
- int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
- assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");
-
- unsigned ElementSizeInBits = 0;
- char Mod;
- unsigned NumVectors;
- std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1);
- if (EltSizeArg >= 0)
- ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
- ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
- }
-
- Out.push_back(std::make_unique<Intrinsic>(
- Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
- TS, ClassS, *this, SVEGuard, SMEGuard));
-
- // Also generate the short-form (e.g. svadd_m) for the given type-spec.
- if (Intrinsic::isOverloadedIntrinsic(Name))
- Out.push_back(std::make_unique<Intrinsic>(
- Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
- ImmChecks, TS, ClassG, *this, SVEGuard, SMEGuard));
- }
-}
-
-void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
- SVEEmitter &Emitter,
- ACLEKind Kind) {
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- for (auto *R : RV)
- createIntrinsic(R, Defs);
-
- // Sort intrinsics in header file by following order/priority:
- // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
- // - Class (is intrinsic overloaded or not)
- // - Intrinsic name
- std::stable_sort(Defs.begin(), Defs.end(),
- [](const std::unique_ptr<Intrinsic> &A,
- const std::unique_ptr<Intrinsic> &B) {
- auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
- return std::make_tuple(
- I->getSVEGuard().str() + I->getSMEGuard().str(),
- (unsigned)I->getClassKind(), I->getName());
- };
- return ToTuple(A) < ToTuple(B);
- });
-
- // Actually emit the intrinsic declarations.
- for (auto &I : Defs)
- I->emitIntrinsic(OS, Emitter, Kind);
-}
-
-void SVEEmitter::createHeader(raw_ostream &OS) {
- OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
- "-----------------------------------===\n"
- " *\n"
- " *\n"
- " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
- "Exceptions.\n"
- " * See https://llvm.org/LICENSE.txt for license information.\n"
- " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
- " *\n"
- " *===-----------------------------------------------------------------"
- "------===\n"
- " */\n\n";
-
- OS << "#ifndef __ARM_SVE_H\n";
- OS << "#define __ARM_SVE_H\n\n";
-
- OS << "#if !defined(__LITTLE_ENDIAN__)\n";
- OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
- OS << "#endif\n";
-
- OS << "#include <stdint.h>\n\n";
- OS << "#ifdef __cplusplus\n";
- OS << "extern \"C\" {\n";
- OS << "#else\n";
- OS << "#include <stdbool.h>\n";
- OS << "#endif\n\n";
-
- OS << "typedef __fp16 float16_t;\n";
- OS << "typedef float float32_t;\n";
- OS << "typedef double float64_t;\n";
-
- OS << "typedef __SVInt8_t svint8_t;\n";
- OS << "typedef __SVInt16_t svint16_t;\n";
- OS << "typedef __SVInt32_t svint32_t;\n";
- OS << "typedef __SVInt64_t svint64_t;\n";
- OS << "typedef __SVUint8_t svuint8_t;\n";
- OS << "typedef __SVUint16_t svuint16_t;\n";
- OS << "typedef __SVUint32_t svuint32_t;\n";
- OS << "typedef __SVUint64_t svuint64_t;\n";
- OS << "typedef __SVFloat16_t svfloat16_t;\n\n";
-
- OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
-
- OS << "#include <arm_bf16.h>\n";
- OS << "#include <arm_vector_types.h>\n";
-
- OS << "typedef __SVFloat32_t svfloat32_t;\n";
- OS << "typedef __SVFloat64_t svfloat64_t;\n";
- OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
- OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
- OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
- OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
- OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
- OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
- OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
- OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
- OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
- OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
- OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
- OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
- OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
- OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
- OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
- OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
- OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
- OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
- OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
- OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
- OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
- OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
- OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
- OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
- OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
- OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
- OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
- OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
- OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
- OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
- OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
- OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
- OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
- OS << "typedef __SVBool_t svbool_t;\n";
- OS << "typedef __clang_svboolx2_t svboolx2_t;\n";
- OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n";
-
- OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
- OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
- OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";
-
- OS << "typedef __SVCount_t svcount_t;\n\n";
-
- OS << "enum svpattern\n";
- OS << "{\n";
- OS << " SV_POW2 = 0,\n";
- OS << " SV_VL1 = 1,\n";
- OS << " SV_VL2 = 2,\n";
- OS << " SV_VL3 = 3,\n";
- OS << " SV_VL4 = 4,\n";
- OS << " SV_VL5 = 5,\n";
- OS << " SV_VL6 = 6,\n";
- OS << " SV_VL7 = 7,\n";
- OS << " SV_VL8 = 8,\n";
- OS << " SV_VL16 = 9,\n";
- OS << " SV_VL32 = 10,\n";
- OS << " SV_VL64 = 11,\n";
- OS << " SV_VL128 = 12,\n";
- OS << " SV_VL256 = 13,\n";
- OS << " SV_MUL4 = 29,\n";
- OS << " SV_MUL3 = 30,\n";
- OS << " SV_ALL = 31\n";
- OS << "};\n\n";
-
- OS << "enum svprfop\n";
- OS << "{\n";
- OS << " SV_PLDL1KEEP = 0,\n";
- OS << " SV_PLDL1STRM = 1,\n";
- OS << " SV_PLDL2KEEP = 2,\n";
- OS << " SV_PLDL2STRM = 3,\n";
- OS << " SV_PLDL3KEEP = 4,\n";
- OS << " SV_PLDL3STRM = 5,\n";
- OS << " SV_PSTL1KEEP = 8,\n";
- OS << " SV_PSTL1STRM = 9,\n";
- OS << " SV_PSTL2KEEP = 10,\n";
- OS << " SV_PSTL2STRM = 11,\n";
- OS << " SV_PSTL3KEEP = 12,\n";
- OS << " SV_PSTL3STRM = 13\n";
- OS << "};\n\n";
-
- OS << "/* Function attributes */\n";
- OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
- "__nodebug__))\n\n";
- OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
- "__nodebug__, __overloadable__))\n\n";
-
- // Add reinterpret functions.
- for (auto [N, Suffix] :
- std::initializer_list<std::pair<unsigned, const char *>>{
- {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
- for (auto ShortForm : {false, true})
- for (const ReinterpretTypeInfo &To : Reinterprets) {
- SVEType ToV(To.BaseType, N);
- for (const ReinterpretTypeInfo &From : Reinterprets) {
- SVEType FromV(From.BaseType, N);
- OS << "__aio "
- "__attribute__((__clang_arm_builtin_alias(__builtin_sve_"
- "reinterpret_"
- << To.Suffix << "_" << From.Suffix << Suffix << ")))\n"
- << ToV.str() << " svreinterpret_" << To.Suffix;
- if (!ShortForm)
- OS << "_" << From.Suffix << Suffix;
- OS << "(" << FromV.str() << " op);\n";
- }
- }
- }
-
- createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);
-
- OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n";
- OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n";
-
- OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n";
- OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n";
- OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n";
- OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n";
-
- OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n";
- OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";
-
- OS << "#ifdef __cplusplus\n";
- OS << "} // extern \"C\"\n";
- OS << "#endif\n\n";
- OS << "#undef __ai\n\n";
- OS << "#undef __aio\n\n";
- OS << "#endif /* __ARM_SVE_H */\n";
-}
-
-void SVEEmitter::createBuiltins(raw_ostream &OS) {
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- for (auto *R : RV)
- createIntrinsic(R, Defs);
-
- // The mappings must be sorted based on BuiltinID.
- llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
- const std::unique_ptr<Intrinsic> &B) {
- return A->getMangledName() < B->getMangledName();
- });
-
- OS << "#ifdef GET_SVE_BUILTINS\n";
- for (auto &Def : Defs) {
- // Only create BUILTINs for non-overloaded intrinsics, as overloaded
- // declarations only live in the header file.
- if (Def->getClassKind() != ClassG) {
- OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
- << Def->getBuiltinTypeStr() << "\", \"n\", \"";
- Def->printGuard(OS);
- OS << "\")\n";
- }
- }
-
- // Add reinterpret functions.
- for (auto [N, Suffix] :
- std::initializer_list<std::pair<unsigned, const char *>>{
- {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
- for (const ReinterpretTypeInfo &To : Reinterprets) {
- SVEType ToV(To.BaseType, N);
- for (const ReinterpretTypeInfo &From : Reinterprets) {
- SVEType FromV(From.BaseType, N);
- OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_"
- << From.Suffix << Suffix << +", \"" << ToV.builtin_str()
- << FromV.builtin_str() << "\", \"n\", \"sme|sve\")\n";
- }
- }
- }
-
- OS << "#endif\n\n";
-}
-
-void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- for (auto *R : RV)
- createIntrinsic(R, Defs);
-
- // The mappings must be sorted based on BuiltinID.
- llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
- const std::unique_ptr<Intrinsic> &B) {
- return A->getMangledName() < B->getMangledName();
- });
-
- OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
- for (auto &Def : Defs) {
- // Builtins only exist for non-overloaded intrinsics, overloaded
- // declarations only live in the header file.
- if (Def->getClassKind() == ClassG)
- continue;
-
- uint64_t Flags = Def->getFlags();
- auto FlagString = std::to_string(Flags);
-
- std::string LLVMName = Def->getMangledLLVMName();
- std::string Builtin = Def->getMangledName();
- if (!LLVMName.empty())
- OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
- << "),\n";
- else
- OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
- }
- OS << "#endif\n\n";
-}
-
-void SVEEmitter::createRangeChecks(raw_ostream &OS) {
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- for (auto *R : RV)
- createIntrinsic(R, Defs);
-
- // The mappings must be sorted based on BuiltinID.
- llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
- const std::unique_ptr<Intrinsic> &B) {
- return A->getMangledName() < B->getMangledName();
- });
-
-
- OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";
-
- // Ensure these are only emitted once.
- std::set<std::string> Emitted;
-
- for (auto &Def : Defs) {
- if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
- Def->getImmChecks().empty())
- continue;
-
- OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
- for (auto &Check : Def->getImmChecks())
- OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
- << Check.getKind() << ", " << Check.getElementSizeInBits() << ", "
- << Check.getBitWidth() << "));\n";
- OS << " break;\n";
-
- Emitted.insert(Def->getMangledName());
- }
-
- OS << "#endif\n\n";
-}
-
-/// Create the SVETypeFlags used in CGBuiltins
-void SVEEmitter::createTypeFlags(raw_ostream &OS) {
- OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
- for (auto &KV : FlagTypes)
- OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
- OS << "#endif\n\n";
-
- OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
- for (auto &KV : EltTypes)
- OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
- OS << "#endif\n\n";
-
- OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
- for (auto &KV : MemEltTypes)
- OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
- OS << "#endif\n\n";
-
- OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
- for (auto &KV : MergeTypes)
- OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
- OS << "#endif\n\n";
-
- OS << "#ifdef LLVM_GET_ARM_INTRIN_IMMCHECKTYPES\n";
- for (auto &KV : ImmCheckTypes)
- OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n";
- OS << "#endif\n\n";
-}
-
-void SVEEmitter::createSMEHeader(raw_ostream &OS) {
- OS << "/*===---- arm_sme.h - ARM SME intrinsics "
- "------===\n"
- " *\n"
- " *\n"
- " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
- "Exceptions.\n"
- " * See https://llvm.org/LICENSE.txt for license information.\n"
- " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
- " *\n"
- " *===-----------------------------------------------------------------"
- "------===\n"
- " */\n\n";
-
- OS << "#ifndef __ARM_SME_H\n";
- OS << "#define __ARM_SME_H\n\n";
-
- OS << "#if !defined(__LITTLE_ENDIAN__)\n";
- OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
- OS << "#endif\n";
-
- OS << "#include <arm_sve.h>\n\n";
- OS << "#include <stddef.h>\n\n";
-
- OS << "/* Function attributes */\n";
- OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
- "__nodebug__))\n\n";
- OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
- "__nodebug__, __overloadable__))\n\n";
-
- OS << "#ifdef __cplusplus\n";
- OS << "extern \"C\" {\n";
- OS << "#endif\n\n";
-
- OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";
-
- OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
- OS << " uint64_t x0, x1;\n";
- OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
- OS << " return x0 & (1ULL << 63);\n";
- OS << "}\n\n";
-
- OS << "__ai bool __arm_in_streaming_mode(void) __arm_streaming_compatible "
- "{\n";
- OS << " uint64_t x0, x1;\n";
- OS << " __builtin_arm_get_sme_state(&x0, &x1);\n";
- OS << " return x0 & 1;\n";
- OS << "}\n\n";
-
- OS << "void *__arm_sc_memcpy(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
- OS << "void *__arm_sc_memmove(void *dest, const void *src, size_t n) __arm_streaming_compatible;\n";
- OS << "void *__arm_sc_memset(void *s, int c, size_t n) __arm_streaming_compatible;\n";
- OS << "void *__arm_sc_memchr(void *s, int c, size_t n) __arm_streaming_compatible;\n\n";
-
- OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
- "__arm_streaming_compatible __arm_out(\"za\") "
- "{ }\n\n";
-
- createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);
-
- OS << "#ifdef __cplusplus\n";
- OS << "} // extern \"C\"\n";
- OS << "#endif\n\n";
- OS << "#undef __ai\n\n";
- OS << "#endif /* __ARM_SME_H */\n";
-}
-
-void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- for (auto *R : RV) {
- createIntrinsic(R, Defs);
- }
-
- // The mappings must be sorted based on BuiltinID.
- llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
- const std::unique_ptr<Intrinsic> &B) {
- return A->getMangledName() < B->getMangledName();
- });
-
- OS << "#ifdef GET_SME_BUILTINS\n";
- for (auto &Def : Defs) {
- // Only create BUILTINs for non-overloaded intrinsics, as overloaded
- // declarations only live in the header file.
- if (Def->getClassKind() != ClassG) {
- OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
- << Def->getBuiltinTypeStr() << "\", \"n\", \"";
- Def->printGuard(OS);
- OS << "\")\n";
- }
- }
-
- OS << "#endif\n\n";
-}
-
-void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- for (auto *R : RV) {
- createIntrinsic(R, Defs);
- }
-
- // The mappings must be sorted based on BuiltinID.
- llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
- const std::unique_ptr<Intrinsic> &B) {
- return A->getMangledName() < B->getMangledName();
- });
-
- OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
- for (auto &Def : Defs) {
- // Builtins only exist for non-overloaded intrinsics, overloaded
- // declarations only live in the header file.
- if (Def->getClassKind() == ClassG)
- continue;
-
- uint64_t Flags = Def->getFlags();
- auto FlagString = std::to_string(Flags);
-
- std::string LLVMName = Def->getLLVMName();
- std::string Builtin = Def->getMangledName();
- if (!LLVMName.empty())
- OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
- << "),\n";
- else
- OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
- }
- OS << "#endif\n\n";
-}
-
-void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- for (auto *R : RV) {
- createIntrinsic(R, Defs);
- }
-
- // The mappings must be sorted based on BuiltinID.
- llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
- const std::unique_ptr<Intrinsic> &B) {
- return A->getMangledName() < B->getMangledName();
- });
-
-
- OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";
-
- // Ensure these are only emitted once.
- std::set<std::string> Emitted;
-
- for (auto &Def : Defs) {
- if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
- Def->getImmChecks().empty())
- continue;
-
- OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
- for (auto &Check : Def->getImmChecks())
- OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
- << Check.getKind() << ", " << Check.getElementSizeInBits() << ", "
- << Check.getBitWidth() << "));\n";
- OS << " break;\n";
-
- Emitted.insert(Def->getMangledName());
- }
-
- OS << "#endif\n\n";
-}
-
-void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- for (auto *R : RV)
- createIntrinsic(R, Defs);
-
- std::map<std::string, std::set<std::string>> IntrinsicsPerState;
- for (auto &Def : Defs) {
- std::string Key;
- auto AddToKey = [&Key](const std::string &S) -> void {
- Key = Key.empty() ? S : (Key + " | " + S);
- };
-
- if (Def->isFlagSet(getEnumValueForFlag("IsInZA")))
- AddToKey("ArmInZA");
- else if (Def->isFlagSet(getEnumValueForFlag("IsOutZA")))
- AddToKey("ArmOutZA");
- else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))
- AddToKey("ArmInOutZA");
-
- if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")))
- AddToKey("ArmInZT0");
- else if (Def->isFlagSet(getEnumValueForFlag("IsOutZT0")))
- AddToKey("ArmOutZT0");
- else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))
- AddToKey("ArmInOutZT0");
-
- if (!Key.empty())
- IntrinsicsPerState[Key].insert(Def->getMangledName());
- }
-
- OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
- for (auto &KV : IntrinsicsPerState) {
- for (StringRef Name : KV.second)
- OS << "case SME::BI__builtin_sme_" << Name << ":\n";
- OS << " return " << KV.first << ";\n";
- }
- OS << "#endif\n\n";
-}
-
-void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
- std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
- SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
- for (auto *R : RV)
- createIntrinsic(R, Defs);
-
- StringRef ExtensionKind;
- switch (Kind) {
- case ACLEKind::SME:
- ExtensionKind = "SME";
- break;
- case ACLEKind::SVE:
- ExtensionKind = "SVE";
- break;
- }
-
- OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";
-
- llvm::StringMap<std::set<std::string>> StreamingMap;
-
- uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");
- uint64_t VerifyRuntimeMode = getEnumValueForFlag("VerifyRuntimeMode");
- uint64_t IsStreamingCompatibleFlag =
- getEnumValueForFlag("IsStreamingCompatible");
-
- for (auto &Def : Defs) {
- if (!Def->isFlagSet(VerifyRuntimeMode) && !Def->getSVEGuard().empty() &&
- !Def->getSMEGuard().empty())
- report_fatal_error("Missing VerifyRuntimeMode flag");
-
- if (Def->isFlagSet(IsStreamingFlag))
- StreamingMap["ArmStreaming"].insert(Def->getMangledName());
- else if (Def->isFlagSet(VerifyRuntimeMode))
- StreamingMap["VerifyRuntimeMode"].insert(Def->getMangledName());
- else if (Def->isFlagSet(IsStreamingCompatibleFlag))
- StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());
- else
- StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());
- }
-
- for (auto BuiltinType : StreamingMap.keys()) {
- for (auto Name : StreamingMap[BuiltinType]) {
- OS << "case " << ExtensionKind << "::BI__builtin_"
- << ExtensionKind.lower() << "_";
- OS << Name << ":\n";
- }
- OS << " BuiltinType = " << BuiltinType << ";\n";
- OS << " break;\n";
- }
-
- OS << "#endif\n\n";
-}
-
-namespace clang {
-void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createHeader(OS);
-}
-
-void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createBuiltins(OS);
-}
-
-void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createCodeGenMap(OS);
-}
-
-void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createRangeChecks(OS);
-}
-
-void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createTypeFlags(OS);
-}
-
-void EmitSveStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);
-}
-
-void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createSMEHeader(OS);
-}
-
-void EmitSmeBuiltins(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createSMEBuiltins(OS);
-}
-
-void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createSMECodeGenMap(OS);
-}
-
-void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createSMERangeChecks(OS);
-}
-
-void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);
-}
-
-void EmitSmeBuiltinZAState(RecordKeeper &Records, raw_ostream &OS) {
- SVEEmitter(Records).createBuiltinZAState(OS);
-}
-} // End namespace clang
More information about the cfe-commits
mailing list