[llvm] b1fba56 - [SVE] Don't require lookup when demangling vector function mappings (#72260)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 23 09:15:51 PST 2023
Author: Graham Hunter
Date: 2023-11-23T17:15:48Z
New Revision: b1fba568f6d4d824554fa22f43ec0f689a265664
URL: https://github.com/llvm/llvm-project/commit/b1fba568f6d4d824554fa22f43ec0f689a265664
DIFF: https://github.com/llvm/llvm-project/commit/b1fba568f6d4d824554fa22f43ec0f689a265664.diff
LOG: [SVE] Don't require lookup when demangling vector function mappings (#72260)
We can determine the VF from a combination of the mangled name (which
indicates the arguments that take vectors) and the element sizes of
the arguments for the scalar function the mapping has been established
for.
The assert when demangling fails has been removed in favour of just
not adding the mapping, which prevents the crash seen in
https://github.com/llvm/llvm-project/issues/71892
This patch also stops using _LLVM_ as an ISA for scalable vector tests,
since there aren't defined rules for the way vector arguments should be
handled (e.g. packed vs. unpacked representation).
Added:
llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll
Modified:
llvm/include/llvm/Analysis/VectorUtils.h
llvm/lib/Analysis/VFABIDemangling.cpp
llvm/lib/Analysis/VectorUtils.cpp
llvm/lib/Transforms/Utils/ModuleUtils.cpp
llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
llvm/unittests/Analysis/VectorFunctionABITest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 7947648aaddd4ea..d54b63fd4f5328f 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -174,13 +174,13 @@ static constexpr char const *_LLVM_Scalarize_ = "_LLVM_Scalarize_";
///
/// \param MangledName -> input string in the format
/// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)].
-/// \param M -> Module used to retrieve informations about the vector
-/// function that are not possible to retrieve from the mangled
-/// name. At the moment, this parameter is needed only to retrieve the
-/// Vectorization Factor of scalable vector functions from their
-/// respective IR declarations.
+/// \param CI -> A call to the scalar function which we're trying to find
+/// a vectorized variant for. This is required to determine the vectorization
+/// factor for scalable vectors, since the mangled name doesn't encode that;
+/// it needs to be derived from the widest element types of vector arguments
+/// or return values.
std::optional<VFInfo> tryDemangleForVFABI(StringRef MangledName,
- const Module &M);
+ const CallInst &CI);
/// Retrieve the `VFParamKind` from a string token.
VFParamKind getVFParamKindFromString(const StringRef Token);
@@ -227,7 +227,7 @@ class VFDatabase {
return;
for (const auto &MangledName : ListOfStrings) {
const std::optional<VFInfo> Shape =
- VFABI::tryDemangleForVFABI(MangledName, *(CI.getModule()));
+ VFABI::tryDemangleForVFABI(MangledName, CI);
// A match is found via scalar and vector names, and also by
// ensuring that the variant described in the attribute has a
// corresponding definition or declaration of the vector
diff --git a/llvm/lib/Analysis/VFABIDemangling.cpp b/llvm/lib/Analysis/VFABIDemangling.cpp
index 1e2d1db4e44b9d6..88f61cfeb9ba4e5 100644
--- a/llvm/lib/Analysis/VFABIDemangling.cpp
+++ b/llvm/lib/Analysis/VFABIDemangling.cpp
@@ -7,9 +7,14 @@
//===----------------------------------------------------------------------===//
#include "llvm/Analysis/VectorUtils.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include <limits>
using namespace llvm;
+#define DEBUG_TYPE "vfabi-demangling"
+
namespace {
/// Utilities for the Vector Function ABI name parser.
@@ -21,8 +26,9 @@ enum class ParseRet {
};
/// Extracts the `<isa>` information from the mangled string, and
-/// sets the `ISA` accordingly.
-ParseRet tryParseISA(StringRef &MangledName, VFISAKind &ISA) {
+/// sets the `ISA` accordingly. If successful, the <isa> token is removed
+/// from the input string `MangledName`.
+static ParseRet tryParseISA(StringRef &MangledName, VFISAKind &ISA) {
if (MangledName.empty())
return ParseRet::Error;
@@ -45,9 +51,9 @@ ParseRet tryParseISA(StringRef &MangledName, VFISAKind &ISA) {
}
/// Extracts the `<mask>` information from the mangled string, and
-/// sets `IsMasked` accordingly. The input string `MangledName` is
-/// left unmodified.
-ParseRet tryParseMask(StringRef &MangledName, bool &IsMasked) {
+/// sets `IsMasked` accordingly. If successful, the <mask> token is removed
+/// from the input string `MangledName`.
+static ParseRet tryParseMask(StringRef &MangledName, bool &IsMasked) {
if (MangledName.consume_front("M")) {
IsMasked = true;
return ParseRet::OK;
@@ -62,20 +68,28 @@ ParseRet tryParseMask(StringRef &MangledName, bool &IsMasked) {
}
/// Extract the `<vlen>` information from the mangled string, and
-/// sets `VF` accordingly. A `<vlen> == "x"` token is interpreted as a scalable
-/// vector length. On success, the `<vlen>` token is removed from
-/// the input string `ParseString`.
-///
-ParseRet tryParseVLEN(StringRef &ParseString, unsigned &VF, bool &IsScalable) {
+/// sets `ParsedVF` accordingly. A `<vlen> == "x"` token is interpreted as a
+/// scalable vector length and the boolean is set to true, otherwise a nonzero
+/// unsigned integer will be directly used as a VF. On success, the `<vlen>`
+/// token is removed from the input string `ParseString`.
+static ParseRet tryParseVLEN(StringRef &ParseString, VFISAKind ISA,
+ std::pair<unsigned, bool> &ParsedVF) {
if (ParseString.consume_front("x")) {
- // Set VF to 0, to be later adjusted to a value grater than zero
- // by looking at the signature of the vector function with
- // `getECFromSignature`.
- VF = 0;
- IsScalable = true;
+ // SVE is the only scalable ISA currently supported.
+ if (ISA != VFISAKind::SVE) {
+ LLVM_DEBUG(dbgs() << "Vector function variant declared with scalable VF "
+ << "but ISA is not SVE\n");
+ return ParseRet::Error;
+ }
+ // We can't determine the VF of a scalable vector by looking at the vlen
+ // string (just 'x'), so say we successfully parsed it but return a 'true'
+ // for the scalable field with an invalid VF field so that we know to look
+ // up the actual VF based on element types from the parameters or return.
+ ParsedVF = {0, true};
return ParseRet::OK;
}
+ unsigned VF = 0;
if (ParseString.consumeInteger(10, VF))
return ParseRet::Error;
@@ -83,7 +97,7 @@ ParseRet tryParseVLEN(StringRef &ParseString, unsigned &VF, bool &IsScalable) {
if (VF == 0)
return ParseRet::Error;
- IsScalable = false;
+ ParsedVF = {VF, false};
return ParseRet::OK;
}
@@ -99,9 +113,9 @@ ParseRet tryParseVLEN(StringRef &ParseString, unsigned &VF, bool &IsScalable) {
///
/// The function expects <token> to be one of "ls", "Rs", "Us" or
/// "Ls".
-ParseRet tryParseLinearTokenWithRuntimeStep(StringRef &ParseString,
- VFParamKind &PKind, int &Pos,
- const StringRef Token) {
+static ParseRet tryParseLinearTokenWithRuntimeStep(StringRef &ParseString,
+ VFParamKind &PKind, int &Pos,
+ const StringRef Token) {
if (ParseString.consume_front(Token)) {
PKind = VFABI::getVFParamKindFromString(Token);
if (ParseString.consumeInteger(10, Pos))
@@ -123,8 +137,9 @@ ParseRet tryParseLinearTokenWithRuntimeStep(StringRef &ParseString,
/// sets `PKind` to the correspondent enum value, sets `StepOrPos` to
/// <number>, and return success. On a syntax error, it return a
/// parsing error. If nothing is parsed, it returns std::nullopt.
-ParseRet tryParseLinearWithRuntimeStep(StringRef &ParseString,
- VFParamKind &PKind, int &StepOrPos) {
+static ParseRet tryParseLinearWithRuntimeStep(StringRef &ParseString,
+ VFParamKind &PKind,
+ int &StepOrPos) {
ParseRet Ret;
// "ls" <RuntimeStepPos>
@@ -162,9 +177,10 @@ ParseRet tryParseLinearWithRuntimeStep(StringRef &ParseString,
///
/// The function expects <token> to be one of "l", "R", "U" or
/// "L".
-ParseRet tryParseCompileTimeLinearToken(StringRef &ParseString,
- VFParamKind &PKind, int &LinearStep,
- const StringRef Token) {
+static ParseRet tryParseCompileTimeLinearToken(StringRef &ParseString,
+ VFParamKind &PKind,
+ int &LinearStep,
+ const StringRef Token) {
if (ParseString.consume_front(Token)) {
PKind = VFABI::getVFParamKindFromString(Token);
const bool Negate = ParseString.consume_front("n");
@@ -187,8 +203,9 @@ ParseRet tryParseCompileTimeLinearToken(StringRef &ParseString,
/// sets `PKind` to the correspondent enum value, sets `LinearStep` to
/// <number>, and return success. On a syntax error, it return a
/// parsing error. If nothing is parsed, it returns std::nullopt.
-ParseRet tryParseLinearWithCompileTimeStep(StringRef &ParseString,
- VFParamKind &PKind, int &StepOrPos) {
+static ParseRet tryParseLinearWithCompileTimeStep(StringRef &ParseString,
+ VFParamKind &PKind,
+ int &StepOrPos) {
// "l" {"n"} <CompileTimeStep>
if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "l") ==
ParseRet::OK)
@@ -220,8 +237,8 @@ ParseRet tryParseLinearWithCompileTimeStep(StringRef &ParseString,
/// sets `PKind` to the correspondent enum value, sets `StepOrPos`
/// accordingly, and return success. On a syntax error, it return a
/// parsing error. If nothing is parsed, it returns std::nullopt.
-ParseRet tryParseParameter(StringRef &ParseString, VFParamKind &PKind,
- int &StepOrPos) {
+static ParseRet tryParseParameter(StringRef &ParseString, VFParamKind &PKind,
+ int &StepOrPos) {
if (ParseString.consume_front("v")) {
PKind = VFParamKind::Vector;
StepOrPos = 0;
@@ -255,7 +272,7 @@ ParseRet tryParseParameter(StringRef &ParseString, VFParamKind &PKind,
/// sets `PKind` to the correspondent enum value, sets `StepOrPos`
/// accordingly, and return success. On a syntax error, it return a
/// parsing error. If nothing is parsed, it returns std::nullopt.
-ParseRet tryParseAlign(StringRef &ParseString, Align &Alignment) {
+static ParseRet tryParseAlign(StringRef &ParseString, Align &Alignment) {
uint64_t Val;
// "a" <number>
if (ParseString.consume_front("a")) {
@@ -273,49 +290,86 @@ ParseRet tryParseAlign(StringRef &ParseString, Align &Alignment) {
return ParseRet::None;
}
-#ifndef NDEBUG
-// Verify the assumtion that all vectors in the signature of a vector
-// function have the same number of elements.
-bool verifyAllVectorsHaveSameWidth(FunctionType *Signature) {
- SmallVector<VectorType *, 2> VecTys;
- if (auto *RetTy = dyn_cast<VectorType>(Signature->getReturnType()))
- VecTys.push_back(RetTy);
- for (auto *Ty : Signature->params())
- if (auto *VTy = dyn_cast<VectorType>(Ty))
- VecTys.push_back(VTy);
-
- if (VecTys.size() <= 1)
- return true;
-
- assert(VecTys.size() > 1 && "Invalid number of elements.");
- const ElementCount EC = VecTys[0]->getElementCount();
- return llvm::all_of(llvm::drop_begin(VecTys), [&EC](VectorType *VTy) {
- return (EC == VTy->getElementCount());
- });
+// Returns the 'natural' VF for a given scalar element type, based on the
+// current architecture.
+//
+// For SVE (currently the only scalable architecture with a defined name
+// mangling), we assume a minimum vector size of 128b and return a VF based on
+// the number of elements of the given type which would fit in such a vector.
+static std::optional<ElementCount> getElementCountForTy(const VFISAKind ISA,
+ const Type *Ty) {
+ // Only AArch64 SVE is supported at present.
+ assert(ISA == VFISAKind::SVE &&
+ "Scalable VF decoding only implemented for SVE\n");
+
+ if (Ty->isIntegerTy(64) || Ty->isDoubleTy() || Ty->isPointerTy())
+ return ElementCount::getScalable(2);
+ if (Ty->isIntegerTy(32) || Ty->isFloatTy())
+ return ElementCount::getScalable(4);
+ if (Ty->isIntegerTy(16) || Ty->is16bitFPTy())
+ return ElementCount::getScalable(8);
+ if (Ty->isIntegerTy(8))
+ return ElementCount::getScalable(16);
+
+ return std::nullopt;
}
-#endif // NDEBUG
-
-// Extract the VectorizationFactor from a given function signature,
-// under the assumtion that all vectors have the same number of
-// elements, i.e. same ElementCount.Min.
-ElementCount getECFromSignature(FunctionType *Signature) {
- assert(verifyAllVectorsHaveSameWidth(Signature) &&
- "Invalid vector signature.");
-
- if (auto *RetTy = dyn_cast<VectorType>(Signature->getReturnType()))
- return RetTy->getElementCount();
- for (auto *Ty : Signature->params())
- if (auto *VTy = dyn_cast<VectorType>(Ty))
- return VTy->getElementCount();
-
- return ElementCount::getFixed(/*Min=*/1);
+
+// Extract the VectorizationFactor from a given function signature, based
+// on the widest scalar element types that will become vector parameters.
+static std::optional<ElementCount>
+getScalableECFromSignature(const FunctionType *Signature, const VFISAKind ISA,
+ const SmallVectorImpl<VFParameter> &Params) {
+ // Start with a very wide EC and drop when we find smaller ECs based on type.
+ ElementCount MinEC =
+ ElementCount::getScalable(std::numeric_limits<unsigned int>::max());
+ for (auto &Param : Params) {
+ // Only vector parameters are used when determining the VF; uniform or
+ // linear are left as scalars, so do not affect VF.
+ if (Param.ParamKind == VFParamKind::Vector) {
+ // If the scalar function doesn't actually have a corresponding argument,
+ // reject the mapping.
+ if (Param.ParamPos >= Signature->getNumParams())
+ return std::nullopt;
+ Type *PTy = Signature->getParamType(Param.ParamPos);
+
+ std::optional<ElementCount> EC = getElementCountForTy(ISA, PTy);
+ // If we have an unknown scalar element type we can't find a reasonable
+ // VF.
+ if (!EC)
+ return std::nullopt;
+
+ // Find the smallest VF, based on the widest scalar type.
+ if (ElementCount::isKnownLT(*EC, MinEC))
+ MinEC = *EC;
+ }
+ }
+
+ // Also check the return type if not void.
+ Type *RetTy = Signature->getReturnType();
+ if (!RetTy->isVoidTy()) {
+ std::optional<ElementCount> ReturnEC = getElementCountForTy(ISA, RetTy);
+ // If we have an unknown scalar element type we can't find a reasonable VF.
+ if (!ReturnEC)
+ return std::nullopt;
+ if (ElementCount::isKnownLT(*ReturnEC, MinEC))
+ MinEC = *ReturnEC;
+ }
+
+ // The SVE Vector function call ABI bases the VF on the widest element types
+ // present, and vector arguments containing types of that width are always
+ // considered to be packed. Arguments with narrower elements are considered
+ // to be unpacked.
+ if (MinEC.getKnownMinValue() < std::numeric_limits<unsigned int>::max())
+ return MinEC;
+
+ return std::nullopt;
}
} // namespace
// Format of the ABI name:
// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)]
std::optional<VFInfo> VFABI::tryDemangleForVFABI(StringRef MangledName,
- const Module &M) {
+ const CallInst &CI) {
const StringRef OriginalName = MangledName;
// Assume there is no custom name <redirection>, and therefore the
// vector name consists of
@@ -338,9 +392,8 @@ std::optional<VFInfo> VFABI::tryDemangleForVFABI(StringRef MangledName,
return std::nullopt;
// Parse the variable size, starting from <vlen>.
- unsigned VF;
- bool IsScalable;
- if (tryParseVLEN(MangledName, VF, IsScalable) != ParseRet::OK)
+ std::pair<unsigned, bool> ParsedVF;
+ if (tryParseVLEN(MangledName, ISA, ParsedVF) != ParseRet::OK)
return std::nullopt;
// Parse the <parameters>.
@@ -374,6 +427,19 @@ std::optional<VFInfo> VFABI::tryDemangleForVFABI(StringRef MangledName,
if (Parameters.empty())
return std::nullopt;
+ // Figure out the number of lanes in vectors for this function variant. This
+ // is easy for fixed length, as the vlen encoding just gives us the value
+ // directly. However, if the vlen mangling indicated that this function
+ // variant expects scalable vectors we need to work it out based on the
+ // demangled parameter types and the scalar function signature.
+ std::optional<ElementCount> EC;
+ if (ParsedVF.second) {
+ EC = getScalableECFromSignature(CI.getFunctionType(), ISA, Parameters);
+ if (!EC)
+ return std::nullopt;
+ } else
+ EC = ElementCount::getFixed(ParsedVF.first);
+
// Check for the <scalarname> and the optional <redirection>, which
// are separated from the prefix with "_"
if (!MangledName.consume_front("_"))
@@ -426,32 +492,7 @@ std::optional<VFInfo> VFABI::tryDemangleForVFABI(StringRef MangledName,
assert(Parameters.back().ParamKind == VFParamKind::GlobalPredicate &&
"The global predicate must be the last parameter");
- // Adjust the VF for scalable signatures. The EC.Min is not encoded
- // in the name of the function, but it is encoded in the IR
- // signature of the function. We need to extract this information
- // because it is needed by the loop vectorizer, which reasons in
- // terms of VectorizationFactor or ElementCount. In particular, we
- // need to make sure that the VF field of the VFShape class is never
- // set to 0.
- if (IsScalable) {
- const Function *F = M.getFunction(VectorName);
- // The declaration of the function must be present in the module
- // to be able to retrieve its signature.
- if (!F)
- return std::nullopt;
- const ElementCount EC = getECFromSignature(F->getFunctionType());
- VF = EC.getKnownMinValue();
- }
-
- // 1. We don't accept a zero lanes vectorization factor.
- // 2. We don't accept the demangling if the vector function is not
- // present in the module.
- if (VF == 0)
- return std::nullopt;
- if (!M.getFunction(VectorName))
- return std::nullopt;
-
- const VFShape Shape({ElementCount::get(VF, IsScalable), Parameters});
+ const VFShape Shape({*EC, Parameters});
return VFInfo({Shape, std::string(ScalarName), std::string(VectorName), ISA});
}
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 4f5db8b7aaf746f..96f39ff7e409ede 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -1466,15 +1466,13 @@ void VFABI::getVectorVariantNames(
S.split(ListAttr, ",");
for (const auto &S : SetVector<StringRef>(ListAttr.begin(), ListAttr.end())) {
-#ifndef NDEBUG
- LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << S << "'\n");
- std::optional<VFInfo> Info =
- VFABI::tryDemangleForVFABI(S, *(CI.getModule()));
- assert(Info && "Invalid name for a VFABI variant.");
- assert(CI.getModule()->getFunction(Info->VectorName) &&
- "Vector function is missing.");
-#endif
- VariantMappings.push_back(std::string(S));
+ std::optional<VFInfo> Info = VFABI::tryDemangleForVFABI(S, CI);
+ if (Info && CI.getModule()->getFunction(Info->VectorName)) {
+ LLVM_DEBUG(dbgs() << "VFABI: Adding mapping '" << S << "' for " << CI
+ << "\n");
+ VariantMappings.push_back(std::string(S));
+ } else
+ LLVM_DEBUG(dbgs() << "VFABI: Invalid mapping '" << S << "'\n");
}
}
diff --git a/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
index d6bb832654cd084..fc42df75875e1e9 100644
--- a/llvm/lib/Transforms/Utils/ModuleUtils.cpp
+++ b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
@@ -346,7 +346,7 @@ void VFABI::setVectorVariantNames(CallInst *CI,
#ifndef NDEBUG
for (const std::string &VariantMapping : VariantMappings) {
LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << VariantMapping << "'\n");
- std::optional<VFInfo> VI = VFABI::tryDemangleForVFABI(VariantMapping, *M);
+ std::optional<VFInfo> VI = VFABI::tryDemangleForVFABI(VariantMapping, *CI);
assert(VI && "Cannot add an invalid VFABI name.");
assert(M->getNamedValue(VI->VectorName) &&
"Cannot add variant to attribute: "
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 19a970fb0716f0f..28962dfba89248a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -744,8 +744,8 @@ declare <vscale x 2 x i64> @foo_uniform(i64, <vscale x 2 x i1>)
declare <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64>, <vscale x 2 x i1>)
declare <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64>)
-attributes #0 = { nounwind "vector-function-abi-variant"="_ZGV_LLVM_Mxv_foo(foo_vector),_ZGV_LLVM_Mxu_foo(foo_uniform)" }
-attributes #1 = { nounwind "vector-function-abi-variant"="_ZGV_LLVM_Mxv_foo(foo_vector)" }
-attributes #2 = { nounwind "vector-function-abi-variant"="_ZGV_LLVM_Nxv_foo(foo_vector_nomask)" }
-attributes #3 = { nounwind "vector-function-abi-variant"="_ZGV_LLVM_Nxv_foo(foo_vector_nomask),_ZGV_LLVM_Mxv_foo(foo_vector)" }
+attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(foo_vector),_ZGVsMxu_foo(foo_uniform)" }
+attributes #1 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(foo_vector)" }
+attributes #2 = { nounwind "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vector_nomask)" }
+attributes #3 = { nounwind "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vector_nomask),_ZGVsMxv_foo(foo_vector)" }
attributes #4 = { "target-features"="+sve" vscale_range(2,16) "no-trapping-math"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
index 69c6f84aac53dd6..84f67310021bf1d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll
@@ -224,9 +224,9 @@ declare <vscale x 2 x i64> @bar_vec(<vscale x 2 x ptr>)
declare <vscale x 2 x double> @sin_vec_nxv2f64(<vscale x 2 x double>)
declare <2 x double> @sin_vec_v2f64(<2 x double>)
-attributes #0 = { "vector-function-abi-variant"="_ZGV_LLVM_Nxv_foo(foo_vec)" }
-attributes #1 = { "vector-function-abi-variant"="_ZGV_LLVM_Nxv_bar(bar_vec)" }
-attributes #2 = { "vector-function-abi-variant"="_ZGV_LLVM_Nxv_llvm.sin.f64(sin_vec_nxv2f64)" }
+attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }
+attributes #1 = { "vector-function-abi-variant"="_ZGVsNxv_bar(bar_vec)" }
+attributes #2 = { "vector-function-abi-variant"="_ZGVsNxv_llvm.sin.f64(sin_vec_nxv2f64)" }
attributes #3 = { "vector-function-abi-variant"="_ZGV_LLVM_N2v_llvm.sin.f64(sin_vec_v2f64)" }
!1 = distinct !{!1, !2, !3}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll
new file mode 100644
index 000000000000000..31cf2c4e2db4cdf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -passes=loop-vectorize,simplifycfg,instcombine -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @test_big_little_params(ptr readonly %a, ptr readonly %b, ptr noalias %c) #0 {
+; CHECK-LABEL: define void @test_big_little_params
+; CHECK-SAME: (ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP0]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP1]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @foo_vector(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i8> [[WIDE_MASKED_LOAD1]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP2]], ptr [[TMP3]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_BODY]], label [[EXIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep_s = getelementptr i32, ptr %a, i64 %iv
+ %load_s = load i32, ptr %gep_s
+ %gep_b = getelementptr i8, ptr %b, i64 %iv
+ %load_b = load i8, ptr %gep_b
+ %call = call i32 @foo_big_little(i32 %load_s, i8 %load_b) #1
+ %arrayidx = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %call, ptr %arrayidx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 1025
+ br i1 %exitcond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @test_little_big_params(ptr readonly %a, ptr readonly %b, ptr noalias %c) #0 {
+; CHECK-LABEL: define void @test_little_big_params
+; CHECK-SAME: (ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x float> @llvm.masked.load.nxv2f32.p0(ptr [[TMP0]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x float> poison)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[TMP1]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 2 x double> @bar_vector(<vscale x 2 x float> [[WIDE_MASKED_LOAD]], <vscale x 2 x double> [[WIDE_MASKED_LOAD1]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP2]], ptr [[TMP3]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP:%.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: for.cond.cleanup:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %gep_f = getelementptr float, ptr %a, i64 %iv
+ %load_f = load float, ptr %gep_f
+ %gep_d = getelementptr double, ptr %b, i64 %iv
+ %load_d = load double, ptr %gep_d
+ %call = call double @bar_little_big(float %load_f, double %load_d) #2
+ %arrayidx = getelementptr inbounds double, ptr %c, i64 %iv
+ store double %call, ptr %arrayidx
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %iv.next, 1025
+ br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+ ret void
+}
+
+;; TODO: Test uniform and linear parameters when they are properly supported,
+;; especially a variant with no vector parameters so the return type
+;; must be used to find the VF.
+
+;; Scalar functions
+declare i32 @foo_big_little(i32, i8)
+declare double @bar_little_big(float, double)
+
+;; Vector function variants
+declare <vscale x 4 x i32> @foo_vector(<vscale x 4 x i32>, <vscale x 4 x i8>, <vscale x 4 x i1>)
+declare <vscale x 2 x double> @bar_vector(<vscale x 2 x float>, <vscale x 2 x double>, <vscale x 2 x i1>)
+
+attributes #0 = { "target-features"="+sve" vscale_range(1,16) }
+attributes #1 = { nounwind "vector-function-abi-variant"="_ZGVsMxvv_foo_big_little(foo_vector)" }
+attributes #2 = { nounwind "vector-function-abi-variant"="_ZGVsMxvv_bar_little_big(bar_vector)" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
index 00e8881426fd8f9..7d095206c6062b4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
@@ -111,5 +111,5 @@ for.cond.cleanup:
declare float @foo(float)
declare <vscale x 4 x float> @foo_vector(<vscale x 4 x float>, <vscale x 4 x i1>)
-attributes #0 = { nounwind "vector-function-abi-variant"="_ZGV_LLVM_Mxv_foo(foo_vector)" }
+attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(foo_vector)" }
attributes #1 = { "target-features"="+sve" vscale_range(1,16) "no-trapping-math"="false" }
diff --git a/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp b/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
index b0b80131bf48f48..09dc15c9e36667b 100644
--- a/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
+++ b/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
@@ -27,15 +27,23 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
// present. We need to make sure we can even invoke
// `getOrInsertFunction` because such method asserts on strings with
// zeroes.
- if (!MangledName.empty() && MangledName.find_first_of(0) == StringRef::npos)
- M->getOrInsertFunction(
- MangledName,
- FunctionType::get(Type::getVoidTy(M->getContext()), false));
- const auto Info = VFABI::tryDemangleForVFABI(MangledName, *M);
+ // TODO: What is this actually testing? That we don't crash?
+ if (!MangledName.empty() && MangledName.find_first_of(0) == StringRef::npos) {
+ FunctionType *FTy =
+ FunctionType::get(Type::getVoidTy(M->getContext()), false);
+ FunctionCallee F = M->getOrInsertFunction(MangledName, FTy);
+ // Fake the arguments to the CallInst.
+ SmallVector<Value *> Args;
+ for (Type *ParamTy : FTy->params()) {
+ Args.push_back(Constant::getNullValue(ParamTy));
+ }
+ std::unique_ptr<CallInst> CI(CallInst::Create(F, Args));
+ const auto Info = VFABI::tryDemangleForVFABI(MangledName, *(CI.get()));
- // Do not optimize away the return value. Inspired by
- // https://github.com/google/benchmark/blob/main/include/benchmark/benchmark.h#L307-L345
- asm volatile("" : : "r,m"(Info) : "memory");
+ // Do not optimize away the return value. Inspired by
+ // https://github.com/google/benchmark/blob/main/include/benchmark/benchmark.h#L307-L345
+ asm volatile("" : : "r,m"(Info) : "memory");
+ }
return 0;
}
diff --git a/llvm/unittests/Analysis/VectorFunctionABITest.cpp b/llvm/unittests/Analysis/VectorFunctionABITest.cpp
index a4c6b2143fc662c..e496d87c06de6bc 100644
--- a/llvm/unittests/Analysis/VectorFunctionABITest.cpp
+++ b/llvm/unittests/Analysis/VectorFunctionABITest.cpp
@@ -25,11 +25,11 @@ class VFABIParserTest : public ::testing::Test {
EXPECT_NE(M.get(), nullptr) << "Loading an invalid module.\n "
<< Err.getMessage() << "\n";
Type *Ty = parseType(IRType, Err, *(M.get()));
- FunctionType *FTy = dyn_cast<FunctionType>(Ty);
+ FTy = dyn_cast<FunctionType>(Ty);
EXPECT_NE(FTy, nullptr) << "Invalid function type string: " << IRType
<< "\n"
<< Err.getMessage() << "\n";
- FunctionCallee F = M->getOrInsertFunction(Name, FTy);
+ F = M->getOrInsertFunction(Name, FTy);
EXPECT_NE(F.getCallee(), nullptr)
<< "The function must be present in the module\n";
// Reset the VFInfo
@@ -40,7 +40,9 @@ class VFABIParserTest : public ::testing::Test {
LLVMContext Ctx;
SMDiagnostic Err;
std::unique_ptr<Module> M;
- // CallInst *CI;
+ FunctionType *FTy;
+ FunctionCallee F;
+
protected:
// Referencies to the parser output field.
ElementCount &VF = Info.Shape.VF;
@@ -65,16 +67,22 @@ class VFABIParserTest : public ::testing::Test {
// generic fixed-length case can use as signature `void()`.
//
bool invokeParser(const StringRef MangledName,
- const StringRef VectorName = "",
+ const StringRef ScalarName = "",
const StringRef IRType = "void()") {
StringRef Name = MangledName;
- if (!VectorName.empty())
- Name = VectorName;
+ if (!ScalarName.empty())
+ Name = ScalarName;
// Reset the VFInfo and the Module to be able to invoke
// `invokeParser` multiple times in the same test.
reset(Name, IRType);
- const auto OptInfo = VFABI::tryDemangleForVFABI(MangledName, *(M.get()));
+ // Fake the arguments to the CallInst.
+ SmallVector<Value *> Args;
+ for (Type *ParamTy : FTy->params()) {
+ Args.push_back(Constant::getNullValue(ParamTy->getScalarType()));
+ }
+ std::unique_ptr<CallInst> CI(CallInst::Create(F, Args));
+ const auto OptInfo = VFABI::tryDemangleForVFABI(MangledName, *(CI.get()));
if (OptInfo) {
Info = *OptInfo;
return true;
@@ -204,10 +212,8 @@ TEST_F(VFABIParserTest, LinearWithCompileTimeNegativeStep) {
}
TEST_F(VFABIParserTest, ParseScalableSVE) {
- EXPECT_TRUE(invokeParser(
- "_ZGVsMxv_sin(custom_vg)", "custom_vg",
- "<vscale x 2 x i32>(<vscale x 2 x i32>, <vscale x 2 x i1>)"));
- EXPECT_EQ(VF, ElementCount::getScalable(2));
+ EXPECT_TRUE(invokeParser("_ZGVsMxv_sin(custom_vg)", "sin", "i32(i32)"));
+ EXPECT_EQ(VF, ElementCount::getScalable(4));
EXPECT_TRUE(IsMasked());
EXPECT_EQ(ISA, VFISAKind::SVE);
EXPECT_EQ(ScalarName, "sin");
@@ -495,12 +501,16 @@ TEST_F(VFABIParserTest, ParseMaskingLLVM) {
}
TEST_F(VFABIParserTest, ParseScalableMaskingLLVM) {
- EXPECT_TRUE(invokeParser(
- "_ZGV_LLVM_Mxv_sin(custom_vector_sin)", "custom_vector_sin",
- "<vscale x 2 x i32> (<vscale x 2 x i32>, <vscale x 2 x i1>)"));
+ EXPECT_FALSE(
+ invokeParser("_ZGV_LLVM_Mxv_sin(custom_vector_sin)", "sin", "i32(i32)"));
+}
+
+TEST_F(VFABIParserTest, ParseScalableMaskingSVE) {
+ EXPECT_TRUE(
+ invokeParser("_ZGVsMxv_sin(custom_vector_sin)", "sin", "i32(i32)"));
EXPECT_TRUE(IsMasked());
- EXPECT_EQ(VF, ElementCount::getScalable(2));
- EXPECT_EQ(ISA, VFISAKind::LLVM);
+ EXPECT_EQ(VF, ElementCount::getScalable(4));
+ EXPECT_EQ(ISA, VFISAKind::SVE);
EXPECT_EQ(Parameters.size(), (unsigned)2);
EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate}));
@@ -508,13 +518,12 @@ TEST_F(VFABIParserTest, ParseScalableMaskingLLVM) {
EXPECT_EQ(VectorName, "custom_vector_sin");
}
-TEST_F(VFABIParserTest, ParseScalableMaskingLLVMSincos) {
- EXPECT_TRUE(invokeParser("_ZGV_LLVM_Mxvl8l8_sincos(custom_vector_sincos)",
- "custom_vector_sincos",
- "void(<vscale x 2 x double>, double *, double *)"));
+TEST_F(VFABIParserTest, ParseScalableMaskingSVESincos) {
+ EXPECT_TRUE(invokeParser("_ZGVsMxvl8l8_sincos(custom_vector_sincos)",
+ "sincos", "void(double, double *, double *)"));
EXPECT_EQ(VF, ElementCount::getScalable(2));
EXPECT_TRUE(IsMasked());
- EXPECT_EQ(ISA, VFISAKind::LLVM);
+ EXPECT_EQ(ISA, VFISAKind::SVE);
EXPECT_EQ(Parameters.size(), (unsigned)4);
EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_Linear, 8}));
@@ -524,6 +533,30 @@ TEST_F(VFABIParserTest, ParseScalableMaskingLLVMSincos) {
EXPECT_EQ(VectorName, "custom_vector_sincos");
}
+// Make sure that we get the correct VF if the return type is wider than any
+// parameter type.
+TEST_F(VFABIParserTest, ParseWiderReturnTypeSVE) {
+ EXPECT_TRUE(
+ invokeParser("_ZGVsMxvv_foo(vector_foo)", "foo", "i64(i32, i32)"));
+ EXPECT_EQ(VF, ElementCount::getScalable(2));
+}
+
+// Make sure we handle void return types.
+TEST_F(VFABIParserTest, ParseVoidReturnTypeSVE) {
+ EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "void(i16)"));
+ EXPECT_EQ(VF, ElementCount::getScalable(8));
+}
+
+// Make sure we reject unsupported parameter types.
+TEST_F(VFABIParserTest, ParseUnsupportedElementTypeSVE) {
+ EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "void(i128)"));
+}
+
+// Make sure we reject unsupported return types
+TEST_F(VFABIParserTest, ParseUnsupportedReturnTypeSVE) {
+ EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "fp128(float)"));
+}
+
class VFABIAttrTest : public testing::Test {
protected:
void SetUp() override {
@@ -581,9 +614,7 @@ TEST_F(VFABIParserTest, ParseScalableRequiresDeclaration) {
// The parser succeds only when the correct function definition of
// `custom_vg` is added to the module.
EXPECT_FALSE(invokeParser(MangledName));
- EXPECT_TRUE(invokeParser(
- MangledName, "custom_vg",
- "<vscale x 4 x double>(<vscale x 4 x double>, <vscale x 4 x i1>)"));
+ EXPECT_TRUE(invokeParser(MangledName, "sin", "double(double)"));
}
TEST_F(VFABIParserTest, ZeroIsInvalidVLEN) {
More information about the llvm-commits
mailing list