[llvm] [llvm][NFC] Refactor AutoUpgrader arm/aarch64 (PR #74145)
Nathan Sidwell via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 22 12:01:43 PST 2023
https://github.com/urnathan updated https://github.com/llvm/llvm-project/pull/74145
>From 73d0dda806511fb8c171ab27b70ab12e5f0cd2a4 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan at acm.org>
Date: Wed, 23 Aug 2023 19:38:04 -0400
Subject: [PATCH 1/4] [llvm][NFC] Refactor AutoUpgrader arm/aarch64
Arm and AArch64 auto upgrade detection is somewhat intertwined. While
there is some comonality in their neon subsets, they're otherwise
pretty independent. Rather than continually check for 'arm.' or
'aarch64' prefixes, this refactors the detection to first consume one
of those prefixes, and then do the common checks (including 'neon.'
detection), and finally do the distinct checks.
---
llvm/lib/IR/AutoUpgrade.cpp | 495 ++++++++++--------
.../Assembler/autoupgrade-thread-pointer.ll | 4 +-
2 files changed, 287 insertions(+), 212 deletions(-)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 6b54047020a05d..09ccd1e2dfd073 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -713,225 +713,300 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
switch (Name[0]) {
default: break;
case 'a': {
- if (Name.starts_with("arm.rbit") || Name.starts_with("aarch64.rbit")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
- F->arg_begin()->getType());
- return true;
- }
- if (Name.starts_with("aarch64.neon.frintn")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::roundeven,
- F->arg_begin()->getType());
- return true;
- }
- if (Name.starts_with("aarch64.neon.rbit")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
- F->arg_begin()->getType());
- return true;
- }
- if (Name == "aarch64.sve.bfdot.lane") {
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::aarch64_sve_bfdot_lane_v2);
- return true;
- }
- if (Name == "aarch64.sve.bfmlalb.lane") {
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::aarch64_sve_bfmlalb_lane_v2);
- return true;
- }
- if (Name == "aarch64.sve.bfmlalt.lane") {
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::aarch64_sve_bfmlalt_lane_v2);
- return true;
- }
- static const Regex LdRegex("^aarch64\\.sve\\.ld[234](.nxv[a-z0-9]+|$)");
- if (LdRegex.match(Name)) {
- Type *ScalarTy =
- dyn_cast<VectorType>(F->getReturnType())->getElementType();
- ElementCount EC =
- dyn_cast<VectorType>(F->arg_begin()->getType())->getElementCount();
- Type *Ty = VectorType::get(ScalarTy, EC);
- Intrinsic::ID ID =
- StringSwitch<Intrinsic::ID>(Name)
- .StartsWith("aarch64.sve.ld2", Intrinsic::aarch64_sve_ld2_sret)
- .StartsWith("aarch64.sve.ld3", Intrinsic::aarch64_sve_ld3_sret)
- .StartsWith("aarch64.sve.ld4", Intrinsic::aarch64_sve_ld4_sret)
- .Default(Intrinsic::not_intrinsic);
- NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Ty);
- return true;
- }
- if (Name.starts_with("aarch64.sve.tuple.get")) {
- Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::vector_extract, Tys);
- return true;
- }
- if (Name.starts_with("aarch64.sve.tuple.set")) {
- auto Args = F->getFunctionType()->params();
- Type *Tys[] = {Args[0], Args[2], Args[1]};
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::vector_insert, Tys);
- return true;
- }
- static const Regex CreateTupleRegex(
- "^aarch64\\.sve\\.tuple\\.create[234](.nxv[a-z0-9]+|$)");
- if (CreateTupleRegex.match(Name)) {
- auto Args = F->getFunctionType()->params();
- Type *Tys[] = {F->getReturnType(), Args[1]};
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::vector_insert, Tys);
- return true;
- }
- if (Name.starts_with("arm.neon.vclz")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
- F->arg_begin()->getType());
- return true;
- }
- if (Name.starts_with("arm.neon.vcnt")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
- F->arg_begin()->getType());
- return true;
- }
- static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
- if (vstRegex.match(Name)) {
- static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
- Intrinsic::arm_neon_vst2,
- Intrinsic::arm_neon_vst3,
- Intrinsic::arm_neon_vst4};
-
- static const Intrinsic::ID StoreLaneInts[] = {
- Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
- Intrinsic::arm_neon_vst4lane
- };
-
- auto fArgs = F->getFunctionType()->params();
- Type *Tys[] = {fArgs[0], fArgs[1]};
- if (!Name.contains("lane"))
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- StoreInts[fArgs.size() - 3], Tys);
- else
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- StoreLaneInts[fArgs.size() - 5], Tys);
- return true;
- }
- if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
- return true;
- }
- if (Name.starts_with("arm.neon.vqadds.")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
- F->arg_begin()->getType());
- return true;
- }
- if (Name.starts_with("arm.neon.vqaddu.")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
- F->arg_begin()->getType());
- return true;
- }
- if (Name.starts_with("arm.neon.vqsubs.")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
- F->arg_begin()->getType());
- return true;
- }
- if (Name.starts_with("arm.neon.vqsubu.")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
- F->arg_begin()->getType());
- return true;
- }
- if (Name.starts_with("aarch64.neon.addp")) {
- if (F->arg_size() != 2)
- break; // Invalid IR.
- VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
- if (Ty && Ty->getElementType()->isFloatingPointTy()) {
+ bool Arm = Name.consume_front("arm.");
+ if (Arm || Name.consume_front("aarch64.")) {
+ // Arm: 'arm.*', !Arm: 'aarch64.*'.
+ if (Name.starts_with("rbit")) {
+ // '(arm|aarch64).rbit'.
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
+ F->arg_begin()->getType());
+ return true;
+ }
+ if (Name == "thread.pointer") {
+ // '(arm|aarch64).thread.pointer'.
NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::aarch64_neon_faddp, Ty);
+ Intrinsic::thread_pointer);
return true;
}
- }
- // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
- // respectively
- if ((Name.starts_with("arm.neon.bfdot.") ||
- Name.starts_with("aarch64.neon.bfdot.")) &&
- Name.ends_with("i8")) {
- Intrinsic::ID IID =
- StringSwitch<Intrinsic::ID>(Name)
- .Cases("arm.neon.bfdot.v2f32.v8i8",
- "arm.neon.bfdot.v4f32.v16i8",
- Intrinsic::arm_neon_bfdot)
- .Cases("aarch64.neon.bfdot.v2f32.v8i8",
- "aarch64.neon.bfdot.v4f32.v16i8",
- Intrinsic::aarch64_neon_bfdot)
- .Default(Intrinsic::not_intrinsic);
- if (IID == Intrinsic::not_intrinsic)
- break;
+ bool Neon = Name.consume_front("neon.");
+ if (Neon) {
+ // '(arm|aarch64).neon.*'.
+ // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
+ // v16i8 respectively.
+ if (Name.consume_front("bfdot.")) {
+ // (arm|aarch64).neon.bfdot.*'.
+ Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+ .Cases("v2f32.v8i8", "v4f32.v16i8",
+ Arm ? Intrinsic::arm_neon_bfdot
+ : Intrinsic::aarch64_neon_bfdot)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
+ assert((OperandWidth == 64 || OperandWidth == 128) &&
+ "Unexpected operand width");
+ LLVMContext &Ctx = F->getParent()->getContext();
+ std::array<Type *, 2> Tys{
+ {F->getReturnType(),
+ FixedVectorType::get(Type::getBFloatTy(Ctx),
+ OperandWidth / 16)}};
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+ return true;
+ }
+ break; // No other '(arm|aarch64).neon.bfdot.*'.
+ }
- size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
- assert((OperandWidth == 64 || OperandWidth == 128) &&
- "Unexpected operand width");
- LLVMContext &Ctx = F->getParent()->getContext();
- std::array<Type *, 2> Tys {{
- F->getReturnType(),
- FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
- }};
- NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
- return true;
- }
+ // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
+ // anymore and accept v8bf16 instead of v16i8.
+ if (Name.consume_front("bfm")) {
+ // (arm|aarch64).neon.bfm*'.
+ if (Name.consume_back(".v4f32.v16i8")) {
+ // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
+ Intrinsic::ID ID =
+ StringSwitch<Intrinsic::ID>(Name)
+ .Case("mla", Arm ? Intrinsic::arm_neon_bfmmla
+ : Intrinsic::aarch64_neon_bfmmla)
+ .Case("lalb", Arm ? Intrinsic::arm_neon_bfmlalb
+ : Intrinsic::aarch64_neon_bfmlalb)
+ .Case("lalt", Arm ? Intrinsic::arm_neon_bfmlalt
+ : Intrinsic::aarch64_neon_bfmlalt)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ std::array<Type *, 0> Tys;
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+ return true;
+ }
+ break; // No other '(arm|aarch64).neon.bfm*.v16i8'.
+ }
+ break; // No other '(arm|aarch64).neon.bfm*.
+ }
+ // Continue on to Aarch64 Neon or Arm Neon.
+ }
+ // Continue on to Arm or Aarch64.
+
+ if (Arm) {
+ // 'arm.*'.
+ if (Neon) {
+ // 'arm.neon.*'.
+ if (Name.consume_front("vclz.")) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
+ F->arg_begin()->getType());
+ return true;
+ }
+ if (Name.starts_with("vcnt")) {
+ // 'arm.neon.vcnt*'.
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
+ F->arg_begin()->getType());
+ return true;
+ }
- // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
- // and accept v8bf16 instead of v16i8
- if ((Name.starts_with("arm.neon.bfm") ||
- Name.starts_with("aarch64.neon.bfm")) &&
- Name.ends_with(".v4f32.v16i8")) {
- Intrinsic::ID IID =
- StringSwitch<Intrinsic::ID>(Name)
- .Case("arm.neon.bfmmla.v4f32.v16i8",
- Intrinsic::arm_neon_bfmmla)
- .Case("arm.neon.bfmlalb.v4f32.v16i8",
- Intrinsic::arm_neon_bfmlalb)
- .Case("arm.neon.bfmlalt.v4f32.v16i8",
- Intrinsic::arm_neon_bfmlalt)
- .Case("aarch64.neon.bfmmla.v4f32.v16i8",
- Intrinsic::aarch64_neon_bfmmla)
- .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
- Intrinsic::aarch64_neon_bfmlalb)
- .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
- Intrinsic::aarch64_neon_bfmlalt)
- .Default(Intrinsic::not_intrinsic);
- if (IID == Intrinsic::not_intrinsic)
- break;
+ if (Name.consume_front("vst")) {
+ // 'arm.neon.vst*'.
+ static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
+ SmallVector<StringRef, 2> Groups;
+ if (vstRegex.match(Name, &Groups)) {
+ static const Intrinsic::ID StoreInts[] = {
+ Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
+ Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
+
+ static const Intrinsic::ID StoreLaneInts[] = {
+ Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
+ Intrinsic::arm_neon_vst4lane};
+
+ auto fArgs = F->getFunctionType()->params();
+ Type *Tys[] = {fArgs[0], fArgs[1]};
+ if (Groups[1].size() == 1)
+ NewFn = Intrinsic::getDeclaration(
+ F->getParent(), StoreInts[fArgs.size() - 3], Tys);
+ else
+ NewFn = Intrinsic::getDeclaration(
+ F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
+ return true;
+ }
+ break; // No other 'arm.neon.vst*'.
+ }
- std::array<Type *, 0> Tys;
- NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
- return true;
- }
+ if (Name.consume_front("vq")) {
+ // 'arm.neon.vq*'.
+ Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+ .StartsWith("adds.", Intrinsic::sadd_sat)
+ .StartsWith("addu.", Intrinsic::uadd_sat)
+ .StartsWith("subs.", Intrinsic::ssub_sat)
+ .StartsWith("subu.", Intrinsic::usub_sat)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
+ F->arg_begin()->getType());
+ return true;
+ }
+ break; // No other 'arm.neon.vq*'.
+ }
+ break; // No other 'arm.neon.*'.
+ }
- if (Name == "arm.mve.vctp64" &&
- cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
- // A vctp64 returning a v4i1 is converted to return a v2i1. Rename the
- // function and deal with it below in UpgradeIntrinsicCall.
- rename(F);
- return true;
+ if (Name.consume_front("mve.")) {
+ // 'arm.mve.*'.
+ if (Name == "vctp64") {
+ if (cast<FixedVectorType>(F->getReturnType())->getNumElements() ==
+ 4) {
+ // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
+ // the function and deal with it below in UpgradeIntrinsicCall.
+ rename(F);
+ return true;
+ }
+ break; // Not 'arm.mve.vctp64'.
+ }
+
+ // These too are changed to accept a v2i1 instead of the old v4i1.
+ if (Name.consume_back(".v4i1")) {
+ // 'arm.mve.*.v4i1'.
+ if (Name.consume_back(".predicated.v2i64.v4i32")) {
+ if (Name == "mull.int" || Name == "vqdmull")
+ return true;
+ break; // No other 'arm.mve.*.predicated.v2i64.v4i32.v4i1'.
+ }
+
+ if (Name.consume_back(".v2i64")) {
+ if (Name.consume_front("vldr.gather.")) {
+ // 'arm.mve.vldr.gather.*.v2i64.v4i1'.
+ if (Name == "base.predicated.v2i64" ||
+ Name == "base.wb.predicated.v2i64" ||
+ Name == "offset.predicated.v2i64.p0i64" ||
+ Name == "offset.predicated.v2i64.p0")
+ return true;
+ break; // No other 'arm.mve.vldr.gather.*.v2i64.v4i1'.
+ }
+ if (Name.consume_front("vstr.scatter.")) {
+ // 'arm.mve.vstr.scatter.*.v2i64.v4i1'.
+ if (Name == "base.predicated.v2i64" ||
+ Name == "base.wb.predicated.v2i64" ||
+ Name == "offset.predicated.p0i64.v2i64" ||
+ Name == "offset.predicated.p0.v2i64")
+ return true;
+ break; // No other 'arm.mve.vstr.scatter.*.v2i64.v4i1'.
+ }
+ break; // No other 'arm.mve.*.v2i64.v4i1'.
+ }
+ break; // No other 'arm.mve.*.v4i1'.
+ }
+ break; // No other 'arm.mve.*'.
+ }
+
+ if (Name.consume_front("cde.vcx")) {
+ // 'arm.cde.vcx*'.
+ if (Name.consume_back(".predicated.v2i64.v4i1")) {
+ // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
+ if (Name == "1q" || Name == "1qa" || Name == "2q" ||
+ Name == "2qa" || Name == "3q" || Name == "3qa")
+ return true;
+ break; // No other 'arm.cde.vcx*.predicated.v2i64.v4i1'.
+ }
+ break; // No other 'arm.cde.vcx*'.
+ }
+ } else {
+ // 'aarch64.*'.
+ if (Neon) {
+ // 'aarch64.neon.*'.
+ Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+ .StartsWith("frintn", Intrinsic::roundeven)
+ .StartsWith("rbit", Intrinsic::bitreverse)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
+ F->arg_begin()->getType());
+ return true;
+ }
+
+ if (Name.starts_with("addp")) {
+ // 'aarch64.neon.addp*'.
+ if (F->arg_size() != 2)
+ break; // Invalid IR.
+ VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
+ if (Ty && Ty->getElementType()->isFloatingPointTy()) {
+ NewFn = Intrinsic::getDeclaration(
+ F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
+ return true;
+ }
+ }
+ break; // No other 'aarch64.neon.*'.
+ }
+ if (Name.consume_front("sve.")) {
+ // 'aarch64.sve.*'.
+ if (Name.consume_front("bf")) {
+ if (Name.consume_back(".lane")) {
+ // 'aarch64.sve.bf*.lane'.
+ Intrinsic::ID ID =
+ StringSwitch<Intrinsic::ID>(Name)
+ .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
+ .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
+ .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+ return true;
+ }
+ break; // No other 'aarch64.sve.bf*.lane'.
+ }
+ break; // No other 'aarch64.sve.bf*'.
+ }
+
+ if (Name.consume_front("ld")) {
+ // 'aarch64.sve.ld*'.
+ static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
+ if (LdRegex.match(Name)) {
+ Type *ScalarTy =
+ dyn_cast<VectorType>(F->getReturnType())->getElementType();
+ ElementCount EC = dyn_cast<VectorType>(F->arg_begin()->getType())
+ ->getElementCount();
+ Type *Ty = VectorType::get(ScalarTy, EC);
+ static const Intrinsic::ID LoadIDs[] = {
+ Intrinsic::aarch64_sve_ld2_sret,
+ Intrinsic::aarch64_sve_ld3_sret,
+ Intrinsic::aarch64_sve_ld4_sret,
+ };
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ LoadIDs[Name[0] - '2'], Ty);
+ return true;
+ }
+ break; // No other 'aarch64.sve.ld*'.
+ }
+
+ if (Name.consume_front("tuple.")) {
+ // 'aarch64.sve.tuple.*'.
+ if (Name.starts_with("get")) {
+ // 'aarch64.sve.tuple.get*'.
+ Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_extract, Tys);
+ return true;
+ }
+
+ if (Name.starts_with("set")) {
+ // 'aarch64.sve.tuple.set*'.
+ auto Args = F->getFunctionType()->params();
+ Type *Tys[] = {Args[0], Args[2], Args[1]};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_insert, Tys);
+ return true;
+ }
+
+ static const Regex CreateTupleRegex(
+ "^create[234](.nxv[a-z0-9]+|$)");
+ if (CreateTupleRegex.match(Name)) {
+ // 'aarch64.sve.tuple.create*'.
+ auto Args = F->getFunctionType()->params();
+ Type *Tys[] = {F->getReturnType(), Args[1]};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_insert, Tys);
+ return true;
+ }
+ break; // No other 'aarch64.sve.tuple.*'.
+ }
+ break; // No other 'aarch64.sve.*'.
+ }
+ }
+ break; // No other 'arm.*', 'aarch64.*'.
}
- // These too are changed to accept a v2i1 insteead of the old v4i1.
- if (Name == "arm.mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
- Name == "arm.mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
- Name == "arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
- Name == "arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
- Name ==
- "arm.mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
- Name == "arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
- Name == "arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
- Name == "arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
- Name ==
- "arm.mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
- Name == "arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
- Name == "arm.cde.vcx1q.predicated.v2i64.v4i1" ||
- Name == "arm.cde.vcx1qa.predicated.v2i64.v4i1" ||
- Name == "arm.cde.vcx2q.predicated.v2i64.v4i1" ||
- Name == "arm.cde.vcx2qa.predicated.v2i64.v4i1" ||
- Name == "arm.cde.vcx3q.predicated.v2i64.v4i1" ||
- Name == "arm.cde.vcx3qa.predicated.v2i64.v4i1")
- return true;
if (Name.consume_front("amdgcn.")) {
if (Name == "alignbit") {
diff --git a/llvm/test/Assembler/autoupgrade-thread-pointer.ll b/llvm/test/Assembler/autoupgrade-thread-pointer.ll
index 2eaabb4e921f0f..b1ed15a7e4ef88 100644
--- a/llvm/test/Assembler/autoupgrade-thread-pointer.ll
+++ b/llvm/test/Assembler/autoupgrade-thread-pointer.ll
@@ -5,14 +5,14 @@ declare ptr @llvm.aarch64.thread.pointer()
declare ptr @llvm.arm.thread.pointer()
define ptr @test1() {
-; CHECK: test1()
+; CHECK-LABEL: define ptr @test1()
; CHECK: call ptr @llvm.thread.pointer()
%1 = call ptr @llvm.aarch64.thread.pointer()
ret ptr %1
}
define ptr @test2() {
-; CHECK: test2()
+; CHECK-LABEL: define ptr @test2()
; CHECK: call ptr @llvm.thread.pointer()
%1 = call ptr @llvm.arm.thread.pointer()
ret ptr %1
>From 6ce94ad53a5d6bd1a6004eaf2267ddbe81ec590c Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan at acm.org>
Date: Sun, 3 Dec 2023 16:32:55 -0500
Subject: [PATCH 2/4] Breakout ARM/aarch64 to separate function
---
llvm/lib/IR/AutoUpgrade.cpp | 318 ++++++++++++++++++++++++++++++++++++
1 file changed, 318 insertions(+)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 09ccd1e2dfd073..d9fa068a75810e 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -621,6 +621,318 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
return false;
}
+// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
+// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
+static bool UpgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
+ StringRef Name,
+ Function *&NewFn) {
+ if (Name.starts_with("rbit")) {
+ // '(arm|aarch64).rbit'.
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
+ F->arg_begin()->getType());
+ return true;
+ }
+
+ if (Name == "thread.pointer") {
+ // '(arm|aarch64).thread.pointer'.
+ NewFn =
+ Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
+ return true;
+ }
+
+ bool Neon = Name.consume_front("neon.");
+ if (Neon) {
+ // '(arm|aarch64).neon.*'.
+ // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
+ // v16i8 respectively.
+ if (Name.consume_front("bfdot.")) {
+ // (arm|aarch64).neon.bfdot.*'.
+ Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+ .Cases("v2f32.v8i8", "v4f32.v16i8",
+ IsArm ? Intrinsic::arm_neon_bfdot
+ : Intrinsic::aarch64_neon_bfdot)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
+ assert((OperandWidth == 64 || OperandWidth == 128) &&
+ "Unexpected operand width");
+ LLVMContext &Ctx = F->getParent()->getContext();
+ std::array<Type *, 2> Tys{
+ {F->getReturnType(),
+ FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+ return true;
+ }
+ return false; // No other '(arm|aarch64).neon.bfdot.*'.
+ }
+
+ // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
+ // anymore and accept v8bf16 instead of v16i8.
+ if (Name.consume_front("bfm")) {
+ // (arm|aarch64).neon.bfm*'.
+ if (Name.consume_back(".v4f32.v16i8")) {
+ // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
+ Intrinsic::ID ID =
+ StringSwitch<Intrinsic::ID>(Name)
+ .Case("mla", IsArm ? Intrinsic::arm_neon_bfmmla
+ : Intrinsic::aarch64_neon_bfmmla)
+ .Case("lalb", IsArm ? Intrinsic::arm_neon_bfmlalb
+ : Intrinsic::aarch64_neon_bfmlalb)
+ .Case("lalt", IsArm ? Intrinsic::arm_neon_bfmlalt
+ : Intrinsic::aarch64_neon_bfmlalt)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ std::array<Type *, 0> Tys;
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+ return true;
+ }
+ return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
+ }
+ return false; // No other '(arm|aarch64).neon.bfm*.
+ }
+ // Continue on to Aarch64 Neon or Arm Neon.
+ }
+ // Continue on to Arm or Aarch64.
+
+ if (IsArm) {
+ // 'arm.*'.
+ if (Neon) {
+ // 'arm.neon.*'.
+ if (Name.consume_front("vclz.")) {
+ // 'arm.neon.vclz.*'.
+ Type *args[2] = {F->arg_begin()->getType(),
+ Type::getInt1Ty(F->getContext())};
+ // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
+ // the end of the name. Change name from 'llvm.arm.neon.vclz.*' to
+ // 'llvm.ctlz.*'.
+ FunctionType *fType =
+ FunctionType::get(F->getReturnType(), args, false);
+ NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
+ "llvm.ctlz." + Name, F->getParent());
+ return true;
+ }
+ if (Name.starts_with("vcnt")) {
+ // 'arm.neon.vcnt*'.
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
+ F->arg_begin()->getType());
+ return true;
+ }
+
+ if (Name.consume_front("vst")) {
+ // 'arm.neon.vst*'.
+ static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
+ SmallVector<StringRef, 2> Groups;
+ if (vstRegex.match(Name, &Groups)) {
+ static const Intrinsic::ID StoreInts[] = {
+ Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
+ Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
+
+ static const Intrinsic::ID StoreLaneInts[] = {
+ Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
+ Intrinsic::arm_neon_vst4lane};
+
+ auto fArgs = F->getFunctionType()->params();
+ Type *Tys[] = {fArgs[0], fArgs[1]};
+ if (Groups[1].size() == 1)
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ StoreInts[fArgs.size() - 3], Tys);
+ else
+ NewFn = Intrinsic::getDeclaration(
+ F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
+ return true;
+ }
+ return false; // No other 'arm.neon.vst*'.
+ }
+
+ if (Name.consume_front("vq")) {
+ // 'arm.neon.vq*'.
+ Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+ .StartsWith("adds.", Intrinsic::sadd_sat)
+ .StartsWith("addu.", Intrinsic::uadd_sat)
+ .StartsWith("subs.", Intrinsic::ssub_sat)
+ .StartsWith("subu.", Intrinsic::usub_sat)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
+ F->arg_begin()->getType());
+ return true;
+ }
+ return false; // No other 'arm.neon.vq*'.
+ }
+ return false; // No other 'arm.neon.*'.
+ }
+
+ if (Name.consume_front("mve.")) {
+ // 'arm.mve.*'.
+ if (Name == "vctp64") {
+ if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
+ // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
+ // the function and deal with it below in UpgradeIntrinsicCall.
+ rename(F);
+ return true;
+ }
+ return false; // Not 'arm.mve.vctp64'.
+ }
+
+ // These too are changed to accept a v2i1 instead of the old v4i1.
+ if (Name.consume_back(".v4i1")) {
+ // 'arm.mve.*.v4i1'.
+ if (Name.consume_back(".predicated.v2i64.v4i32")) {
+ // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
+ if (Name == "mull.int" || Name == "vqdmull")
+ return true;
+ return false; // No other 'arm.mve.*.predicated.v2i64.v4i32.v4i1'.
+ }
+
+ if (Name.consume_back(".v2i64")) {
+ // 'arm.mve.*.v2i64.v4i1'
+ bool IsGather = Name.consume_front("vldr.gather.");
+ if (IsGather || Name.consume_front("vstr.scatter.")) {
+ if (Name.consume_front("base.")) {
+ Name.consume_front("wb.");
+ if (Name == "predicated.v2i64")
+ // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
+ // predicated.v2i64.v2i64.v4i1'.
+ return true;
+ // No other 'arm.(vldr.gather|vstr.scatter).base.*.v2i64.v4i1'
+ return false;
+ }
+
+ if (Name.consume_front("offset.predicated.")) {
+ if (Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
+ Name == (IsGather ? "v2i64.p0" : "p0.v2i64"))
+ return true;
+ // No other 'arm.(vldr.gather|vstr.scatter).offset
+ // .predicated.*.v2i64.v4i1'
+ return false;
+ }
+
+ // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
+ return false;
+ }
+
+ return false; // No other 'arm.mve.*.v2i64.v4i1'.
+ }
+ return false; // No other 'arm.mve.*.v4i1'.
+ }
+ return false; // No other 'arm.mve.*'.
+ }
+
+ if (Name.consume_front("cde.vcx")) {
+ // 'arm.cde.vcx*'.
+ if (Name.consume_back(".predicated.v2i64.v4i1")) {
+ // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
+ if (Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
+ Name == "3q" || Name == "3qa")
+ return true;
+ return false; // No other 'arm.cde.vcx*.predicated.v2i64.v4i1'.
+ }
+ return false; // No other 'arm.cde.vcx*'.
+ }
+ } else {
+ // 'aarch64.*'.
+ if (Neon) {
+ // 'aarch64.neon.*'.
+ Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+ .StartsWith("frintn", Intrinsic::roundeven)
+ .StartsWith("rbit", Intrinsic::bitreverse)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
+ F->arg_begin()->getType());
+ return true;
+ }
+
+ if (Name.starts_with("addp")) {
+ // 'aarch64.neon.addp*'.
+ if (F->arg_size() != 2)
+ return false; // Invalid IR.
+ VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
+ if (Ty && Ty->getElementType()->isFloatingPointTy()) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::aarch64_neon_faddp, Ty);
+ return true;
+ }
+ }
+ return false; // No other 'aarch64.neon.*'.
+ }
+ if (Name.consume_front("sve.")) {
+ // 'aarch64.sve.*'.
+ if (Name.consume_front("bf")) {
+ if (Name.consume_back(".lane")) {
+ // 'aarch64.sve.bf*.lane'.
+ Intrinsic::ID ID =
+ StringSwitch<Intrinsic::ID>(Name)
+ .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
+ .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
+ .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
+ return true;
+ }
+ return false; // No other 'aarch64.sve.bf*.lane'.
+ }
+ return false; // No other 'aarch64.sve.bf*'.
+ }
+
+ if (Name.consume_front("ld")) {
+ // 'aarch64.sve.ld*'.
+ static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
+ if (LdRegex.match(Name)) {
+ Type *ScalarTy =
+ dyn_cast<VectorType>(F->getReturnType())->getElementType();
+ ElementCount EC = dyn_cast<VectorType>(F->arg_begin()->getType())
+ ->getElementCount();
+ Type *Ty = VectorType::get(ScalarTy, EC);
+ static const Intrinsic::ID LoadIDs[] = {
+ Intrinsic::aarch64_sve_ld2_sret,
+ Intrinsic::aarch64_sve_ld3_sret,
+ Intrinsic::aarch64_sve_ld4_sret,
+ };
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ LoadIDs[Name[0] - '2'], Ty);
+ return true;
+ }
+ return false; // No other 'aarch64.sve.ld*'.
+ }
+
+ if (Name.consume_front("tuple.")) {
+ // 'aarch64.sve.tuple.*'.
+ if (Name.starts_with("get")) {
+ // 'aarch64.sve.tuple.get*'.
+ Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_extract, Tys);
+ return true;
+ }
+
+ if (Name.starts_with("set")) {
+ // 'aarch64.sve.tuple.set*'.
+ auto Args = F->getFunctionType()->params();
+ Type *Tys[] = {Args[0], Args[2], Args[1]};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_insert, Tys);
+ return true;
+ }
+
+ static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
+ if (CreateTupleRegex.match(Name)) {
+ // 'aarch64.sve.tuple.create*'.
+ auto Args = F->getFunctionType()->params();
+ Type *Tys[] = {F->getReturnType(), Args[1]};
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::vector_insert, Tys);
+ return true;
+ }
+ return false; // No other 'aarch64.sve.tuple.*'.
+ }
+ return false; // No other 'aarch64.sve.*'.
+ }
+ }
+ return false; // No other 'arm.*', 'aarch64.*'.
+}
+
static Intrinsic::ID ShouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
if (Name.consume_front("abs."))
return StringSwitch<Intrinsic::ID>(Name)
@@ -713,6 +1025,12 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
switch (Name[0]) {
default: break;
case 'a': {
+ bool IsArm = Name.consume_front("arm.");
+ if (IsArm || Name.consume_front("aarch64.")) {
+ if (UpgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
+ return true;
+ break;
+ }
bool Arm = Name.consume_front("arm.");
if (Arm || Name.consume_front("aarch64.")) {
// Arm: 'arm.*', !Arm: 'aarch64.*'.
>From 25cdd7d9addaf129c1aff72c2aab0efdca08bbaf Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan at acm.org>
Date: Fri, 8 Dec 2023 14:39:47 -0500
Subject: [PATCH 3/4] Comments #1
---
llvm/lib/IR/AutoUpgrade.cpp | 338 ++----------------------------------
1 file changed, 11 insertions(+), 327 deletions(-)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index d9fa068a75810e..cfa87c073f18b9 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -682,8 +682,7 @@ static bool UpgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
: Intrinsic::aarch64_neon_bfmlalt)
.Default(Intrinsic::not_intrinsic);
if (ID != Intrinsic::not_intrinsic) {
- std::array<Type *, 0> Tys;
- NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
return true;
}
return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
@@ -698,22 +697,16 @@ static bool UpgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
// 'arm.*'.
if (Neon) {
// 'arm.neon.*'.
- if (Name.consume_front("vclz.")) {
- // 'arm.neon.vclz.*'.
- Type *args[2] = {F->arg_begin()->getType(),
- Type::getInt1Ty(F->getContext())};
- // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
- // the end of the name. Change name from 'llvm.arm.neon.vclz.*' to
- // 'llvm.ctlz.*'.
- FunctionType *fType =
- FunctionType::get(F->getReturnType(), args, false);
- NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
- "llvm.ctlz." + Name, F->getParent());
- return true;
- }
- if (Name.starts_with("vcnt")) {
- // 'arm.neon.vcnt*'.
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
+ Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
+ .StartsWith("vclz.", Intrinsic::ctlz)
+ .StartsWith("vcnt.", Intrinsic::ctpop)
+ .StartsWith("vqadds.", Intrinsic::sadd_sat)
+ .StartsWith("vqaddu.", Intrinsic::uadd_sat)
+ .StartsWith("vqsubs.", Intrinsic::ssub_sat)
+ .StartsWith("vqsubu.", Intrinsic::usub_sat)
+ .Default(Intrinsic::not_intrinsic);
+ if (ID != Intrinsic::not_intrinsic) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
F->arg_begin()->getType());
return true;
}
@@ -744,21 +737,6 @@ static bool UpgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
return false; // No other 'arm.neon.vst*'.
}
- if (Name.consume_front("vq")) {
- // 'arm.neon.vq*'.
- Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
- .StartsWith("adds.", Intrinsic::sadd_sat)
- .StartsWith("addu.", Intrinsic::uadd_sat)
- .StartsWith("subs.", Intrinsic::ssub_sat)
- .StartsWith("subu.", Intrinsic::usub_sat)
- .Default(Intrinsic::not_intrinsic);
- if (ID != Intrinsic::not_intrinsic) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
- F->arg_begin()->getType());
- return true;
- }
- return false; // No other 'arm.neon.vq*'.
- }
return false; // No other 'arm.neon.*'.
}
@@ -1031,300 +1009,6 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
return true;
break;
}
- bool Arm = Name.consume_front("arm.");
- if (Arm || Name.consume_front("aarch64.")) {
- // Arm: 'arm.*', !Arm: 'aarch64.*'.
- if (Name.starts_with("rbit")) {
- // '(arm|aarch64).rbit'.
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
- F->arg_begin()->getType());
- return true;
- }
- if (Name == "thread.pointer") {
- // '(arm|aarch64).thread.pointer'.
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::thread_pointer);
- return true;
- }
-
- bool Neon = Name.consume_front("neon.");
- if (Neon) {
- // '(arm|aarch64).neon.*'.
- // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
- // v16i8 respectively.
- if (Name.consume_front("bfdot.")) {
- // (arm|aarch64).neon.bfdot.*'.
- Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
- .Cases("v2f32.v8i8", "v4f32.v16i8",
- Arm ? Intrinsic::arm_neon_bfdot
- : Intrinsic::aarch64_neon_bfdot)
- .Default(Intrinsic::not_intrinsic);
- if (ID != Intrinsic::not_intrinsic) {
- size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
- assert((OperandWidth == 64 || OperandWidth == 128) &&
- "Unexpected operand width");
- LLVMContext &Ctx = F->getParent()->getContext();
- std::array<Type *, 2> Tys{
- {F->getReturnType(),
- FixedVectorType::get(Type::getBFloatTy(Ctx),
- OperandWidth / 16)}};
- NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
- return true;
- }
- break; // No other '(arm|aarch64).neon.bfdot.*'.
- }
-
- // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
- // anymore and accept v8bf16 instead of v16i8.
- if (Name.consume_front("bfm")) {
- // (arm|aarch64).neon.bfm*'.
- if (Name.consume_back(".v4f32.v16i8")) {
- // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
- Intrinsic::ID ID =
- StringSwitch<Intrinsic::ID>(Name)
- .Case("mla", Arm ? Intrinsic::arm_neon_bfmmla
- : Intrinsic::aarch64_neon_bfmmla)
- .Case("lalb", Arm ? Intrinsic::arm_neon_bfmlalb
- : Intrinsic::aarch64_neon_bfmlalb)
- .Case("lalt", Arm ? Intrinsic::arm_neon_bfmlalt
- : Intrinsic::aarch64_neon_bfmlalt)
- .Default(Intrinsic::not_intrinsic);
- if (ID != Intrinsic::not_intrinsic) {
- std::array<Type *, 0> Tys;
- NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
- return true;
- }
- break; // No other '(arm|aarch64).neon.bfm*.v16i8'.
- }
- break; // No other '(arm|aarch64).neon.bfm*.
- }
- // Continue on to Aarch64 Neon or Arm Neon.
- }
- // Continue on to Arm or Aarch64.
-
- if (Arm) {
- // 'arm.*'.
- if (Neon) {
- // 'arm.neon.*'.
- if (Name.consume_front("vclz.")) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
- F->arg_begin()->getType());
- return true;
- }
- if (Name.starts_with("vcnt")) {
- // 'arm.neon.vcnt*'.
- NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
- F->arg_begin()->getType());
- return true;
- }
-
- if (Name.consume_front("vst")) {
- // 'arm.neon.vst*'.
- static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
- SmallVector<StringRef, 2> Groups;
- if (vstRegex.match(Name, &Groups)) {
- static const Intrinsic::ID StoreInts[] = {
- Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
- Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
-
- static const Intrinsic::ID StoreLaneInts[] = {
- Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
- Intrinsic::arm_neon_vst4lane};
-
- auto fArgs = F->getFunctionType()->params();
- Type *Tys[] = {fArgs[0], fArgs[1]};
- if (Groups[1].size() == 1)
- NewFn = Intrinsic::getDeclaration(
- F->getParent(), StoreInts[fArgs.size() - 3], Tys);
- else
- NewFn = Intrinsic::getDeclaration(
- F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
- return true;
- }
- break; // No other 'arm.neon.vst*'.
- }
-
- if (Name.consume_front("vq")) {
- // 'arm.neon.vq*'.
- Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
- .StartsWith("adds.", Intrinsic::sadd_sat)
- .StartsWith("addu.", Intrinsic::uadd_sat)
- .StartsWith("subs.", Intrinsic::ssub_sat)
- .StartsWith("subu.", Intrinsic::usub_sat)
- .Default(Intrinsic::not_intrinsic);
- if (ID != Intrinsic::not_intrinsic) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
- F->arg_begin()->getType());
- return true;
- }
- break; // No other 'arm.neon.vq*'.
- }
- break; // No other 'arm.neon.*'.
- }
-
- if (Name.consume_front("mve.")) {
- // 'arm.mve.*'.
- if (Name == "vctp64") {
- if (cast<FixedVectorType>(F->getReturnType())->getNumElements() ==
- 4) {
- // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
- // the function and deal with it below in UpgradeIntrinsicCall.
- rename(F);
- return true;
- }
- break; // Not 'arm.mve.vctp64'.
- }
-
- // These too are changed to accept a v2i1 instead of the old v4i1.
- if (Name.consume_back(".v4i1")) {
- // 'arm.mve.*.v4i1'.
- if (Name.consume_back(".predicated.v2i64.v4i32")) {
- if (Name == "mull.int" || Name == "vqdmull")
- return true;
- break; // No other 'arm.mve.*.predicated.v2i64.v4i32.v4i1'.
- }
-
- if (Name.consume_back(".v2i64")) {
- if (Name.consume_front("vldr.gather.")) {
- // 'arm.mve.vldr.gather.*.v2i64.v4i1'.
- if (Name == "base.predicated.v2i64" ||
- Name == "base.wb.predicated.v2i64" ||
- Name == "offset.predicated.v2i64.p0i64" ||
- Name == "offset.predicated.v2i64.p0")
- return true;
- break; // No other 'arm.mve.vldr.gather.*.v2i64.v4i1'.
- }
- if (Name.consume_front("vstr.scatter.")) {
- // 'arm.mve.vstr.scatter.*.v2i64.v4i1'.
- if (Name == "base.predicated.v2i64" ||
- Name == "base.wb.predicated.v2i64" ||
- Name == "offset.predicated.p0i64.v2i64" ||
- Name == "offset.predicated.p0.v2i64")
- return true;
- break; // No other 'arm.mve.vstr.scatter.*.v2i64.v4i1'.
- }
- break; // No other 'arm.mve.*.v2i64.v4i1'.
- }
- break; // No other 'arm.mve.*.v4i1'.
- }
- break; // No other 'arm.mve.*'.
- }
-
- if (Name.consume_front("cde.vcx")) {
- // 'arm.cde.vcx*'.
- if (Name.consume_back(".predicated.v2i64.v4i1")) {
- // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
- if (Name == "1q" || Name == "1qa" || Name == "2q" ||
- Name == "2qa" || Name == "3q" || Name == "3qa")
- return true;
- break; // No other 'arm.cde.vcx*.predicated.v2i64.v4i1'.
- }
- break; // No other 'arm.cde.vcx*'.
- }
- } else {
- // 'aarch64.*'.
- if (Neon) {
- // 'aarch64.neon.*'.
- Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
- .StartsWith("frintn", Intrinsic::roundeven)
- .StartsWith("rbit", Intrinsic::bitreverse)
- .Default(Intrinsic::not_intrinsic);
- if (ID != Intrinsic::not_intrinsic) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), ID,
- F->arg_begin()->getType());
- return true;
- }
-
- if (Name.starts_with("addp")) {
- // 'aarch64.neon.addp*'.
- if (F->arg_size() != 2)
- break; // Invalid IR.
- VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
- if (Ty && Ty->getElementType()->isFloatingPointTy()) {
- NewFn = Intrinsic::getDeclaration(
- F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
- return true;
- }
- }
- break; // No other 'aarch64.neon.*'.
- }
- if (Name.consume_front("sve.")) {
- // 'aarch64.sve.*'.
- if (Name.consume_front("bf")) {
- if (Name.consume_back(".lane")) {
- // 'aarch64.sve.bf*.lane'.
- Intrinsic::ID ID =
- StringSwitch<Intrinsic::ID>(Name)
- .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
- .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
- .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
- .Default(Intrinsic::not_intrinsic);
- if (ID != Intrinsic::not_intrinsic) {
- NewFn = Intrinsic::getDeclaration(F->getParent(), ID);
- return true;
- }
- break; // No other 'aarch64.sve.bf*.lane'.
- }
- break; // No other 'aarch64.sve.bf*'.
- }
-
- if (Name.consume_front("ld")) {
- // 'aarch64.sve.ld*'.
- static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
- if (LdRegex.match(Name)) {
- Type *ScalarTy =
- dyn_cast<VectorType>(F->getReturnType())->getElementType();
- ElementCount EC = dyn_cast<VectorType>(F->arg_begin()->getType())
- ->getElementCount();
- Type *Ty = VectorType::get(ScalarTy, EC);
- static const Intrinsic::ID LoadIDs[] = {
- Intrinsic::aarch64_sve_ld2_sret,
- Intrinsic::aarch64_sve_ld3_sret,
- Intrinsic::aarch64_sve_ld4_sret,
- };
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- LoadIDs[Name[0] - '2'], Ty);
- return true;
- }
- break; // No other 'aarch64.sve.ld*'.
- }
-
- if (Name.consume_front("tuple.")) {
- // 'aarch64.sve.tuple.*'.
- if (Name.starts_with("get")) {
- // 'aarch64.sve.tuple.get*'.
- Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::vector_extract, Tys);
- return true;
- }
-
- if (Name.starts_with("set")) {
- // 'aarch64.sve.tuple.set*'.
- auto Args = F->getFunctionType()->params();
- Type *Tys[] = {Args[0], Args[2], Args[1]};
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::vector_insert, Tys);
- return true;
- }
-
- static const Regex CreateTupleRegex(
- "^create[234](.nxv[a-z0-9]+|$)");
- if (CreateTupleRegex.match(Name)) {
- // 'aarch64.sve.tuple.create*'.
- auto Args = F->getFunctionType()->params();
- Type *Tys[] = {F->getReturnType(), Args[1]};
- NewFn = Intrinsic::getDeclaration(F->getParent(),
- Intrinsic::vector_insert, Tys);
- return true;
- }
- break; // No other 'aarch64.sve.tuple.*'.
- }
- break; // No other 'aarch64.sve.*'.
- }
- }
- break; // No other 'arm.*', 'aarch64.*'.
- }
if (Name.consume_front("amdgcn.")) {
if (Name == "alignbit") {
>From b05493e2d5eaeae4e4379d1eafbee80f4d58d3bb Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan at acm.org>
Date: Fri, 15 Dec 2023 13:54:46 -0500
Subject: [PATCH 4/4] Return bool conditions
---
llvm/lib/IR/AutoUpgrade.cpp | 38 +++++++++++++------------------------
1 file changed, 13 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index cfa87c073f18b9..3a3b41fb786c2d 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -755,35 +755,25 @@ static bool UpgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
// These too are changed to accept a v2i1 instead of the old v4i1.
if (Name.consume_back(".v4i1")) {
// 'arm.mve.*.v4i1'.
- if (Name.consume_back(".predicated.v2i64.v4i32")) {
+ if (Name.consume_back(".predicated.v2i64.v4i32"))
// 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
- if (Name == "mull.int" || Name == "vqdmull")
- return true;
- return false; // No other 'arm.mve.*.predicated.v2i64.v4i32.v4i1'.
- }
+ return Name == "mull.int" || Name == "vqdmull";
if (Name.consume_back(".v2i64")) {
// 'arm.mve.*.v2i64.v4i1'
bool IsGather = Name.consume_front("vldr.gather.");
if (IsGather || Name.consume_front("vstr.scatter.")) {
if (Name.consume_front("base.")) {
+ // Optional 'wb.' prefix.
Name.consume_front("wb.");
- if (Name == "predicated.v2i64")
- // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
- // predicated.v2i64.v2i64.v4i1'.
- return true;
- // No other 'arm.(vldr.gather|vstr.scatter).base.*.v2i64.v4i1'
- return false;
+ // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
+ // predicated.v2i64.v2i64.v4i1'.
+ return Name == "predicated.v2i64";
}
- if (Name.consume_front("offset.predicated.")) {
- if (Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
- Name == (IsGather ? "v2i64.p0" : "p0.v2i64"))
- return true;
- // No other 'arm.(vldr.gather|vstr.scatter).offset
- // .predicated.*.v2i64.v4i1'
- return false;
- }
+ if (Name.consume_front("offset.predicated."))
+ return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
+ Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
// No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
return false;
@@ -798,13 +788,11 @@ static bool UpgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
if (Name.consume_front("cde.vcx")) {
// 'arm.cde.vcx*'.
- if (Name.consume_back(".predicated.v2i64.v4i1")) {
+ if (Name.consume_back(".predicated.v2i64.v4i1"))
// 'arm.cde.vcx*.predicated.v2i64.v4i1'.
- if (Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
- Name == "3q" || Name == "3qa")
- return true;
- return false; // No other 'arm.cde.vcx*.predicated.v2i64.v4i1'.
- }
+ return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
+ Name == "3q" || Name == "3qa";
+
return false; // No other 'arm.cde.vcx*'.
}
} else {
More information about the llvm-commits
mailing list