[llvm] [VPlan] Support struct return types for widen intrinsics (NFC). (PR #165218)
Shih-Po Hung via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 6 06:04:21 PST 2025
https://github.com/arcbbb updated https://github.com/llvm/llvm-project/pull/165218
>From 6a70859129f3f048b413f23e68a810f26429e26d Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Mon, 27 Oct 2025 02:01:42 -0700
Subject: [PATCH 1/2] [VPlan] Support struct return types for widen intrinsics.
NFC.
---
llvm/include/llvm/Analysis/VectorUtils.h | 5 ++++
llvm/lib/Analysis/VectorUtils.cpp | 14 ++++++++++
.../Transforms/Vectorize/LoopVectorize.cpp | 17 +++++++++++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 27 ++++++++++++++++---
4 files changed, 59 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index ce94906ee7c00..f0c064f5490a7 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -164,6 +164,11 @@ LLVM_ABI bool
isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx,
const TargetTransformInfo *TTI);
+/// Identifies if the vector form of the intrinsic that returns a struct has
+/// a scalar element at the struct element index \p RetIdx.
+LLVM_ABI bool isVectorIntrinsicWithStructReturnScalarAtField(Intrinsic::ID ID,
+ int RetIdx);
+
/// Identifies if the vector form of the intrinsic that returns a struct is
/// overloaded at the struct element index \p RetIdx. /// \p TTI is used to
/// consider target specific intrinsics, if no target specific intrinsics
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 091d94843698c..1f2fc80501cba 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -217,6 +217,16 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
}
}
+bool llvm::isVectorIntrinsicWithStructReturnScalarAtField(Intrinsic::ID ID,
+ int RetIdx) {
+ switch (ID) {
+ case Intrinsic::vp_load_ff:
+ return RetIdx == 1;
+ default:
+ return false;
+ }
+}
+
bool llvm::isVectorIntrinsicWithStructReturnOverloadAtField(
Intrinsic::ID ID, int RetIdx, const TargetTransformInfo *TTI) {
@@ -224,6 +234,10 @@ bool llvm::isVectorIntrinsicWithStructReturnOverloadAtField(
return TTI->isTargetIntrinsicWithStructReturnOverloadAtField(ID, RetIdx);
switch (ID) {
+ case Intrinsic::modf:
+ case Intrinsic::sincos:
+ case Intrinsic::sincospi:
+ return false;
case Intrinsic::frexp:
return RetIdx == 0 || RetIdx == 1;
default:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index facb0fabdf57e..93419100cea35 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4100,7 +4100,22 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
Type *ScalarTy = TypeInfo.inferScalarType(ToCheck);
if (!Visited.insert({ScalarTy}).second)
continue;
- Type *WideTy = toVectorizedTy(ScalarTy, VF);
+ Type *WideTy;
+ if (auto *WI = dyn_cast<VPWidenIntrinsicRecipe>(&R);
+ WI && ScalarTy->isStructTy()) {
+ auto *StructTy = cast<StructType>(ScalarTy);
+ SmallVector<Type *, 2> Tys;
+ for (unsigned I = 0, E = StructTy->getNumElements(); I != E; ++I) {
+ Type *ElementTy = StructTy->getStructElementType(I);
+ if (!isVectorIntrinsicWithStructReturnScalarAtField(
+ WI->getVectorIntrinsicID(), I))
+ ElementTy = toVectorizedTy(ElementTy, VF);
+ Tys.push_back(ElementTy);
+ }
+ WideTy = StructType::create(Tys);
+ } else
+ WideTy = toVectorizedTy(ScalarTy, VF);
+
if (any_of(getContainedTypes(WideTy), WillGenerateTargetVectors))
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 931a5b7582c4e..7cb15914fb2ce 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1738,7 +1738,16 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
SmallVector<Type *, 2> TysForDecl;
// Add return type if intrinsic is overloaded on it.
- if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1, State.TTI))
+ if (ResultTy->isStructTy()) {
+ auto *StructTy = cast<StructType>(ResultTy);
+ for (unsigned I = 0, E = StructTy->getNumElements(); I != E; ++I) {
+ if (isVectorIntrinsicWithStructReturnOverloadAtField(VectorIntrinsicID, I,
+ State.TTI))
+ TysForDecl.push_back(
+ toVectorizedTy(StructTy->getStructElementType(I), State.VF));
+ }
+ } else if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1,
+ State.TTI))
TysForDecl.push_back(VectorType::get(getResultType(), State.VF));
SmallVector<Value *, 4> Args;
for (const auto &I : enumerate(operands())) {
@@ -1802,8 +1811,20 @@ static InstructionCost getCostForIntrinsics(Intrinsic::ID ID,
Arguments.push_back(V);
}
- Type *ScalarRetTy = Ctx.Types.inferScalarType(&R);
- Type *RetTy = VF.isVector() ? toVectorizedTy(ScalarRetTy, VF) : ScalarRetTy;
+ Type *RetTy = Ctx.Types.inferScalarType(&R);
+
+ if (VF.isVector() && RetTy->isStructTy()) {
+ auto *StructTy = cast<StructType>(RetTy);
+ SmallVector<Type *> Tys;
+ for (unsigned I = 0, E = StructTy->getNumElements(); I != E; ++I) {
+ Type *ElementTy = StructTy->getStructElementType(I);
+ if (!isVectorIntrinsicWithStructReturnScalarAtField(ID, I))
+ ElementTy = toVectorizedTy(ElementTy, VF);
+ Tys.push_back(ElementTy);
+ }
+ RetTy = StructType::get(StructTy->getContext(), Tys);
+ } else if (VF.isVector())
+ RetTy = toVectorizedTy(RetTy, VF);
SmallVector<Type *> ParamTys;
for (const VPValue *Op : Operands) {
ParamTys.push_back(VF.isVector()
>From e2415bbd3510bf1f037d8a42b6ea9b3ce33c8e08 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Thu, 6 Nov 2025 05:56:00 -0800
Subject: [PATCH 2/2] Extend support in toVectorizedStructTy
---
llvm/include/llvm/IR/VectorTypeUtils.h | 10 +++++---
llvm/lib/IR/VectorTypeUtils.cpp | 24 ++++++++++++++-----
.../Transforms/Vectorize/LoopVectorize.cpp | 19 ++++-----------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 17 +++----------
4 files changed, 32 insertions(+), 38 deletions(-)
diff --git a/llvm/include/llvm/IR/VectorTypeUtils.h b/llvm/include/llvm/IR/VectorTypeUtils.h
index e3d7fadad6089..53fba9db09b1d 100644
--- a/llvm/include/llvm/IR/VectorTypeUtils.h
+++ b/llvm/include/llvm/IR/VectorTypeUtils.h
@@ -31,7 +31,11 @@ inline Type *toVectorTy(Type *Scalar, unsigned VF) {
/// Note:
/// - If \p EC is scalar, \p StructTy is returned unchanged
/// - Only unpacked literal struct types are supported
-LLVM_ABI Type *toVectorizedStructTy(StructType *StructTy, ElementCount EC);
+/// vector types.
+/// - If IID (Intrinsic::ID) is provided, only fields that are vector types
+/// are widened.
+LLVM_ABI Type *toVectorizedStructTy(StructType *StructTy, ElementCount EC,
+ unsigned IID = 0);
/// A helper for converting structs of vector types to structs of scalar types.
/// Note: Only unpacked literal struct types are supported.
@@ -52,9 +56,9 @@ LLVM_ABI bool canVectorizeStructTy(StructType *StructTy);
/// - If the incoming type is void, we return void
/// - If \p EC is scalar, \p Ty is returned unchanged
/// - Only unpacked literal struct types are supported
-inline Type *toVectorizedTy(Type *Ty, ElementCount EC) {
+inline Type *toVectorizedTy(Type *Ty, ElementCount EC, unsigned IID = 0) {
if (StructType *StructTy = dyn_cast<StructType>(Ty))
- return toVectorizedStructTy(StructTy, EC);
+ return toVectorizedStructTy(StructTy, EC, IID);
return toVectorTy(Ty, EC);
}
diff --git a/llvm/lib/IR/VectorTypeUtils.cpp b/llvm/lib/IR/VectorTypeUtils.cpp
index 62e39aab90079..744fe2a704a10 100644
--- a/llvm/lib/IR/VectorTypeUtils.cpp
+++ b/llvm/lib/IR/VectorTypeUtils.cpp
@@ -8,23 +8,35 @@
#include "llvm/IR/VectorTypeUtils.h"
#include "llvm/ADT/SmallVectorExtras.h"
+#include "llvm/Analysis/VectorUtils.h"
using namespace llvm;
/// A helper for converting structs of scalar types to structs of vector types.
/// Note: Only unpacked literal struct types are supported.
-Type *llvm::toVectorizedStructTy(StructType *StructTy, ElementCount EC) {
+Type *llvm::toVectorizedStructTy(StructType *StructTy, ElementCount EC,
+ unsigned IID) {
if (EC.isScalar())
return StructTy;
assert(isUnpackedStructLiteral(StructTy) &&
"expected unpacked struct literal");
assert(all_of(StructTy->elements(), VectorType::isValidElementType) &&
"expected all element types to be valid vector element types");
- return StructType::get(
- StructTy->getContext(),
- map_to_vector(StructTy->elements(), [&](Type *ElTy) -> Type * {
- return VectorType::get(ElTy, EC);
- }));
+ if (IID != 0) {
+ SmallVector<Type *, 2> Tys;
+ for (unsigned I = 0, E = StructTy->getNumElements(); I != E; ++I) {
+ Type *ElTy = StructTy->getStructElementType(I);
+ if (!isVectorIntrinsicWithStructReturnScalarAtField(IID, I))
+ ElTy = toVectorizedTy(ElTy, EC);
+ Tys.push_back(ElTy);
+ }
+ return StructType::create(Tys);
+ } else
+ return StructType::get(
+ StructTy->getContext(),
+ map_to_vector(StructTy->elements(), [&](Type *ElTy) -> Type * {
+ return VectorType::get(ElTy, EC);
+ }));
}
/// A helper for converting structs of vector types to structs of scalar types.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 93419100cea35..1f1ed8eb3d60d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4100,21 +4100,10 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
Type *ScalarTy = TypeInfo.inferScalarType(ToCheck);
if (!Visited.insert({ScalarTy}).second)
continue;
- Type *WideTy;
- if (auto *WI = dyn_cast<VPWidenIntrinsicRecipe>(&R);
- WI && ScalarTy->isStructTy()) {
- auto *StructTy = cast<StructType>(ScalarTy);
- SmallVector<Type *, 2> Tys;
- for (unsigned I = 0, E = StructTy->getNumElements(); I != E; ++I) {
- Type *ElementTy = StructTy->getStructElementType(I);
- if (!isVectorIntrinsicWithStructReturnScalarAtField(
- WI->getVectorIntrinsicID(), I))
- ElementTy = toVectorizedTy(ElementTy, VF);
- Tys.push_back(ElementTy);
- }
- WideTy = StructType::create(Tys);
- } else
- WideTy = toVectorizedTy(ScalarTy, VF);
+ unsigned IID = 0;
+ if (auto *WI = dyn_cast<VPWidenIntrinsicRecipe>(&R))
+ WI->getVectorIntrinsicID();
+ Type *WideTy = toVectorizedTy(ScalarTy, VF, IID);
if (any_of(getContainedTypes(WideTy), WillGenerateTargetVectors))
return true;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 7cb15914fb2ce..ba7a832f70874 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1811,20 +1811,9 @@ static InstructionCost getCostForIntrinsics(Intrinsic::ID ID,
Arguments.push_back(V);
}
- Type *RetTy = Ctx.Types.inferScalarType(&R);
-
- if (VF.isVector() && RetTy->isStructTy()) {
- auto *StructTy = cast<StructType>(RetTy);
- SmallVector<Type *> Tys;
- for (unsigned I = 0, E = StructTy->getNumElements(); I != E; ++I) {
- Type *ElementTy = StructTy->getStructElementType(I);
- if (!isVectorIntrinsicWithStructReturnScalarAtField(ID, I))
- ElementTy = toVectorizedTy(ElementTy, VF);
- Tys.push_back(ElementTy);
- }
- RetTy = StructType::get(StructTy->getContext(), Tys);
- } else if (VF.isVector())
- RetTy = toVectorizedTy(RetTy, VF);
+ Type *ScalarRetTy = Ctx.Types.inferScalarType(&R);
+ Type *RetTy =
+ VF.isVector() ? toVectorizedTy(ScalarRetTy, VF, ID) : ScalarRetTy;
SmallVector<Type *> ParamTys;
for (const VPValue *Op : Operands) {
ParamTys.push_back(VF.isVector()
More information about the llvm-commits
mailing list