[llvm] [LV] Add initial support for vectorizing literal struct return values (PR #109833)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 9 16:27:11 PST 2025
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/109833
>From 6e45274da4778c8b947ad6330eaeacb6342252f1 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 20 Sep 2024 13:52:48 +0000
Subject: [PATCH 01/10] [LV] Add initial support for vectorizing literal struct
return values
This patch adds initial support for vectorizing literal struct return
values. Currently, this is limited to the case where the struct is
homogeneous (all elements have the same type) and not packed. The users
of the call also must all be `extractvalue` instructions.
The intended use case for this is vectorizing intrinsics such as:
```
declare { float, float } @llvm.sincos.f32(float %x)
```
Mapping them to structure-returning library calls such as:
```
declare { <4 x float>, <4 x i32> } @Sleef_sincosf4_u10advsimd(<4 x float>)
```
Or their widened form (such as `@llvm.sincos.v4f32` in this case).
Implementing this required two main changes:
1. Supporting widening `extractvalue`
2. Adding support for vectorized struct types in LV
* This is mostly limited to parts of the cost model and scalarization
Since the supported use case is narrow, the required changes are
relatively small.
---
.../Vectorize/LoopVectorizationLegality.h | 10 --
.../Vectorize/LoopVectorizationLegality.cpp | 13 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 76 ++++++-----
llvm/lib/Transforms/Vectorize/VPlan.cpp | 27 ++--
.../Transforms/Vectorize/VPlanAnalysis.cpp | 7 +
llvm/lib/Transforms/Vectorize/VPlanHelpers.h | 7 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 14 +-
.../AArch64/scalable-struct-return.ll | 34 +++--
.../Transforms/LoopVectorize/struct-return.ll | 73 ++++++++---
.../vplan-widen-struct-return.ll | 122 ++++++++++++++++++
10 files changed, 291 insertions(+), 92 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/vplan-widen-struct-return.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 3c5cf1ebe6ba2ed..e959d93b57275f5 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -416,10 +416,6 @@ class LoopVectorizationLegality {
/// has a vectorized variant available.
bool hasVectorCallVariants() const { return VecCallVariantsFound; }
- /// Returns true if there is at least one function call in the loop which
- /// returns a struct type and needs to be vectorized.
- bool hasStructVectorCall() const { return StructVecCallFound; }
-
unsigned getNumStores() const { return LAI->getNumStores(); }
unsigned getNumLoads() const { return LAI->getNumLoads(); }
@@ -639,12 +635,6 @@ class LoopVectorizationLegality {
/// the use of those function variants.
bool VecCallVariantsFound = false;
- /// If we find a call (to be vectorized) that returns a struct type, record
- /// that so we can bail out until this is supported.
- /// TODO: Remove this flag once vectorizing calls with struct returns is
- /// supported.
- bool StructVecCallFound = false;
-
/// Keep track of all the countable and uncountable exiting blocks if
/// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index e3599315e224f3d..420cbc5384ce470 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -954,7 +954,7 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
if (CI && !VFDatabase::getMappings(*CI).empty())
VecCallVariantsFound = true;
- auto CanWidenInstructionTy = [this](Instruction const &Inst) {
+ auto CanWidenInstructionTy = [](Instruction const &Inst) {
Type *InstTy = Inst.getType();
if (!isa<StructType>(InstTy))
return canVectorizeTy(InstTy);
@@ -962,15 +962,8 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
// For now, we only recognize struct values returned from calls where
// all users are extractvalue as vectorizable. All element types of the
// struct must be types that can be widened.
- if (isa<CallInst>(Inst) && canWidenCallReturnType(InstTy) &&
- all_of(Inst.users(), IsaPred<ExtractValueInst>)) {
- // TODO: Remove the `StructVecCallFound` flag once vectorizing calls
- // with struct returns is supported.
- StructVecCallFound = true;
- return true;
- }
-
- return false;
+ return isa<CallInst>(Inst) && canWidenCallReturnType(InstTy) &&
+ all_of(Inst.users(), IsaPred<ExtractValueInst>);
};
// Check that the instruction return type is vectorizable.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 660a6ef574576b0..49feea4066e3fa0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2389,7 +2389,9 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
VPReplicateRecipe *RepRecipe,
const VPLane &Lane,
VPTransformState &State) {
- assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");
+ assert((!Instr->getType()->isAggregateType() ||
+ canVectorizeTy(Instr->getType())) &&
+ "Expected vectorizable or non-aggregate type.");
// Does this instruction return a value ?
bool IsVoidRetTy = Instr->getType()->isVoidTy();
@@ -2894,10 +2896,10 @@ LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
return ScalarCallCost;
}
-static Type *maybeVectorizeType(Type *Elt, ElementCount VF) {
- if (VF.isScalar() || (!Elt->isIntOrPtrTy() && !Elt->isFloatingPointTy()))
- return Elt;
- return VectorType::get(Elt, VF);
+static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
+ if (VF.isScalar() || !canVectorizeTy(Ty))
+ return Ty;
+ return toVectorizedTy(Ty, VF);
}
InstructionCost
@@ -3646,9 +3648,8 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
// ExtractValue instructions must be uniform, because the operands are
// known to be loop-invariant.
- if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
- assert(IsOutOfScope(EVI->getAggregateOperand()) &&
- "Expected aggregate value to be loop invariant");
+ auto *EVI = dyn_cast<ExtractValueInst>(&I);
+ if (EVI && IsOutOfScope(EVI->getAggregateOperand())) {
AddToWorklistIfAllowed(EVI);
continue;
}
@@ -4513,8 +4514,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
llvm_unreachable("unhandled recipe");
}
- auto WillWiden = [&TTI, VF](Type *ScalarTy) {
- Type *VectorTy = toVectorTy(ScalarTy, VF);
+ auto WillWiden = [&TTI, VF](Type *VectorTy) {
unsigned NumLegalParts = TTI.getNumberOfParts(VectorTy);
if (!NumLegalParts)
return false;
@@ -4545,7 +4545,8 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
Type *ScalarTy = TypeInfo.inferScalarType(ToCheck);
if (!Visited.insert({ScalarTy}).second)
continue;
- if (WillWiden(ScalarTy))
+ Type *WideTy = toVectorizedTy(ScalarTy, VF);
+ if (any_of(getContainedTypes(WideTy), WillWiden))
return true;
}
}
@@ -5503,10 +5504,13 @@ InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
// Compute the scalarization overhead of needed insertelement instructions
// and phi nodes.
if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
- ScalarCost += TTI.getScalarizationOverhead(
- cast<VectorType>(toVectorTy(I->getType(), VF)),
- APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ true,
- /*Extract*/ false, CostKind);
+ Type *WideTy = toVectorizedTy(I->getType(), VF);
+ for (Type *VectorTy : getContainedTypes(WideTy)) {
+ ScalarCost += TTI.getScalarizationOverhead(
+ cast<VectorType>(VectorTy), APInt::getAllOnes(VF.getFixedValue()),
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind);
+ }
ScalarCost +=
VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
}
@@ -6004,13 +6008,17 @@ LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
return 0;
InstructionCost Cost = 0;
- Type *RetTy = toVectorTy(I->getType(), VF);
+ Type *RetTy = toVectorizedTy(I->getType(), VF);
if (!RetTy->isVoidTy() &&
- (!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore()))
- Cost += TTI.getScalarizationOverhead(
- cast<VectorType>(RetTy), APInt::getAllOnes(VF.getKnownMinValue()),
- /*Insert*/ true,
- /*Extract*/ false, CostKind);
+ (!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore())) {
+
+ for (Type *VectorTy : getContainedTypes(RetTy)) {
+ Cost += TTI.getScalarizationOverhead(
+ cast<VectorType>(VectorTy), APInt::getAllOnes(VF.getKnownMinValue()),
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind);
+ }
+ }
// Some targets keep addresses scalar.
if (isa<LoadInst>(I) && !TTI.prefersVectorizedAddressing())
@@ -6268,9 +6276,9 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) {
bool MaskRequired = Legal->isMaskRequired(CI);
// Compute corresponding vector type for return value and arguments.
- Type *RetTy = toVectorTy(ScalarRetTy, VF);
+ Type *RetTy = toVectorizedTy(ScalarRetTy, VF);
for (Type *ScalarTy : ScalarTys)
- Tys.push_back(toVectorTy(ScalarTy, VF));
+ Tys.push_back(toVectorizedTy(ScalarTy, VF));
// An in-loop reduction using an fmuladd intrinsic is a special case;
// we don't want the normal cost for that intrinsic.
@@ -6460,7 +6468,7 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
HasSingleCopyAfterVectorization(I, VF));
VectorTy = RetTy;
} else
- VectorTy = toVectorTy(RetTy, VF);
+ VectorTy = toVectorizedTy(RetTy, VF);
if (VF.isVector() && VectorTy->isVectorTy() &&
!TTI.getNumberOfParts(VectorTy))
@@ -8582,7 +8590,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
case Instruction::Shl:
case Instruction::Sub:
case Instruction::Xor:
- case Instruction::Freeze:
+ case Instruction::Freeze: {
SmallVector<VPValue *> NewOps(Operands);
if (Instruction::isBinaryOp(I->getOpcode())) {
// The legacy cost model uses SCEV to check if some of the operands are
@@ -8607,6 +8615,15 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
+ }
+ case Instruction::ExtractValue: {
+ SmallVector<VPValue *> NewOps(Operands);
+ Type *I32Ty = IntegerType::getInt32Ty(I->getContext());
+ for (unsigned Idx : cast<ExtractValueInst>(I)->getIndices())
+ NewOps.push_back(
+ Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
+ return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
+ }
};
}
@@ -9888,7 +9905,7 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
VectorType::get(UI->getType(), State.VF));
State.set(this, Poison);
}
- State.packScalarIntoVectorValue(this, *State.Lane);
+ State.packScalarIntoVectorizedValue(this, *State.Lane);
}
return;
}
@@ -10405,13 +10422,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
return false;
}
- if (LVL.hasStructVectorCall()) {
- reportVectorizationFailure("Auto-vectorization of calls that return struct "
- "types is not yet supported",
- "StructCallVectorizationUnsupported", ORE, L);
- return false;
- }
-
// Entrance to the VPlan-native vectorization path. Outer loops are processed
// here. They may require CFG and instruction level transformations before
// even evaluating whether vectorization is profitable. Since we cannot modify
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 5a88ebeffb18ba1..c52cbca27de3ba0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -336,10 +336,10 @@ Value *VPTransformState::get(VPValue *Def, bool NeedsScalar) {
} else {
// Initialize packing with insertelements to start from undef.
assert(!VF.isScalable() && "VF is assumed to be non scalable.");
- Value *Undef = PoisonValue::get(VectorType::get(LastInst->getType(), VF));
+ Value *Undef = PoisonValue::get(toVectorizedTy(LastInst->getType(), VF));
set(Def, Undef);
for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
- packScalarIntoVectorValue(Def, Lane);
+ packScalarIntoVectorizedValue(Def, Lane);
VectorValue = get(Def);
}
Builder.restoreIP(OldIP);
@@ -392,13 +392,24 @@ void VPTransformState::setDebugLocFrom(DebugLoc DL) {
Builder.SetCurrentDebugLocation(DIL);
}
-void VPTransformState::packScalarIntoVectorValue(VPValue *Def,
- const VPLane &Lane) {
+void VPTransformState::packScalarIntoVectorizedValue(VPValue *Def,
+ const VPLane &Lane) {
Value *ScalarInst = get(Def, Lane);
- Value *VectorValue = get(Def);
- VectorValue = Builder.CreateInsertElement(VectorValue, ScalarInst,
- Lane.getAsRuntimeExpr(Builder, VF));
- set(Def, VectorValue);
+ Value *WideValue = get(Def);
+ Value *LaneExpr = Lane.getAsRuntimeExpr(Builder, VF);
+ if (auto *StructTy = dyn_cast<StructType>(WideValue->getType())) {
+ // We must handle each element of a vectorized struct type.
+ for (unsigned I = 0, E = StructTy->getNumElements(); I != E; I++) {
+ Value *ScalarValue = Builder.CreateExtractValue(ScalarInst, I);
+ Value *VectorValue = Builder.CreateExtractValue(WideValue, I);
+ VectorValue =
+ Builder.CreateInsertElement(VectorValue, ScalarValue, LaneExpr);
+ WideValue = Builder.CreateInsertValue(WideValue, VectorValue, I);
+ }
+ } else {
+ WideValue = Builder.CreateInsertElement(WideValue, ScalarInst, LaneExpr);
+ }
+ set(Def, WideValue);
}
BasicBlock *VPBasicBlock::createEmptyBasicBlock(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 71fb6d42116cfe4..d7bc949d6a85bdd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -125,6 +125,13 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenRecipe *R) {
case Instruction::FNeg:
case Instruction::Freeze:
return inferScalarType(R->getOperand(0));
+ case Instruction::ExtractValue: {
+ assert(R->getNumOperands() == 2 && "expected single level extractvalue");
+ auto *StructTy = cast<StructType>(inferScalarType(R->getOperand(0)));
+ auto *CI = cast<ConstantInt>(R->getOperand(1)->getLiveInIRValue());
+ unsigned Idx = CI->getZExtValue();
+ return StructTy->getTypeAtIndex(Idx);
+ }
default:
break;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 74713daf904f047..cd1ad9bec91f762 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -241,7 +241,7 @@ struct VPTransformState {
set(Def, V, VPLane(0));
return;
}
- assert((VF.isScalar() || V->getType()->isVectorTy()) &&
+ assert((VF.isScalar() || isVectorizedTy(V->getType())) &&
"scalar values must be stored as (0, 0)");
Data.VPV2Vector[Def] = V;
}
@@ -290,8 +290,9 @@ struct VPTransformState {
/// Set the debug location in the builder using the debug location \p DL.
void setDebugLocFrom(DebugLoc DL);
- /// Construct the vector value of a scalarized value \p V one lane at a time.
- void packScalarIntoVectorValue(VPValue *Def, const VPLane &Lane);
+ /// Construct the vectorized value of a scalarized value \p V one lane at a
+ /// time.
+ void packScalarIntoVectorizedValue(VPValue *Def, const VPLane &Lane);
/// Hold state information used when constructing the CFG of the output IR,
/// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c84a93d7398f73b..cc9ae05c40348f0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1129,7 +1129,7 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
Arguments.push_back(V);
}
- Type *RetTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);
+ Type *RetTy = toVectorizedTy(Ctx.Types.inferScalarType(this), VF);
SmallVector<Type *> ParamTys;
for (unsigned I = 0; I != getNumOperands(); ++I)
ParamTys.push_back(
@@ -1435,6 +1435,15 @@ void VPWidenRecipe::execute(VPTransformState &State) {
State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
break;
}
+ case Instruction::ExtractValue: {
+ assert(getNumOperands() == 2 && "expected single level extractvalue");
+ Value *Op = State.get(getOperand(0));
+ auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
+ unsigned Idx = CI->getZExtValue();
+ Value *Extract = Builder.CreateExtractValue(Op, Idx);
+ State.set(this, Extract);
+ break;
+ }
case Instruction::Freeze: {
Value *Op = State.get(getOperand(0));
@@ -1536,6 +1545,9 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
Ctx.CostKind);
}
+ case Instruction::ExtractValue:
+ return Ctx.TTI.getInstructionCost(cast<Instruction>(getUnderlyingValue()),
+ TTI::TCK_RecipThroughput);
case Instruction::ICmp:
case Instruction::FCmp: {
Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
index 77781f95b0858ea..2fde624624ee9a2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
@@ -1,15 +1,18 @@
-; RUN: opt < %s -mattr=+sve -passes=loop-vectorize -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -pass-remarks-analysis=loop-vectorize 2>%t | FileCheck %s
-; RUN: cat %t | FileCheck --check-prefix=CHECK-REMARKS %s
+; RUN: opt < %s -mattr=+sve -passes=loop-vectorize -force-vector-interleave=1 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
; Tests basic vectorization of scalable homogeneous struct literal returns.
-; TODO: Support vectorization in this case.
-; CHECK-REMARKS: remark: {{.*}} loop not vectorized: Auto-vectorization of calls that return struct types is not yet supported
define void @struct_return_f32_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_f32_widen
-; CHECK-NOT: vector.body:
+; CHECK-SAME: (ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]])
+; CHECK: vector.body:
+; CHECK: [[WIDE_CALL:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @scalable_vec_masked_foo(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; CHECK: [[WIDE_A:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[WIDE_CALL]], 0
+; CHECK: [[WIDE_B:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[WIDE_CALL]], 1
+; CHECK: call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[WIDE_A]], ptr {{%.*}}, i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK: call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[WIDE_B]], ptr {{%.*}}, i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
entry:
br label %for.body
@@ -32,11 +35,15 @@ exit:
ret void
}
-; TODO: Support vectorization in this case.
-; CHECK-REMARKS: remark: {{.*}} loop not vectorized: Auto-vectorization of calls that return struct types is not yet supported
define void @struct_return_f64_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_f64_widen
-; CHECK-NOT: vector.body:
+; CHECK-SAME: (ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]])
+; CHECK: vector.body:
+; CHECK: [[WIDE_CALL:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @scalable_vec_masked_bar(<vscale x 2 x double> [[WIDE_MASKED_LOAD:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; CHECK: [[WIDE_A:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[WIDE_CALL]], 0
+; CHECK: [[WIDE_B:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[WIDE_CALL]], 1
+; CHECK: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[WIDE_A]], ptr {{%.*}}, i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK: call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[WIDE_B]], ptr {{%.*}}, i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
entry:
br label %for.body
@@ -59,11 +66,16 @@ exit:
ret void
}
-; TODO: Support vectorization in this case.
-; CHECK-REMARKS: remark: {{.*}} loop not vectorized: Auto-vectorization of calls that return struct types is not yet supported
define void @struct_return_f32_widen_rt_checks(ptr %in, ptr writeonly %out_a, ptr writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_f32_widen_rt_checks
-; CHECK-NOT: vector.body:
+; CHECK-SAME: (ptr [[IN:%.*]], ptr writeonly [[OUT_A:%.*]], ptr writeonly [[OUT_B:%.*]])
+; CHECK: entry:
+; CHECK: br i1 false, label %scalar.ph, label %vector.memcheck
+; CHECK: vector.memcheck:
+; CHECK: vector.body:
+; CHECK: call { <vscale x 4 x float>, <vscale x 4 x float> } @scalable_vec_masked_foo(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; CHECK: for.body:
+; CHECK: call { float, float } @foo(float [[LOAD:%.*]])
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return.ll b/llvm/test/Transforms/LoopVectorize/struct-return.ll
index 9f98e8af2e98c58..d8336e231cd4526 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return.ll
@@ -1,15 +1,20 @@
-; RUN: opt < %s -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S -pass-remarks-analysis=loop-vectorize 2>%t | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S -pass-remarks=loop-vectorize -pass-remarks-analysis=loop-vectorize 2>%t | FileCheck %s
; RUN: cat %t | FileCheck --check-prefix=CHECK-REMARKS %s
target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
; Tests basic vectorization of homogeneous struct literal returns.
-; TODO: Support vectorization in this case.
-; CHECK-REMARKS: remark: {{.*}} loop not vectorized: Auto-vectorization of calls that return struct types is not yet supported
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
define void @struct_return_f32_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_f32_widen
-; CHECK-NOT: vector.body:
+; CHECK-SAME: (ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]])
+; CHECK: vector.body:
+; CHECK: [[WIDE_CALL:%.*]] = call { <2 x float>, <2 x float> } @fixed_vec_foo(<2 x float> [[WIDE_LOAD:%.*]])
+; CHECK: [[WIDE_A:%.*]] = extractvalue { <2 x float>, <2 x float> } [[WIDE_CALL]], 0
+; CHECK: [[WIDE_B:%.*]] = extractvalue { <2 x float>, <2 x float> } [[WIDE_CALL]], 1
+; CHECK: store <2 x float> [[WIDE_A]], ptr {{%.*}}, align 4
+; CHECK: store <2 x float> [[WIDE_B]], ptr {{%.*}}, align 4
entry:
br label %for.body
@@ -32,11 +37,16 @@ exit:
ret void
}
-; TODO: Support vectorization in this case.
-; CHECK-REMARKS: remark: {{.*}} loop not vectorized: Auto-vectorization of calls that return struct types is not yet supported
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
define void @struct_return_f64_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_f64_widen
-; CHECK-NOT: vector.body:
+; CHECK-SAME: (ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]])
+; CHECK: vector.body:
+; CHECK: [[WIDE_CALL:%.*]] = call { <2 x double>, <2 x double> } @fixed_vec_bar(<2 x double> [[WIDE_LOAD:%.*]])
+; CHECK: [[WIDE_A:%.*]] = extractvalue { <2 x double>, <2 x double> } [[WIDE_CALL]], 0
+; CHECK: [[WIDE_B:%.*]] = extractvalue { <2 x double>, <2 x double> } [[WIDE_CALL]], 1
+; CHECK: store <2 x double> [[WIDE_A]], ptr {{%.*}}, align 8
+; CHECK: store <2 x double> [[WIDE_B]], ptr {{%.*}}, align 8
entry:
br label %for.body
@@ -59,11 +69,36 @@ exit:
ret void
}
-; TODO: Support vectorization in this case.
-; CHECK-REMARKS: remark: {{.*}} loop not vectorized: Auto-vectorization of calls that return struct types is not yet supported
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
+; Note: Later instcombines reduce this down quite a lot.
define void @struct_return_f32_replicate(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_f32_replicate
-; CHECK-NOT: vector.body:
+; CHECK-SAME: (ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]])
+; CHECK: vector.body:
+; CHECK: [[CALL_LANE_0:%.*]] = tail call { float, float } @foo(float {{%.*}})
+; CHECK: [[CALL_LANE_1:%.*]] = tail call { float, float } @foo(float {{%.*}})
+; // Lane 0
+; CHECK: [[A_0:%.*]] = extractvalue { float, float } [[CALL_LANE_0]], 0
+; CHECK: [[VEC_A_0:%.*]] = insertelement <2 x float> poison, float [[A_0]], i32 0
+; CHECK: [[WIDE_A_0:%.*]] = insertvalue { <2 x float>, <2 x float> } poison, <2 x float> [[VEC_A_0]], 0
+; CHECK: [[B_0:%.*]] = extractvalue { float, float } [[CALL_LANE_0]], 1
+; CHECK: [[UNDEF_B_0:%.*]] = extractvalue { <2 x float>, <2 x float> } [[WIDE_A_0]], 1
+; CHECK: [[VEC_B_0:%.*]] = insertelement <2 x float> [[UNDEF_B_0]], float [[B_0]], i32 0
+; CHECK: [[WIDE_0:%.*]] = insertvalue { <2 x float>, <2 x float> } [[WIDE_A_0]], <2 x float> [[VEC_B_0]], 1
+; // Lane 1
+; CHECK: [[A_1:%.*]] = extractvalue { float, float } [[CALL_LANE_1]], 0
+; CHECK: [[VEC_A_0_EXT:%.*]] = extractvalue { <2 x float>, <2 x float> } [[WIDE_0]], 0
+; CHECK: [[VEC_A:%.*]] = insertelement <2 x float> [[VEC_A_0_EXT]], float [[A_1]], i32 1
+; CHECK: [[WIDE_A:%.*]] = insertvalue { <2 x float>, <2 x float> } [[WIDE_0]], <2 x float> [[VEC_A]], 0
+; CHECK: [[B_1:%.*]] = extractvalue { float, float } [[CALL_LANE_1]], 1
+; CHECK: [[VEC_B_0_EXT:%.*]] = extractvalue { <2 x float>, <2 x float> } [[WIDE_A]], 1
+; CHECK: [[VEC_B:%.*]] = insertelement <2 x float> [[VEC_B_0_EXT]], float [[B_1]], i32 1
+; CHECK: [[WIDE:%.*]] = insertvalue { <2 x float>, <2 x float> } [[WIDE_A]], <2 x float> [[VEC_B]], 1
+; // Store wide values:
+; CHECK: [[VEC_A_EXT:%.*]] = extractvalue { <2 x float>, <2 x float> } [[WIDE]], 0
+; CHECK: [[VEC_B_EXT:%.*]] = extractvalue { <2 x float>, <2 x float> } [[WIDE]], 1
+; CHECK: store <2 x float> [[VEC_A_EXT]], ptr {{%.*}}, align 4
+; CHECK: store <2 x float> [[VEC_B_EXT]], ptr {{%.*}}, align 4
entry:
br label %for.body
@@ -87,11 +122,17 @@ exit:
ret void
}
-; TODO: Support vectorization in this case.
-; CHECK-REMARKS: remark: {{.*}} loop not vectorized: Auto-vectorization of calls that return struct types is not yet supported
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
define void @struct_return_f32_widen_rt_checks(ptr %in, ptr writeonly %out_a, ptr writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_f32_widen_rt_checks
-; CHECK-NOT: vector.body:
+; CHECK-SAME: (ptr [[IN:%.*]], ptr writeonly [[OUT_A:%.*]], ptr writeonly [[OUT_B:%.*]])
+; CHECK: entry:
+; CHECK: br i1 false, label %scalar.ph, label %vector.memcheck
+; CHECK: vector.memcheck:
+; CHECK: vector.body:
+; CHECK: call { <2 x float>, <2 x float> } @fixed_vec_foo(<2 x float> [[WIDE_LOAD:%.*]])
+; CHECK: for.body:
+; CHECK call { float, float } @foo(float [[LOAD:%.*]])
entry:
br label %for.body
@@ -143,11 +184,11 @@ exit:
ret void
}
-; TODO: Support vectorization in this case.
-; CHECK-REMARKS: remark: {{.*}} loop not vectorized: Auto-vectorization of calls that return struct types is not yet supported
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
define void @struct_return_i32_three_results_widen(ptr noalias %in, ptr noalias writeonly %out_a) {
; CHECK-LABEL: define void @struct_return_i32_three_results_widen
-; CHECK-NOT: vector.body:
+; CHECK: vector.body:
+; CHECK: call { <2 x i32>, <2 x i32>, <2 x i32> } @fixed_vec_qux(<2 x i32> [[WIDE_LOAD:%.*]])
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-struct-return.ll b/llvm/test/Transforms/LoopVectorize/vplan-widen-struct-return.ll
new file mode 100644
index 000000000000000..17a707df8956d3a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan-widen-struct-return.ll
@@ -0,0 +1,122 @@
+; REQUIRES: asserts
+; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output -S 2>&1 | FileCheck %s
+
+define void @struct_return_f32_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: LV: Checking a loop in 'struct_return_f32_widen'
+; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
+; CHECK-NEXT: Live-in vp<%0> = VF * UF
+; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%in>, vp<%3>
+; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%in_val> = load vp<%4>
+; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%in_val>) (using library function: fixed_vec_foo)
+; CHECK-NEXT: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-NEXT: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%out_a>, vp<%3>
+; CHECK-NEXT: vp<%5> = vector-pointer ir<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<%5>, ir<%extract_a>
+; CHECK-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%out_b>, vp<%3>
+; CHECK-NEXT: vp<%6> = vector-pointer ir<%arrayidx4>
+; CHECK-NEXT: WIDEN store vp<%6>, ir<%extract_b>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
+ %in_val = load float, ptr %arrayidx, align 4
+ %call = tail call { float, float } @foo(float %in_val) #0
+ %extract_a = extractvalue { float, float } %call, 0
+ %extract_b = extractvalue { float, float } %call, 1
+ %arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
+ store float %extract_a, ptr %arrayidx2, align 4
+ %arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
+ store float %extract_b, ptr %arrayidx4, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @struct_return_f32_replicate(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: LV: Checking a loop in 'struct_return_f32_replicate'
+; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
+; CHECK-NEXT: Live-in vp<%0> = VF * UF
+; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in ir<1024> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%in>, vp<%3>
+; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%in_val> = load vp<%4>
+; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-NEXT: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-NEXT: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%out_a>, vp<%3>
+; CHECK-NEXT: vp<%5> = vector-pointer ir<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<%5>, ir<%extract_a>
+; CHECK-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%out_b>, vp<%3>
+; CHECK-NEXT: vp<%6> = vector-pointer ir<%arrayidx4>
+; CHECK-NEXT: WIDEN store vp<%6>, ir<%extract_b>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
+ %in_val = load float, ptr %arrayidx, align 4
+ ; #3 does not have a fixed-size vector mapping (so replication is used)
+ %call = tail call { float, float } @foo(float %in_val) #1
+ %extract_a = extractvalue { float, float } %call, 0
+ %extract_b = extractvalue { float, float } %call, 1
+ %arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
+ store float %extract_a, ptr %arrayidx2, align 4
+ %arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
+ store float %extract_b, ptr %arrayidx4, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+
+declare { float, float } @foo(float)
+
+declare { <2 x float>, <2 x float> } @fixed_vec_foo(<2 x float>)
+declare { <vscale x 4 x float>, <vscale x 4 x float> } @scalable_vec_masked_foo(<vscale x 4 x float>, <vscale x 4 x i1>)
+
+attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_foo(fixed_vec_foo)" }
+attributes #1 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(scalable_vec_masked_foo)" }
>From 178b2c334a222e4b0b0df0172c62a2500a9bfaf1 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 15 Jan 2025 13:06:34 +0000
Subject: [PATCH 02/10] Fixups
---
.../Transforms/Vectorize/LoopVectorize.cpp | 11 +-
.../Transforms/Vectorize/VPlanAnalysis.cpp | 3 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 +-
.../AArch64/struct-return-cost.ll | 133 ++++++++++++++++++
4 files changed, 142 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 49feea4066e3fa0..298a9d3b98e8cbb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3648,10 +3648,13 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
// ExtractValue instructions must be uniform, because the operands are
// known to be loop-invariant.
- auto *EVI = dyn_cast<ExtractValueInst>(&I);
- if (EVI && IsOutOfScope(EVI->getAggregateOperand())) {
- AddToWorklistIfAllowed(EVI);
- continue;
+ if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
+ if (IsOutOfScope(EVI->getAggregateOperand())) {
+ AddToWorklistIfAllowed(EVI);
+ continue;
+ }
+ assert(isa<CallInst>(EVI->getAggregateOperand()) &&
+ "Expected aggregate value to be call return value");
}
// If there's no pointer operand, there's nothing to do.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index d7bc949d6a85bdd..bf61251fc913396 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -129,8 +129,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPWidenRecipe *R) {
assert(R->getNumOperands() == 2 && "expected single level extractvalue");
auto *StructTy = cast<StructType>(inferScalarType(R->getOperand(0)));
auto *CI = cast<ConstantInt>(R->getOperand(1)->getLiveInIRValue());
- unsigned Idx = CI->getZExtValue();
- return StructTy->getTypeAtIndex(Idx);
+ return StructTy->getTypeAtIndex(CI->getZExtValue());
}
default:
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cc9ae05c40348f0..b98912a24811dab 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1439,8 +1439,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
assert(getNumOperands() == 2 && "expected single level extractvalue");
Value *Op = State.get(getOperand(0));
auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
- unsigned Idx = CI->getZExtValue();
- Value *Extract = Builder.CreateExtractValue(Op, Idx);
+ Value *Extract = Builder.CreateExtractValue(Op, CI->getZExtValue());
State.set(this, Extract);
break;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
new file mode 100644
index 000000000000000..fc411112323a7c1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
@@ -0,0 +1,133 @@
+; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg < %s -S -o - | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-COST
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+; CHECK-COST-LABEL: struct_return_widen
+; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val) #0
+; CHECK-COST: Cost of 10 for VF 2: WIDEN-CALL ir<%call> = call @foo(ir<%in_val>) (using library function: fixed_vec_foo)
+; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+
+define void @struct_return_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @struct_return_widen(
+; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds half, ptr [[IN]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x half>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x half>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = call { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half> [[WIDE_LOAD]])
+; CHECK-NEXT: [[TMP3:%.*]] = call { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half> [[WIDE_LOAD1]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP2]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP3]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP2]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP3]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds half, ptr [[OUT_A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 4
+; CHECK-NEXT: store <2 x half> [[TMP4]], ptr [[TMP8]], align 4
+; CHECK-NEXT: store <2 x half> [[TMP5]], ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds half, ptr [[OUT_B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP10]], i64 4
+; CHECK-NEXT: store <2 x half> [[TMP6]], ptr [[TMP10]], align 4
+; CHECK-NEXT: store <2 x half> [[TMP7]], ptr [[TMP11]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds half, ptr %in, i64 %iv
+ %in_val = load half, ptr %arrayidx, align 4
+ %call = tail call { half, half } @foo(half %in_val) #0
+ %extract_a = extractvalue { half, half } %call, 0
+ %extract_b = extractvalue { half, half } %call, 1
+ %arrayidx2 = getelementptr inbounds half, ptr %out_a, i64 %iv
+ store half %extract_a, ptr %arrayidx2, align 4
+ %arrayidx4 = getelementptr inbounds half, ptr %out_b, i64 %iv
+ store half %extract_b, ptr %arrayidx4, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; CHECK-COST-LABEL: struct_return_replicate
+; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val) #0
+; CHECK-COST: Cost of 26 for VF 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+
+define void @struct_return_replicate(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @struct_return_replicate(
+; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds half, ptr [[IN]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x half>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x half> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call { half, half } @foo(half [[TMP1]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x half> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = tail call { half, half } @foo(half [[TMP3]]) #[[ATTR0]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { half, half } [[TMP2]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x half> poison, half [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { half, half } [[TMP2]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x half> poison, half [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { half, half } [[TMP4]], 0
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x half> [[TMP6]], half [[TMP9]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { half, half } [[TMP4]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x half> [[TMP8]], half [[TMP11]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds half, ptr [[OUT_A]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x half> [[TMP10]], ptr [[TMP13]], align 4
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds half, ptr [[OUT_B]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x half> [[TMP12]], ptr [[TMP14]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds half, ptr %in, i64 %iv
+ %in_val = load half, ptr %arrayidx, align 4
+ ; #3 does not have a fixed-size vector mapping (so replication is used)
+ %call = tail call { half, half } @foo(half %in_val) #1
+ %extract_a = extractvalue { half, half } %call, 0
+ %extract_b = extractvalue { half, half } %call, 1
+ %arrayidx2 = getelementptr inbounds half, ptr %out_a, i64 %iv
+ store half %extract_a, ptr %arrayidx2, align 4
+ %arrayidx4 = getelementptr inbounds half, ptr %out_b, i64 %iv
+ store half %extract_b, ptr %arrayidx4, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+declare { half, half } @foo(half)
+
+declare { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half>)
+declare { <vscale x 4 x half>, <vscale x 4 x half> } @scalable_vec_masked_foo(<vscale x 4 x half>, <vscale x 4 x i1>)
+
+attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_foo(fixed_vec_foo)" }
+attributes #1 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(scalable_vec_masked_foo)" }
>From c0f9813aac1dd878ac4fe8bb658d051a4c6916e6 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 15 Jan 2025 13:46:13 +0000
Subject: [PATCH 03/10] Fixups
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 298a9d3b98e8cbb..889f4f9b64df9aa 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4517,7 +4517,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
llvm_unreachable("unhandled recipe");
}
- auto WillWiden = [&TTI, VF](Type *VectorTy) {
+ auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
unsigned NumLegalParts = TTI.getNumberOfParts(VectorTy);
if (!NumLegalParts)
return false;
@@ -4529,7 +4529,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
// explicitly ask TTI about the register class uses for each part.
return NumLegalParts <= VF.getKnownMinValue();
}
- // Two or more parts that share a register - are vectorized.
+ // Two or more elements that share a register - are vectorized.
return NumLegalParts < VF.getKnownMinValue();
};
@@ -4549,7 +4549,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
if (!Visited.insert({ScalarTy}).second)
continue;
Type *WideTy = toVectorizedTy(ScalarTy, VF);
- if (any_of(getContainedTypes(WideTy), WillWiden))
+ if (any_of(getContainedTypes(WideTy), WillGenerateTargetVectors))
return true;
}
}
>From cc7e82ae71530b86cfe43e5da4855e946d8f7065 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 15 Jan 2025 15:07:07 +0000
Subject: [PATCH 04/10] Fixups
Change-Id: I687099076728ddb73466cfd7cf1a8307f064b4f7
---
.../Transforms/Vectorize/LoopVectorize.cpp | 13 ++++---
.../AArch64/struct-return-cost.ll | 30 +++++++--------
.../Transforms/LoopVectorize/struct-return.ll | 38 ++++++++++++++++++-
3 files changed, 60 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 889f4f9b64df9aa..3d3ed25e9eee217 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5524,15 +5524,18 @@ InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
// overhead.
for (Use &U : I->operands())
if (auto *J = dyn_cast<Instruction>(U.get())) {
- assert(VectorType::isValidElementType(J->getType()) &&
+ assert(canVectorizeTy(J->getType()) &&
"Instruction has non-scalar type");
if (CanBeScalarized(J))
Worklist.push_back(J);
else if (needsExtract(J, VF)) {
- ScalarCost += TTI.getScalarizationOverhead(
- cast<VectorType>(toVectorTy(J->getType(), VF)),
- APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ false,
- /*Extract*/ true, CostKind);
+ Type *WideTy = toVectorizedTy(J->getType(), VF);
+ for (Type *VectorTy : getContainedTypes(WideTy)) {
+ ScalarCost += TTI.getScalarizationOverhead(
+ cast<VectorType>(VectorTy),
+ APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ false,
+ /*Extract*/ true, CostKind);
+ }
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
index fc411112323a7c1..d787b6787ffcc09 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
@@ -20,8 +20,8 @@ define void @struct_return_widen(ptr noalias %in, ptr noalias writeonly %out_a,
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds half, ptr [[IN]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x half>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x half>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x half>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x half>, ptr [[TMP1]], align 2
; CHECK-NEXT: [[TMP2:%.*]] = call { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half> [[WIDE_LOAD]])
; CHECK-NEXT: [[TMP3:%.*]] = call { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half> [[WIDE_LOAD1]])
; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP2]], 0
@@ -30,12 +30,12 @@ define void @struct_return_widen(ptr noalias %in, ptr noalias writeonly %out_a,
; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP3]], 1
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds half, ptr [[OUT_A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 4
-; CHECK-NEXT: store <2 x half> [[TMP4]], ptr [[TMP8]], align 4
-; CHECK-NEXT: store <2 x half> [[TMP5]], ptr [[TMP9]], align 4
+; CHECK-NEXT: store <2 x half> [[TMP4]], ptr [[TMP8]], align 2
+; CHECK-NEXT: store <2 x half> [[TMP5]], ptr [[TMP9]], align 2
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds half, ptr [[OUT_B]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP10]], i64 4
-; CHECK-NEXT: store <2 x half> [[TMP6]], ptr [[TMP10]], align 4
-; CHECK-NEXT: store <2 x half> [[TMP7]], ptr [[TMP11]], align 4
+; CHECK-NEXT: store <2 x half> [[TMP6]], ptr [[TMP10]], align 2
+; CHECK-NEXT: store <2 x half> [[TMP7]], ptr [[TMP11]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -48,14 +48,14 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds half, ptr %in, i64 %iv
- %in_val = load half, ptr %arrayidx, align 4
+ %in_val = load half, ptr %arrayidx, align 2
%call = tail call { half, half } @foo(half %in_val) #0
%extract_a = extractvalue { half, half } %call, 0
%extract_b = extractvalue { half, half } %call, 1
%arrayidx2 = getelementptr inbounds half, ptr %out_a, i64 %iv
- store half %extract_a, ptr %arrayidx2, align 4
+ store half %extract_a, ptr %arrayidx2, align 2
%arrayidx4 = getelementptr inbounds half, ptr %out_b, i64 %iv
- store half %extract_b, ptr %arrayidx4, align 4
+ store half %extract_b, ptr %arrayidx4, align 2
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %exit, label %for.body
@@ -78,7 +78,7 @@ define void @struct_return_replicate(ptr noalias %in, ptr noalias writeonly %out
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds half, ptr [[IN]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x half>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x half>, ptr [[TMP0]], align 2
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x half> [[WIDE_LOAD]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = tail call { half, half } @foo(half [[TMP1]]) #[[ATTR0:[0-9]+]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x half> [[WIDE_LOAD]], i64 1
@@ -92,9 +92,9 @@ define void @struct_return_replicate(ptr noalias %in, ptr noalias writeonly %out
; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { half, half } [[TMP4]], 1
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x half> [[TMP8]], half [[TMP11]], i64 1
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds half, ptr [[OUT_A]], i64 [[INDEX]]
-; CHECK-NEXT: store <2 x half> [[TMP10]], ptr [[TMP13]], align 4
+; CHECK-NEXT: store <2 x half> [[TMP10]], ptr [[TMP13]], align 2
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds half, ptr [[OUT_B]], i64 [[INDEX]]
-; CHECK-NEXT: store <2 x half> [[TMP12]], ptr [[TMP14]], align 4
+; CHECK-NEXT: store <2 x half> [[TMP12]], ptr [[TMP14]], align 2
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -107,15 +107,15 @@ entry:
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds half, ptr %in, i64 %iv
- %in_val = load half, ptr %arrayidx, align 4
+ %in_val = load half, ptr %arrayidx, align 2
; #3 does not have a fixed-size vector mapping (so replication is used)
%call = tail call { half, half } @foo(half %in_val) #1
%extract_a = extractvalue { half, half } %call, 0
%extract_b = extractvalue { half, half } %call, 1
%arrayidx2 = getelementptr inbounds half, ptr %out_a, i64 %iv
- store half %extract_a, ptr %arrayidx2, align 4
+ store half %extract_a, ptr %arrayidx2, align 2
%arrayidx4 = getelementptr inbounds half, ptr %out_b, i64 %iv
- store half %extract_b, ptr %arrayidx4, align 4
+ store half %extract_b, ptr %arrayidx4, align 2
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %exit, label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return.ll b/llvm/test/Transforms/LoopVectorize/struct-return.ll
index d8336e231cd4526..1b2a809a552d8f1 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return.ll
@@ -208,6 +208,40 @@ exit:
ret void
}
+; Test crafted to exercise computePredInstDiscount with struct results
+; (mainly it does not crash).
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
+define void @scalarized_predicated_struct_return(ptr %a) optsize {
+; CHECK-LABEL: define void @scalarized_predicated_struct_return
+; CHECK: vector.body:
+; CHECK: pred.store.if:
+; CHECK: tail call { i64, i64 } @bar_i64(i64 %5)
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv
+ %in_val = load i64, ptr %arrayidx, align 8
+ %sgt_zero = icmp sgt i64 %in_val, 0
+ br i1 %sgt_zero, label %if.then, label %for.inc
+
+if.then:
+ %call = tail call { i64, i64 } @bar_i64(i64 %in_val) #6
+ %extract_a = extractvalue { i64, i64 } %call, 0
+ %div = udiv i64 %extract_a, %in_val
+ store i64 %div, ptr %arrayidx, align 8
+ br label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
; Negative test. Widening structs of vectors is not supported.
; CHECK-REMARKS-COUNT: remark: {{.*}} loop not vectorized: instruction return type cannot be vectorized
define void @negative_struct_of_vectors(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
@@ -431,13 +465,14 @@ declare { [2 x float] } @foo_arrays(float)
declare { float, [1 x float] } @foo_one_non_widenable_element(float)
declare { <1 x float>, <1 x float> } @foo_vectors(<1 x float>)
declare { i32, i32, i32 } @qux(i32)
+declare { i64, i64 } @bar_i64(i64)
declare { <2 x float>, <2 x float> } @fixed_vec_foo(<2 x float>)
declare { <2 x double>, <2 x double> } @fixed_vec_bar(<2 x double>)
declare { <2 x float>, <2 x i32> } @fixed_vec_baz(<2 x float>)
declare { <2 x i32>, <2 x i32>, <2 x i32> } @fixed_vec_qux(<2 x i32>)
-
declare { <vscale x 4 x float>, <vscale x 4 x float> } @scalable_vec_masked_foo(<vscale x 4 x float>, <vscale x 4 x i1>)
+declare { <vscale x 4 x i64>, <vscale x 4 x i64> } @scalable_vec_masked_bar_i64(<vscale x 4 x i64>, <vscale x 4 x i1>)
attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_foo(fixed_vec_foo)" }
attributes #1 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_bar(fixed_vec_bar)" }
@@ -445,3 +480,4 @@ attributes #2 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_baz(fixed_vec
attributes #3 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(scalable_vec_masked_foo)" }
attributes #4 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_bar_named(fixed_vec_bar)" }
attributes #5 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_qux(fixed_vec_qux)" }
+attributes #6 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_bar_i64(scalable_vec_masked_bar_i64)" }
>From 6f1e15cf9dc83e72338d140b6b6669ebc4814703 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 17 Jan 2025 15:35:21 +0000
Subject: [PATCH 05/10] Fixups
---
.../Transforms/Vectorize/LoopVectorize.cpp | 7 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 +-
.../AArch64/struct-return-cost.ll | 141 ++++++++++--------
.../vplan-widen-struct-return.ll | 62 ++++----
4 files changed, 117 insertions(+), 96 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3d3ed25e9eee217..d7e83781d25ae46 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8625,9 +8625,10 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(Operands);
Type *I32Ty = IntegerType::getInt32Ty(I->getContext());
- for (unsigned Idx : cast<ExtractValueInst>(I)->getIndices())
- NewOps.push_back(
- Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
+ auto *EVI = cast<ExtractValueInst>(I);
+ assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
+ unsigned Idx = EVI->getIndices()[0];
+ NewOps.push_back(Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
}
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b98912a24811dab..cd4c6c5921d1d96 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1545,8 +1545,7 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
Ctx.CostKind);
}
case Instruction::ExtractValue:
- return Ctx.TTI.getInstructionCost(cast<Instruction>(getUnderlyingValue()),
- TTI::TCK_RecipThroughput);
+ return 0;
case Instruction::ICmp:
case Instruction::FCmp: {
Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
index d787b6787ffcc09..8459753025be611 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
@@ -1,12 +1,13 @@
-; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg < %s -S -o - | FileCheck %s --check-prefix=CHECK
-; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=CHECK-COST
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "(:|@)" --version 5
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize < %s -S -o - 2>%t | FileCheck %s
+; RUN: cat %t | FileCheck %s --check-prefix=CHECK-COST
; REQUIRES: asserts
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"
; CHECK-COST-LABEL: struct_return_widen
-; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val) #0
+; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val)
; CHECK-COST: Cost of 10 for VF 2: WIDEN-CALL ir<%call> = call @foo(ir<%in_val>) (using library function: fixed_vec_foo)
; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
@@ -14,33 +15,16 @@ target triple = "aarch64--linux-gnu"
define void @struct_return_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_widen(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds half, ptr [[IN]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 4
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x half>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x half>, ptr [[TMP1]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = call { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half> [[WIDE_LOAD]])
-; CHECK-NEXT: [[TMP3:%.*]] = call { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half> [[WIDE_LOAD1]])
-; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP2]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP3]], 0
-; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP2]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <2 x half>, <2 x half> } [[TMP3]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds half, ptr [[OUT_A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 4
-; CHECK-NEXT: store <2 x half> [[TMP4]], ptr [[TMP8]], align 2
-; CHECK-NEXT: store <2 x half> [[TMP5]], ptr [[TMP9]], align 2
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds half, ptr [[OUT_B]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP10]], i64 4
-; CHECK-NEXT: store <2 x half> [[TMP6]], ptr [[TMP10]], align 2
-; CHECK-NEXT: store <2 x half> [[TMP7]], ptr [[TMP11]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[VECTOR_PH:.*:]]
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[TMP2:%.*]] = call { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half> [[WIDE_LOAD:%.*]])
+; CHECK: [[TMP3:%.*]] = call { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half> [[WIDE_LOAD1:%.*]])
+; CHECK: [[MIDDLE_BLOCK:.*:]]
+; CHECK: [[SCALAR_PH:.*:]]
+; CHECK: [[FOR_BODY:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { half, half } @foo(half [[IN_VAL:%.*]]) #[[ATTR2:[0-9]+]]
+; CHECK: [[EXIT:.*:]]
;
entry:
br label %for.body
@@ -65,7 +49,7 @@ exit:
}
; CHECK-COST-LABEL: struct_return_replicate
-; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val) #0
+; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val)
; CHECK-COST: Cost of 26 for VF 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
@@ -73,33 +57,16 @@ exit:
define void @struct_return_replicate(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_replicate(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds half, ptr [[IN]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x half>, ptr [[TMP0]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x half> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = tail call { half, half } @foo(half [[TMP1]]) #[[ATTR0:[0-9]+]]
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x half> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP4:%.*]] = tail call { half, half } @foo(half [[TMP3]]) #[[ATTR0]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { half, half } [[TMP2]], 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x half> poison, half [[TMP5]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { half, half } [[TMP2]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x half> poison, half [[TMP7]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { half, half } [[TMP4]], 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x half> [[TMP6]], half [[TMP9]], i64 1
-; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { half, half } [[TMP4]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x half> [[TMP8]], half [[TMP11]], i64 1
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds half, ptr [[OUT_A]], i64 [[INDEX]]
-; CHECK-NEXT: store <2 x half> [[TMP10]], ptr [[TMP13]], align 2
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds half, ptr [[OUT_B]], i64 [[INDEX]]
-; CHECK-NEXT: store <2 x half> [[TMP12]], ptr [[TMP14]], align 2
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[VECTOR_PH:.*:]]
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[TMP4:%.*]] = tail call { half, half } @foo(half [[TMP3:%.*]]) #[[ATTR3:[0-9]+]]
+; CHECK: [[TMP6:%.*]] = tail call { half, half } @foo(half [[TMP5:%.*]]) #[[ATTR3]]
+; CHECK: [[MIDDLE_BLOCK:.*:]]
+; CHECK: [[SCALAR_PH:.*:]]
+; CHECK: [[FOR_BODY:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { half, half } @foo(half [[IN_VAL:%.*]]) #[[ATTR3]]
+; CHECK: [[EXIT:.*:]]
;
entry:
br label %for.body
@@ -108,7 +75,7 @@ for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds half, ptr %in, i64 %iv
%in_val = load half, ptr %arrayidx, align 2
- ; #3 does not have a fixed-size vector mapping (so replication is used)
+ ; #1 does not have a fixed-size vector mapping (so replication is used)
%call = tail call { half, half } @foo(half %in_val) #1
%extract_a = extractvalue { half, half } %call, 0
%extract_b = extractvalue { half, half } %call, 1
@@ -124,10 +91,64 @@ exit:
ret void
}
+; CHECK-COST-LABEL: struct_return_scalable
+; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val)
+; CHECK-COST: Cost of 26 for VF 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 1: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 10 for VF vscale x 8: WIDEN-CALL ir<%call> = call @foo(ir<%in_val>, ir<true>) (using library function: scalable_vec_masked_foo)
+
+define void @struct_return_scalable(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) #2 {
+; CHECK-LABEL: define void @struct_return_scalable(
+; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK: [[VECTOR_PH:.*:]]
+; CHECK: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK: [[TMP12:%.*]] = call { <vscale x 8 x half>, <vscale x 8 x half> } @scalable_vec_masked_foo(<vscale x 8 x half> [[WIDE_LOAD:%.*]], <vscale x 8 x i1> splat (i1 true))
+; CHECK: [[TMP13:%.*]] = call { <vscale x 8 x half>, <vscale x 8 x half> } @scalable_vec_masked_foo(<vscale x 8 x half> [[WIDE_LOAD1:%.*]], <vscale x 8 x i1> splat (i1 true))
+; CHECK: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK: [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK: [[MIDDLE_BLOCK:.*:]]
+; CHECK: [[SCALAR_PH:.*:]]
+; CHECK: [[FOR_BODY:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { half, half } @foo(half [[IN_VAL:%.*]]) #[[ATTR3]]
+; CHECK: [[EXIT:.*:]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds half, ptr %in, i64 %iv
+ %in_val = load half, ptr %arrayidx, align 2
+ %call = tail call { half, half } @foo(half %in_val) #1
+ %extract_a = extractvalue { half, half } %call, 0
+ %extract_b = extractvalue { half, half } %call, 1
+ %arrayidx2 = getelementptr inbounds half, ptr %out_a, i64 %iv
+ store half %extract_a, ptr %arrayidx2, align 2
+ %arrayidx4 = getelementptr inbounds half, ptr %out_b, i64 %iv
+ store half %extract_b, ptr %arrayidx4, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+
declare { half, half } @foo(half)
declare { <2 x half>, <2 x half> } @fixed_vec_foo(<2 x half>)
-declare { <vscale x 4 x half>, <vscale x 4 x half> } @scalable_vec_masked_foo(<vscale x 4 x half>, <vscale x 4 x i1>)
+declare { <vscale x 8 x half>, <vscale x 8 x half> } @scalable_vec_masked_foo(<vscale x 8 x half>, <vscale x 8 x i1>)
attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_foo(fixed_vec_foo)" }
attributes #1 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(scalable_vec_masked_foo)" }
+attributes #2 = { "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-struct-return.ll b/llvm/test/Transforms/LoopVectorize/vplan-widen-struct-return.ll
index 17a707df8956d3a..bb61398ae5a6db3 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-widen-struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-widen-struct-return.ll
@@ -1,11 +1,11 @@
; REQUIRES: asserts
-; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output -S 2>&1 | FileCheck %s
define void @struct_return_f32_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: LV: Checking a loop in 'struct_return_f32_widen'
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
-; CHECK-NEXT: Live-in vp<%0> = VF * UF
-; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -16,22 +16,22 @@ define void @struct_return_f32_widen(ptr noalias %in, ptr noalias writeonly %out
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%in>, vp<%3>
-; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%in_val> = load vp<%4>
+; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%in>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[IN_VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%in_val> = load vp<[[IN_VEC_PTR]]>
; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%in_val>) (using library function: fixed_vec_foo)
; CHECK-NEXT: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
; CHECK-NEXT: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%out_a>, vp<%3>
-; CHECK-NEXT: vp<%5> = vector-pointer ir<%arrayidx2>
-; CHECK-NEXT: WIDEN store vp<%5>, ir<%extract_a>
-; CHECK-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%out_b>, vp<%3>
-; CHECK-NEXT: vp<%6> = vector-pointer ir<%arrayidx4>
-; CHECK-NEXT: WIDEN store vp<%6>, ir<%extract_b>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%out_a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[OUT_A_VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<[[OUT_A_VEC_PTR]]>, ir<%extract_a>
+; CHECK-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%out_b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[OUT_B_VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx4>
+; CHECK-NEXT: WIDEN store vp<[[OUT_B_VEC_PTR]]>, ir<%extract_b>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VTC]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
entry:
@@ -59,8 +59,8 @@ exit:
define void @struct_return_f32_replicate(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: LV: Checking a loop in 'struct_return_f32_replicate'
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
-; CHECK-NEXT: Live-in vp<%0> = VF * UF
-; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
; CHECK-NEXT: Live-in ir<1024> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -71,22 +71,22 @@ define void @struct_return_f32_replicate(ptr noalias %in, ptr noalias writeonly
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%in>, vp<%3>
-; CHECK-NEXT: vp<%4> = vector-pointer ir<%arrayidx>
-; CHECK-NEXT: WIDEN ir<%in_val> = load vp<%4>
+; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%in>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[IN_VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx>
+; CHECK-NEXT: WIDEN ir<%in_val> = load vp<[[IN_VEC_PTR]]>
; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%in_val>)
; CHECK-NEXT: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
; CHECK-NEXT: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%out_a>, vp<%3>
-; CHECK-NEXT: vp<%5> = vector-pointer ir<%arrayidx2>
-; CHECK-NEXT: WIDEN store vp<%5>, ir<%extract_a>
-; CHECK-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%out_b>, vp<%3>
-; CHECK-NEXT: vp<%6> = vector-pointer ir<%arrayidx4>
-; CHECK-NEXT: WIDEN store vp<%6>, ir<%extract_b>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
-; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%out_a>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[OUT_A_VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx2>
+; CHECK-NEXT: WIDEN store vp<[[OUT_A_VEC_PTR]]>, ir<%extract_a>
+; CHECK-NEXT: CLONE ir<%arrayidx4> = getelementptr inbounds ir<%out_b>, vp<[[STEPS]]>
+; CHECK-NEXT: vp<[[OUT_B_VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx4>
+; CHECK-NEXT: WIDEN store vp<[[OUT_B_VEC_PTR]]>, ir<%extract_b>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VTC]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
entry:
>From 4b1534e43e46c3d6a3bc03dc1cf5ed6ed3bc397f Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 21 Jan 2025 13:48:43 +0000
Subject: [PATCH 06/10] Fixups
---
.../llvm/Analysis/TargetTransformInfo.h | 21 +++++++++++++++++++
.../llvm/Analysis/TargetTransformInfoImpl.h | 7 +++++++
llvm/lib/Analysis/TargetTransformInfo.cpp | 12 +++++++++++
.../Transforms/Vectorize/LoopVectorize.cpp | 4 ++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 10 +++++++--
5 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 08ab4ee2ec1cf73..8c6c49ebce8c238 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1473,6 +1473,15 @@ class TargetTransformInfo {
TTI::TargetCostKind CostKind,
unsigned Index = -1) const;
+ /// \return The expected of aggregate inserts and extracts. Use an empty
+ /// ArrayRef to indicate that there is no information on the indices. This is
+ /// used when the instruction is not available; a typical use case is to
+ /// provision the cost of vectorization/scalarization in vectorizer passes.
+ InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
+ TTI::TargetCostKind CostKind,
+ ArrayRef<unsigned> Indices,
+ Value *AggDef = nullptr) const;
+
/// \return The cost of replication shuffle of \p VF elements typed \p EltTy
/// \p ReplicationFactor times.
///
@@ -2223,6 +2232,11 @@ class TargetTransformInfo::Concept {
const APInt &DemandedDstElts,
TTI::TargetCostKind CostKind) = 0;
+ virtual InstructionCost
+ getInsertExtractValueCost(unsigned Opcode, Type *AggType,
+ TTI::TargetCostKind CostKind,
+ ArrayRef<unsigned> Indices, Value *AggDef) = 0;
+
virtual InstructionCost
getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace, TTI::TargetCostKind CostKind,
@@ -2950,6 +2964,13 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
return Impl.getReplicationShuffleCost(EltTy, ReplicationFactor, VF,
DemandedDstElts, CostKind);
}
+ InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
+ TTI::TargetCostKind CostKind,
+ ArrayRef<unsigned> Indices,
+ Value *AggDef) override {
+ return Impl.getInsertExtractValueCost(Opcode, AggType, CostKind, Indices,
+ AggDef);
+ }
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 5128c6b86a5f00c..1faebde81b493df 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -745,6 +745,13 @@ class TargetTransformInfoImplBase {
return 1;
}
+ InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
+ TTI::TargetCostKind CostKind,
+ ArrayRef<unsigned> Indices,
+ Value *AggDef) const {
+ return 0;
+ }
+
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index dc066099bdc1d33..7a04ee6ea358908 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1113,6 +1113,18 @@ TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
return Cost;
}
+InstructionCost TargetTransformInfo::getInsertExtractValueCost(
+ unsigned Opcode, Type *AggType, TTI::TargetCostKind CostKind,
+ ArrayRef<unsigned> Indices, Value *AggDef) const {
+ assert((Opcode == Instruction::InsertValue ||
+ Opcode == Instruction::ExtractValue) &&
+ "Expecting Opcode to be insertvalue/extractvalue.");
+ InstructionCost Cost = TTIImpl->getInsertExtractValueCost(
+ Opcode, AggType, CostKind, Indices, AggDef);
+ assert(Cost >= 0 && "TTI should not produce negative costs!");
+ return Cost;
+}
+
InstructionCost TargetTransformInfo::getReplicationShuffleCost(
Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
TTI::TargetCostKind CostKind) const {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d7e83781d25ae46..0faf30df4455e8c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3646,13 +3646,13 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
}
}
- // ExtractValue instructions must be uniform, because the operands are
- // known to be loop-invariant.
if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
if (IsOutOfScope(EVI->getAggregateOperand())) {
AddToWorklistIfAllowed(EVI);
continue;
}
+ // Only ExtractValue instructions where the aggregate value comes from a
+ // call are allowed to be non-uniform.
assert(isa<CallInst>(EVI->getAggregateOperand()) &&
"Expected aggregate value to be call return value");
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index cd4c6c5921d1d96..e27f378f383d76f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1544,8 +1544,14 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
Ctx.CostKind);
}
- case Instruction::ExtractValue:
- return 0;
+ case Instruction::ExtractValue: {
+ assert(getNumOperands() == 2 && "expected single level extractvalue");
+ Type *VectorizedTy = toVectorizedTy(Ctx.Types.inferScalarType(this), VF);
+ auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
+ return Ctx.TTI.getInsertExtractValueCost(
+ Instruction::ExtractValue, VectorizedTy, Ctx.CostKind,
+ static_cast<unsigned>(CI->getZExtValue()));
+ }
case Instruction::ICmp:
case Instruction::FCmp: {
Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
>From e5af9fcf1a94d25cf656a4761f686675da51e318 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 27 Jan 2025 14:25:22 +0000
Subject: [PATCH 07/10] More cost checks
---
.../AArch64/struct-return-cost.ll | 45 +++++++++++++++++++
1 file changed, 45 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
index 8459753025be611..c72149324373431 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
@@ -8,9 +8,20 @@ target triple = "aarch64--linux-gnu"
; CHECK-COST-LABEL: struct_return_widen
; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val)
+; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %extract_a = extractvalue { half, half } %call, 0
+; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %extract_b = extractvalue { half, half } %call, 1
+;
; CHECK-COST: Cost of 10 for VF 2: WIDEN-CALL ir<%call> = call @foo(ir<%in_val>) (using library function: fixed_vec_foo)
+; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
define void @struct_return_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_widen(
@@ -50,9 +61,20 @@ exit:
; CHECK-COST-LABEL: struct_return_replicate
; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val)
+; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %extract_a = extractvalue { half, half } %call, 0
+; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %extract_b = extractvalue { half, half } %call, 1
+;
; CHECK-COST: Cost of 26 for VF 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
define void @struct_return_replicate(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_replicate(
@@ -93,13 +115,36 @@ exit:
; CHECK-COST-LABEL: struct_return_scalable
; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { half, half } @foo(half %in_val)
+; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %extract_a = extractvalue { half, half } %call, 0
+; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %extract_b = extractvalue { half, half } %call, 1
+;
; CHECK-COST: Cost of 26 for VF 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of Invalid for VF vscale x 1: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF vscale x 1: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF vscale x 1: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of Invalid for VF vscale x 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF vscale x 2: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF vscale x 2: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of Invalid for VF vscale x 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
+; CHECK-COST: Cost of 0 for VF vscale x 4: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF vscale x 4: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+;
; CHECK-COST: Cost of 10 for VF vscale x 8: WIDEN-CALL ir<%call> = call @foo(ir<%in_val>, ir<true>) (using library function: scalable_vec_masked_foo)
+; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
+; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
define void @struct_return_scalable(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) #2 {
; CHECK-LABEL: define void @struct_return_scalable(
>From 2ee07ab68494a0838d97f68526bce73ccfeb0199 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 28 Jan 2025 12:05:20 +0000
Subject: [PATCH 08/10] Add comment
---
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 1faebde81b493df..db6bbd2f8cb357e 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -749,6 +749,12 @@ class TargetTransformInfoImplBase {
TTI::TargetCostKind CostKind,
ArrayRef<unsigned> Indices,
Value *AggDef) const {
+ // Extract/insert values are generally assumed to be free (as the aggregates
+ // will be removed e.g. by SROA). A target may want to override this to cost
+ // an extract operation differently based on the producer (AggDef). E.g. if
+ // AggDef is a call, the result may be returned via the stack (so the
+ // extract acts like a load). However, generally, that cost is included in
+ // the producer's cost.
return 0;
}
>From af8febd2d647c46490e0e0acca5059e3b4e83096 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 31 Jan 2025 13:39:27 +0000
Subject: [PATCH 09/10] Fixups
---
.../llvm/Analysis/TargetTransformInfo.h | 23 +++++++-----------
.../llvm/Analysis/TargetTransformInfoImpl.h | 24 +++++++++----------
llvm/lib/Analysis/TargetTransformInfo.cpp | 6 ++---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 ++-----
4 files changed, 24 insertions(+), 37 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 8c6c49ebce8c238..31630a4550b8cdb 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1473,14 +1473,11 @@ class TargetTransformInfo {
TTI::TargetCostKind CostKind,
unsigned Index = -1) const;
- /// \return The expected of aggregate inserts and extracts. Use an empty
- /// ArrayRef to indicate that there is no information on the indices. This is
+ /// \return The expected of aggregate inserts and extracts. This is
/// used when the instruction is not available; a typical use case is to
/// provision the cost of vectorization/scalarization in vectorizer passes.
- InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
- TTI::TargetCostKind CostKind,
- ArrayRef<unsigned> Indices,
- Value *AggDef = nullptr) const;
+ InstructionCost getInsertExtractValueCost(unsigned Opcode,
+ TTI::TargetCostKind CostKind) const;
/// \return The cost of replication shuffle of \p VF elements typed \p EltTy
/// \p ReplicationFactor times.
@@ -2233,9 +2230,7 @@ class TargetTransformInfo::Concept {
TTI::TargetCostKind CostKind) = 0;
virtual InstructionCost
- getInsertExtractValueCost(unsigned Opcode, Type *AggType,
- TTI::TargetCostKind CostKind,
- ArrayRef<unsigned> Indices, Value *AggDef) = 0;
+ getInsertExtractValueCost(unsigned Opcode, TTI::TargetCostKind CostKind) = 0;
virtual InstructionCost
getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
@@ -2964,12 +2959,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
return Impl.getReplicationShuffleCost(EltTy, ReplicationFactor, VF,
DemandedDstElts, CostKind);
}
- InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
- TTI::TargetCostKind CostKind,
- ArrayRef<unsigned> Indices,
- Value *AggDef) override {
- return Impl.getInsertExtractValueCost(Opcode, AggType, CostKind, Indices,
- AggDef);
+ InstructionCost
+ getInsertExtractValueCost(unsigned Opcode,
+ TTI::TargetCostKind CostKind) override {
+ return Impl.getInsertExtractValueCost(Opcode, CostKind);
}
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index db6bbd2f8cb357e..a8d6dd18266bbb1 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -745,17 +745,15 @@ class TargetTransformInfoImplBase {
return 1;
}
- InstructionCost getInsertExtractValueCost(unsigned Opcode, Type *AggType,
- TTI::TargetCostKind CostKind,
- ArrayRef<unsigned> Indices,
- Value *AggDef) const {
- // Extract/insert values are generally assumed to be free (as the aggregates
- // will be removed e.g. by SROA). A target may want to override this to cost
- // an extract operation differently based on the producer (AggDef). E.g. if
- // AggDef is a call, the result may be returned via the stack (so the
- // extract acts like a load). However, generally, that cost is included in
- // the producer's cost.
- return 0;
+ InstructionCost
+ getInsertExtractValueCost(unsigned Opcode,
+ TTI::TargetCostKind CostKind) const {
+ // Note: The `insertvalue` cost here is chosen to match the default case of
+ // getInstructionCost() -- as pior to adding this helper `insertvalue` was
+ // not handled.
+ if (Opcode == Instruction::InsertValue)
+ return CostKind == TTI::TCK_RecipThroughput ? -1 : TTI::TCC_Basic;
+ return TTI::TCC_Free;
}
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
@@ -1319,9 +1317,11 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
case Instruction::PHI:
case Instruction::Switch:
return TargetTTI->getCFInstrCost(Opcode, CostKind, I);
- case Instruction::ExtractValue:
case Instruction::Freeze:
return TTI::TCC_Free;
+ case Instruction::ExtractValue:
+ case Instruction::InsertValue:
+ return TargetTTI->getInsertExtractValueCost(Opcode, CostKind);
case Instruction::Alloca:
if (cast<AllocaInst>(U)->isStaticAlloca())
return TTI::TCC_Free;
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 7a04ee6ea358908..1ca9a16b1811276 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1114,13 +1114,11 @@ TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
}
InstructionCost TargetTransformInfo::getInsertExtractValueCost(
- unsigned Opcode, Type *AggType, TTI::TargetCostKind CostKind,
- ArrayRef<unsigned> Indices, Value *AggDef) const {
+ unsigned Opcode, TTI::TargetCostKind CostKind) const {
assert((Opcode == Instruction::InsertValue ||
Opcode == Instruction::ExtractValue) &&
"Expecting Opcode to be insertvalue/extractvalue.");
- InstructionCost Cost = TTIImpl->getInsertExtractValueCost(
- Opcode, AggType, CostKind, Indices, AggDef);
+ InstructionCost Cost = TTIImpl->getInsertExtractValueCost(Opcode, CostKind);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e27f378f383d76f..410fd0c55831ce9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1545,12 +1545,8 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
Ctx.CostKind);
}
case Instruction::ExtractValue: {
- assert(getNumOperands() == 2 && "expected single level extractvalue");
- Type *VectorizedTy = toVectorizedTy(Ctx.Types.inferScalarType(this), VF);
- auto *CI = cast<ConstantInt>(getOperand(1)->getLiveInIRValue());
- return Ctx.TTI.getInsertExtractValueCost(
- Instruction::ExtractValue, VectorizedTy, Ctx.CostKind,
- static_cast<unsigned>(CI->getZExtValue()));
+ return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,
+ Ctx.CostKind);
}
case Instruction::ICmp:
case Instruction::FCmp: {
>From 8ddfd70a2a448a0521dd5c45fe5de6f2d5145bd2 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 10 Feb 2025 00:26:53 +0000
Subject: [PATCH 10/10] Fix typo
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 31630a4550b8cdb..9048481b4918927 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1473,7 +1473,7 @@ class TargetTransformInfo {
TTI::TargetCostKind CostKind,
unsigned Index = -1) const;
- /// \return The expected of aggregate inserts and extracts. This is
+ /// \return The expected cost of aggregate inserts and extracts. This is
/// used when the instruction is not available; a typical use case is to
/// provision the cost of vectorization/scalarization in vectorizer passes.
InstructionCost getInsertExtractValueCost(unsigned Opcode,
More information about the llvm-commits
mailing list