[llvm-branch-commits] [llvm] 5b10d2d - release/23.x: [LV] Have ExtractValue match the shape of the operand. (#229927)
Tobias Hieta via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 16 04:46:56 PST 2026
Author: Florian Hahn
Date: 2026-11-16T13:46:15+01:00
New Revision: 5b10d2d74929eae78912771222c9855af97496f4
URL: https://github.com/llvm/llvm-project/commit/5b10d2d74929eae78912771222c9855af97496f4
DIFF: https://github.com/llvm/llvm-project/commit/5b10d2d74929eae78912771222c9855af97496f4.diff
LOG: release/23.x: [LV] Have ExtractValue match the shape of the operand. (#229927)
Previously, BuildStructVector would create a vector of a wide struct
type, where each element is widened.
This can cause issues, if a transform turns an extractvalue of a wide
call into a single-scalar (e.g. makeScalarizationDecisions), because we
currently cannot extract lane 0 from a struct of vectors.
Instead of relying on BuildStructVector, we can instead have the
ExtractValue match the shape of its operand (either widen if the operand
is wide, or replicated if the operand is scalar). If a vector value is
needed, we can directly use the existing BuildVector.
This fixes a number of crashes in the newly added tests, including
https://github.com/llvm/llvm-project/issues/229609.
Depends on https://github.com/llvm/llvm-project/pull/229925
PR: https://github.com/llvm/llvm-project/pull/229927
Added:
llvm/test/Transforms/LoopVectorize/AArch64/struct-return-extractvalue.ll
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
llvm/test/Transforms/LoopVectorize/VPlan/vplan-widen-struct-return.ll
llvm/test/Transforms/LoopVectorize/extract-value-widen.ll
llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
llvm/test/Transforms/LoopVectorize/struct-return.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d4cc8f3a717e2d..671f85cd18a4c8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6472,6 +6472,19 @@ VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
VPI->getOpcode()) &&
"Should have been handled prior to this!");
+ // An extractvalue takes the shape of its operand: we cannot extract a lane
+ // from a widened struct, nor build a struct from replicated lanes.
+ if (VPI->getOpcode() == Instruction::ExtractValue &&
+ !vputils::isSingleScalar(VPI->getOperand(0)))
+ if (VPRecipeBase *OpR = VPI->getOperand(0)->getDefiningRecipe())
+ if (!isa<VPReplicateRecipe>(OpR))
+ return tryToWiden(VPI);
+ if (VPI->getOpcode() == Instruction::ExtractValue)
+ return new VPReplicateRecipe(Instr, VPI->operandsWithoutMask(),
+ vputils::isSingleScalar(VPI->getOperand(0)),
+ /*Mask=*/nullptr, *VPI, *VPI,
+ VPI->getDebugLoc());
+
if (!shouldWiden(Instr, Range))
return nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 76bfa8bf6192d4..75cd097d25f777 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1250,10 +1250,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// 0 or successor 1 based on condition 0.
BranchOnTwoConds,
Broadcast,
- /// Given operands of (the same) struct type, creates a struct of fixed-
- /// width vectors each containing a struct field of all operands. The
- /// number of operands matches the element count of every vector.
- BuildStructVector,
/// Creates a fixed-width vector containing all operands. The number of
/// operands matches the vector element count.
BuildVector,
@@ -1412,7 +1408,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
unsigned getOpcode() const { return Opcode; }
/// Add \p Op as operand of this VPInstruction. Only supported for AnyOf,
- /// ComputeReductionResult, BuildVector, BuildStructVector, ExtractLane,
+ /// ComputeReductionResult, BuildVector, ExtractLane,
/// ExtractLastActive, FirstActiveLane, LastActiveLane.
void addOperand(VPValue *Op);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 7e7d0f9b6a280f..97f1a84e120f31 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -344,13 +344,6 @@ inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
return m_VPInstruction<VPInstruction::BuildVector>();
}
-/// BuildStructVector matches only its opcode, w/o matching its operands as the
-/// number of operands is not fixed.
-inline VPInstruction_match<VPInstruction::BuildStructVector>
-m_BuildStructVector() {
- return m_VPInstruction<VPInstruction::BuildStructVector>();
-}
-
template <typename Op0_t>
inline VPInstruction_match<Instruction::Freeze, Op0_t>
m_Freeze(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 9aea7d11a6aaae..5470376e5c9a44 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -576,8 +576,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
bool AllOperandsSameType =
Instruction::isBinaryOp(Opcode) ||
is_contained({VPInstruction::FirstOrderRecurrenceSplice,
- VPInstruction::BuildVector,
- VPInstruction::BuildStructVector},
+ VPInstruction::BuildVector},
Opcode);
if (AllOperandsSameType)
for (unsigned Idx = 1; Idx != Operands.size(); ++Idx)
@@ -674,7 +673,6 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::AtomicCmpXchg:
case Instruction::Fence:
case VPInstruction::AnyOf:
- case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
case VPInstruction::Intrinsic:
case VPInstruction::CanonicalIVIncrementForPart:
@@ -882,28 +880,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateVectorSplat(
State.VF, State.get(getOperand(0), /*IsScalar*/ true), "broadcast");
}
- case VPInstruction::BuildStructVector: {
- // For struct types, we need to build a new 'wide' struct type, where each
- // element is widened, i.e., we create a struct of vectors.
- auto *StructTy = cast<StructType>(getOperand(0)->getScalarType());
- Value *Res = PoisonValue::get(toVectorizedTy(StructTy, State.VF));
- for (const auto &[LaneIndex, Op] : enumerate(operands())) {
- for (unsigned FieldIndex = 0; FieldIndex != StructTy->getNumElements();
- FieldIndex++) {
- Value *ScalarValue =
- Builder.CreateExtractValue(State.get(Op, true), FieldIndex);
- Value *VectorValue = Builder.CreateExtractValue(Res, FieldIndex);
- VectorValue =
- Builder.CreateInsertElement(VectorValue, ScalarValue, LaneIndex);
- Res = Builder.CreateInsertValue(Res, VectorValue, FieldIndex);
- }
- }
- return Res;
- }
case VPInstruction::BuildVector: {
- auto *ScalarTy = getOperand(0)->getScalarType();
- auto NumOfElements = ElementCount::getFixed(getNumOperands());
- Value *Res = PoisonValue::get(toVectorizedTy(ScalarTy, NumOfElements));
+ Value *Res = PoisonValue::get(
+ FixedVectorType::get(getScalarType(), getNumOperands()));
for (const auto &[Idx, Op] : enumerate(operands()))
Res = Builder.CreateInsertElement(Res, State.get(Op, true),
Builder.getInt32(Idx));
@@ -1532,7 +1511,6 @@ void VPInstruction::addOperand(VPValue *Op) {
break;
case VPInstruction::ComputeReductionResult:
case VPInstruction::BuildVector:
- case VPInstruction::BuildStructVector:
assert(Ty == getOperand(0)->getScalarType() &&
"appended operand must match operand 0's scalar type");
break;
@@ -1576,8 +1554,7 @@ void VPInstruction::execute(VPTransformState &State) {
bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
(vputils::onlyFirstLaneUsed(this) ||
isVectorToScalar() || isSingleScalar());
- assert((((GeneratedValue->getType()->isVectorTy() ||
- GeneratedValue->getType()->isStructTy()) ==
+ assert(((GeneratedValue->getType()->isVectorTy() ==
!GeneratesPerFirstLaneOnly) ||
State.VF.isScalar()) &&
"scalar value but not only first lane defined");
@@ -1613,7 +1590,6 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::BranchOnTwoConds:
case VPInstruction::BranchOnCount:
case VPInstruction::Broadcast:
- case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
case VPInstruction::CalculateTripCountMinusVF:
case VPInstruction::CanonicalIVIncrementForPart:
@@ -1681,10 +1657,9 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::ReductionStartVector:
case VPInstruction::ResumeForEpilogue:
return true;
- case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
- // Before replicating by VF, Build(Struct)Vector uses all lanes of the
- // operand, after replicating its operands only the first lane is used.
+ // Before replicating by VF, BuildVector uses all lanes of the operand,
+ // after replicating its operands only the first lane is used.
// Before replicating, it will have only a single operand.
return getNumOperands() > 1;
case VPInstruction::PtrAdd:
@@ -1764,9 +1739,6 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::Broadcast:
O << "broadcast";
break;
- case VPInstruction::BuildStructVector:
- O << "buildstructvector";
- break;
case VPInstruction::BuildVector:
O << "buildvector";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c6948169e1b57f..4e93e979a14d7e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -551,11 +551,24 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
VPBlockUtils::insertTwoBlocksAfter(Pred, Exiting, Entry);
VPBlockUtils::connectBlocks(Pred, Exiting);
- if (!PredRecipe->user_empty()) {
- auto *PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask,
- RecipeWithoutMask->getDebugLoc());
+ PredRecipe->replaceAllUsesWith(RecipeWithoutMask);
+ // Move the extracts of a struct result into the region, so only scalars
+ // leave it.
+ SmallVector<VPUser *> Defs = {RecipeWithoutMask};
+ if (RecipeWithoutMask->getScalarType()->isStructTy())
+ Defs = to_vector(RecipeWithoutMask->users());
+ for (VPUser *U : Defs) {
+ auto *Def = cast<VPReplicateRecipe>(U);
+ assert((Def == RecipeWithoutMask ||
+ Def->getOpcode() == Instruction::ExtractValue) &&
+ "only extracts can use a predicated struct");
+ Def->moveBefore(*Pred, Pred->end());
+ if (Def->user_empty())
+ continue;
+ auto *PHIRecipe = new VPPredInstPHIRecipe(Def, Def->getDebugLoc());
Exiting->appendRecipe(PHIRecipe);
- PredRecipe->replaceAllUsesWith(PHIRecipe);
+ Def->replaceAllUsesWith(PHIRecipe);
+ PHIRecipe->setOperand(0, Def);
}
PredRecipe->eraseFromParent();
return Region;
@@ -906,8 +919,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
continue;
// TODO: Support scalarizing ExtractValue.
- if (match(Def,
- m_Binary<Instruction::ExtractValue>(m_VPValue(), m_VPValue())))
+ if (isa<ExtractValueInst>(Def->getUnderlyingInstr()))
continue;
auto *Clone = VPBuilder::createSingleScalarOp(
@@ -5502,7 +5514,7 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
vp_depth_first_shallow(Plan.getEntry()));
auto VPBBsInsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(LoopRegion->getEntry()));
- // Materialize Build(Struct)Vector for all replicating VPReplicateRecipes,
+ // Materialize BuildVector for all replicating VPReplicateRecipes,
// VPScalarIVStepsRecipe and VPInstructions, excluding ones in replicate
// regions. Those are not materialized explicitly yet.
// TODO: materialize build vectors for replicating recipes in replicating
@@ -5525,11 +5537,7 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
none_of(DefR->users(), UsesVectorOrInsideReplicateRegion))
continue;
- Type *ScalarTy = DefR->getScalarType();
- unsigned Opcode = ScalarTy->isStructTy()
- ? VPInstruction::BuildStructVector
- : VPInstruction::BuildVector;
- auto *BuildVector = new VPInstruction(Opcode, {DefR});
+ auto *BuildVector = new VPInstruction(VPInstruction::BuildVector, {DefR});
BuildVector->insertAfter(DefR);
DefR->replaceUsesWithIf(
@@ -7519,6 +7527,11 @@ void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (VPI->mayHaveSideEffects())
continue;
+ // ExtractValue takes the shape of its aggregate, decided in
+ // tryToCreateWidenNonPhiRecipe.
+ if (VPI->getOpcode() == Instruction::ExtractValue)
+ continue;
+
// We want to drop the mask operand, verify we can safely do that.
if (VPI->isMasked() && !VPI->isSafeToSpeculativelyExecute())
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 85375625d34b53..a0d77343b53a95 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -455,7 +455,7 @@ struct VPlanTransforms {
static void materializeBackedgeTakenCount(VPlan &Plan,
VPBasicBlock *VectorPH);
- /// Add explicit Build[Struct]Vector recipes to Pack multiple scalar values
+ /// Add explicit BuildVector recipes to Pack multiple scalar values
/// into vectors and Unpack recipes to extract scalars from vectors as
/// needed.
static void materializePacksAndUnpacks(VPlan &Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 594c201d5a8cb7..ba2c9158fa4752 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -767,8 +767,7 @@ static void processLaneForReplicateRegion(VPlan &Plan, Type *IdxTy,
assert(vputils::onlyFirstLaneUsed(OldPhi) &&
"VPPhis expected to have only first lane used");
auto *BVUser = dyn_cast_or_null<VPInstruction>(OldPhi->getSingleUser());
- if (BVUser && match(BVUser, m_CombineOr(m_BuildVector(),
- m_BuildStructVector()))) {
+ if (BVUser && match(BVUser, m_BuildVector())) {
assert(BVUser->getOperand(0) == OldPhi &&
"Unexpected first operand of build vector user");
BVUser->setOperand(Lane, NewPhi);
@@ -809,12 +808,12 @@ static void dissolveReplicateRegion(VPRegionBlock *Region, ElementCount VF,
return;
}
- // Create a BuildVector or BuildStructVector in successor block for every
- // VPPhi in (first lane's) exiting block having vector uses. All their
+ // Create a BuildVector in successor block for every VPPhi in (first lane's)
+ // exiting block having vector uses. All their
// operands are initialized to poison and will be replaced when processing
// each clone, except for the operand of the first lane which set here.
// BuildVectors are recorded to be replaced later by chains of insert-element
- // and widen phi's.
+ // and widen phis.
unsigned NumLanes = VF.getFixedValue();
SmallVector<VPInstruction *> BuildVectors;
for (auto &R : FirstLaneExiting->phis()) {
@@ -823,14 +822,10 @@ static void dissolveReplicateRegion(VPRegionBlock *Region, ElementCount VF,
continue;
Type *ScalarTy = Phi->getScalarType();
- bool IsStruct = isa<StructType>(ScalarTy);
VPValue *Poison = Plan.getPoison(ScalarTy);
SmallVector<VPValue *> BVOps(NumLanes, Poison);
- auto *BV = new VPInstruction(IsStruct ? VPInstruction::BuildStructVector
- : VPInstruction::BuildVector,
- BVOps);
- if (!IsStruct)
- BuildVectors.push_back(BV);
+ auto *BV = new VPInstruction(VPInstruction::BuildVector, BVOps);
+ BuildVectors.push_back(BV);
Phi->replaceAllUsesWith(BV);
BV->setOperand(0, Phi);
BV->insertBefore(*Successor, Successor->getFirstNonPhi());
@@ -973,12 +968,11 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
// operand, to have all LaneDefs as its operands.
for (VPUser *U : to_vector(DefR->users())) {
auto *VPI = dyn_cast<VPInstruction>(U);
- if (!VPI || (VPI->getOpcode() != VPInstruction::BuildVector &&
- VPI->getOpcode() != VPInstruction::BuildStructVector))
+ if (!VPI || VPI->getOpcode() != VPInstruction::BuildVector)
continue;
assert(VPI->getNumOperands() == 1 &&
- "Build(Struct)Vector must have a single operand before "
- "replicating by VF");
+ "BuildVector must have a single operand before replicating by "
+ "VF");
VPI->setOperand(0, LaneDefs[0]);
for (VPValue *LaneDef : drop_begin(LaneDefs))
VPI->addOperand(LaneDef);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index d57385d54645cd..693c557c76f18b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -31,6 +31,7 @@ bool vputils::propagatesFirstLaneUse(const VPUser *U, const VPValue *Op) {
return false;
unsigned Opcode = VPI->getOpcode();
switch (Opcode) {
+ case Instruction::ExtractValue:
case Instruction::FCmp:
case Instruction::ICmp:
case Instruction::Select:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
index 80d4036fd1bc09..5b38b55fa31e09 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
@@ -16,12 +16,12 @@ target triple = "aarch64--linux-gnu"
; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
;
; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of 0 for VF 4: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of 0 for VF 4: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of 0 for VF 8: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of 0 for VF 8: REPLICATE ir<%extract_b> = extractvalue ir<%call>
define void @struct_return_widen(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_widen(
@@ -62,16 +62,16 @@ exit:
; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %extract_b = extractvalue { half, half } %call, 1
;
; CHECK-COST: Cost of 26 for VF 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of 0 for VF 2: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of 0 for VF 2: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of 0 for VF 4: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of 0 for VF 4: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of 0 for VF 8: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of 0 for VF 8: REPLICATE ir<%extract_b> = extractvalue ir<%call>
define void @struct_return_replicate(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @struct_return_replicate(
@@ -79,8 +79,8 @@ define void @struct_return_replicate(ptr noalias %in, ptr noalias writeonly %out
; CHECK: [[ENTRY:.*:]]
; CHECK: [[VECTOR_PH:.*:]]
; CHECK: [[VECTOR_BODY:.*:]]
-; CHECK: [[TMP3:%.*]] = tail call { half, half } @foo(half [[TMP1:%.*]]) #[[ATTR2:[0-9]+]]
-; CHECK: [[TMP4:%.*]] = tail call { half, half } @foo(half [[TMP2:%.*]]) #[[ATTR2]]
+; CHECK: [[TMP2:%.*]] = tail call { half, half } @foo(half [[TMP1:%.*]]) #[[ATTR2:[0-9]+]]
+; CHECK: [[TMP4:%.*]] = tail call { half, half } @foo(half [[TMP3:%.*]]) #[[ATTR2]]
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[EXIT:.*:]]
;
@@ -113,28 +113,28 @@ exit:
; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %extract_b = extractvalue { half, half } %call, 1
;
; CHECK-COST: Cost of 26 for VF 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF 2: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of 0 for VF 2: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of 0 for VF 2: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of 58 for VF 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF 4: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of 0 for VF 4: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of 0 for VF 4: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of 122 for VF 8: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF 8: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of 0 for VF 8: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of 0 for VF 8: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of Invalid for VF vscale x 1: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF vscale x 1: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF vscale x 1: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of Invalid for VF vscale x 1: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of Invalid for VF vscale x 1: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of Invalid for VF vscale x 2: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF vscale x 2: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF vscale x 2: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of Invalid for VF vscale x 2: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of Invalid for VF vscale x 2: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of Invalid for VF vscale x 4: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-COST: Cost of 0 for VF vscale x 4: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-COST: Cost of 0 for VF vscale x 4: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-COST: Cost of Invalid for VF vscale x 4: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-COST: Cost of Invalid for VF vscale x 4: REPLICATE ir<%extract_b> = extractvalue ir<%call>
;
; CHECK-COST: Cost of 10 for VF vscale x 8: WIDEN-CALL ir<%call> = call @foo(ir<%in_val>, ir<true>) (using library function: scalable_vec_masked_foo)
; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-extractvalue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-extractvalue.ll
new file mode 100644
index 00000000000000..b150d74b5bcaab
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-extractvalue.ll
@@ -0,0 +1,349 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "middle.block:" --version 6
+; RUN: opt -passes=loop-vectorize -S %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; The call is scalarized and the sum is only used by a predicated store, so
+; its extractvalue is sunk into the replicate region, apart from the call.
+define void @extractvalue_sunk_apart_from_scalarized_call(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @extractvalue_sunk_apart_from_scalarized_call(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP3:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[INDEX]], i64 1)
+; CHECK-NEXT: [[TMP4:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[TMP0]], i64 1)
+; CHECK-NEXT: [[TMP5:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[TMP1]], i64 1)
+; CHECK-NEXT: [[TMP6:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[TMP2]], i64 1)
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i1> poison, i1 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i1> [[TMP9]], i1 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i1> poison, i1 [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i1> [[TMP13]], i1 [[TMP12]], i32 1
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[TMP15]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP15]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP16]], align 8
+; CHECK-NEXT: [[TMP17:%.*]] = zext <2 x i1> [[TMP10]] to <2 x i64>
+; CHECK-NEXT: [[TMP18:%.*]] = zext <2 x i1> [[TMP14]] to <2 x i64>
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[TMP19]], i64 2
+; CHECK-NEXT: store <2 x i64> [[TMP17]], ptr [[TMP19]], align 8
+; CHECK-NEXT: store <2 x i64> [[TMP18]], ptr [[TMP20]], align 8
+; CHECK-NEXT: [[TMP21:%.*]] = icmp sgt <2 x i64> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP22:%.*]] = icmp sgt <2 x i64> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <2 x i1> [[TMP21]], i64 0
+; CHECK-NEXT: br i1 [[TMP23]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP24:%.*]] = extractvalue { i64, i1 } [[TMP3]], 0
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP24]]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: store i64 [[TMP26]], ptr [[TMP25]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <2 x i1> [[TMP21]], i64 1
+; CHECK-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; CHECK: [[PRED_STORE_IF2]]:
+; CHECK-NEXT: [[TMP28:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP28]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: store i64 [[TMP30]], ptr [[TMP29]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; CHECK: [[PRED_STORE_CONTINUE3]]:
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x i1> [[TMP22]], i64 0
+; CHECK-NEXT: br i1 [[TMP31]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; CHECK: [[PRED_STORE_IF4]]:
+; CHECK-NEXT: [[TMP32:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP32]]
+; CHECK-NEXT: [[TMP34:%.*]] = extractelement <2 x i64> [[WIDE_LOAD1]], i64 0
+; CHECK-NEXT: store i64 [[TMP34]], ptr [[TMP33]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
+; CHECK: [[PRED_STORE_CONTINUE5]]:
+; CHECK-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP22]], i64 1
+; CHECK-NEXT: br i1 [[TMP35]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
+; CHECK: [[PRED_STORE_IF6]]:
+; CHECK-NEXT: [[TMP36:%.*]] = extractvalue { i64, i1 } [[TMP6]], 0
+; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = extractelement <2 x i64> [[WIDE_LOAD1]], i64 1
+; CHECK-NEXT: store i64 [[TMP38]], ptr [[TMP37]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
+; CHECK: [[PRED_STORE_CONTINUE7]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %res = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %iv, i64 1)
+ %sum = extractvalue { i64, i1 } %res, 0
+ %ov = extractvalue { i64, i1 } %res, 1
+ %gep.a = getelementptr i64, ptr %a, i64 %iv
+ %l.a = load i64, ptr %gep.a
+ %ov.ext = zext i1 %ov to i64
+ %gep.b = getelementptr i64, ptr %b, i64 %iv
+ store i64 %ov.ext, ptr %gep.b
+ %cmp = icmp sgt i64 %l.a, 0
+ br i1 %cmp, label %then, label %loop.latch
+
+then:
+ %gep.c = getelementptr i64, ptr %c, i64 %sum
+ store i64 %l.a, ptr %gep.c
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop.header
+
+exit:
+ ret void
+}
+
+; The loop-invariant call is scalarized and hoisted out of the loop, with the
+; product used as a single scalar and the overflow bit used by all lanes.
+define void @extractvalue_of_hoisted_scalarized_call(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) {
+; CHECK-LABEL: define void @extractvalue_of_hoisted_scalarized_call(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[N]], i64 3)
+; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { i64, i1 } [[TMP0]], 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP0]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP0]], 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i1> poison, i1 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i1> [[TMP15]], i1 [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = zext <2 x i1> [[TMP5]] to <2 x i64>
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i64 2
+; CHECK-NEXT: store <2 x i64> [[TMP6]], ptr [[TMP7]], align 8
+; CHECK-NEXT: store <2 x i64> [[TMP6]], ptr [[TMP8]], align 8
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP1]], [[INDEX]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP10]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP11]], align 8
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[TMP12]], i64 2
+; CHECK-NEXT: store <2 x i64> [[WIDE_LOAD]], ptr [[TMP12]], align 8
+; CHECK-NEXT: store <2 x i64> [[WIDE_LOAD1]], ptr [[TMP13]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %res = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %n, i64 3)
+ %mul = extractvalue { i64, i1 } %res, 0
+ %ov = extractvalue { i64, i1 } %res, 1
+ %ov.ext = zext i1 %ov to i64
+ %gep.b = getelementptr i64, ptr %b, i64 %iv
+ store i64 %ov.ext, ptr %gep.b
+ %idx = add i64 %mul, %iv
+ %gep.a = getelementptr i64, ptr %a, i64 %idx
+ %l.a = load i64, ptr %gep.a
+ %gep.c = getelementptr i64, ptr %c, i64 %iv
+ store i64 %l.a, ptr %gep.c
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The scalarized call's product is only used as the base of an interleave group, so
+; its extractvalue is narrowed to a single scalar.
+define void @narrow_extractvalue_of_scalarized_call(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) {
+; CHECK-LABEL: define void @narrow_extractvalue_of_scalarized_call(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[INDEX]], i64 2)
+; CHECK-NEXT: [[TMP5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[TMP1]], i64 2)
+; CHECK-NEXT: [[TMP6:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[TMP2]], i64 2)
+; CHECK-NEXT: [[TMP7:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 [[TMP3]], i64 2)
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT: [[TMP27:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { i64, i1 } [[TMP7]], 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i1> poison, i1 [[TMP9]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i1> [[TMP15]], i1 [[TMP10]], i32 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i1> [[TMP13]], i1 [[TMP27]], i32 2
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i1> [[TMP14]], i1 [[TMP12]], i32 3
+; CHECK-NEXT: [[TMP17:%.*]] = zext <4 x i1> [[TMP16]] to <4 x i32>
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP17]], ptr [[TMP18]], align 4
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP19]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP20:%.*]] = add <4 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: store <4 x i32> [[TMP20]], ptr [[TMP21]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %res = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %iv, i64 2)
+ %mul = extractvalue { i64, i1 } %res, 0
+ %ov = extractvalue { i64, i1 } %res, 1
+ %ov.ext = zext i1 %ov to i32
+ %gep.b = getelementptr i32, ptr %b, i64 %iv
+ store i32 %ov.ext, ptr %gep.b
+ %gep.a.0 = getelementptr i32, ptr %a, i64 %mul
+ %l.0 = load i32, ptr %gep.a.0
+ %mul.1 = add i64 %mul, 1
+ %gep.a.1 = getelementptr i32, ptr %a, i64 %mul.1
+ %l.1 = load i32, ptr %gep.a.1
+ %sum = add i32 %l.0, %l.1
+ %gep.c = getelementptr i32, ptr %c, i64 %iv
+ store i32 %sum, ptr %gep.c
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; Both extracts are only used in the predicated block, so the call is sunk
+; together with them.
+define void @extractvalues_sunk_with_scalarized_call(ptr noalias %a, ptr noalias %c, i64 %n) {
+; CHECK-LABEL: define void @extractvalues_sunk_with_scalarized_call(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[TMP1]], i64 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <2 x i64> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt <2 x i64> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP3]], i64 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[INDEX]], i64 1)
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { i64, i1 } [[TMP6]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = zext i1 [[TMP7]] to i64
+; CHECK-NEXT: store i64 [[TMP10]], ptr [[TMP9]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP3]], i64 1
+; CHECK-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
+; CHECK: [[PRED_STORE_IF2]]:
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[TMP12]], i64 1)
+; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { i64, i1 } [[TMP13]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { i64, i1 } [[TMP13]], 1
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = zext i1 [[TMP14]] to i64
+; CHECK-NEXT: store i64 [[TMP17]], ptr [[TMP16]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; CHECK: [[PRED_STORE_CONTINUE3]]:
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: br i1 [[TMP18]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
+; CHECK: [[PRED_STORE_IF4]]:
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP20:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[TMP19]], i64 1)
+; CHECK-NEXT: [[TMP22:%.*]] = extractvalue { i64, i1 } [[TMP20]], 0
+; CHECK-NEXT: [[TMP21:%.*]] = extractvalue { i64, i1 } [[TMP20]], 1
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = zext i1 [[TMP21]] to i64
+; CHECK-NEXT: store i64 [[TMP24]], ptr [[TMP23]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE5]]
+; CHECK: [[PRED_STORE_CONTINUE5]]:
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
+; CHECK-NEXT: br i1 [[TMP25]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
+; CHECK: [[PRED_STORE_IF6]]:
+; CHECK-NEXT: [[TMP26:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP27:%.*]] = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 [[TMP26]], i64 1)
+; CHECK-NEXT: [[TMP29:%.*]] = extractvalue { i64, i1 } [[TMP27]], 0
+; CHECK-NEXT: [[TMP28:%.*]] = extractvalue { i64, i1 } [[TMP27]], 1
+; CHECK-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP29]]
+; CHECK-NEXT: [[TMP31:%.*]] = zext i1 [[TMP28]] to i64
+; CHECK-NEXT: store i64 [[TMP31]], ptr [[TMP30]], align 8
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE7]]
+; CHECK: [[PRED_STORE_CONTINUE7]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %res = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %iv, i64 1)
+ %sum = extractvalue { i64, i1 } %res, 0
+ %ov = extractvalue { i64, i1 } %res, 1
+ %gep.a = getelementptr i64, ptr %a, i64 %iv
+ %l.a = load i64, ptr %gep.a
+ %cmp = icmp sgt i64 %l.a, 0
+ br i1 %cmp, label %then, label %loop.latch
+
+then:
+ %ov.ext = zext i1 %ov to i64
+ %gep.c = getelementptr i64, ptr %c, i64 %sum
+ store i64 %ov.ext, ptr %gep.c
+ br label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop.header, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.vectorize.width", i32 2}
+!2 = !{!"llvm.loop.vectorize.enable"}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-widen-struct-return.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-widen-struct-return.ll
index ffdb5f41c2de53..e55a0478870eb5 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-widen-struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-widen-struct-return.ll
@@ -111,8 +111,8 @@ define void @struct_return_f32_replicate(ptr noalias %in, ptr noalias writeonly
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>
; CHECK-NEXT: WIDEN ir<%in_val> = load vp<[[VP5]]>
; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%in_val>)
-; CHECK-NEXT: WIDEN ir<%extract_a> = extractvalue ir<%call>, ir<0>
-; CHECK-NEXT: WIDEN ir<%extract_b> = extractvalue ir<%call>, ir<1>
+; CHECK-NEXT: REPLICATE ir<%extract_a> = extractvalue ir<%call>
+; CHECK-NEXT: REPLICATE ir<%extract_b> = extractvalue ir<%call>
; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%out_a>, vp<[[VP4]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>
; CHECK-NEXT: WIDEN store vp<[[VP6]]>, ir<%extract_a>
diff --git a/llvm/test/Transforms/LoopVectorize/extract-value-widen.ll b/llvm/test/Transforms/LoopVectorize/extract-value-widen.ll
index 2e8b99e31a6a15..760292b94251dc 100644
--- a/llvm/test/Transforms/LoopVectorize/extract-value-widen.ll
+++ b/llvm/test/Transforms/LoopVectorize/extract-value-widen.ll
@@ -42,3 +42,54 @@ loop.header:
exit:
ret void
}
+
+; The overflow bit needs all lanes, so the call is widened to a struct of
+; vectors, while only the first lane of the sum is used. The sum must be
+; extracted from the struct with a wide extractvalue.
+define void @extract_first_lane_from_widened_struct(ptr %p, ptr noalias %q) {
+; CHECK-LABEL: define void @extract_first_lane_from_widened_struct(
+; CHECK-SAME: ptr [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> [[VEC_IND]], <2 x i64> splat (i64 1))
+; CHECK-NEXT: [[TMP1:%.*]] = extractvalue { <2 x i64>, <2 x i1> } [[TMP0]], 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i1> } [[TMP0]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <2 x i1> [[TMP2]] to <2 x i64>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[Q]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
+; CHECK-NEXT: store <2 x i64> zeroinitializer, ptr [[TMP6]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %res = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %iv, i64 1)
+ %iv.next = extractvalue { i64, i1 } %res, 0
+ %ov = extractvalue { i64, i1 } %res, 1
+ %ov.ext = zext i1 %ov to i64
+ %gep.q = getelementptr i64, ptr %q, i64 %iv
+ store i64 %ov.ext, ptr %gep.q, align 8
+ %gep.p = getelementptr i64, ptr %p, i64 %iv.next
+ store i64 0, ptr %gep.p, align 8
+ %ec = icmp eq i64 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll b/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
index 92da4ca6639697..bd29041311458f 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
@@ -22,23 +22,15 @@ define void @struct_return_1xi64_replicate(ptr noalias %in, ptr noalias writeonl
; VF4-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 3
; VF4-NEXT: [[TMP9:%.*]] = tail call { i64 } @fn1(float [[TMP4]]) #[[ATTR0]]
; VF4-NEXT: [[TMP10:%.*]] = extractvalue { i64 } [[TMP3]], 0
-; VF4-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
-; VF4-NEXT: [[TMP12:%.*]] = insertvalue { <4 x i64> } poison, <4 x i64> [[TMP11]], 0
; VF4-NEXT: [[TMP13:%.*]] = extractvalue { i64 } [[TMP5]], 0
-; VF4-NEXT: [[TMP14:%.*]] = extractvalue { <4 x i64> } [[TMP12]], 0
-; VF4-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP13]], i64 1
-; VF4-NEXT: [[TMP16:%.*]] = insertvalue { <4 x i64> } [[TMP12]], <4 x i64> [[TMP15]], 0
; VF4-NEXT: [[TMP17:%.*]] = extractvalue { i64 } [[TMP7]], 0
-; VF4-NEXT: [[TMP18:%.*]] = extractvalue { <4 x i64> } [[TMP16]], 0
-; VF4-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP17]], i64 2
-; VF4-NEXT: [[TMP20:%.*]] = insertvalue { <4 x i64> } [[TMP16]], <4 x i64> [[TMP19]], 0
; VF4-NEXT: [[TMP21:%.*]] = extractvalue { i64 } [[TMP9]], 0
-; VF4-NEXT: [[TMP22:%.*]] = extractvalue { <4 x i64> } [[TMP20]], 0
-; VF4-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP21]], i64 3
-; VF4-NEXT: [[TMP24:%.*]] = insertvalue { <4 x i64> } [[TMP20]], <4 x i64> [[TMP23]], 0
-; VF4-NEXT: [[TMP25:%.*]] = extractvalue { <4 x i64> } [[TMP24]], 0
+; VF4-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; VF4-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
+; VF4-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP17]], i32 2
+; VF4-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP21]], i32 3
; VF4-NEXT: [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[OUT_A]], i64 [[INDEX]]
-; VF4-NEXT: store <4 x i64> [[TMP25]], ptr [[TMP26]], align 4
+; VF4-NEXT: store <4 x i64> [[TMP23]], ptr [[TMP26]], align 4
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; VF4-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -57,29 +49,21 @@ define void @struct_return_1xi64_replicate(ptr noalias %in, ptr noalias writeonl
; VF2IC2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP0]], align 4
; VF2IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x float>, ptr [[TMP2]], align 4
; VF2IC2-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 0
-; VF2IC2-NEXT: [[TMP4:%.*]] = tail call { i64 } @fn1(float [[TMP3]]) #[[ATTR0:[0-9]+]]
+; VF2IC2-NEXT: [[TMP15:%.*]] = tail call { i64 } @fn1(float [[TMP3]]) #[[ATTR0:[0-9]+]]
; VF2IC2-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 1
; VF2IC2-NEXT: [[TMP6:%.*]] = tail call { i64 } @fn1(float [[TMP5]]) #[[ATTR0]]
-; VF2IC2-NEXT: [[TMP7:%.*]] = extractvalue { i64 } [[TMP4]], 0
-; VF2IC2-NEXT: [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i64 0
-; VF2IC2-NEXT: [[TMP9:%.*]] = insertvalue { <2 x i64> } poison, <2 x i64> [[TMP8]], 0
-; VF2IC2-NEXT: [[TMP10:%.*]] = extractvalue { i64 } [[TMP6]], 0
-; VF2IC2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x i64> } [[TMP9]], 0
-; VF2IC2-NEXT: [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i64 1
-; VF2IC2-NEXT: [[TMP13:%.*]] = insertvalue { <2 x i64> } [[TMP9]], <2 x i64> [[TMP12]], 0
; VF2IC2-NEXT: [[TMP14:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 0
-; VF2IC2-NEXT: [[TMP15:%.*]] = tail call { i64 } @fn1(float [[TMP14]]) #[[ATTR0]]
+; VF2IC2-NEXT: [[TMP17:%.*]] = tail call { i64 } @fn1(float [[TMP14]]) #[[ATTR0]]
; VF2IC2-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 1
-; VF2IC2-NEXT: [[TMP17:%.*]] = tail call { i64 } @fn1(float [[TMP16]]) #[[ATTR0]]
+; VF2IC2-NEXT: [[TMP9:%.*]] = tail call { i64 } @fn1(float [[TMP16]]) #[[ATTR0]]
; VF2IC2-NEXT: [[TMP18:%.*]] = extractvalue { i64 } [[TMP15]], 0
-; VF2IC2-NEXT: [[TMP19:%.*]] = insertelement <2 x i64> poison, i64 [[TMP18]], i64 0
-; VF2IC2-NEXT: [[TMP20:%.*]] = insertvalue { <2 x i64> } poison, <2 x i64> [[TMP19]], 0
+; VF2IC2-NEXT: [[TMP11:%.*]] = extractvalue { i64 } [[TMP6]], 0
+; VF2IC2-NEXT: [[TMP12:%.*]] = insertelement <2 x i64> poison, i64 [[TMP18]], i32 0
+; VF2IC2-NEXT: [[TMP25:%.*]] = insertelement <2 x i64> [[TMP12]], i64 [[TMP11]], i32 1
; VF2IC2-NEXT: [[TMP21:%.*]] = extractvalue { i64 } [[TMP17]], 0
-; VF2IC2-NEXT: [[TMP22:%.*]] = extractvalue { <2 x i64> } [[TMP20]], 0
-; VF2IC2-NEXT: [[TMP23:%.*]] = insertelement <2 x i64> [[TMP22]], i64 [[TMP21]], i64 1
-; VF2IC2-NEXT: [[TMP24:%.*]] = insertvalue { <2 x i64> } [[TMP20]], <2 x i64> [[TMP23]], 0
-; VF2IC2-NEXT: [[TMP25:%.*]] = extractvalue { <2 x i64> } [[TMP13]], 0
-; VF2IC2-NEXT: [[TMP26:%.*]] = extractvalue { <2 x i64> } [[TMP24]], 0
+; VF2IC2-NEXT: [[TMP20:%.*]] = extractvalue { i64 } [[TMP9]], 0
+; VF2IC2-NEXT: [[TMP19:%.*]] = insertelement <2 x i64> poison, i64 [[TMP21]], i32 0
+; VF2IC2-NEXT: [[TMP26:%.*]] = insertelement <2 x i64> [[TMP19]], i64 [[TMP20]], i32 1
; VF2IC2-NEXT: [[TMP27:%.*]] = getelementptr inbounds i64, ptr [[OUT_A]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP29:%.*]] = getelementptr inbounds i64, ptr [[TMP27]], i64 2
; VF2IC2-NEXT: store <2 x i64> [[TMP25]], ptr [[TMP27]], align 4
@@ -128,42 +112,25 @@ define void @struct_return_2xf32_replicate(ptr noalias %in, ptr noalias writeonl
; VF4-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 3
; VF4-NEXT: [[TMP9:%.*]] = tail call { float, float } @fn2(float [[TMP4]]) #[[ATTR1]]
; VF4-NEXT: [[TMP10:%.*]] = extractvalue { float, float } [[TMP3]], 0
-; VF4-NEXT: [[TMP11:%.*]] = insertelement <4 x float> poison, float [[TMP10]], i64 0
-; VF4-NEXT: [[TMP12:%.*]] = insertvalue { <4 x float>, <4 x float> } poison, <4 x float> [[TMP11]], 0
-; VF4-NEXT: [[TMP13:%.*]] = extractvalue { float, float } [[TMP3]], 1
-; VF4-NEXT: [[TMP14:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP12]], 1
-; VF4-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP13]], i64 0
-; VF4-NEXT: [[TMP16:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP12]], <4 x float> [[TMP15]], 1
; VF4-NEXT: [[TMP17:%.*]] = extractvalue { float, float } [[TMP5]], 0
-; VF4-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP16]], 0
-; VF4-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[TMP17]], i64 1
-; VF4-NEXT: [[TMP20:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP16]], <4 x float> [[TMP19]], 0
-; VF4-NEXT: [[TMP21:%.*]] = extractvalue { float, float } [[TMP5]], 1
-; VF4-NEXT: [[TMP22:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP20]], 1
-; VF4-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP22]], float [[TMP21]], i64 1
-; VF4-NEXT: [[TMP24:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP20]], <4 x float> [[TMP23]], 1
; VF4-NEXT: [[TMP25:%.*]] = extractvalue { float, float } [[TMP7]], 0
-; VF4-NEXT: [[TMP26:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP24]], 0
-; VF4-NEXT: [[TMP27:%.*]] = insertelement <4 x float> [[TMP26]], float [[TMP25]], i64 2
-; VF4-NEXT: [[TMP28:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP24]], <4 x float> [[TMP27]], 0
-; VF4-NEXT: [[TMP29:%.*]] = extractvalue { float, float } [[TMP7]], 1
-; VF4-NEXT: [[TMP30:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP28]], 1
-; VF4-NEXT: [[TMP31:%.*]] = insertelement <4 x float> [[TMP30]], float [[TMP29]], i64 2
-; VF4-NEXT: [[TMP32:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP28]], <4 x float> [[TMP31]], 1
; VF4-NEXT: [[TMP33:%.*]] = extractvalue { float, float } [[TMP9]], 0
-; VF4-NEXT: [[TMP34:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP32]], 0
-; VF4-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP34]], float [[TMP33]], i64 3
-; VF4-NEXT: [[TMP36:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP32]], <4 x float> [[TMP35]], 0
+; VF4-NEXT: [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP10]], i32 0
+; VF4-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP17]], i32 1
+; VF4-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP25]], i32 2
+; VF4-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP15]], float [[TMP33]], i32 3
+; VF4-NEXT: [[TMP20:%.*]] = extractvalue { float, float } [[TMP3]], 1
+; VF4-NEXT: [[TMP18:%.*]] = extractvalue { float, float } [[TMP5]], 1
+; VF4-NEXT: [[TMP19:%.*]] = extractvalue { float, float } [[TMP7]], 1
; VF4-NEXT: [[TMP37:%.*]] = extractvalue { float, float } [[TMP9]], 1
-; VF4-NEXT: [[TMP38:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP36]], 1
-; VF4-NEXT: [[TMP39:%.*]] = insertelement <4 x float> [[TMP38]], float [[TMP37]], i64 3
-; VF4-NEXT: [[TMP40:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP36]], <4 x float> [[TMP39]], 1
-; VF4-NEXT: [[TMP41:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP40]], 0
-; VF4-NEXT: [[TMP42:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP40]], 1
+; VF4-NEXT: [[TMP21:%.*]] = insertelement <4 x float> poison, float [[TMP20]], i32 0
+; VF4-NEXT: [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP18]], i32 1
+; VF4-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP22]], float [[TMP19]], i32 2
+; VF4-NEXT: [[TMP39:%.*]] = insertelement <4 x float> [[TMP23]], float [[TMP37]], i32 3
; VF4-NEXT: [[TMP43:%.*]] = getelementptr inbounds float, ptr [[OUT_A]], i64 [[INDEX]]
-; VF4-NEXT: store <4 x float> [[TMP41]], ptr [[TMP43]], align 4
+; VF4-NEXT: store <4 x float> [[TMP35]], ptr [[TMP43]], align 4
; VF4-NEXT: [[TMP45:%.*]] = getelementptr inbounds float, ptr [[OUT_B]], i64 [[INDEX]]
-; VF4-NEXT: store <4 x float> [[TMP42]], ptr [[TMP45]], align 4
+; VF4-NEXT: store <4 x float> [[TMP39]], ptr [[TMP45]], align 4
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF4-NEXT: [[TMP47:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; VF4-NEXT: br i1 [[TMP47]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -185,52 +152,34 @@ define void @struct_return_2xf32_replicate(ptr noalias %in, ptr noalias writeonl
; VF2IC2-NEXT: [[TMP4:%.*]] = tail call { float, float } @fn2(float [[TMP3]]) #[[ATTR1:[0-9]+]]
; VF2IC2-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 1
; VF2IC2-NEXT: [[TMP6:%.*]] = tail call { float, float } @fn2(float [[TMP5]]) #[[ATTR1]]
+; VF2IC2-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 0
+; VF2IC2-NEXT: [[TMP23:%.*]] = tail call { float, float } @fn2(float [[TMP9]]) #[[ATTR1]]
+; VF2IC2-NEXT: [[TMP8:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 1
+; VF2IC2-NEXT: [[TMP25:%.*]] = tail call { float, float } @fn2(float [[TMP8]]) #[[ATTR1]]
; VF2IC2-NEXT: [[TMP7:%.*]] = extractvalue { float, float } [[TMP4]], 0
-; VF2IC2-NEXT: [[TMP8:%.*]] = insertelement <2 x float> poison, float [[TMP7]], i64 0
-; VF2IC2-NEXT: [[TMP9:%.*]] = insertvalue { <2 x float>, <2 x float> } poison, <2 x float> [[TMP8]], 0
-; VF2IC2-NEXT: [[TMP10:%.*]] = extractvalue { float, float } [[TMP4]], 1
-; VF2IC2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP9]], 1
-; VF2IC2-NEXT: [[TMP12:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP10]], i64 0
-; VF2IC2-NEXT: [[TMP13:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP9]], <2 x float> [[TMP12]], 1
; VF2IC2-NEXT: [[TMP14:%.*]] = extractvalue { float, float } [[TMP6]], 0
-; VF2IC2-NEXT: [[TMP15:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP13]], 0
-; VF2IC2-NEXT: [[TMP16:%.*]] = insertelement <2 x float> [[TMP15]], float [[TMP14]], i64 1
-; VF2IC2-NEXT: [[TMP17:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP13]], <2 x float> [[TMP16]], 0
-; VF2IC2-NEXT: [[TMP18:%.*]] = extractvalue { float, float } [[TMP6]], 1
-; VF2IC2-NEXT: [[TMP19:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP17]], 1
-; VF2IC2-NEXT: [[TMP20:%.*]] = insertelement <2 x float> [[TMP19]], float [[TMP18]], i64 1
-; VF2IC2-NEXT: [[TMP21:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP17]], <2 x float> [[TMP20]], 1
-; VF2IC2-NEXT: [[TMP22:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 0
-; VF2IC2-NEXT: [[TMP23:%.*]] = tail call { float, float } @fn2(float [[TMP22]]) #[[ATTR1]]
-; VF2IC2-NEXT: [[TMP24:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 1
-; VF2IC2-NEXT: [[TMP25:%.*]] = tail call { float, float } @fn2(float [[TMP24]]) #[[ATTR1]]
+; VF2IC2-NEXT: [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP7]], i32 0
+; VF2IC2-NEXT: [[TMP41:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP14]], i32 1
; VF2IC2-NEXT: [[TMP26:%.*]] = extractvalue { float, float } [[TMP23]], 0
-; VF2IC2-NEXT: [[TMP27:%.*]] = insertelement <2 x float> poison, float [[TMP26]], i64 0
-; VF2IC2-NEXT: [[TMP28:%.*]] = insertvalue { <2 x float>, <2 x float> } poison, <2 x float> [[TMP27]], 0
-; VF2IC2-NEXT: [[TMP29:%.*]] = extractvalue { float, float } [[TMP23]], 1
-; VF2IC2-NEXT: [[TMP30:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP28]], 1
-; VF2IC2-NEXT: [[TMP31:%.*]] = insertelement <2 x float> [[TMP30]], float [[TMP29]], i64 0
-; VF2IC2-NEXT: [[TMP32:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP28]], <2 x float> [[TMP31]], 1
-; VF2IC2-NEXT: [[TMP33:%.*]] = extractvalue { float, float } [[TMP25]], 0
-; VF2IC2-NEXT: [[TMP34:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP32]], 0
-; VF2IC2-NEXT: [[TMP35:%.*]] = insertelement <2 x float> [[TMP34]], float [[TMP33]], i64 1
-; VF2IC2-NEXT: [[TMP36:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP32]], <2 x float> [[TMP35]], 0
+; VF2IC2-NEXT: [[TMP15:%.*]] = extractvalue { float, float } [[TMP25]], 0
+; VF2IC2-NEXT: [[TMP16:%.*]] = insertelement <2 x float> poison, float [[TMP26]], i32 0
+; VF2IC2-NEXT: [[TMP42:%.*]] = insertelement <2 x float> [[TMP16]], float [[TMP15]], i32 1
+; VF2IC2-NEXT: [[TMP18:%.*]] = extractvalue { float, float } [[TMP4]], 1
+; VF2IC2-NEXT: [[TMP33:%.*]] = extractvalue { float, float } [[TMP6]], 1
+; VF2IC2-NEXT: [[TMP20:%.*]] = insertelement <2 x float> poison, float [[TMP18]], i32 0
+; VF2IC2-NEXT: [[TMP35:%.*]] = insertelement <2 x float> [[TMP20]], float [[TMP33]], i32 1
+; VF2IC2-NEXT: [[TMP22:%.*]] = extractvalue { float, float } [[TMP23]], 1
; VF2IC2-NEXT: [[TMP37:%.*]] = extractvalue { float, float } [[TMP25]], 1
-; VF2IC2-NEXT: [[TMP38:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP36]], 1
-; VF2IC2-NEXT: [[TMP39:%.*]] = insertelement <2 x float> [[TMP38]], float [[TMP37]], i64 1
-; VF2IC2-NEXT: [[TMP40:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP36]], <2 x float> [[TMP39]], 1
-; VF2IC2-NEXT: [[TMP41:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP21]], 0
-; VF2IC2-NEXT: [[TMP42:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP40]], 0
-; VF2IC2-NEXT: [[TMP43:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP21]], 1
-; VF2IC2-NEXT: [[TMP44:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP40]], 1
+; VF2IC2-NEXT: [[TMP24:%.*]] = insertelement <2 x float> poison, float [[TMP22]], i32 0
+; VF2IC2-NEXT: [[TMP39:%.*]] = insertelement <2 x float> [[TMP24]], float [[TMP37]], i32 1
; VF2IC2-NEXT: [[TMP45:%.*]] = getelementptr inbounds float, ptr [[OUT_A]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP47:%.*]] = getelementptr inbounds float, ptr [[TMP45]], i64 2
; VF2IC2-NEXT: store <2 x float> [[TMP41]], ptr [[TMP45]], align 4
; VF2IC2-NEXT: store <2 x float> [[TMP42]], ptr [[TMP47]], align 4
; VF2IC2-NEXT: [[TMP48:%.*]] = getelementptr inbounds float, ptr [[OUT_B]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP50:%.*]] = getelementptr inbounds float, ptr [[TMP48]], i64 2
-; VF2IC2-NEXT: store <2 x float> [[TMP43]], ptr [[TMP48]], align 4
-; VF2IC2-NEXT: store <2 x float> [[TMP44]], ptr [[TMP50]], align 4
+; VF2IC2-NEXT: store <2 x float> [[TMP35]], ptr [[TMP48]], align 4
+; VF2IC2-NEXT: store <2 x float> [[TMP39]], ptr [[TMP50]], align 4
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; VF2IC2-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; VF2IC2-NEXT: br i1 [[TMP51]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -279,59 +228,33 @@ define void @struct_return_3xi32_replicate(ptr noalias %in, ptr noalias writeonl
; VF4-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3
; VF4-NEXT: [[TMP9:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP4]]) #[[ATTR2]]
; VF4-NEXT: [[TMP10:%.*]] = extractvalue { i32, i32, i32 } [[TMP3]], 0
-; VF4-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i64 0
-; VF4-NEXT: [[TMP12:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP11]], 0
-; VF4-NEXT: [[TMP13:%.*]] = extractvalue { i32, i32, i32 } [[TMP3]], 1
-; VF4-NEXT: [[TMP14:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP12]], 1
-; VF4-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP13]], i64 0
-; VF4-NEXT: [[TMP16:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP12]], <4 x i32> [[TMP15]], 1
-; VF4-NEXT: [[TMP17:%.*]] = extractvalue { i32, i32, i32 } [[TMP3]], 2
-; VF4-NEXT: [[TMP18:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP16]], 2
-; VF4-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP17]], i64 0
-; VF4-NEXT: [[TMP20:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP16]], <4 x i32> [[TMP19]], 2
; VF4-NEXT: [[TMP21:%.*]] = extractvalue { i32, i32, i32 } [[TMP5]], 0
-; VF4-NEXT: [[TMP22:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP20]], 0
-; VF4-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP21]], i64 1
-; VF4-NEXT: [[TMP24:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP20]], <4 x i32> [[TMP23]], 0
-; VF4-NEXT: [[TMP25:%.*]] = extractvalue { i32, i32, i32 } [[TMP5]], 1
-; VF4-NEXT: [[TMP26:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP24]], 1
-; VF4-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP25]], i64 1
-; VF4-NEXT: [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP24]], <4 x i32> [[TMP27]], 1
-; VF4-NEXT: [[TMP29:%.*]] = extractvalue { i32, i32, i32 } [[TMP5]], 2
-; VF4-NEXT: [[TMP30:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP28]], 2
-; VF4-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP29]], i64 1
-; VF4-NEXT: [[TMP32:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP28]], <4 x i32> [[TMP31]], 2
; VF4-NEXT: [[TMP33:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 0
-; VF4-NEXT: [[TMP34:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP32]], 0
-; VF4-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP33]], i64 2
-; VF4-NEXT: [[TMP36:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP32]], <4 x i32> [[TMP35]], 0
-; VF4-NEXT: [[TMP37:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 1
-; VF4-NEXT: [[TMP38:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP36]], 1
-; VF4-NEXT: [[TMP39:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP37]], i64 2
-; VF4-NEXT: [[TMP40:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP36]], <4 x i32> [[TMP39]], 1
-; VF4-NEXT: [[TMP41:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 2
-; VF4-NEXT: [[TMP42:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP40]], 2
-; VF4-NEXT: [[TMP43:%.*]] = insertelement <4 x i32> [[TMP42]], i32 [[TMP41]], i64 2
-; VF4-NEXT: [[TMP44:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP40]], <4 x i32> [[TMP43]], 2
; VF4-NEXT: [[TMP45:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 0
-; VF4-NEXT: [[TMP46:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP44]], 0
-; VF4-NEXT: [[TMP47:%.*]] = insertelement <4 x i32> [[TMP46]], i32 [[TMP45]], i64 3
-; VF4-NEXT: [[TMP48:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP44]], <4 x i32> [[TMP47]], 0
-; VF4-NEXT: [[TMP49:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 1
-; VF4-NEXT: [[TMP50:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP48]], 1
-; VF4-NEXT: [[TMP51:%.*]] = insertelement <4 x i32> [[TMP50]], i32 [[TMP49]], i64 3
-; VF4-NEXT: [[TMP52:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP48]], <4 x i32> [[TMP51]], 1
-; VF4-NEXT: [[TMP53:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 2
-; VF4-NEXT: [[TMP54:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP52]], 2
-; VF4-NEXT: [[TMP55:%.*]] = insertelement <4 x i32> [[TMP54]], i32 [[TMP53]], i64 3
-; VF4-NEXT: [[TMP56:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP52]], <4 x i32> [[TMP55]], 2
-; VF4-NEXT: [[TMP57:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP56]], 0
+; VF4-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i32 0
+; VF4-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP21]], i32 1
+; VF4-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP33]], i32 2
+; VF4-NEXT: [[TMP57:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP45]], i32 3
; VF4-NEXT: [[TMP58:%.*]] = getelementptr inbounds i32, ptr [[DST_A]], i64 [[INDEX]]
; VF4-NEXT: store <4 x i32> [[TMP57]], ptr [[TMP58]], align 4
-; VF4-NEXT: [[TMP60:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP56]], 1
+; VF4-NEXT: [[TMP18:%.*]] = extractvalue { i32, i32, i32 } [[TMP3]], 1
+; VF4-NEXT: [[TMP19:%.*]] = extractvalue { i32, i32, i32 } [[TMP5]], 1
+; VF4-NEXT: [[TMP20:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 1
+; VF4-NEXT: [[TMP25:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 1
+; VF4-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> poison, i32 [[TMP18]], i32 0
+; VF4-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP19]], i32 1
+; VF4-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i32 2
+; VF4-NEXT: [[TMP60:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP25]], i32 3
; VF4-NEXT: [[TMP61:%.*]] = getelementptr inbounds i32, ptr [[DST_B]], i64 [[INDEX]]
; VF4-NEXT: store <4 x i32> [[TMP60]], ptr [[TMP61]], align 4
-; VF4-NEXT: [[TMP63:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP56]], 2
+; VF4-NEXT: [[TMP27:%.*]] = extractvalue { i32, i32, i32 } [[TMP3]], 2
+; VF4-NEXT: [[TMP28:%.*]] = extractvalue { i32, i32, i32 } [[TMP5]], 2
+; VF4-NEXT: [[TMP29:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 2
+; VF4-NEXT: [[TMP30:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 2
+; VF4-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> poison, i32 [[TMP27]], i32 0
+; VF4-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 1
+; VF4-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 2
+; VF4-NEXT: [[TMP63:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP30]], i32 3
; VF4-NEXT: [[TMP64:%.*]] = getelementptr inbounds i32, ptr [[DST_C]], i64 [[INDEX]]
; VF4-NEXT: store <4 x i32> [[TMP63]], ptr [[TMP64]], align 4
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -352,73 +275,45 @@ define void @struct_return_3xi32_replicate(ptr noalias %in, ptr noalias writeonl
; VF2IC2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
; VF2IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
; VF2IC2-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 0
-; VF2IC2-NEXT: [[TMP4:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP3]]) #[[ATTR2:[0-9]+]]
+; VF2IC2-NEXT: [[TMP31:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP3]]) #[[ATTR2:[0-9]+]]
; VF2IC2-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i64 1
-; VF2IC2-NEXT: [[TMP6:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP5]]) #[[ATTR2]]
-; VF2IC2-NEXT: [[TMP7:%.*]] = extractvalue { i32, i32, i32 } [[TMP4]], 0
-; VF2IC2-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
-; VF2IC2-NEXT: [[TMP9:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } poison, <2 x i32> [[TMP8]], 0
-; VF2IC2-NEXT: [[TMP10:%.*]] = extractvalue { i32, i32, i32 } [[TMP4]], 1
-; VF2IC2-NEXT: [[TMP11:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP9]], 1
-; VF2IC2-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP10]], i64 0
-; VF2IC2-NEXT: [[TMP13:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP9]], <2 x i32> [[TMP12]], 1
-; VF2IC2-NEXT: [[TMP14:%.*]] = extractvalue { i32, i32, i32 } [[TMP4]], 2
-; VF2IC2-NEXT: [[TMP15:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP13]], 2
-; VF2IC2-NEXT: [[TMP16:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP14]], i64 0
-; VF2IC2-NEXT: [[TMP17:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP13]], <2 x i32> [[TMP16]], 2
-; VF2IC2-NEXT: [[TMP18:%.*]] = extractvalue { i32, i32, i32 } [[TMP6]], 0
-; VF2IC2-NEXT: [[TMP19:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP17]], 0
-; VF2IC2-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP18]], i64 1
-; VF2IC2-NEXT: [[TMP21:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP17]], <2 x i32> [[TMP20]], 0
-; VF2IC2-NEXT: [[TMP22:%.*]] = extractvalue { i32, i32, i32 } [[TMP6]], 1
-; VF2IC2-NEXT: [[TMP23:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP21]], 1
-; VF2IC2-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP22]], i64 1
-; VF2IC2-NEXT: [[TMP25:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP21]], <2 x i32> [[TMP24]], 1
-; VF2IC2-NEXT: [[TMP26:%.*]] = extractvalue { i32, i32, i32 } [[TMP6]], 2
-; VF2IC2-NEXT: [[TMP27:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP25]], 2
-; VF2IC2-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TMP26]], i64 1
-; VF2IC2-NEXT: [[TMP29:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP25]], <2 x i32> [[TMP28]], 2
+; VF2IC2-NEXT: [[TMP33:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP5]]) #[[ATTR2]]
; VF2IC2-NEXT: [[TMP30:%.*]] = extractelement <2 x i32> [[WIDE_LOAD1]], i64 0
-; VF2IC2-NEXT: [[TMP31:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP30]]) #[[ATTR2]]
+; VF2IC2-NEXT: [[TMP7:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP30]]) #[[ATTR2]]
; VF2IC2-NEXT: [[TMP32:%.*]] = extractelement <2 x i32> [[WIDE_LOAD1]], i64 1
-; VF2IC2-NEXT: [[TMP33:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP32]]) #[[ATTR2]]
+; VF2IC2-NEXT: [[TMP9:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP32]]) #[[ATTR2]]
; VF2IC2-NEXT: [[TMP34:%.*]] = extractvalue { i32, i32, i32 } [[TMP31]], 0
-; VF2IC2-NEXT: [[TMP35:%.*]] = insertelement <2 x i32> poison, i32 [[TMP34]], i64 0
-; VF2IC2-NEXT: [[TMP36:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } poison, <2 x i32> [[TMP35]], 0
-; VF2IC2-NEXT: [[TMP37:%.*]] = extractvalue { i32, i32, i32 } [[TMP31]], 1
-; VF2IC2-NEXT: [[TMP38:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP36]], 1
-; VF2IC2-NEXT: [[TMP39:%.*]] = insertelement <2 x i32> [[TMP38]], i32 [[TMP37]], i64 0
-; VF2IC2-NEXT: [[TMP40:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP36]], <2 x i32> [[TMP39]], 1
-; VF2IC2-NEXT: [[TMP41:%.*]] = extractvalue { i32, i32, i32 } [[TMP31]], 2
-; VF2IC2-NEXT: [[TMP42:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP40]], 2
-; VF2IC2-NEXT: [[TMP43:%.*]] = insertelement <2 x i32> [[TMP42]], i32 [[TMP41]], i64 0
-; VF2IC2-NEXT: [[TMP44:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP40]], <2 x i32> [[TMP43]], 2
; VF2IC2-NEXT: [[TMP45:%.*]] = extractvalue { i32, i32, i32 } [[TMP33]], 0
-; VF2IC2-NEXT: [[TMP46:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP44]], 0
-; VF2IC2-NEXT: [[TMP47:%.*]] = insertelement <2 x i32> [[TMP46]], i32 [[TMP45]], i64 1
-; VF2IC2-NEXT: [[TMP48:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP44]], <2 x i32> [[TMP47]], 0
-; VF2IC2-NEXT: [[TMP49:%.*]] = extractvalue { i32, i32, i32 } [[TMP33]], 1
-; VF2IC2-NEXT: [[TMP50:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP48]], 1
-; VF2IC2-NEXT: [[TMP51:%.*]] = insertelement <2 x i32> [[TMP50]], i32 [[TMP49]], i64 1
-; VF2IC2-NEXT: [[TMP52:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP48]], <2 x i32> [[TMP51]], 1
-; VF2IC2-NEXT: [[TMP53:%.*]] = extractvalue { i32, i32, i32 } [[TMP33]], 2
-; VF2IC2-NEXT: [[TMP54:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP52]], 2
-; VF2IC2-NEXT: [[TMP55:%.*]] = insertelement <2 x i32> [[TMP54]], i32 [[TMP53]], i64 1
-; VF2IC2-NEXT: [[TMP56:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP52]], <2 x i32> [[TMP55]], 2
-; VF2IC2-NEXT: [[TMP57:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP29]], 0
-; VF2IC2-NEXT: [[TMP58:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP56]], 0
+; VF2IC2-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> poison, i32 [[TMP34]], i32 0
+; VF2IC2-NEXT: [[TMP57:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[TMP45]], i32 1
+; VF2IC2-NEXT: [[TMP14:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 0
+; VF2IC2-NEXT: [[TMP53:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 0
+; VF2IC2-NEXT: [[TMP16:%.*]] = insertelement <2 x i32> poison, i32 [[TMP14]], i32 0
+; VF2IC2-NEXT: [[TMP55:%.*]] = insertelement <2 x i32> [[TMP16]], i32 [[TMP53]], i32 1
; VF2IC2-NEXT: [[TMP59:%.*]] = getelementptr inbounds i32, ptr [[DST_A]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP61:%.*]] = getelementptr inbounds i32, ptr [[TMP59]], i64 2
; VF2IC2-NEXT: store <2 x i32> [[TMP57]], ptr [[TMP59]], align 4
-; VF2IC2-NEXT: store <2 x i32> [[TMP58]], ptr [[TMP61]], align 4
-; VF2IC2-NEXT: [[TMP62:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP29]], 1
-; VF2IC2-NEXT: [[TMP63:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP56]], 1
+; VF2IC2-NEXT: store <2 x i32> [[TMP55]], ptr [[TMP61]], align 4
+; VF2IC2-NEXT: [[TMP20:%.*]] = extractvalue { i32, i32, i32 } [[TMP31]], 1
+; VF2IC2-NEXT: [[TMP21:%.*]] = extractvalue { i32, i32, i32 } [[TMP33]], 1
+; VF2IC2-NEXT: [[TMP22:%.*]] = insertelement <2 x i32> poison, i32 [[TMP20]], i32 0
+; VF2IC2-NEXT: [[TMP62:%.*]] = insertelement <2 x i32> [[TMP22]], i32 [[TMP21]], i32 1
+; VF2IC2-NEXT: [[TMP24:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 1
+; VF2IC2-NEXT: [[TMP25:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 1
+; VF2IC2-NEXT: [[TMP26:%.*]] = insertelement <2 x i32> poison, i32 [[TMP24]], i32 0
+; VF2IC2-NEXT: [[TMP63:%.*]] = insertelement <2 x i32> [[TMP26]], i32 [[TMP25]], i32 1
; VF2IC2-NEXT: [[TMP64:%.*]] = getelementptr inbounds i32, ptr [[DST_B]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP66:%.*]] = getelementptr inbounds i32, ptr [[TMP64]], i64 2
; VF2IC2-NEXT: store <2 x i32> [[TMP62]], ptr [[TMP64]], align 4
; VF2IC2-NEXT: store <2 x i32> [[TMP63]], ptr [[TMP66]], align 4
-; VF2IC2-NEXT: [[TMP67:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP29]], 2
-; VF2IC2-NEXT: [[TMP68:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP56]], 2
+; VF2IC2-NEXT: [[TMP37:%.*]] = extractvalue { i32, i32, i32 } [[TMP31]], 2
+; VF2IC2-NEXT: [[TMP38:%.*]] = extractvalue { i32, i32, i32 } [[TMP33]], 2
+; VF2IC2-NEXT: [[TMP39:%.*]] = insertelement <2 x i32> poison, i32 [[TMP37]], i32 0
+; VF2IC2-NEXT: [[TMP67:%.*]] = insertelement <2 x i32> [[TMP39]], i32 [[TMP38]], i32 1
+; VF2IC2-NEXT: [[TMP40:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 2
+; VF2IC2-NEXT: [[TMP35:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 2
+; VF2IC2-NEXT: [[TMP36:%.*]] = insertelement <2 x i32> poison, i32 [[TMP40]], i32 0
+; VF2IC2-NEXT: [[TMP68:%.*]] = insertelement <2 x i32> [[TMP36]], i32 [[TMP35]], i32 1
; VF2IC2-NEXT: [[TMP69:%.*]] = getelementptr inbounds i32, ptr [[DST_C]], i64 [[INDEX]]
; VF2IC2-NEXT: [[TMP71:%.*]] = getelementptr inbounds i32, ptr [[TMP69]], i64 2
; VF2IC2-NEXT: store <2 x i32> [[TMP67]], ptr [[TMP69]], align 4
@@ -469,66 +364,42 @@ define void @struct_return_2xf32_replicate_predicated(ptr %a) {
; VF4-NEXT: br i1 [[TMP2]], label %[[PRED_CALL_IF:.*]], label %[[PRED_CALL_CONTINUE:.*]]
; VF4: [[PRED_CALL_IF]]:
; VF4-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 0
-; VF4-NEXT: [[TMP13:%.*]] = tail call { float, float } @fn2(float [[TMP3]]) #[[ATTR3:[0-9]+]]
+; VF4-NEXT: [[TMP4:%.*]] = tail call { float, float } @fn2(float [[TMP3]]) #[[ATTR3:[0-9]+]]
+; VF4-NEXT: [[TMP5:%.*]] = extractvalue { float, float } [[TMP4]], 0
+; VF4-NEXT: [[TMP6:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
; VF4-NEXT: br label %[[PRED_CALL_CONTINUE]]
; VF4: [[PRED_CALL_CONTINUE]]:
-; VF4-NEXT: [[TMP4:%.*]] = phi { float, float } [ poison, %[[VECTOR_BODY]] ], [ [[TMP13]], %[[PRED_CALL_IF]] ]
+; VF4-NEXT: [[TMP7:%.*]] = phi <4 x float> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_CALL_IF]] ]
; VF4-NEXT: [[TMP55:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
; VF4-NEXT: br i1 [[TMP55]], label %[[PRED_CALL_IF1:.*]], label %[[PRED_CALL_CONTINUE2:.*]]
; VF4: [[PRED_CALL_IF1]]:
; VF4-NEXT: [[TMP14:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 1
-; VF4-NEXT: [[TMP17:%.*]] = tail call { float, float } @fn2(float [[TMP14]]) #[[ATTR3]]
+; VF4-NEXT: [[TMP10:%.*]] = tail call { float, float } @fn2(float [[TMP14]]) #[[ATTR3]]
+; VF4-NEXT: [[TMP11:%.*]] = extractvalue { float, float } [[TMP10]], 0
+; VF4-NEXT: [[TMP12:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP11]], i64 1
; VF4-NEXT: br label %[[PRED_CALL_CONTINUE2]]
; VF4: [[PRED_CALL_CONTINUE2]]:
-; VF4-NEXT: [[TMP15:%.*]] = phi { float, float } [ poison, %[[PRED_CALL_CONTINUE]] ], [ [[TMP17]], %[[PRED_CALL_IF1]] ]
+; VF4-NEXT: [[TMP13:%.*]] = phi <4 x float> [ [[TMP7]], %[[PRED_CALL_CONTINUE]] ], [ [[TMP12]], %[[PRED_CALL_IF1]] ]
; VF4-NEXT: [[TMP59:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
; VF4-NEXT: br i1 [[TMP59]], label %[[PRED_CALL_IF3:.*]], label %[[PRED_CALL_CONTINUE4:.*]]
; VF4: [[PRED_CALL_IF3]]:
; VF4-NEXT: [[TMP18:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 2
-; VF4-NEXT: [[TMP12:%.*]] = tail call { float, float } @fn2(float [[TMP18]]) #[[ATTR3]]
+; VF4-NEXT: [[TMP16:%.*]] = tail call { float, float } @fn2(float [[TMP18]]) #[[ATTR3]]
+; VF4-NEXT: [[TMP17:%.*]] = extractvalue { float, float } [[TMP16]], 0
+; VF4-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP17]], i64 2
; VF4-NEXT: br label %[[PRED_CALL_CONTINUE4]]
; VF4: [[PRED_CALL_CONTINUE4]]:
-; VF4-NEXT: [[TMP27:%.*]] = phi { float, float } [ poison, %[[PRED_CALL_CONTINUE2]] ], [ [[TMP12]], %[[PRED_CALL_IF3]] ]
+; VF4-NEXT: [[TMP42:%.*]] = phi <4 x float> [ [[TMP13]], %[[PRED_CALL_CONTINUE2]] ], [ [[TMP19]], %[[PRED_CALL_IF3]] ]
; VF4-NEXT: [[TMP63:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
; VF4-NEXT: br i1 [[TMP63]], label %[[PRED_CALL_IF5:.*]], label %[[PRED_CALL_CONTINUE6:.*]]
; VF4: [[PRED_CALL_IF5]]:
; VF4-NEXT: [[TMP24:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 3
-; VF4-NEXT: [[TMP25:%.*]] = tail call { float, float } @fn2(float [[TMP24]]) #[[ATTR3]]
-; VF4-NEXT: br label %[[PRED_CALL_CONTINUE6]]
-; VF4: [[PRED_CALL_CONTINUE6]]:
-; VF4-NEXT: [[TMP39:%.*]] = phi { float, float } [ poison, %[[PRED_CALL_CONTINUE4]] ], [ [[TMP25]], %[[PRED_CALL_IF5]] ]
-; VF4-NEXT: [[TMP5:%.*]] = extractvalue { float, float } [[TMP4]], 0
-; VF4-NEXT: [[TMP6:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0
-; VF4-NEXT: [[TMP7:%.*]] = insertvalue { <4 x float>, <4 x float> } poison, <4 x float> [[TMP6]], 0
-; VF4-NEXT: [[TMP8:%.*]] = extractvalue { float, float } [[TMP4]], 1
-; VF4-NEXT: [[TMP9:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP7]], 1
-; VF4-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP8]], i64 0
-; VF4-NEXT: [[TMP11:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP7]], <4 x float> [[TMP10]], 1
-; VF4-NEXT: [[TMP16:%.*]] = extractvalue { float, float } [[TMP15]], 0
-; VF4-NEXT: [[TMP26:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP11]], 0
-; VF4-NEXT: [[TMP29:%.*]] = insertelement <4 x float> [[TMP26]], float [[TMP16]], i64 1
-; VF4-NEXT: [[TMP19:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP11]], <4 x float> [[TMP29]], 0
-; VF4-NEXT: [[TMP20:%.*]] = extractvalue { float, float } [[TMP15]], 1
-; VF4-NEXT: [[TMP21:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP19]], 1
-; VF4-NEXT: [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP20]], i64 1
-; VF4-NEXT: [[TMP23:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP19]], <4 x float> [[TMP22]], 1
-; VF4-NEXT: [[TMP28:%.*]] = extractvalue { float, float } [[TMP27]], 0
-; VF4-NEXT: [[TMP36:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP23]], 0
-; VF4-NEXT: [[TMP37:%.*]] = insertelement <4 x float> [[TMP36]], float [[TMP28]], i64 2
-; VF4-NEXT: [[TMP31:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP23]], <4 x float> [[TMP37]], 0
-; VF4-NEXT: [[TMP32:%.*]] = extractvalue { float, float } [[TMP27]], 1
-; VF4-NEXT: [[TMP33:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP31]], 1
-; VF4-NEXT: [[TMP34:%.*]] = insertelement <4 x float> [[TMP33]], float [[TMP32]], i64 2
-; VF4-NEXT: [[TMP35:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP31]], <4 x float> [[TMP34]], 1
+; VF4-NEXT: [[TMP39:%.*]] = tail call { float, float } @fn2(float [[TMP24]]) #[[ATTR3]]
; VF4-NEXT: [[TMP40:%.*]] = extractvalue { float, float } [[TMP39]], 0
-; VF4-NEXT: [[TMP42:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP35]], 0
; VF4-NEXT: [[TMP48:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP40]], i64 3
-; VF4-NEXT: [[TMP43:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP35]], <4 x float> [[TMP48]], 0
-; VF4-NEXT: [[TMP44:%.*]] = extractvalue { float, float } [[TMP39]], 1
-; VF4-NEXT: [[TMP45:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP43]], 1
-; VF4-NEXT: [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP44]], i64 3
-; VF4-NEXT: [[TMP47:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP43]], <4 x float> [[TMP46]], 1
-; VF4-NEXT: [[TMP49:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP47]], 0
+; VF4-NEXT: br label %[[PRED_CALL_CONTINUE6]]
+; VF4: [[PRED_CALL_CONTINUE6]]:
+; VF4-NEXT: [[TMP49:%.*]] = phi <4 x float> [ [[TMP42]], %[[PRED_CALL_CONTINUE4]] ], [ [[TMP48]], %[[PRED_CALL_IF5]] ]
; VF4-NEXT: [[TMP50:%.*]] = fdiv <4 x float> [[TMP49]], [[WIDE_LOAD]]
; VF4-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; VF4: [[PRED_STORE_IF]]:
@@ -594,8 +465,8 @@ define void @struct_return_2xf32_replicate_predicated(ptr %a) {
; VF2IC2: [[PRED_STORE_IF2]]:
; VF2IC2-NEXT: [[TMP13:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 1
; VF2IC2-NEXT: [[TMP14:%.*]] = tail call { float, float } @fn2(float [[TMP13]]) #[[ATTR3]]
-; VF2IC2-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 1
; VF2IC2-NEXT: [[TMP16:%.*]] = extractvalue { float, float } [[TMP14]], 0
+; VF2IC2-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 1
; VF2IC2-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]]
; VF2IC2-NEXT: [[TMP19:%.*]] = fdiv float [[TMP16]], [[TMP13]]
; VF2IC2-NEXT: store float [[TMP19]], ptr [[TMP17]], align 8
@@ -606,8 +477,8 @@ define void @struct_return_2xf32_replicate_predicated(ptr %a) {
; VF2IC2: [[PRED_STORE_IF4]]:
; VF2IC2-NEXT: [[TMP21:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 0
; VF2IC2-NEXT: [[TMP22:%.*]] = tail call { float, float } @fn2(float [[TMP21]]) #[[ATTR3]]
-; VF2IC2-NEXT: [[TMP23:%.*]] = add i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP24:%.*]] = extractvalue { float, float } [[TMP22]], 0
+; VF2IC2-NEXT: [[TMP23:%.*]] = add i64 [[INDEX]], 2
; VF2IC2-NEXT: [[TMP25:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP23]]
; VF2IC2-NEXT: [[TMP27:%.*]] = fdiv float [[TMP24]], [[TMP21]]
; VF2IC2-NEXT: store float [[TMP27]], ptr [[TMP25]], align 8
@@ -618,8 +489,8 @@ define void @struct_return_2xf32_replicate_predicated(ptr %a) {
; VF2IC2: [[PRED_STORE_IF6]]:
; VF2IC2-NEXT: [[TMP29:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 1
; VF2IC2-NEXT: [[TMP30:%.*]] = tail call { float, float } @fn2(float [[TMP29]]) #[[ATTR3]]
-; VF2IC2-NEXT: [[TMP31:%.*]] = add i64 [[INDEX]], 3
; VF2IC2-NEXT: [[TMP32:%.*]] = extractvalue { float, float } [[TMP30]], 0
+; VF2IC2-NEXT: [[TMP31:%.*]] = add i64 [[INDEX]], 3
; VF2IC2-NEXT: [[TMP33:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP31]]
; VF2IC2-NEXT: [[TMP35:%.*]] = fdiv float [[TMP32]], [[TMP29]]
; VF2IC2-NEXT: store float [[TMP35]], ptr [[TMP33]], align 8
@@ -656,6 +527,132 @@ exit:
ret void
}
+define void @struct_return_predicated_extractvalue(i1 %c, ptr noalias %a, ptr noalias %p, ptr noalias %q) {
+; VF4-LABEL: define void @struct_return_predicated_extractvalue(
+; VF4-SAME: i1 [[C:%.*]], ptr noalias [[A:%.*]], ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; VF4-NEXT: [[ENTRY:.*:]]
+; VF4-NEXT: br label %[[VECTOR_PH:.*]]
+; VF4: [[VECTOR_PH]]:
+; VF4-NEXT: [[TMP0:%.*]] = xor i1 [[C]], true
+; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF4: [[VECTOR_BODY]]:
+; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; VF4-NEXT: [[TMP42:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
+; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP42]], align 4
+; VF4-NEXT: [[TMP43:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 0
+; VF4-NEXT: [[TMP3:%.*]] = tail call { float, float } @fn2(float [[TMP43]]) #[[ATTR3]]
+; VF4-NEXT: [[TMP44:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 1
+; VF4-NEXT: [[TMP5:%.*]] = tail call { float, float } @fn2(float [[TMP44]]) #[[ATTR3]]
+; VF4-NEXT: [[TMP45:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 2
+; VF4-NEXT: [[TMP7:%.*]] = tail call { float, float } @fn2(float [[TMP45]]) #[[ATTR3]]
+; VF4-NEXT: [[TMP46:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i64 3
+; VF4-NEXT: [[TMP9:%.*]] = tail call { float, float } @fn2(float [[TMP46]]) #[[ATTR3]]
+; VF4-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4: [[PRED_STORE_IF]]:
+; VF4-NEXT: [[TMP37:%.*]] = extractvalue { float, float } [[TMP3]], 1
+; VF4-NEXT: store float [[TMP37]], ptr [[Q]], align 4
+; VF4-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VF4: [[PRED_STORE_CONTINUE]]:
+; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4: [[PRED_STORE_IF1]]:
+; VF4-NEXT: [[TMP38:%.*]] = extractvalue { float, float } [[TMP5]], 1
+; VF4-NEXT: store float [[TMP38]], ptr [[Q]], align 4
+; VF4-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VF4: [[PRED_STORE_CONTINUE2]]:
+; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4: [[PRED_STORE_IF3]]:
+; VF4-NEXT: [[TMP39:%.*]] = extractvalue { float, float } [[TMP7]], 1
+; VF4-NEXT: store float [[TMP39]], ptr [[Q]], align 4
+; VF4-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; VF4: [[PRED_STORE_CONTINUE4]]:
+; VF4-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; VF4: [[PRED_STORE_IF5]]:
+; VF4-NEXT: [[TMP40:%.*]] = extractvalue { float, float } [[TMP9]], 1
+; VF4-NEXT: store float [[TMP40]], ptr [[Q]], align 4
+; VF4-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; VF4: [[PRED_STORE_CONTINUE6]]:
+; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; VF4-NEXT: br i1 [[TMP41]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF4: [[MIDDLE_BLOCK]]:
+;
+; VF2IC2-LABEL: define void @struct_return_predicated_extractvalue(
+; VF2IC2-SAME: i1 [[C:%.*]], ptr noalias [[A:%.*]], ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
+; VF2IC2-NEXT: [[ENTRY:.*:]]
+; VF2IC2-NEXT: br label %[[VECTOR_PH:.*]]
+; VF2IC2: [[VECTOR_PH]]:
+; VF2IC2-NEXT: [[TMP0:%.*]] = xor i1 [[C]], true
+; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
+; VF2IC2: [[VECTOR_BODY]]:
+; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; VF2IC2-NEXT: [[TMP42:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
+; VF2IC2-NEXT: [[TMP43:%.*]] = getelementptr float, ptr [[TMP42]], i64 2
+; VF2IC2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP42]], align 4
+; VF2IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x float>, ptr [[TMP43]], align 4
+; VF2IC2-NEXT: [[TMP44:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 0
+; VF2IC2-NEXT: [[TMP4:%.*]] = tail call { float, float } @fn2(float [[TMP44]]) #[[ATTR3]]
+; VF2IC2-NEXT: [[TMP45:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i64 1
+; VF2IC2-NEXT: [[TMP6:%.*]] = tail call { float, float } @fn2(float [[TMP45]]) #[[ATTR3]]
+; VF2IC2-NEXT: [[TMP46:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 0
+; VF2IC2-NEXT: [[TMP8:%.*]] = tail call { float, float } @fn2(float [[TMP46]]) #[[ATTR3]]
+; VF2IC2-NEXT: [[TMP47:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i64 1
+; VF2IC2-NEXT: [[TMP10:%.*]] = tail call { float, float } @fn2(float [[TMP47]]) #[[ATTR3]]
+; VF2IC2-NEXT: store float 0.000000e+00, ptr [[P]], align 4
+; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF2IC2: [[PRED_STORE_IF]]:
+; VF2IC2-NEXT: [[TMP37:%.*]] = extractvalue { float, float } [[TMP4]], 1
+; VF2IC2-NEXT: store float [[TMP37]], ptr [[Q]], align 4
+; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VF2IC2: [[PRED_STORE_CONTINUE]]:
+; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF2IC2: [[PRED_STORE_IF1]]:
+; VF2IC2-NEXT: [[TMP38:%.*]] = extractvalue { float, float } [[TMP6]], 1
+; VF2IC2-NEXT: store float [[TMP38]], ptr [[Q]], align 4
+; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; VF2IC2: [[PRED_STORE_CONTINUE2]]:
+; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF2IC2: [[PRED_STORE_IF3]]:
+; VF2IC2-NEXT: [[TMP39:%.*]] = extractvalue { float, float } [[TMP8]], 1
+; VF2IC2-NEXT: store float [[TMP39]], ptr [[Q]], align 4
+; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; VF2IC2: [[PRED_STORE_CONTINUE4]]:
+; VF2IC2-NEXT: br i1 [[TMP0]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; VF2IC2: [[PRED_STORE_IF5]]:
+; VF2IC2-NEXT: [[TMP40:%.*]] = extractvalue { float, float } [[TMP10]], 1
+; VF2IC2-NEXT: store float [[TMP40]], ptr [[Q]], align 4
+; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; VF2IC2: [[PRED_STORE_CONTINUE6]]:
+; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF2IC2-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; VF2IC2-NEXT: br i1 [[TMP41]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF2IC2: [[MIDDLE_BLOCK]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %iv.next, %latch ], [ 0, %entry ]
+ %gep = getelementptr float, ptr %a, i64 %iv
+ %x = load float, ptr %gep
+ %call = tail call { float, float } @fn2(float %x) #3
+ store float 0.0, ptr %p
+ br i1 %c, label %latch, label %if
+
+if:
+ %val = extractvalue { float, float } %call, 1
+ store float %val, ptr %q
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
declare { i64 } @fn1(float)
declare { float, float } @fn2(float)
declare { i32, i32, i32 } @fn3(i32)
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return.ll b/llvm/test/Transforms/LoopVectorize/struct-return.ll
index b0b3c369e91617..92d05d9f4c9340 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return.ll
@@ -282,34 +282,22 @@ define void @scalarized_predicated_struct_return(ptr %a) {
; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP10:%.*]] = tail call { i64, i64 } @bar_i64(i64 [[TMP3]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = tail call { i64, i64 } @bar_i64(i64 [[TMP3]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { i64, i64 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> poison, i64 [[TMP5]], i64 0
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP4:%.*]] = phi { i64, i64 } [ poison, %[[VECTOR_BODY]] ], [ [[TMP10]], %[[PRED_STORE_IF]] ]
+; CHECK-NEXT: [[TMP32:%.*]] = phi <2 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_STORE_IF]] ]
; CHECK-NEXT: [[TMP29:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
; CHECK-NEXT: br i1 [[TMP29]], label %[[PRED_CALL_IF1:.*]], label %[[PRED_CALL_CONTINUE2:.*]]
; CHECK: [[PRED_CALL_IF1]]:
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP24:%.*]] = tail call { i64, i64 } @bar_i64(i64 [[TMP11]]) #[[ATTR3]]
-; CHECK-NEXT: br label %[[PRED_CALL_CONTINUE2]]
-; CHECK: [[PRED_CALL_CONTINUE2]]:
-; CHECK-NEXT: [[TMP12:%.*]] = phi { i64, i64 } [ poison, %[[PRED_STORE_CONTINUE]] ], [ [[TMP24]], %[[PRED_CALL_IF1]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = extractvalue { i64, i64 } [[TMP4]], 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> poison, i64 [[TMP5]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertvalue { <2 x i64>, <2 x i64> } poison, <2 x i64> [[TMP6]], 0
-; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { i64, i64 } [[TMP4]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP7]], 1
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i64 0
-; CHECK-NEXT: [[TMP30:%.*]] = insertvalue { <2 x i64>, <2 x i64> } [[TMP7]], <2 x i64> [[TMP14]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = tail call { i64, i64 } @bar_i64(i64 [[TMP11]]) #[[ATTR3]]
; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { i64, i64 } [[TMP12]], 0
-; CHECK-NEXT: [[TMP32:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP30]], 0
; CHECK-NEXT: [[TMP33:%.*]] = insertelement <2 x i64> [[TMP32]], i64 [[TMP13]], i64 1
-; CHECK-NEXT: [[TMP19:%.*]] = insertvalue { <2 x i64>, <2 x i64> } [[TMP30]], <2 x i64> [[TMP33]], 0
-; CHECK-NEXT: [[TMP20:%.*]] = extractvalue { i64, i64 } [[TMP12]], 1
-; CHECK-NEXT: [[TMP21:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP19]], 1
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x i64> [[TMP21]], i64 [[TMP20]], i64 1
-; CHECK-NEXT: [[TMP23:%.*]] = insertvalue { <2 x i64>, <2 x i64> } [[TMP19]], <2 x i64> [[TMP22]], 1
-; CHECK-NEXT: [[TMP25:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP23]], 0
+; CHECK-NEXT: br label %[[PRED_CALL_CONTINUE2]]
+; CHECK: [[PRED_CALL_CONTINUE2]]:
+; CHECK-NEXT: [[TMP25:%.*]] = phi <2 x i64> [ [[TMP32]], %[[PRED_STORE_CONTINUE]] ], [ [[TMP33]], %[[PRED_CALL_IF1]] ]
; CHECK-NEXT: [[TMP26:%.*]] = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> [[TMP25]], <2 x i64> [[WIDE_LOAD]], <2 x i1> [[TMP1]])
; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE1:.*]]
; CHECK: [[PRED_STORE_IF2]]:
@@ -720,6 +708,207 @@ exit:
ret void
}
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
+define void @extractvalue_invariant_multi_index(ptr noalias writeonly %out, { { float, float } } %agg) {
+; CHECK-LABEL: define void @extractvalue_invariant_multi_index(
+; CHECK-SAME: ptr noalias writeonly [[OUT:%.*]], { { float, float } } [[AGG:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = extractvalue { { float, float } } [[AGG]], 0, 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x float> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %extract = extractvalue { { float, float } } %agg, 0, 1
+ %arrayidx = getelementptr inbounds float, ptr %out, i64 %iv
+ store float %extract, ptr %arrayidx, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
+define void @extractvalue_invariant_array(ptr noalias writeonly %out, [2 x float] %agg) {
+; CHECK-LABEL: define void @extractvalue_invariant_array(
+; CHECK-SAME: ptr noalias writeonly [[OUT:%.*]], [2 x float] [[AGG:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = extractvalue [2 x float] [[AGG]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x float> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %extract = extractvalue [2 x float] %agg, 1
+ %arrayidx = getelementptr inbounds float, ptr %out, i64 %iv
+ store float %extract, ptr %arrayidx, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
+define void @struct_return_replicate_extractvalue_cse(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @struct_return_replicate_extractvalue_cse(
+; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[IN]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call { i64, i64 } @bar_i64(i64 [[TMP1]]) #[[ATTR3]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = tail call { i64, i64 } @bar_i64(i64 [[TMP3]]) #[[ATTR3]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { i64, i64 } [[TMP2]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = extractvalue { i64, i64 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i64> poison, i64 [[TMP9]], i32 0
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i64> [[TMP7]], i64 [[TMP16]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { i64, i64 } [[TMP2]], 0
+; CHECK-NEXT: [[TMP10:%.*]] = extractvalue { i64, i64 } [[TMP4]], 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP13]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i32 1
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[OUT_A]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i64> [[TMP18]], ptr [[TMP22]], align 8
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[OUT_B]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i64> [[TMP12]], ptr [[TMP23]], align 8
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i64, ptr %in, i64 %iv
+ %in_val = load i64, ptr %arrayidx, align 8
+ %call = tail call { i64, i64 } @bar_i64(i64 %in_val) #6
+ %extract_a = extractvalue { i64, i64 } %call, 0
+ %extract_b = extractvalue { i64, i64 } %call, 0
+ %arrayidx2 = getelementptr inbounds i64, ptr %out_a, i64 %iv
+ store i64 %extract_a, ptr %arrayidx2, align 8
+ %arrayidx4 = getelementptr inbounds i64, ptr %out_b, i64 %iv
+ store i64 %extract_b, ptr %arrayidx4, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; Tail-folded predicated call whose extracts feed a first-order recurrence and
+; live-outs.
+; CHECK-REMARKS: remark: {{.*}} vectorized loop
+define float @struct_return_tail_folded_predicated_for_and_live_outs(ptr noalias writeonly %out, i64 %n) {
+; CHECK-LABEL: define float @struct_return_tail_folded_predicated_for_and_live_outs(
+; CHECK-SAME: ptr noalias writeonly [[OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[EXIT:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP33:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x float> [ <float poison, float 0.000000e+00>, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = uitofp <2 x i64> [[VEC_IND]] to <2 x float>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = tail call { float, float } @foo(float [[TMP4]]) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = tail call { float, float } @foo(float [[TMP8]]) #[[ATTR5]]
+; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { float, float } [[TMP10]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractvalue { float, float } [[TMP5]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x float> poison, float [[TMP18]], i32 0
+; CHECK-NEXT: [[TMP26]] = insertelement <2 x float> [[TMP7]], float [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <2 x float> [[VECTOR_RECUR]], <2 x float> [[TMP26]], <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { float, float } [[TMP10]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { float, float } [[TMP5]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x float> poison, float [[TMP15]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x float> [[TMP12]], float [[TMP11]], i32 1
+; CHECK-NEXT: [[TMP14:%.*]] = fsub <2 x float> [[TMP27]], [[TMP13]]
+; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[TMP33]]
+; CHECK-NEXT: store <2 x float> [[TMP14]], ptr [[TMP34]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP33]], 2
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP38]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT1:label %.*]], label %[[EXIT]]
+; CHECK: [[EXIT]]:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %for = phi float [ 0.0, %entry ], [ %extract_a, %for.body ]
+ %in_val = uitofp i64 %iv to float
+ %call = tail call { float, float } @foo(float %in_val) #3
+ %extract_a = extractvalue { float, float } %call, 0
+ %extract_b = extractvalue { float, float } %call, 1
+ %sub = fsub float %for, %extract_b
+ %arrayidx = getelementptr inbounds float, ptr %out, i64 %iv
+ store float %sub, ptr %arrayidx, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %n
+ br i1 %exitcond.not, label %exit, label %for.body, !llvm.loop !0
+
+exit:
+ %res = fadd float %extract_a, %extract_b
+ ret float %res
+}
+
declare { float, float } @foo(float)
declare { double, double } @bar(double)
declare { float, i32 } @baz(float)
@@ -745,3 +934,6 @@ attributes #3 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_foo(scalable_
attributes #4 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_bar_named(fixed_vec_bar)" }
attributes #5 = { nounwind "vector-function-abi-variant"="_ZGVnN2v_qux(fixed_vec_qux)" }
attributes #6 = { nounwind "vector-function-abi-variant"="_ZGVsMxv_bar_i64(scalable_vec_masked_bar_i64)" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.predicate.enable"}
More information about the llvm-branch-commits
mailing list