[llvm] [LV] Add support for widening loads/stores to a VF multiple (PR #217670)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 08:55:02 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Benjamin Maxwell (MacDue)
<details>
<summary>Changes</summary>
This patch adds support for widening loads and stores by a "VF multiple" that must divide the UF. It is currently limited to unmasked operations, but we plan to extend it to masked loops via the wide active lane mask.
For now, this is driven by a new TTI hook,
`getPreferredVFMultipleForMemoryOp`. A small VPlan transform uses that hook to set the VF multiple on `VPWidenLoadRecipe` and `VPWidenStoreRecipe`.
When VPlan unrolling encounters a load or store with a VF multiple greater than 1, it inserts extracts for the unroll parts of widened loads and concatenates multiple unroll parts for widened stores.
On AArch64 this is used to target multi-vector load/store instructions.
---
Patch is 61.02 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/217670.diff
15 Files Affected:
- (modified) llvm/include/llvm/Analysis/TargetTransformInfo.h (+9)
- (modified) llvm/include/llvm/Analysis/TargetTransformInfoImpl.h (+8)
- (modified) llvm/lib/Analysis/TargetTransformInfo.cpp (+7)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+31)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h (+4)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+2)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (+22-5)
- (modified) llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h (+36-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+27-1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+41)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+5)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp (+66)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/multi-vector-mem-ops.ll (+374)
- (added) llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-printing-multi-vector-mem-ops.ll (+151)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll (+1)
``````````diff
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 973ed8820691d..9591424b3ca03 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -979,6 +979,15 @@ class TargetTransformInfo {
unsigned Opcode1,
const SmallBitVector &OpcodeMask) const;
+ /// Return the preferred multiple of VF to use for a contiguous load/store.
+ /// Returning 1 leaves the operation at VF. The returned value must divide UF.
+ ///
+ /// \p Opcode must be either Instruction::Load or Instruction::Store.
+ LLVM_ABI unsigned
+ getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataType,
+ ElementCount VF, unsigned UF,
+ bool IsMasked = false) const;
+
/// Return true if we should be enabling ordered reductions for the target.
LLVM_ABI bool enableOrderedReductions() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 1cda5ff84d29f..80b6c4b88d952 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -422,6 +422,14 @@ class LLVM_ABI TargetTransformInfoImplBase {
return false;
}
+ virtual unsigned getPreferredVFMultipleForMemoryOp(unsigned Opcode,
+ Type *DataType,
+ ElementCount VF,
+ unsigned UF,
+ bool IsMasked) const {
+ return 1;
+ }
+
virtual bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
Align Alignment,
unsigned AddrSpace) const {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 42afdf67a5f93..dba0e8513c0cc 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -547,6 +547,13 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
+unsigned TargetTransformInfo::getPreferredVFMultipleForMemoryOp(
+ unsigned Opcode, Type *DataType, ElementCount VF, unsigned UF,
+ bool IsMasked) const {
+ return TTIImpl->getPreferredVFMultipleForMemoryOp(Opcode, DataType, VF, UF,
+ IsMasked);
+}
+
bool TargetTransformInfo::isLegalInterleavedAccessType(
VectorType *VTy, unsigned Factor, Align Alignment,
unsigned AddrSpace) const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 2e28901cc5127..a79b1a80bda07 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5835,6 +5835,37 @@ bool AArch64TTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
(ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2());
}
+unsigned
+AArch64TTIImpl::getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataTy,
+ ElementCount VF, unsigned UF,
+ bool IsMasked) const {
+ assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
+ "expected load/store opcode");
+ if (IsMasked)
+ return 1; // TODO: Support masked multi-vector loads/stores.
+
+ if (!ST->enableSubRegLiveness())
+ return 1;
+
+ if ((Opcode != Instruction::Load && Opcode != Instruction::Store) ||
+ !ST->hasSVE2p1() || !VF.isScalable() || !isPowerOf2_32(UF))
+ return 1;
+
+ unsigned VectorWidth = VF.getKnownMinValue() * DL.getTypeSizeInBits(DataTy);
+ if (VectorWidth % 128 != 0)
+ return 1;
+
+ for (unsigned TargetWidth : {512u, 256u}) {
+ if (TargetWidth % VectorWidth == 0) {
+ unsigned Scale = TargetWidth / VectorWidth;
+ if (Scale <= UF)
+ return Scale;
+ }
+ }
+
+ return 1;
+}
+
unsigned
AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF,
bool HasUnorderedReductions) const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index cd71b1179bd45..8363d8d33c48d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -279,6 +279,10 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) const override;
+ unsigned getPreferredVFMultipleForMemoryOp(unsigned Opcode, Type *DataType,
+ ElementCount VF, unsigned UF,
+ bool IsMasked) const override;
+
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE) const override;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c12a4f562f600..ba6a6c9111000 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5933,6 +5933,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
*PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
// TODO: Move to VPlan transform stage once the transition to the VPlan-based
// cost model is complete for better cost estimates.
+ RUN_VPLAN_PASS(VPlanTransforms::scaleMemoryAccessesByUF, BestVPlan, BestVF,
+ BestUF, CM.TTI);
RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 4bb18e46c64ad..106aded9846fc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1253,6 +1253,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
WideActiveLaneMask,
// Extracts each unrolled part of a (VF * UF) widened vector/mask.
ExtractVectorForPart,
+ // Concatenates its unrolled part operands into one widened vector.
+ ConcatVectorParts,
ExplicitVectorLength,
// Represents the incoming loop-invariant alias-mask. All memory accesses
// in the loop must stay within the active lanes.
@@ -3753,6 +3755,10 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
/// Whether the memory access is masked.
bool IsMasked = false;
+ /// Multiple of VF used to widen this memory operation. The final operation
+ /// loads or stores VF * VFMultiple elements
+ unsigned VFMultiple = 1;
+
void setMask(VPValue *Mask) {
assert(!IsMasked && "cannot re-set mask");
if (!Mask)
@@ -3799,6 +3805,12 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const;
Instruction &getIngredient() const { return Ingredient; }
+
+ /// Set the VF multiple for this memory operation.
+ void setVFMultiple(unsigned VFMultiple) { this->VFMultiple = VFMultiple; }
+
+ /// Returns the VF multiple of this memory operation.
+ unsigned getVFMultiple() const { return VFMultiple; }
};
/// A recipe for widening load operations, using the address to load from and an
@@ -3814,8 +3826,11 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPSingleDefRecipe,
}
VPWidenLoadRecipe *clone() override {
- return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
- getMask(), Consecutive, *this, getDebugLoc());
+ auto *R =
+ new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(), getMask(),
+ Consecutive, *this, getDebugLoc());
+ R->setVFMultiple(VFMultiple);
+ return R;
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadSC);
@@ -3913,9 +3928,11 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPRecipeBase,
}
VPWidenStoreRecipe *clone() override {
- return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
- getStoredValue(), getMask(), Consecutive,
- *this, getDebugLoc());
+ auto *R = new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
+ getStoredValue(), getMask(), Consecutive,
+ *this, getDebugLoc());
+ R->setVFMultiple(VFMultiple);
+ return R;
}
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenStoreSC);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 648ffa2b11a4f..6b124070e0f39 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -296,7 +296,10 @@ struct Recipe_match {
// Check for recipes that do not have opcodes.
if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||
std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||
- std::is_same_v<RecipeTy, VPVectorEndPointerRecipe>)
+ std::is_same_v<RecipeTy, VPVectorEndPointerRecipe> ||
+ std::is_same_v<RecipeTy, VPVectorPointerRecipe> ||
+ std::is_same_v<RecipeTy, VPWidenLoadRecipe> ||
+ std::is_same_v<RecipeTy, VPWidenStoreRecipe>)
return DefR;
else
return DefR && DefR->getOpcode() == Opcode;
@@ -988,6 +991,38 @@ m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
}
+template <typename Op0_t, typename Op1_t>
+using VectorPointerRecipe_match =
+ Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
+ /*Commutative*/ false, VPVectorPointerRecipe>;
+
+template <typename Op0_t, typename Op1_t>
+VectorPointerRecipe_match<Op0_t, Op1_t> m_VecPtr(const Op0_t &Op0,
+ const Op1_t &Op1) {
+ return VectorPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
+}
+
+template <typename Op0_t>
+using VPWidenLoadRecipe_match =
+ Recipe_match<std::tuple<Op0_t>, 0,
+ /*Commutative*/ false, VPWidenLoadRecipe>;
+
+template <typename Op0_t>
+VPWidenLoadRecipe_match<Op0_t> m_WidenLoad(const Op0_t &Op0) {
+ return VPWidenLoadRecipe_match<Op0_t>(Op0);
+}
+
+template <typename Op0_t, typename Op1_t>
+using VPWidenStoreRecipe_match =
+ Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
+ /*Commutative*/ false, VPWidenStoreRecipe>;
+
+template <typename Op0_t, typename Op1_t>
+VPWidenStoreRecipe_match<Op0_t, Op1_t> m_WidenStore(const Op0_t &Op0,
+ const Op1_t &Op1) {
+ return VPWidenStoreRecipe_match<Op0_t, Op1_t>(Op0, Op1);
+}
+
template <typename Op0_t, typename Op1_t>
using VectorEndPointerRecipe_match =
Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b5524948c1bd9..4faf7506638fd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -566,6 +566,9 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
return StructTy->getTypeAtIndex(
cast<VPConstantInt>(Operands[1])->getZExtValue());
}
+ case VPInstruction::ConcatVectorParts:
+ case VPInstruction::ExtractVectorForPart:
+ return Op0Ty;
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
case VPInstruction::NumActiveLanes:
@@ -691,6 +694,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::LastActiveLane:
case VPInstruction::ExtractLane:
case VPInstruction::ExtractLastActive:
+ case VPInstruction::ConcatVectorParts:
// Cannot determine the number of operands from the opcode.
return -1u;
}
@@ -1119,6 +1123,19 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateExtractVector(
DstTy, Src, Builder.getInt64(State.VF.getKnownMinValue() * Part), Name);
}
+ case VPInstruction::ConcatVectorParts: {
+ unsigned VectorOps = getNumOperands();
+ auto *WideDataTy = VectorType::get(
+ getScalarType(), State.VF.multiplyCoefficientBy(VectorOps));
+ Value *WideData = PoisonValue::get(WideDataTy);
+
+ for (unsigned I = 0; I < VectorOps; ++I) {
+ Value *Part = State.get(getOperand(I));
+ WideData = Builder.CreateInsertVector(WideDataTy, WideData, Part,
+ I * State.VF.getKnownMinValue());
+ }
+ return WideData;
+ }
default:
llvm_unreachable("Unsupported opcode for instruction");
}
@@ -1643,6 +1660,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::IncomingAliasMask:
+ case VPInstruction::ConcatVectorParts:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::FirstActiveLane:
@@ -1777,6 +1795,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
break;
+ case VPInstruction::ConcatVectorParts:
+ O << "concat-vector-parts";
+ break;
case VPInstruction::ExplicitVectorLength:
O << "EXPLICIT-VECTOR-LENGTH";
break;
@@ -4210,7 +4231,8 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
void VPWidenLoadRecipe::execute(VPTransformState &State) {
Type *ScalarDataTy = getScalarType();
- auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
+ auto *DataTy =
+ VectorType::get(ScalarDataTy, State.VF.multiplyCoefficientBy(VFMultiple));
bool CreateGather = !isConsecutive();
auto &Builder = State.Builder;
@@ -4240,6 +4262,8 @@ void VPWidenLoadRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
O << Indent << "WIDEN ";
printAsOperand(O, SlotTracker);
O << " = load ";
+ if (VFMultiple > 1)
+ O << "x" << VFMultiple << ' ';
printOperands(O, SlotTracker);
}
#endif
@@ -4327,6 +4351,8 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
void VPWidenStoreRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "WIDEN store ";
+ if (VFMultiple > 1)
+ O << "x" << VFMultiple << ' ';
printOperands(O, SlotTracker);
}
#endif
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e75e9cf4e9103..8ce384c4f74cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3867,6 +3867,47 @@ void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
}
}
+void VPlanTransforms::scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
+ unsigned UF,
+ const TargetTransformInfo &TTI) {
+ if (UF == 1)
+ return;
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ uint64_t Stride;
+ VPValue *StoredValue = nullptr;
+ auto m_ConstantStrideVecPtr =
+ m_VecPtr(m_VPValue(), m_ConstantInt(Stride));
+ if ((!match(&R, m_WidenLoad(m_ConstantStrideVecPtr)) &&
+ !match(&R, m_WidenStore(m_ConstantStrideVecPtr,
+ m_VPValue(StoredValue)))) ||
+ Stride != 1)
+ continue;
+
+ auto *MemOp = cast<VPWidenMemoryRecipe>(&R);
+ if (!MemOp->isConsecutive())
+ continue;
+
+ // TODO: Support masked loads/stores. This requires widening the header
+ // mask to the same factor as the memory operation.
+ assert(!MemOp->isMasked() && "Masked accesses are not supported yet");
+
+ Type *AccessType = StoredValue ? StoredValue->getScalarType()
+ : R.getVPSingleValue()->getScalarType();
+ unsigned Opcode = isa<VPWidenLoadRecipe>(MemOp->getAsRecipe())
+ ? Instruction::Load
+ : Instruction::Store;
+ unsigned ScaleFactor = TTI.getPreferredVFMultipleForMemoryOp(
+ Opcode, AccessType, VF, UF, /*IsMasked=*/false);
+ assert((ScaleFactor != 0 && UF % ScaleFactor == 0) &&
+ "ScaleFactor must divide UF");
+ MemOp->setVFMultiple(ScaleFactor);
+ }
+ }
+}
+
/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e1832cd2de27a..fbb2c22894158 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -435,6 +435,11 @@ struct VPlanTransforms {
static void sinkPredicatedStores(VPlan &Plan, PredicatedScalarEvolution &PSE,
const Loop *L);
+ /// Widens memory operations by a factor of UF based on a target hook.
+ /// This allows targets to use wider memory operations when profitable.
+ static void scaleMemoryAccessesByUF(VPlan &Plan, ElementCount VF, unsigned UF,
+ const TargetTransformInfo &TTI);
+
// Materialize vector trip counts for constants early if it can simply be
// computed as (Original TC / VF * UF) * VF * UF.
static void
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 21ea2a6b74ff5..2c31a947e32af 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -72,6 +72,9 @@ class UnrollState {
return Plan.getConstantInt(CanIVIntTy, Part);
}
+ /// Unroll a VPWidenLoadRecipe or VPWidenStoreRecipe with a VFMultiple > 1.
+ void unrollMemOpWithVFMultiple(VPRecipeBase &R, unsigned VFMultiple);
+
public:
UnrollState(VPlan &Plan, unsigned UF) : Plan(Plan), UF(UF) {}
@@ -288,6 +291,62 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
}
}
+void UnrollState::unrollMemOpWithVFMultiple(VPRecipeBase &R,
+ unsigned VFMultiple) {
+ assert(VFMultiple > 1 && UF % VFMultiple == 0);
+ SmallVector<VPRecipeBase *, 4> Groups(UF / VFMultiple, nullptr);
+ Groups[0] = &R;
+
+ // A memory op with a VFMultiple is widened to VF * VFMultiple elements, so
+ // after unrolling by UF we materialize UF / VFMultiple such ops, each
+ // covering VFMultiple unroll parts.
+ VPBuilder Builder = VPBuilder::getToInsertAfter(&R);
+ for (unsigned Group = 1; Group < Groups.size(); ++Group) {
+ auto *Copy = Builder.insert(R.clone());
+ remapOperands(Copy, Group * VFMultiple);
+ Groups[Group] = Copy;
+ }
+
+ if (auto *Store = dyn_cast<VPWidenStoreRecipe>(&R)) {
+ VPValue *StoredValue = Store->getStoredValue();
+ for (unsigned Group = 0; Group < Groups.size(); ++Group) {
+ VPRecipeBase *Store = Groups[Group];
+ Builder.setInsertPoint(Store);
+ SmallVector<VPValue *, 4> Parts;
+ // We need to concatenate VFMultiple parts to form the stored value.
+ for (unsigned Part = 0; Part < VFMultiple; ++Part)
+ Parts.push_back(
+ getValueForPart(StoredValue, Group * VFMultiple + Part));
+ auto *Concat =
+ Builder.createNaryOp(VPInstruction::ConcatVectorParts, Parts);
+ Groups[Group]->set...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/217670
More information about the llvm-commits
mailing list