[llvm] [VectorCombine] Support simplification to scalar store for multiple insertelt (PR #132820)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 01:56:09 PDT 2026
================
@@ -1954,70 +1954,230 @@ static Align computeAlignmentAfterScalarization(Align VectorAlignment,
return commonAlignment(VectorAlignment, DL.getTypeStoreSize(ScalarType));
}
-// Combine patterns like:
-// %0 = load <4 x i32>, <4 x i32>* %a
-// %1 = insertelement <4 x i32> %0, i32 %b, i32 1
-// store <4 x i32> %1, <4 x i32>* %a
-// to:
-// %0 = bitcast <4 x i32>* %a to i32*
-// %1 = getelementptr inbounds i32, i32* %0, i64 0, i64 1
-// store i32 %b, i32* %1
-bool VectorCombine::foldSingleElementStore(Instruction &I) {
+/// Fold a vector store fed by a single-use insertelement chain into scalar
+/// stores.
+///
+/// Before:
+///
+/// %p --> vector load --> insert %x, lane 1 --> insert %y, lane 3
+/// |
+/// v
+/// vector store to %p
+///
+/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
+/// Stored value: [ old | x | old | y ] (one vector store)
+///
+/// After:
+///
+/// +--> GEP(%p, lane 1) --> store %x
+/// %p -------------+
+/// +--> GEP(%p, lane 3) --> store %y
+///
+/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
+/// Scalar stores: x y
+/// store at 1 store at 3
+///
+/// Step 1. Gate:
+/// target supports vector-element GEP addressing
+///
+/// Step 2. Trace:
+/// vector store <-- insertelement <-- ... <-- insertelement <-- load
+///
+/// Steps 3-5. Validate:
+/// reject unprofitable full overwrites; require simple accesses, a
+/// common address/block, no memory write in between, and scalarizable
+/// indices.
+bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
+ // Step 1: The target must support addressing a vector element with a GEP.
if (!TTI.allowVectorElementIndexingUsingGEP())
return false;
+
auto *SI = cast<StoreInst>(&I);
if (!SI->isSimple() || !isa<VectorType>(SI->getValueOperand()->getType()))
return false;
- // TODO: Combine more complicated patterns (multiple insert) by referencing
- // TargetTransformInfo.
- Instruction *Source;
- Value *NewElement;
- Value *Idx;
- if (!match(SI->getValueOperand(),
- m_InsertElt(m_Instruction(Source), m_Value(NewElement),
- m_Value(Idx))))
- return false;
-
- if (auto *Load = dyn_cast<LoadInst>(Source)) {
- auto VecTy = cast<VectorType>(SI->getValueOperand()->getType());
- Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts();
- // Don't optimize for atomic/volatile load or store. Ensure memory is not
- // modified between, vector type matches store size, and index is inbounds.
- if (!Load->isSimple() || Load->getParent() != SI->getParent() ||
- !DL->typeSizeEqualsStoreSize(Load->getType()->getScalarType()) ||
- SrcAddr != SI->getPointerOperand()->stripPointerCasts())
- return false;
+ // Step 2: Collect a single-use insertelement chain, starting at the vector
+ // store and walking back to the candidate load.
+ Value *Source = SI->getValueOperand();
+ SmallVector<std::pair<Value *, Value *>, 4> InsertElements;
+ Value *Base = Source;
+ while (auto *Insert = dyn_cast<InsertElementInst>(Base)) {
+ if (!Insert->hasOneUse())
+ break;
+ Value *InsertVal = Insert->getOperand(1);
+ Value *Idx = Insert->getOperand(2);
+ InsertElements.push_back({InsertVal, Idx});
+ Base = Insert->getOperand(0);
+ }
- if (isMemModifiedBetween(Load->getIterator(), SI->getIterator(),
- MemoryLocation::get(SI), AA))
- return false;
+ if (InsertElements.empty())
+ return false;
+
+ // The backwards walk collected the inserts in reverse program order. Restore
+ // it now so later scalar stores preserve writes to duplicate/equal indices.
+ std::reverse(InsertElements.begin(), InsertElements.end());
+ auto *Load = dyn_cast<LoadInst>(Base);
+ if (!Load)
+ return false;
+ auto VecTy = cast<VectorType>(SI->getValueOperand()->getType());
+
+ // Step 3: Avoid replacing a complete overwrite with scalar stores when every
+ // lane receives the same value; keeping the vector operation is preferable.
+ if (auto *FVT = dyn_cast<FixedVectorType>(VecTy)) {
+ if (InsertElements.size() == FVT->getNumElements()) {
+ Value *FirstVal = InsertElements.front().first;
+ if (all_of(InsertElements,
+ [FirstVal](const auto &Elt) { return Elt.first == FirstVal; }))
+ return false;
+ }
+ }
+ Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts();
+ // Step 4: Establish the load/store update is legal: both accesses are simple,
+ // have the same base address and block, have scalar elements whose type size
+ // equals their store size, and no intervening operation modifies the updated
+ // memory.
+ if (!Load->isSimple() || Load->getParent() != SI->getParent() ||
+ !DL->typeSizeEqualsStoreSize(Load->getType()->getScalarType()) ||
+ SrcAddr != SI->getPointerOperand()->stripPointerCasts())
+ return false;
+
+ if (isMemModifiedBetween(Load->getIterator(), SI->getIterator(),
+ MemoryLocation::get(SI), AA))
+ return false;
+
+ // Step 5: Validate every index before changing IR. A safe-with-freeze result
+ // is recorded by ScalarizationResult, so discard it until profitability is
+ // known; otherwise a rejected candidate could leave a freeze behind.
+ for (auto [InsertVal, Idx] : InsertElements) {
auto ScalarizableIdx =
- canScalarizeAccess(VecTy, Idx, SQ.getWithInstruction(Load));
+ canScalarizeAccess(VecTy, Idx, SQ.getWithInstruction(&I));
if (ScalarizableIdx.isUnsafe())
return false;
- // Ensure we add the load back to the worklist BEFORE its users so they can
- // erased in the correct order.
- Worklist.push(Load);
+ // We are only checking legality here. Do not mutate IR before the
+ // profitability check, but also do not leave a pending ToFreeze behind.
+ ScalarizableIdx.discard();
+ }
+
+ InstructionCost OldCost = TTI.getMemoryOpCost(
+ Instruction::Store, SI->getValueOperand()->getType(), SI->getAlign(),
+ SI->getPointerAddressSpace(), CostKind);
+
+ if (Load->hasOneUse())
+ OldCost += TTI.getMemoryOpCost(Instruction::Load, Load->getType(),
+ Load->getAlign(),
+ Load->getPointerAddressSpace(), CostKind);
+
+ for (auto [InsertVal, Idx] : InsertElements) {
+ int Index = -1;
+ if (auto *CIdx = dyn_cast<ConstantInt>(Idx))
+ Index = CIdx->getZExtValue();
+
+ OldCost += TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
+ CostKind, Index);
+ }
+
+ bool HasKnownStride =
+ InsertElements.size() > 1 && all_of(InsertElements, [](const auto &Elt) {
+ return isa<ConstantInt>(Elt.second);
+ });
+
+ InstructionCost NewCost = 0;
+ if (HasKnownStride) {
+ // getPointersChainCost requires concrete pointer values. Create detached
+ // GEPs for the cost query so the profitability check does not mutate IR.
+ // ScalarGEPs owns them through unique_value, so they are deleted
+ // automatically at the end of this scope.
+ SmallVector<unique_value, 4> ScalarGEPs;
+ SmallVector<const Value *, 4> ScalarPtrs;
+ for (const auto &Elt : InsertElements) {
+ Value *Idx = Elt.second;
+ SmallVector<Value *, 2> GEPIndices = {ConstantInt::get(Idx->getType(), 0),
+ Idx};
+ auto *GEP = GetElementPtrInst::CreateInBounds(
+ VecTy, SI->getPointerOperand(), GEPIndices);
+ ScalarPtrs.push_back(GEP);
+ ScalarGEPs.emplace_back(GEP);
+ }
+
+ bool IsUnitStride = true;
+ uint64_t PrevIdx =
+ cast<ConstantInt>(InsertElements.front().second)->getZExtValue();
+ for (const auto &Elt : drop_begin(InsertElements)) {
+ Value *Idx = Elt.second;
+ uint64_t CurrentIdx = cast<ConstantInt>(Idx)->getZExtValue();
+ if (CurrentIdx != PrevIdx + 1) {
+ IsUnitStride = false;
+ break;
+ }
+ PrevIdx = CurrentIdx;
+ }
+
+ auto PtrsInfo = IsUnitStride ? TTI::PointersChainInfo::getUnitStride()
+ : TTI::PointersChainInfo::getKnownStride();
+ NewCost +=
+ TTI.getPointersChainCost(ScalarPtrs, ScalarPtrs.front(), PtrsInfo,
+ VecTy->getElementType(), CostKind);
+ } else {
+ // Without a known stride, cost each prospective GEP independently.
+ // getGEPCost accepts the base and indices directly, so no detached GEP
+ // instructions are needed for this query.
+ for (auto [InsertVal, Idx] : InsertElements) {
+ SmallVector<const Value *, 2> GEPIndices = {
+ ConstantInt::get(Idx->getType(), 0), Idx};
+ NewCost += TTI.getGEPCost(VecTy, SI->getPointerOperand(), GEPIndices,
+ InsertVal->getType(), CostKind);
+ }
----------------
ParkHanbum wrote:
I think we are.
https://github.com/llvm/llvm-project/pull/132820
More information about the llvm-commits
mailing list