[llvm] fb1a751 - [VectorCombine] Support simplification to scalar store for multiple insertelt (#132820)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 21:50:06 PDT 2026
Author: hanbeom
Date: 2026-08-24T13:50:01+09:00
New Revision: fb1a751f758d79a723796a00bd6565cd7fa48e94
URL: https://github.com/llvm/llvm-project/commit/fb1a751f758d79a723796a00bd6565cd7fa48e94
DIFF: https://github.com/llvm/llvm-project/commit/fb1a751f758d79a723796a00bd6565cd7fa48e94.diff
LOG: [VectorCombine] Support simplification to scalar store for multiple insertelt (#132820)
VectorCombine already folds a load–insertelement–store sequence
containing
a single insertion into a scalar store. Extend this fold to chains of
insertelement instructions:
V0 = load vector from p
V1 = insertelement V0, x1, i1
...
Vn = insertelement V(n-1), xn, in
store Vn to p
=>
store x1 to element i1 of the vector at p
...
store xn to element in of the vector at p
The scalar stores are emitted in insertion order, so later insertions
overwrite earlier ones when their indices are equal, including when the
equality is only known at run time.
Proof: https://alive2.llvm.org/ce/z/QTspTf
Added:
llvm/test/Transforms/VectorCombine/AArch64/load-insert-store.ll
llvm/test/Transforms/VectorCombine/X86/load-insert-store.ll
Modified:
llvm/lib/Transforms/Vectorize/VectorCombine.cpp
llvm/test/Transforms/VectorCombine/load-insert-store.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index b3a545f388d17..6d1ee34d8be8b 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -132,7 +132,7 @@ class VectorCombine {
bool foldExtractedCmps(Instruction &I);
bool foldSelectsFromBitcast(Instruction &I);
bool foldBinopOfReductions(Instruction &I);
- bool foldSingleElementStore(Instruction &I);
+ bool foldInsertElementsToStores(Instruction &I);
bool scalarizeLoad(Instruction &I);
bool scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy, Value *Ptr);
bool scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy, Value *Ptr);
@@ -1921,46 +1921,103 @@ static Align computeAlignmentAfterScalarization(Align VectorAlignment,
return commonAlignment(VectorAlignment, DL.getTypeStoreSize(ScalarType));
}
-// Combine patterns like:
-// %0 = load <4 x i32>, <4 x i32>* %a
-// %1 = insertelement <4 x i32> %0, i32 %b, i32 1
-// store <4 x i32> %1, <4 x i32>* %a
-// to:
-// %0 = bitcast <4 x i32>* %a to i32*
-// %1 = getelementptr inbounds i32, i32* %0, i64 0, i64 1
-// store i32 %b, i32* %1
-bool VectorCombine::foldSingleElementStore(Instruction &I) {
+/// Fold a vector store fed by a single-use insertelement chain into scalar
+/// stores.
+///
+/// Before:
+///
+/// %p --> vector load --> insert %x, lane 1 --> insert %y, lane 3
+/// |
+/// v
+/// vector store to %p
+///
+/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
+/// Stored value: [ old | x | old | y ] (one vector store)
+///
+/// After:
+///
+/// +--> GEP(%p, lane 1) --> store %x
+/// %p -------------+
+/// +--> GEP(%p, lane 3) --> store %y
+///
+/// Vector lanes: [ 0 ] [ 1 ] [ 2 ] [ 3 ]
+/// Scalar stores: x y
+/// store at 1 store at 3
+///
+/// Step 1. Gate:
+/// target supports vector-element GEP addressing
+///
+/// Step 2. Trace:
+/// vector store <-- insertelement <-- ... <-- insertelement <-- load
+///
+/// Steps 3-5. Validate:
+/// reject unprofitable full overwrites; require simple accesses, a
+/// common address/block, no memory write in between, and scalarizable
+/// indices.
+bool VectorCombine::foldInsertElementsToStores(Instruction &I) {
+ // Step 1: The target must support addressing a vector element with a GEP.
if (!TTI.allowVectorElementIndexingUsingGEP())
return false;
+
auto *SI = cast<StoreInst>(&I);
if (!SI->isSimple() || !isa<VectorType>(SI->getValueOperand()->getType()))
return false;
- // TODO: Combine more complicated patterns (multiple insert) by referencing
- // TargetTransformInfo.
- Instruction *Source;
- Value *NewElement;
- Value *Idx;
- if (!match(SI->getValueOperand(),
- m_InsertElt(m_Instruction(Source), m_Value(NewElement),
- m_Value(Idx))))
- return false;
-
- if (auto *Load = dyn_cast<LoadInst>(Source)) {
- auto VecTy = cast<VectorType>(SI->getValueOperand()->getType());
- Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts();
- // Don't optimize for atomic/volatile load or store. Ensure memory is not
- // modified between, vector type matches store size, and index is inbounds.
- if (!Load->isSimple() || Load->getParent() != SI->getParent() ||
- !DL->typeSizeEqualsStoreSize(Load->getType()->getScalarType()) ||
- SrcAddr != SI->getPointerOperand()->stripPointerCasts())
- return false;
+ // Step 2: Collect a single-use insertelement chain, starting at the vector
+ // store and walking back to the candidate load.
+ Value *Source = SI->getValueOperand();
+ SmallVector<std::pair<Value *, Value *>, 4> InsertElements;
+ Value *Base = Source;
+ while (auto *Insert = dyn_cast<InsertElementInst>(Base)) {
+ if (!Insert->hasOneUse())
+ break;
+ Value *InsertVal = Insert->getOperand(1);
+ Value *Idx = Insert->getOperand(2);
+ InsertElements.push_back({InsertVal, Idx});
+ Base = Insert->getOperand(0);
+ }
- if (isMemModifiedBetween(Load->getIterator(), SI->getIterator(),
- MemoryLocation::get(SI), AA))
- return false;
+ if (InsertElements.empty())
+ return false;
+
+ // The backwards walk collected the inserts in reverse program order. Restore
+ // it now so later scalar stores preserve writes to duplicate/equal indices.
+ std::reverse(InsertElements.begin(), InsertElements.end());
+ auto *Load = dyn_cast<LoadInst>(Base);
+ if (!Load)
+ return false;
+ auto *VecTy = cast<VectorType>(SI->getValueOperand()->getType());
+
+ // Step 3: Avoid replacing a complete overwrite with scalar stores when every
+ // lane receives the same value; keeping the vector operation is preferable.
+ if (auto *FVT = dyn_cast<FixedVectorType>(VecTy)) {
+ if (InsertElements.size() == FVT->getNumElements()) {
+ Value *FirstVal = InsertElements.front().first;
+ if (all_of(InsertElements,
+ [FirstVal](const auto &Elt) { return Elt.first == FirstVal; }))
+ return false;
+ }
+ }
+ Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts();
+ // Step 4: Establish the load/store update is legal: both accesses are simple,
+ // have the same base address and block, have scalar elements whose type size
+ // equals their store size, and no intervening operation modifies the updated
+ // memory.
+ if (!Load->isSimple() || Load->getParent() != SI->getParent() ||
+ !DL->typeSizeEqualsStoreSize(Load->getType()->getScalarType()) ||
+ SrcAddr != SI->getPointerOperand()->stripPointerCasts())
+ return false;
+
+ if (isMemModifiedBetween(Load->getIterator(), SI->getIterator(),
+ MemoryLocation::get(SI), AA))
+ return false;
+
+ // Step 5: Validate every index before changing IR. A safe-with-freeze result
+ // is recorded by ScalarizationResult, so discard it until profitability is
+ // known; otherwise a rejected candidate could leave a freeze behind.
+ for (auto [InsertVal, Idx] : InsertElements) {
auto ScalarizableIdx =
- canScalarizeAccess(VecTy, Idx, SQ.getWithInstruction(Load));
+ canScalarizeAccess(VecTy, Idx, SQ.getWithInstruction(&I));
if (ScalarizableIdx.isUnsafe())
return false;
@@ -1971,31 +2028,102 @@ bool VectorCombine::foldSingleElementStore(Instruction &I) {
return false;
}
- // Ensure we add the load back to the worklist BEFORE its users so they can
- // erased in the correct order.
- Worklist.push(Load);
+ // We are only checking legality here. Do not mutate IR before the
+ // profitability check, but also do not leave a pending ToFreeze behind.
+ ScalarizableIdx.discard();
+ }
+
+ InstructionCost OldCost = TTI.getMemoryOpCost(
+ Instruction::Store, SI->getValueOperand()->getType(), SI->getAlign(),
+ SI->getPointerAddressSpace(), CostKind);
+
+ if (Load->hasOneUse())
+ OldCost += TTI.getMemoryOpCost(Instruction::Load, Load->getType(),
+ Load->getAlign(),
+ Load->getPointerAddressSpace(), CostKind);
+
+ for (auto [InsertVal, Idx] : InsertElements) {
+ int Index = -1;
+ if (auto *CIdx = dyn_cast<ConstantInt>(Idx))
+ Index = CIdx->getZExtValue();
+
+ OldCost += TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
+ CostKind, Index);
+ }
+
+ InstructionCost NewCost = 0;
+ // This transform replaces insertelement operations on a single vector with
+ // GEPs and scalar stores, so assume constant-index GEP offsets stay within
+ // addressing-mode ranges that getGEPCost considers TCC_Free. Cost only GEPs
+ // with dynamic indices.
+ for (auto [InsertVal, Idx] : InsertElements) {
+ if (isa<ConstantInt>(Idx))
+ continue;
+ const Value *GEPIndices[] = {ConstantInt::get(Idx->getType(), 0), Idx};
+ NewCost += TTI.getGEPCost(VecTy, SI->getPointerOperand(), GEPIndices,
+ InsertVal->getType(), CostKind);
+ }
+
+ for (auto [InsertVal, Idx] : InsertElements) {
+ Align ScalarOpAlignment = computeAlignmentAfterScalarization(
+ std::max(SI->getAlign(), Load->getAlign()), InsertVal->getType(), Idx,
+ *DL);
+
+ NewCost += TTI.getMemoryOpCost(Instruction::Store, InsertVal->getType(),
+ ScalarOpAlignment,
+ SI->getPointerAddressSpace(), CostKind);
+ }
+
+ LLVM_DEBUG(dbgs() << "Found an insert-elements vector store scalarization "
+ "candidate: "
+ << I << "\n"
+ << " NumInserts: " << InsertElements.size() << "\n"
+ << " OldCost: " << OldCost << " vs NewCost: " << NewCost
+ << "\n");
+
+ if (OldCost <= NewCost)
+ return false;
+
+ for (auto [InsertVal, Idx] : InsertElements) {
+ auto ScalarizableIdx =
+ canScalarizeAccess(VecTy, Idx, SQ.getWithInstruction(&I));
+ assert(!ScalarizableIdx.isUnsafe() && "already checked above");
if (ScalarizableIdx.isSafeWithFreeze())
ScalarizableIdx.freeze(Builder, *cast<Instruction>(Idx));
- Value *GEPIdx = materializeScalarizedGEPIndex(Idx, GEPIndex, Builder);
+ }
+
+ Worklist.push(Load);
+ StoreInst *LastStore = nullptr;
+ for (auto [InsertVal, Idx] : InsertElements) {
+ auto ScalarizableIdx =
+ canScalarizeAccess(VecTy, Idx, SQ.getWithInstruction(&I));
+ if (ScalarizableIdx.isUnsafe())
+ return false;
+
+ IntegerType *GEPIndexTy =
+ getScalarizedGEPIndexInfo(VecTy, Idx, SI->getPointerOperandType(), *DL);
+
+ Value *GEPIdx = materializeScalarizedGEPIndex(Idx, GEPIndexTy, Builder);
Value *GEP = Builder.CreateInBoundsGEP(
SI->getValueOperand()->getType(), SI->getPointerOperand(),
{ConstantInt::get(GEPIdx->getType(), 0), GEPIdx});
- StoreInst *NSI = Builder.CreateStore(NewElement, GEP);
- NSI->copyMetadata(*SI);
+
+ LastStore = Builder.CreateStore(InsertVal, GEP);
+ LastStore->copyMetadata(*SI);
+
// The new GEP may change the pointer operand, so !invariant.group cannot
// be transferred to the scalar store.
- NSI->setMetadata(LLVMContext::MD_invariant_group, nullptr);
+ LastStore->setMetadata(LLVMContext::MD_invariant_group, nullptr);
Align ScalarOpAlignment = computeAlignmentAfterScalarization(
- std::max(SI->getAlign(), Load->getAlign()), NewElement->getType(), Idx,
+ std::max(SI->getAlign(), Load->getAlign()), InsertVal->getType(), Idx,
*DL);
- NSI->setAlignment(ScalarOpAlignment);
- replaceValue(I, *NSI);
- eraseInstruction(I);
- return true;
+ LastStore->setAlignment(ScalarOpAlignment);
}
- return false;
+ replaceValue(I, *LastStore);
+ eraseInstruction(I);
+ return true;
}
/// Try to scalarize vector loads feeding extractelement or bitcast
@@ -6782,7 +6910,7 @@ bool VectorCombine::run() {
return true;
if (Opcode == Instruction::Store)
- if (foldSingleElementStore(I))
+ if (foldInsertElementsToStores(I))
return true;
// If this is an early pipeline invocation of this pass, we are done.
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/load-insert-store.ll b/llvm/test/Transforms/VectorCombine/AArch64/load-insert-store.ll
new file mode 100644
index 0000000000000..f9649db0b9bd8
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AArch64/load-insert-store.ll
@@ -0,0 +1,420 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Run the complete multi-insert suite for both AArch64 byte orders.
+; RUN: opt -S -passes=vector-combine -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=vector-combine -mtriple=aarch64_be-unknown-linux-gnu < %s | FileCheck %s
+
+declare void @llvm.assume(i1)
+
+; Use real little- and big-endian targets to verify that an insertelement lane
+; is scalarized to the same memory element on both byte orders.
+define void @insert_store2(ptr %p, i16 %x, i16 %y) {
+; CHECK-LABEL: @insert_store2(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[X:%.*]], ptr [[GEP0]], align 1
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[Y:%.*]], ptr [[GEP1]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %p, align 1
+ %insert0 = insertelement <8 x i16> %load, i16 %x, i32 6
+ %insert1 = insertelement <8 x i16> %insert0, i16 %y, i32 7
+ store <8 x i16> %insert1, ptr %p, align 1
+ ret void
+}
+
+define void @insert_store2_same_value(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store2_same_value(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 6
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 7
+ store <8 x i16> %vec2, ptr %q, align 1
+ ret void
+}
+
+define void @insert_store2_duplicate_
diff erent_values(ptr %p, i16 %a, i16 %b) {
+; CHECK-LABEL: @insert_store2_duplicate_
diff erent_values(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[A:%.*]], ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[B:%.*]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %v = load <8 x i16>, ptr %p
+ %v1 = insertelement <8 x i16> %v, i16 %a, i32 3
+ %v2 = insertelement <8 x i16> %v1, i16 %b, i32 3
+ store <8 x i16> %v2, ptr %p
+ ret void
+}
+
+define void @insert_store3(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store3(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 5
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 5
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 6
+ %vec3 = insertelement <8 x i16> %vec2, i16 %s, i32 7
+ store <8 x i16> %vec3, ptr %q, align 1
+ ret void
+}
+
+define void @insert_store8(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store8(
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[Q:%.*]], align 16
+; CHECK-NEXT: [[VEC1:%.*]] = insertelement <8 x i16> [[TMP0]], i16 [[S:%.*]], i32 0
+; CHECK-NEXT: [[VEC2:%.*]] = insertelement <8 x i16> [[VEC1]], i16 [[S]], i32 1
+; CHECK-NEXT: [[VEC3:%.*]] = insertelement <8 x i16> [[VEC2]], i16 [[S]], i32 2
+; CHECK-NEXT: [[VEC4:%.*]] = insertelement <8 x i16> [[VEC3]], i16 [[S]], i32 3
+; CHECK-NEXT: [[VEC5:%.*]] = insertelement <8 x i16> [[VEC4]], i16 [[S]], i32 4
+; CHECK-NEXT: [[VEC6:%.*]] = insertelement <8 x i16> [[VEC5]], i16 [[S]], i32 5
+; CHECK-NEXT: [[VEC7:%.*]] = insertelement <8 x i16> [[VEC6]], i16 [[S]], i32 6
+; CHECK-NEXT: [[VEC8:%.*]] = insertelement <8 x i16> [[VEC7]], i16 [[S]], i32 7
+; CHECK-NEXT: store <8 x i16> [[VEC8]], ptr [[Q]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 0
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 1
+ %vec3 = insertelement <8 x i16> %vec2, i16 %s, i32 2
+ %vec4 = insertelement <8 x i16> %vec3, i16 %s, i32 3
+ %vec5 = insertelement <8 x i16> %vec4, i16 %s, i32 4
+ %vec6 = insertelement <8 x i16> %vec5, i16 %s, i32 5
+ %vec7 = insertelement <8 x i16> %vec6, i16 %s, i32 6
+ %vec8 = insertelement <8 x i16> %vec7, i16 %s, i32 7
+ store <8 x i16> %vec8, ptr %q, align 1
+ ret void
+}
+
+define void @insert_store_multi_nonconst_index_freeze(ptr %p, i8 %s, i8 %x, i8 %y) {
+; CHECK-LABEL: @insert_store_multi_nonconst_index_freeze(
+; CHECK-NEXT: [[S_FROZEN:%.*]] = freeze i8 [[S:%.*]]
+; CHECK-NEXT: [[IDX:%.*]] = and i8 [[S_FROZEN]], 1
+; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i8 [[IDX]] to i64
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x i8>, ptr [[P:%.*]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT: store i8 [[X:%.*]], ptr [[GEP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i8>, ptr [[P]], i64 0, i64 2
+; CHECK-NEXT: store i8 [[Y:%.*]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %ld = load <4 x i8>, ptr %p, align 4
+ %idx = and i8 %s, 1
+ %ins0 = insertelement <4 x i8> %ld, i8 %x, i8 %idx
+ %ins1 = insertelement <4 x i8> %ins0, i8 %y, i32 2
+ store <4 x i8> %ins1, ptr %p, align 4
+ ret void
+}
+
+define void @insert_store_gap(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_gap(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 2
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 5
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 2
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 5
+ store <8 x i16> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_reverse(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_reverse(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 5
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 7
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 6
+ %vec3 = insertelement <8 x i16> %vec2, i16 %s, i32 5
+ store <8 x i16> %vec3, ptr %q
+ ret void
+}
+
+; A duplicate lane must still be written in insertion order on both targets.
+define void @insert_store_duplicate(ptr %p, i16 %x, i16 %y) {
+; CHECK-LABEL: @insert_store_duplicate(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[X:%.*]], ptr [[GEP0]], align 1
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[Y:%.*]], ptr [[GEP1]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %p, align 1
+ %insert0 = insertelement <8 x i16> %load, i16 %x, i32 3
+ %insert1 = insertelement <8 x i16> %insert0, i16 %y, i32 3
+ store <8 x i16> %insert1, ptr %p, align 1
+ ret void
+}
+
+define void @insert_store_duplicate_same_value(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_duplicate_same_value(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 3
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 3
+ store <8 x i16> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_i32(ptr %q, i32 zeroext %s) {
+; CHECK-LABEL: @insert_store_i32(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[Q:%.*]], i64 0, i64 2
+; CHECK-NEXT: store i32 [[S:%.*]], ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[Q]], i64 0, i64 3
+; CHECK-NEXT: store i32 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: ret void
+;
+ %load = load <4 x i32>, ptr %q
+ %vec1 = insertelement <4 x i32> %load, i32 %s, i32 2
+ %vec2 = insertelement <4 x i32> %vec1, i32 %s, i32 3
+ store <4 x i32> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_i8(ptr %q, i8 zeroext %s) {
+; CHECK-LABEL: @insert_store_i8(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <16 x i8>, ptr [[Q:%.*]], i64 0, i64 8
+; CHECK-NEXT: store i8 [[S:%.*]], ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[Q]], i64 0, i64 9
+; CHECK-NEXT: store i8 [[S]], ptr [[TMP1]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <16 x i8>, ptr %q
+ %vec1 = insertelement <16 x i8> %load, i8 %s, i32 8
+ %vec2 = insertelement <16 x i8> %vec1, i8 %s, i32 9
+ store <16 x i8> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_alignment(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_alignment(
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[TMP0]], i64 0, i64 4
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 8
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q, align 16
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 0
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 4
+ store <8 x i16> %vec2, ptr %q, align 16
+ ret void
+}
+
+define void @insert_store_size(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_size(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <16 x i16>, ptr [[Q:%.*]], i64 0, i64 8
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i16>, ptr [[Q]], i64 0, i64 12
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 8
+; CHECK-NEXT: ret void
+;
+ %load = load <16 x i16>, ptr %q
+ %vec1 = insertelement <16 x i16> %load, i16 %s, i32 8
+ %vec2 = insertelement <16 x i16> %vec1, i16 %s, i32 12
+ store <16 x i16> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_nonconst4(ptr %q, i8 zeroext %s, i32 %idx1, i32 %idx2, i32 %idx3, i32 %idx4) {
+; CHECK-LABEL: @insert_store_nonconst4(
+; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[Q:%.*]], align 16
+; CHECK-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[TMP0]], i8 [[S:%.*]], i32 [[IDX1:%.*]]
+; CHECK-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[S]], i32 [[IDX2:%.*]]
+; CHECK-NEXT: [[VECINS3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[S]], i32 [[IDX3:%.*]]
+; CHECK-NEXT: [[VECINS4:%.*]] = insertelement <16 x i8> [[VECINS3]], i8 [[S]], i32 [[IDX4:%.*]]
+; CHECK-NEXT: store <16 x i8> [[VECINS4]], ptr [[Q]], align 16
+; CHECK-NEXT: ret void
+;
+ %load = load <16 x i8>, ptr %q
+ %vecins1 = insertelement <16 x i8> %load, i8 %s, i32 %idx1
+ %vecins2 = insertelement <16 x i8> %vecins1, i8 %s, i32 %idx2
+ %vecins3 = insertelement <16 x i8> %vecins2, i8 %s, i32 %idx3
+ %vecins4 = insertelement <16 x i8> %vecins3, i8 %s, i32 %idx4
+ store <16 x i8> %vecins4, ptr %q
+ ret void
+}
+
+define void @insert_store_vscale_nonconst2(ptr %q, i8 zeroext %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_vscale_nonconst2(
+; CHECK-NEXT: [[TMP0:%.*]] = load <vscale x 16 x i8>, ptr [[Q:%.*]], align 16
+; CHECK-NEXT: [[VECINS1:%.*]] = insertelement <vscale x 16 x i8> [[TMP0]], i8 [[S:%.*]], i32 [[IDX1:%.*]]
+; CHECK-NEXT: [[VECINS2:%.*]] = insertelement <vscale x 16 x i8> [[VECINS1]], i8 [[S]], i32 [[IDX2:%.*]]
+; CHECK-NEXT: store <vscale x 16 x i8> [[VECINS2]], ptr [[Q]], align 16
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 16 x i8>, ptr %q
+ %vecins1 = insertelement <vscale x 16 x i8> %load, i8 %s, i32 %idx1
+ %vecins2 = insertelement <vscale x 16 x i8> %vecins1, i8 %s, i32 %idx2
+ store <vscale x 16 x i8> %vecins2, ptr %q
+ ret void
+}
+
+define void @insert_store_nonconst_large_alignment2(ptr %q, i32 zeroext %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_nonconst_large_alignment2(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[IDX1:%.*]], 4
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[IDX2:%.*]], 4
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT: [[IDX1_GEPIDX:%.*]] = zext i32 [[IDX1]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[Q:%.*]], i64 0, i64 [[IDX1_GEPIDX]]
+; CHECK-NEXT: store i32 [[S:%.*]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[IDX2_GEPIDX:%.*]] = zext i32 [[IDX2]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[Q]], i64 0, i64 [[IDX2_GEPIDX]]
+; CHECK-NEXT: store i32 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: ret void
+;
+ %cmp1 = icmp ult i32 %idx1, 4
+ %cmp2 = icmp ult i32 %idx2, 4
+ call void @llvm.assume(i1 %cmp1)
+ call void @llvm.assume(i1 %cmp2)
+ %i = load <4 x i32>, ptr %q, align 128
+ %vecins1 = insertelement <4 x i32> %i, i32 %s, i32 %idx1
+ %vecins2 = insertelement <4 x i32> %vecins1, i32 %s, i32 %idx2
+ store <4 x i32> %vecins2, ptr %q, align 128
+ ret void
+}
+
+define void @insert_store_nonconst_align_maximum_8_2(ptr %q, i64 %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_nonconst_align_maximum_8_2(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[IDX1:%.*]], 2
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[IDX2:%.*]], 2
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT: [[IDX1_GEPIDX:%.*]] = zext i32 [[IDX1]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q:%.*]], i64 0, i64 [[IDX1_GEPIDX]]
+; CHECK-NEXT: store i64 [[S:%.*]], ptr [[TMP1]], align 8
+; CHECK-NEXT: [[IDX2_GEPIDX:%.*]] = zext i32 [[IDX2]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q]], i64 0, i64 [[IDX2_GEPIDX]]
+; CHECK-NEXT: store i64 [[S]], ptr [[TMP2]], align 8
+; CHECK-NEXT: ret void
+;
+ %cmp1 = icmp ult i32 %idx1, 2
+ %cmp2 = icmp ult i32 %idx2, 2
+ call void @llvm.assume(i1 %cmp1)
+ call void @llvm.assume(i1 %cmp2)
+ %i = load <8 x i64>, ptr %q, align 8
+ %vecins1 = insertelement <8 x i64> %i, i64 %s, i32 %idx1
+ %vecins2 = insertelement <8 x i64> %vecins1, i64 %s, i32 %idx2
+ store <8 x i64> %vecins2, ptr %q, align 8
+ ret void
+}
+
+define void @insert_store_nonconst_align_maximum_4_2(ptr %q, i64 %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_nonconst_align_maximum_4_2(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[IDX1:%.*]], 2
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[IDX2:%.*]], 2
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT: [[IDX1_GEPIDX:%.*]] = zext i32 [[IDX1]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q:%.*]], i64 0, i64 [[IDX1_GEPIDX]]
+; CHECK-NEXT: store i64 [[S:%.*]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[IDX2_GEPIDX:%.*]] = zext i32 [[IDX2]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q]], i64 0, i64 [[IDX2_GEPIDX]]
+; CHECK-NEXT: store i64 [[S]], ptr [[TMP2]], align 4
+; CHECK-NEXT: ret void
+;
+ %cmp1 = icmp ult i32 %idx1, 2
+ %cmp2 = icmp ult i32 %idx2, 2
+ call void @llvm.assume(i1 %cmp1)
+ call void @llvm.assume(i1 %cmp2)
+ %i = load <8 x i64>, ptr %q, align 4
+ %vecins1 = insertelement <8 x i64> %i, i64 %s, i32 %idx1
+ %vecins2 = insertelement <8 x i64> %vecins1, i64 %s, i32 %idx2
+ store <8 x i64> %vecins2, ptr %q, align 4
+ ret void
+}
+
+define void @insert_store_nonconst_align_larger_2(ptr %q, i64 %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_nonconst_align_larger_2(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[IDX1:%.*]], 2
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[IDX2:%.*]], 2
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT: [[IDX1_GEPIDX:%.*]] = zext i32 [[IDX1]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q:%.*]], i64 0, i64 [[IDX1_GEPIDX]]
+; CHECK-NEXT: store i64 [[S:%.*]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[IDX2_GEPIDX:%.*]] = zext i32 [[IDX2]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q]], i64 0, i64 [[IDX2_GEPIDX]]
+; CHECK-NEXT: store i64 [[S]], ptr [[TMP2]], align 4
+; CHECK-NEXT: ret void
+;
+ %cmp1 = icmp ult i32 %idx1, 2
+ %cmp2 = icmp ult i32 %idx2, 2
+ call void @llvm.assume(i1 %cmp1)
+ call void @llvm.assume(i1 %cmp2)
+ %i = load <8 x i64>, ptr %q, align 4
+ %vecins1 = insertelement <8 x i64> %i, i64 %s, i32 %idx1
+ %vecins2 = insertelement <8 x i64> %vecins1, i64 %s, i32 %idx2
+ store <8 x i64> %vecins2, ptr %q, align 2
+ ret void
+}
+
+define void @insert_store_dynamic_indices_may_alias(ptr %p, i16 %a, i16 %b, i32 %i, i32 %j) {
+; CHECK-LABEL: @insert_store_dynamic_indices_may_alias(
+; CHECK-NEXT: [[CI:%.*]] = icmp ult i32 [[I:%.*]], 8
+; CHECK-NEXT: [[CJ:%.*]] = icmp ult i32 [[J:%.*]], 8
+; CHECK-NEXT: call void @llvm.assume(i1 [[CI]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CJ]])
+; CHECK-NEXT: [[I_GEPIDX:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 [[I_GEPIDX]]
+; CHECK-NEXT: store i16 [[A:%.*]], ptr [[TMP1]], align 2
+; CHECK-NEXT: [[J_GEPIDX:%.*]] = zext i32 [[J]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 [[J_GEPIDX]]
+; CHECK-NEXT: store i16 [[B:%.*]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %ci = icmp ult i32 %i, 8
+ %cj = icmp ult i32 %j, 8
+ call void @llvm.assume(i1 %ci)
+ call void @llvm.assume(i1 %cj)
+
+ %v = load <8 x i16>, ptr %p
+ %v1 = insertelement <8 x i16> %v, i16 %a, i32 %i
+ %v2 = insertelement <8 x i16> %v1, i16 %b, i32 %j
+ store <8 x i16> %v2, ptr %p
+ ret void
+}
+
+; counter-example
+define void @crash_counter_exam(ptr %p, i32 %x, i32 %arg) {
+; CHECK-LABEL: @crash_counter_exam(
+; CHECK-NEXT: [[IDX:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[IDX]], 4
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P:%.*]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT: store i32 [[X:%.*]], ptr [[TMP1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ld = load <4 x i32>, ptr %p, align 16
+ %idx = freeze i32 %arg
+ %cmp = icmp ult i32 %idx, 4
+ call void @llvm.assume(i1 %cmp)
+ %ins = insertelement <4 x i32> %ld, i32 %x, i32 %idx
+ store <4 x i32> %ins, ptr %p, align 16
+ ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-insert-store.ll b/llvm/test/Transforms/VectorCombine/X86/load-insert-store.ll
new file mode 100644
index 0000000000000..9476ecbe2b135
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/load-insert-store.ll
@@ -0,0 +1,422 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Run the multi-insert suite for each supported x86-64 CPU level.
+; RUN: opt -S -passes=vector-combine -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64 < %s | FileCheck %s
+; RUN: opt -S -passes=vector-combine -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v2 < %s | FileCheck %s
+; RUN: opt -S -passes=vector-combine -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 < %s | FileCheck %s
+; RUN: opt -S -passes=vector-combine -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v4 < %s | FileCheck %s
+
+declare void @llvm.assume(i1)
+
+; Scalarization profitability is target-dependent. Check it for each x86-64
+; microarchitecture level that this transform is expected to support.
+define void @insert_store2(ptr %p, i16 %x, i16 %y) {
+; CHECK-LABEL: @insert_store2(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[X:%.*]], ptr [[GEP0]], align 1
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[Y:%.*]], ptr [[GEP1]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %p, align 1
+ %insert0 = insertelement <8 x i16> %load, i16 %x, i32 6
+ %insert1 = insertelement <8 x i16> %insert0, i16 %y, i32 7
+ store <8 x i16> %insert1, ptr %p, align 1
+ ret void
+}
+
+define void @insert_store2_same_value(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store2_same_value(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 6
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 7
+ store <8 x i16> %vec2, ptr %q, align 1
+ ret void
+}
+
+define void @insert_store2_duplicate_
diff erent_values(ptr %p, i16 %a, i16 %b) {
+; CHECK-LABEL: @insert_store2_duplicate_
diff erent_values(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[A:%.*]], ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[B:%.*]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %v = load <8 x i16>, ptr %p
+ %v1 = insertelement <8 x i16> %v, i16 %a, i32 3
+ %v2 = insertelement <8 x i16> %v1, i16 %b, i32 3
+ store <8 x i16> %v2, ptr %p
+ ret void
+}
+
+define void @insert_store3(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store3(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 5
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 5
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 6
+ %vec3 = insertelement <8 x i16> %vec2, i16 %s, i32 7
+ store <8 x i16> %vec3, ptr %q, align 1
+ ret void
+}
+
+define void @insert_store8(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store8(
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[Q:%.*]], align 16
+; CHECK-NEXT: [[VEC1:%.*]] = insertelement <8 x i16> [[TMP0]], i16 [[S:%.*]], i32 0
+; CHECK-NEXT: [[VEC2:%.*]] = insertelement <8 x i16> [[VEC1]], i16 [[S]], i32 1
+; CHECK-NEXT: [[VEC3:%.*]] = insertelement <8 x i16> [[VEC2]], i16 [[S]], i32 2
+; CHECK-NEXT: [[VEC4:%.*]] = insertelement <8 x i16> [[VEC3]], i16 [[S]], i32 3
+; CHECK-NEXT: [[VEC5:%.*]] = insertelement <8 x i16> [[VEC4]], i16 [[S]], i32 4
+; CHECK-NEXT: [[VEC6:%.*]] = insertelement <8 x i16> [[VEC5]], i16 [[S]], i32 5
+; CHECK-NEXT: [[VEC7:%.*]] = insertelement <8 x i16> [[VEC6]], i16 [[S]], i32 6
+; CHECK-NEXT: [[VEC8:%.*]] = insertelement <8 x i16> [[VEC7]], i16 [[S]], i32 7
+; CHECK-NEXT: store <8 x i16> [[VEC8]], ptr [[Q]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 0
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 1
+ %vec3 = insertelement <8 x i16> %vec2, i16 %s, i32 2
+ %vec4 = insertelement <8 x i16> %vec3, i16 %s, i32 3
+ %vec5 = insertelement <8 x i16> %vec4, i16 %s, i32 4
+ %vec6 = insertelement <8 x i16> %vec5, i16 %s, i32 5
+ %vec7 = insertelement <8 x i16> %vec6, i16 %s, i32 6
+ %vec8 = insertelement <8 x i16> %vec7, i16 %s, i32 7
+ store <8 x i16> %vec8, ptr %q, align 1
+ ret void
+}
+
+define void @insert_store_multi_nonconst_index_freeze(ptr %p, i8 %s, i8 %x, i8 %y) {
+; CHECK-LABEL: @insert_store_multi_nonconst_index_freeze(
+; CHECK-NEXT: [[S_FROZEN:%.*]] = freeze i8 [[S:%.*]]
+; CHECK-NEXT: [[IDX:%.*]] = and i8 [[S_FROZEN]], 1
+; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i8 [[IDX]] to i64
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x i8>, ptr [[P:%.*]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT: store i8 [[X:%.*]], ptr [[GEP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i8>, ptr [[P]], i64 0, i64 2
+; CHECK-NEXT: store i8 [[Y:%.*]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %ld = load <4 x i8>, ptr %p, align 4
+ %idx = and i8 %s, 1
+ %ins0 = insertelement <4 x i8> %ld, i8 %x, i8 %idx
+ %ins1 = insertelement <4 x i8> %ins0, i8 %y, i32 2
+ store <4 x i8> %ins1, ptr %p, align 4
+ ret void
+}
+
+define void @insert_store_gap(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_gap(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 2
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 5
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 2
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 5
+ store <8 x i16> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_reverse(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_reverse(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 5
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 7
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 6
+ %vec3 = insertelement <8 x i16> %vec2, i16 %s, i32 5
+ store <8 x i16> %vec3, ptr %q
+ ret void
+}
+
+; Keep the program order of stores when two inserts select the same element.
+define void @insert_store_duplicate(ptr %p, i16 %x, i16 %y) {
+; CHECK-LABEL: @insert_store_duplicate(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[X:%.*]], ptr [[GEP0]], align 1
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[Y:%.*]], ptr [[GEP1]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %p, align 1
+ %insert0 = insertelement <8 x i16> %load, i16 %x, i32 3
+ %insert1 = insertelement <8 x i16> %insert0, i16 %y, i32 3
+ store <8 x i16> %insert1, ptr %p, align 1
+ ret void
+}
+
+define void @insert_store_duplicate_same_value(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_duplicate_same_value(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 3
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 3
+ store <8 x i16> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_i32(ptr %q, i32 zeroext %s) {
+; CHECK-LABEL: @insert_store_i32(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[Q:%.*]], i64 0, i64 2
+; CHECK-NEXT: store i32 [[S:%.*]], ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[Q]], i64 0, i64 3
+; CHECK-NEXT: store i32 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: ret void
+;
+ %load = load <4 x i32>, ptr %q
+ %vec1 = insertelement <4 x i32> %load, i32 %s, i32 2
+ %vec2 = insertelement <4 x i32> %vec1, i32 %s, i32 3
+ store <4 x i32> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_i8(ptr %q, i8 zeroext %s) {
+; CHECK-LABEL: @insert_store_i8(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <16 x i8>, ptr [[Q:%.*]], i64 0, i64 8
+; CHECK-NEXT: store i8 [[S:%.*]], ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i8>, ptr [[Q]], i64 0, i64 9
+; CHECK-NEXT: store i8 [[S]], ptr [[TMP1]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <16 x i8>, ptr %q
+ %vec1 = insertelement <16 x i8> %load, i8 %s, i32 8
+ %vec2 = insertelement <16 x i8> %vec1, i8 %s, i32 9
+ store <16 x i8> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_alignment(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_alignment(
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[TMP0]], i64 0, i64 4
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 8
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q, align 16
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 0
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 4
+ store <8 x i16> %vec2, ptr %q, align 16
+ ret void
+}
+
+define void @insert_store_size(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store_size(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <16 x i16>, ptr [[Q:%.*]], i64 0, i64 8
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i16>, ptr [[Q]], i64 0, i64 12
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 8
+; CHECK-NEXT: ret void
+;
+ %load = load <16 x i16>, ptr %q
+ %vec1 = insertelement <16 x i16> %load, i16 %s, i32 8
+ %vec2 = insertelement <16 x i16> %vec1, i16 %s, i32 12
+ store <16 x i16> %vec2, ptr %q
+ ret void
+}
+
+define void @insert_store_nonconst4(ptr %q, i8 zeroext %s, i32 %idx1, i32 %idx2, i32 %idx3, i32 %idx4) {
+; CHECK-LABEL: @insert_store_nonconst4(
+; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[Q:%.*]], align 16
+; CHECK-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[TMP0]], i8 [[S:%.*]], i32 [[IDX1:%.*]]
+; CHECK-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[S]], i32 [[IDX2:%.*]]
+; CHECK-NEXT: [[VECINS3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[S]], i32 [[IDX3:%.*]]
+; CHECK-NEXT: [[VECINS4:%.*]] = insertelement <16 x i8> [[VECINS3]], i8 [[S]], i32 [[IDX4:%.*]]
+; CHECK-NEXT: store <16 x i8> [[VECINS4]], ptr [[Q]], align 16
+; CHECK-NEXT: ret void
+;
+ %load = load <16 x i8>, ptr %q
+ %vecins1 = insertelement <16 x i8> %load, i8 %s, i32 %idx1
+ %vecins2 = insertelement <16 x i8> %vecins1, i8 %s, i32 %idx2
+ %vecins3 = insertelement <16 x i8> %vecins2, i8 %s, i32 %idx3
+ %vecins4 = insertelement <16 x i8> %vecins3, i8 %s, i32 %idx4
+ store <16 x i8> %vecins4, ptr %q
+ ret void
+}
+
+define void @insert_store_vscale_nonconst2(ptr %q, i8 zeroext %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_vscale_nonconst2(
+; CHECK-NEXT: [[TMP0:%.*]] = load <vscale x 16 x i8>, ptr [[Q:%.*]], align 16
+; CHECK-NEXT: [[VECINS1:%.*]] = insertelement <vscale x 16 x i8> [[TMP0]], i8 [[S:%.*]], i32 [[IDX1:%.*]]
+; CHECK-NEXT: [[VECINS2:%.*]] = insertelement <vscale x 16 x i8> [[VECINS1]], i8 [[S]], i32 [[IDX2:%.*]]
+; CHECK-NEXT: store <vscale x 16 x i8> [[VECINS2]], ptr [[Q]], align 16
+; CHECK-NEXT: ret void
+;
+ %load = load <vscale x 16 x i8>, ptr %q
+ %vecins1 = insertelement <vscale x 16 x i8> %load, i8 %s, i32 %idx1
+ %vecins2 = insertelement <vscale x 16 x i8> %vecins1, i8 %s, i32 %idx2
+ store <vscale x 16 x i8> %vecins2, ptr %q
+ ret void
+}
+
+define void @insert_store_nonconst_large_alignment2(ptr %q, i32 zeroext %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_nonconst_large_alignment2(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[IDX1:%.*]], 4
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[IDX2:%.*]], 4
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT: [[IDX1_GEPIDX:%.*]] = zext i32 [[IDX1]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[Q:%.*]], i64 0, i64 [[IDX1_GEPIDX]]
+; CHECK-NEXT: store i32 [[S:%.*]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[IDX2_GEPIDX:%.*]] = zext i32 [[IDX2]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[Q]], i64 0, i64 [[IDX2_GEPIDX]]
+; CHECK-NEXT: store i32 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: ret void
+;
+ %cmp1 = icmp ult i32 %idx1, 4
+ %cmp2 = icmp ult i32 %idx2, 4
+ call void @llvm.assume(i1 %cmp1)
+ call void @llvm.assume(i1 %cmp2)
+ %i = load <4 x i32>, ptr %q, align 128
+ %vecins1 = insertelement <4 x i32> %i, i32 %s, i32 %idx1
+ %vecins2 = insertelement <4 x i32> %vecins1, i32 %s, i32 %idx2
+ store <4 x i32> %vecins2, ptr %q, align 128
+ ret void
+}
+
+define void @insert_store_nonconst_align_maximum_8_2(ptr %q, i64 %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_nonconst_align_maximum_8_2(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[IDX1:%.*]], 2
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[IDX2:%.*]], 2
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT: [[IDX1_GEPIDX:%.*]] = zext i32 [[IDX1]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q:%.*]], i64 0, i64 [[IDX1_GEPIDX]]
+; CHECK-NEXT: store i64 [[S:%.*]], ptr [[TMP1]], align 8
+; CHECK-NEXT: [[IDX2_GEPIDX:%.*]] = zext i32 [[IDX2]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q]], i64 0, i64 [[IDX2_GEPIDX]]
+; CHECK-NEXT: store i64 [[S]], ptr [[TMP2]], align 8
+; CHECK-NEXT: ret void
+;
+ %cmp1 = icmp ult i32 %idx1, 2
+ %cmp2 = icmp ult i32 %idx2, 2
+ call void @llvm.assume(i1 %cmp1)
+ call void @llvm.assume(i1 %cmp2)
+ %i = load <8 x i64>, ptr %q, align 8
+ %vecins1 = insertelement <8 x i64> %i, i64 %s, i32 %idx1
+ %vecins2 = insertelement <8 x i64> %vecins1, i64 %s, i32 %idx2
+ store <8 x i64> %vecins2, ptr %q, align 8
+ ret void
+}
+
+define void @insert_store_nonconst_align_maximum_4_2(ptr %q, i64 %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_nonconst_align_maximum_4_2(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[IDX1:%.*]], 2
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[IDX2:%.*]], 2
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT: [[IDX1_GEPIDX:%.*]] = zext i32 [[IDX1]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q:%.*]], i64 0, i64 [[IDX1_GEPIDX]]
+; CHECK-NEXT: store i64 [[S:%.*]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[IDX2_GEPIDX:%.*]] = zext i32 [[IDX2]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q]], i64 0, i64 [[IDX2_GEPIDX]]
+; CHECK-NEXT: store i64 [[S]], ptr [[TMP2]], align 4
+; CHECK-NEXT: ret void
+;
+ %cmp1 = icmp ult i32 %idx1, 2
+ %cmp2 = icmp ult i32 %idx2, 2
+ call void @llvm.assume(i1 %cmp1)
+ call void @llvm.assume(i1 %cmp2)
+ %i = load <8 x i64>, ptr %q, align 4
+ %vecins1 = insertelement <8 x i64> %i, i64 %s, i32 %idx1
+ %vecins2 = insertelement <8 x i64> %vecins1, i64 %s, i32 %idx2
+ store <8 x i64> %vecins2, ptr %q, align 4
+ ret void
+}
+
+define void @insert_store_nonconst_align_larger_2(ptr %q, i64 %s, i32 %idx1, i32 %idx2) {
+; CHECK-LABEL: @insert_store_nonconst_align_larger_2(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[IDX1:%.*]], 2
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[IDX2:%.*]], 2
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP1]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP2]])
+; CHECK-NEXT: [[IDX1_GEPIDX:%.*]] = zext i32 [[IDX1]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q:%.*]], i64 0, i64 [[IDX1_GEPIDX]]
+; CHECK-NEXT: store i64 [[S:%.*]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[IDX2_GEPIDX:%.*]] = zext i32 [[IDX2]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i64>, ptr [[Q]], i64 0, i64 [[IDX2_GEPIDX]]
+; CHECK-NEXT: store i64 [[S]], ptr [[TMP2]], align 4
+; CHECK-NEXT: ret void
+;
+ %cmp1 = icmp ult i32 %idx1, 2
+ %cmp2 = icmp ult i32 %idx2, 2
+ call void @llvm.assume(i1 %cmp1)
+ call void @llvm.assume(i1 %cmp2)
+ %i = load <8 x i64>, ptr %q, align 4
+ %vecins1 = insertelement <8 x i64> %i, i64 %s, i32 %idx1
+ %vecins2 = insertelement <8 x i64> %vecins1, i64 %s, i32 %idx2
+ store <8 x i64> %vecins2, ptr %q, align 2
+ ret void
+}
+
+define void @insert_store_dynamic_indices_may_alias(ptr %p, i16 %a, i16 %b, i32 %i, i32 %j) {
+; CHECK-LABEL: @insert_store_dynamic_indices_may_alias(
+; CHECK-NEXT: [[CI:%.*]] = icmp ult i32 [[I:%.*]], 8
+; CHECK-NEXT: [[CJ:%.*]] = icmp ult i32 [[J:%.*]], 8
+; CHECK-NEXT: call void @llvm.assume(i1 [[CI]])
+; CHECK-NEXT: call void @llvm.assume(i1 [[CJ]])
+; CHECK-NEXT: [[I_GEPIDX:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 [[I_GEPIDX]]
+; CHECK-NEXT: store i16 [[A:%.*]], ptr [[TMP1]], align 2
+; CHECK-NEXT: [[J_GEPIDX:%.*]] = zext i32 [[J]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 [[J_GEPIDX]]
+; CHECK-NEXT: store i16 [[B:%.*]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %ci = icmp ult i32 %i, 8
+ %cj = icmp ult i32 %j, 8
+ call void @llvm.assume(i1 %ci)
+ call void @llvm.assume(i1 %cj)
+
+ %v = load <8 x i16>, ptr %p
+ %v1 = insertelement <8 x i16> %v, i16 %a, i32 %i
+ %v2 = insertelement <8 x i16> %v1, i16 %b, i32 %j
+ store <8 x i16> %v2, ptr %p
+ ret void
+}
+
+; counter-example
+define void @crash_counter_exam(ptr %p, i32 %x, i32 %arg) {
+; CHECK-LABEL: @crash_counter_exam(
+; CHECK-NEXT: [[IDX:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[IDX]], 4
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[IDX_GEPIDX:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[P:%.*]], i64 0, i64 [[IDX_GEPIDX]]
+; CHECK-NEXT: store i32 [[X:%.*]], ptr [[TMP1]], align 4
+; CHECK-NEXT: ret void
+;
+ %ld = load <4 x i32>, ptr %p, align 16
+ %idx = freeze i32 %arg
+ %cmp = icmp ult i32 %idx, 4
+ call void @llvm.assume(i1 %cmp)
+ %ins = insertelement <4 x i32> %ld, i32 %x, i32 %idx
+ store <4 x i32> %ins, ptr %p, align 16
+ ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/load-insert-store.ll b/llvm/test/Transforms/VectorCombine/load-insert-store.ll
index 22808dc84da16..75f5ff03a1f7e 100644
--- a/llvm/test/Transforms/VectorCombine/load-insert-store.ll
+++ b/llvm/test/Transforms/VectorCombine/load-insert-store.ll
@@ -1,4 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; Verify scalar stores use the same vector-element address for both byte orders.
; RUN: opt -S -passes=vector-combine -data-layout=e < %s | FileCheck %s
; RUN: opt -S -passes=vector-combine -data-layout=E < %s | FileCheck %s
@@ -23,8 +24,7 @@ define void @insert_store_across_synchronization(ptr %q, i32 %s) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[Q:%.*]], align 16
; CHECK-NEXT: call void @may_synchronize()
-; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x i32> [[V]], i32 [[S:%.*]],
-; CHECK-SAME: i32 1
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x i32> [[V]], i32 [[S:%.*]], i32 1
; CHECK-NEXT: store <4 x i32> [[VECINS]], ptr [[Q]], align 16
; CHECK-NEXT: ret void
;
@@ -36,6 +36,81 @@ entry:
ret void
}
+define void @insert_store2(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store2(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 6
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 7
+ store <8 x i16> %vec2, ptr %q, align 1
+ ret void
+}
+
+define void @insert_store2_duplicate_
diff erent_values(ptr %p, i16 %a, i16 %b) {
+; CHECK-LABEL: @insert_store2_duplicate_
diff erent_values(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[A:%.*]], ptr [[TMP1]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P]], i64 0, i64 3
+; CHECK-NEXT: store i16 [[B:%.*]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %v = load <8 x i16>, ptr %p
+ %v1 = insertelement <8 x i16> %v, i16 %a, i32 3
+ %v2 = insertelement <8 x i16> %v1, i16 %b, i32 3
+ store <8 x i16> %v2, ptr %p
+ ret void
+}
+
+define void @insert_store3(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store3(
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q:%.*]], i64 0, i64 5
+; CHECK-NEXT: store i16 [[S:%.*]], ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 6
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP1]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i16>, ptr [[Q]], i64 0, i64 7
+; CHECK-NEXT: store i16 [[S]], ptr [[TMP2]], align 2
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 5
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 6
+ %vec3 = insertelement <8 x i16> %vec2, i16 %s, i32 7
+ store <8 x i16> %vec3, ptr %q, align 1
+ ret void
+}
+
+define void @insert_store8(ptr %q, i16 zeroext %s) {
+; CHECK-LABEL: @insert_store8(
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[Q:%.*]], align 16
+; CHECK-NEXT: [[VEC1:%.*]] = insertelement <8 x i16> [[TMP0]], i16 [[S:%.*]], i32 0
+; CHECK-NEXT: [[VEC2:%.*]] = insertelement <8 x i16> [[VEC1]], i16 [[S]], i32 1
+; CHECK-NEXT: [[VEC3:%.*]] = insertelement <8 x i16> [[VEC2]], i16 [[S]], i32 2
+; CHECK-NEXT: [[VEC4:%.*]] = insertelement <8 x i16> [[VEC3]], i16 [[S]], i32 3
+; CHECK-NEXT: [[VEC5:%.*]] = insertelement <8 x i16> [[VEC4]], i16 [[S]], i32 4
+; CHECK-NEXT: [[VEC6:%.*]] = insertelement <8 x i16> [[VEC5]], i16 [[S]], i32 5
+; CHECK-NEXT: [[VEC7:%.*]] = insertelement <8 x i16> [[VEC6]], i16 [[S]], i32 6
+; CHECK-NEXT: [[VEC8:%.*]] = insertelement <8 x i16> [[VEC7]], i16 [[S]], i32 7
+; CHECK-NEXT: store <8 x i16> [[VEC8]], ptr [[Q]], align 1
+; CHECK-NEXT: ret void
+;
+ %load = load <8 x i16>, ptr %q
+ %vec1 = insertelement <8 x i16> %load, i16 %s, i32 0
+ %vec2 = insertelement <8 x i16> %vec1, i16 %s, i32 1
+ %vec3 = insertelement <8 x i16> %vec2, i16 %s, i32 2
+ %vec4 = insertelement <8 x i16> %vec3, i16 %s, i32 3
+ %vec5 = insertelement <8 x i16> %vec4, i16 %s, i32 4
+ %vec6 = insertelement <8 x i16> %vec5, i16 %s, i32 5
+ %vec7 = insertelement <8 x i16> %vec6, i16 %s, i32 6
+ %vec8 = insertelement <8 x i16> %vec7, i16 %s, i32 7
+ store <8 x i16> %vec8, ptr %q, align 1
+ ret void
+}
+
define void @insert_store_i16_align1(ptr %q, i16 zeroext %s) {
; CHECK-LABEL: @insert_store_i16_align1(
; CHECK-NEXT: entry:
More information about the llvm-commits
mailing list