[llvm] [LSV] Support vectorization of mixed-type contiguous accesses (PR #177908)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Sun Mar 29 14:10:33 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/177908
>From 95082c57bb42d76fc1581639a189b12081f3079e Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Sat, 20 Dec 2025 02:32:15 -0500
Subject: [PATCH] [LoadStoreVectorizer] Support vectorization of mixed-type
contiguous accesses
Relaxes the LoadStoreVectorizer to support merging contiguous loads and
stores of different element types.
Key changes:
- Removes ElementSize from the EquivalenceClass key, allowing
instructions with different types (e.g., i32 and i16) to be grouped
together if they share the same underlying object.
- Calculates the vector element type using the Greatest Common Divisor
(GCD) of all scalar sizes in the chain.
- Updates vectorization logic to handle type mismatches:
- Loads: Generates a vector load of the GCD type, then reconstructs
original values using shuffles/extracts and bitcasts.
- Stores: Bitcasts values to the GCD type (splitting if necessary) and
inserts them into the vector to be stored.
Adds extensive tests covering mixed integer sizes (i64/i32/i16/i8),
floating point mixes, and mixed load/store scenarios.
Resolves #97715
---
.../Vectorize/LoadStoreVectorizer.cpp | 357 ++++++++++++++----
.../AMDGPU/merge-vectors.ll | 7 +-
2 files changed, 292 insertions(+), 72 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 89ba9227d8952..8d8e0666a1d2c 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -98,6 +98,7 @@
#include "llvm/Support/Alignment.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/ModRef.h"
@@ -126,21 +127,16 @@ namespace {
// Equivalence class key, the initial tuple by which we group loads/stores.
// Loads/stores with different EqClassKeys are never merged.
-//
-// (We could in theory remove element-size from the this tuple. We'd just need
-// to fix up the vector packing/unpacking code.)
using EqClassKey =
std::tuple<const Value * /* result of getUnderlyingObject() */,
unsigned /* AddrSpace */,
- unsigned /* Load/Store element size bits */,
char /* IsLoad; char b/c bool can't be a DenseMap key */
>;
[[maybe_unused]] llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
const EqClassKey &K) {
- const auto &[UnderlyingObject, AddrSpace, ElementSize, IsLoad] = K;
+ const auto &[UnderlyingObject, AddrSpace, IsLoad] = K;
return OS << (IsLoad ? "load" : "store") << " of " << *UnderlyingObject
- << " of element size " << ElementSize << " bits in addrspace "
- << AddrSpace;
+ << " in addrspace " << AddrSpace;
}
// A Chain is a set of instructions such that:
@@ -298,6 +294,24 @@ class Vectorizer {
/// Discards any length-1 subchains.
std::vector<Chain> splitChainByAlignment(Chain &C);
+ /// Splits out pointer-typed elements whose scalar size exceeds the GCD
+ /// element size of the chain, so that pointers are never vectorized at a
+ /// width smaller than their native size. Non-pointer elements and pointers
+ /// whose size matches the GCD remain in the main chain.
+ std::vector<Chain> splitChainByPointerWidth(Chain &C);
+
+ /// Inserts a cast or a series of casts to convert \p V to \p NewTy.
+ ///
+ /// This function handles cases where the bit-widths of the source and
+ /// destination types differ by first converting the value to a scalar
+ /// integer of the original width, resizing it via zero-extension or
+ /// truncation, and then casting it to the final destination type.
+ ///
+ /// \param V The value to be cast.
+ /// \param NewTy The target type for the value.
+ /// \returns The resulting casted Value.
+ Value *insertCast(Value *V, Type *NewTy);
+
/// Converts the instrs in the chain into a single vectorized load or store.
/// Adds the old scalar loads/stores to ToErase.
bool vectorizeChain(Chain &C);
@@ -313,9 +327,14 @@ class Vectorizer {
Instruction *ContextInst,
unsigned Depth);
+
/// Gets the element type of the vector that the chain will load or store.
/// This is nontrivial because the chain may contain elements of different
/// types; e.g. it's legal to have a chain that contains both i32 and float.
+ /// The element type is determined by taking the GCD of the bitwidths of the
+ /// chain elements. If all elements are floating-point typed, then the
+ /// appropriate FP type is returned. Otherwise, int type is returned with
+ /// the GCD as the bitwidth.
Type *getChainElemTy(const Chain &C);
/// Determines whether ChainElem can be moved up (if IsLoad) or down (if
@@ -555,10 +574,11 @@ bool Vectorizer::runOnChain(Chain &C) {
// all at worst O(n lg n) in the number of instructions, and splitting chains
// doesn't change the number of instrs. So the whole loop nest is O(n lg n).)
bool Changed = false;
- for (auto &C : splitChainByMayAliasInstrs(C))
- for (auto &C : splitChainByContiguity(C))
- for (auto &C : splitChainByAlignment(C))
- Changed |= vectorizeChain(C);
+ for (auto &C : splitChainByPointerWidth(C))
+ for (auto &C : splitChainByMayAliasInstrs(C))
+ for (auto &C : splitChainByContiguity(C))
+ for (auto &C : splitChainByAlignment(C))
+ Changed |= vectorizeChain(C);
return Changed;
}
@@ -793,29 +813,56 @@ std::vector<Chain> Vectorizer::splitChainByContiguity(Chain &C) {
Type *Vectorizer::getChainElemTy(const Chain &C) {
assert(!C.empty());
- // The rules are:
- // - If there are any pointer types in the chain, use an integer type.
- // - Prefer an integer type if it appears in the chain.
- // - Otherwise, use the first type in the chain.
- //
- // The rule about pointer types is a simplification when we merge e.g. a load
- // of a ptr and a double. There's no direct conversion from a ptr to a
- // double; it requires a ptrtoint followed by a bitcast.
- //
- // It's unclear to me if the other rules have any practical effect, but we do
- // it to match this pass's previous behavior.
- if (any_of(C, [](const ChainElem &E) {
- return getLoadStoreType(E.Inst)->getScalarType()->isPointerTy();
- })) {
- return Type::getIntNTy(
- F.getContext(),
- DL.getTypeSizeInBits(getLoadStoreType(C[0].Inst)->getScalarType()));
+ unsigned GCDSize = 0;
+ SmallVector<Type *, 8> OriginalTypes;
+
+ // Compute the GCD of the bitwidths of the chain element types.
+ bool AllFPTy = false;
+ for (const ChainElem &E : C) {
+ Type *Ty = getLoadStoreType(E.Inst)->getScalarType();
+ AllFPTy |= Ty->isFloatingPointTy();
+ OriginalTypes.push_back(Ty);
+ unsigned Sz = DL.getTypeSizeInBits(Ty);
+ if (GCDSize == 0)
+ GCDSize = Sz;
+ else
+ GCDSize = std::gcd(GCDSize, Sz);
}
- for (const ChainElem &E : C)
- if (Type *T = getLoadStoreType(E.Inst)->getScalarType(); T->isIntegerTy())
- return T;
- return getLoadStoreType(C[0].Inst)->getScalarType();
+ // If all chain elements are floating-point typed, return
+ // the FP type of size `GCDSize`. This step is necessary to
+ // avoid unnecessary casts from int to float.
+ if (AllFPTy) {
+ switch (GCDSize) {
+ case 16:
+ // Check if we should use bfloat or half
+ if (llvm::any_of(C, [](const ChainElem &E) {
+ return getLoadStoreType(E.Inst)->getScalarType()->isBFloatTy();
+ })) {
+ // If there's a mix of bfloat and other 16-bit types (like half),
+ // it's safer to use an integer type to avoid invalid bitcasts.
+ if (llvm::any_of(C, [](const ChainElem &E) {
+ Type *Ty = getLoadStoreType(E.Inst)->getScalarType();
+ return !Ty->isBFloatTy() && Ty->isFloatingPointTy();
+ }))
+ return Type::getInt16Ty(F.getContext());
+ return Type::getBFloatTy(F.getContext());
+ }
+ return Type::getHalfTy(F.getContext());
+ case 32:
+ return Type::getFloatTy(F.getContext());
+ case 64:
+ return Type::getDoubleTy(F.getContext());
+ case 80:
+ return Type::getX86_FP80Ty(F.getContext());
+ case 128:
+ return Type::getFP128Ty(F.getContext());
+ default:
+ llvm_unreachable("Unsupported floating point type size");
+ }
+ }
+
+ return Type::getIntNTy(F.getContext(), GCDSize);
}
std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
@@ -1075,6 +1122,123 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
return Ret;
}
+std::vector<Chain> Vectorizer::splitChainByPointerWidth(Chain &C) {
+ if (C.empty())
+ return {};
+
+ sortChainInOffsetOrder(C);
+
+ LLVM_DEBUG({
+ dbgs() << "LSV: splitChainByPointerWidth considering chain:\n";
+ dumpChain(C);
+ });
+
+ unsigned GCDSize = 0;
+ for (const ChainElem &E : C) {
+ Type *Ty = getLoadStoreType(E.Inst)->getScalarType();
+ unsigned Sz = DL.getTypeSizeInBits(Ty);
+ GCDSize = GCDSize == 0 ? Sz : std::gcd(GCDSize, Sz);
+ }
+
+ // Find the best-aligned element in the original chain so we can propagate
+ // alignment to sub-chains after splitting.
+ Align BestAlign = Align(1);
+ APInt BestAlignOffset = C[0].OffsetFromLeader;
+ for (const ChainElem &E : C) {
+ Align A = getLoadStoreAlignment(E.Inst);
+ if (A > BestAlign) {
+ BestAlign = A;
+ BestAlignOffset = E.OffsetFromLeader;
+ }
+ }
+
+ Chain MainChain;
+ DenseMap<unsigned, Chain> PtrGroups;
+
+ for (ChainElem &E : C) {
+ Type *ScalarTy = getLoadStoreType(E.Inst)->getScalarType();
+ unsigned Sz = DL.getTypeSizeInBits(ScalarTy);
+ if (ScalarTy->isPointerTy() && Sz > GCDSize) {
+ PtrGroups[Sz].push_back(E);
+ } else {
+ MainChain.push_back(E);
+ }
+ }
+
+ std::vector<Chain> Ret;
+ if (MainChain.size() > 1)
+ Ret.push_back(std::move(MainChain));
+ for (auto &[Size, Group] : PtrGroups) {
+ if (Group.size() > 1)
+ Ret.push_back(std::move(Group));
+ }
+
+ LLVM_DEBUG({
+ dbgs() << "LSV: splitChainByPointerWidth produced " << Ret.size()
+ << " chains\n";
+ for (const auto &Chain : Ret) {
+ dbgs() << " Chain:\n";
+ dumpChain(Chain);
+ }
+ });
+
+ return Ret;
+}
+
+Value *Vectorizer::insertCast(Value *V, Type *NewTy) {
+ auto *VTy = V->getType();
+ assert(VTy != NewTy && "Types should mismatch in this function.");
+
+ unsigned VTySize = DL.getTypeSizeInBits(VTy);
+ unsigned NewTySize = DL.getTypeSizeInBits(NewTy);
+
+ if (VTySize == NewTySize)
+ return Builder.CreateBitOrPointerCast(V, NewTy);
+
+ // Mismatched bitwidth: resize via integer and then cast to final type.
+ // This preserves the bits, which is generally what's needed for memory
+ // vectorization.
+
+ // 1. Get raw bits as a scalar integer of source width.
+ Value *VInt = V;
+ if (VTy->isPtrOrPtrVectorTy()) {
+ VInt = Builder.CreatePtrToInt(V, DL.getIntPtrType(VTy));
+ } else if (VTy->isFPOrFPVectorTy()) {
+ VInt = Builder.CreateBitCast(V, Type::getIntNTy(F.getContext(), VTySize));
+ }
+
+ // If VInt is a vector (e.g. from PtrToInt), flatten it to a scalar integer
+ // so we can perform ZExt/Trunc.
+ if (VInt->getType()->isVectorTy())
+ VInt =
+ Builder.CreateBitCast(VInt, Type::getIntNTy(F.getContext(), VTySize));
+
+ // 2. Resize
+ Value *ResizedInt = Builder.CreateZExtOrTrunc(
+ VInt, Type::getIntNTy(F.getContext(), NewTySize));
+
+ // 3. Convert to destination type
+ if (NewTy->isIntOrIntVectorTy()) {
+ if (NewTy->isVectorTy())
+ return Builder.CreateBitCast(ResizedInt, NewTy);
+ return ResizedInt;
+ }
+
+ if (NewTy->isPtrOrPtrVectorTy()) {
+ if (NewTy->isVectorTy()) {
+ Value *IntVec =
+ Builder.CreateBitCast(ResizedInt, DL.getIntPtrType(NewTy));
+ return Builder.CreateIntToPtr(IntVec, NewTy);
+ }
+ return Builder.CreateIntToPtr(ResizedInt, NewTy);
+ }
+
+ if (NewTy->isFPOrFPVectorTy())
+ return Builder.CreateBitCast(ResizedInt, NewTy);
+
+ return Builder.CreateBitOrPointerCast(V, NewTy);
+}
+
bool Vectorizer::vectorizeChain(Chain &C) {
if (C.size() < 2)
return false;
@@ -1095,6 +1259,7 @@ bool Vectorizer::vectorizeChain(Chain &C) {
});
Type *VecElemTy = getChainElemTy(C);
+ unsigned VecElemSize = DL.getTypeSizeInBits(VecElemTy);
bool IsLoadChain = isa<LoadInst>(C[0].Inst);
unsigned AS = getLoadStoreAddressSpace(C[0].Inst);
unsigned BytesAdded = DL.getTypeStoreSize(getLoadStoreType(&*C[0].Inst));
@@ -1111,6 +1276,42 @@ bool Vectorizer::vectorizeChain(Chain &C) {
}
assert(8 * ChainBytes % DL.getTypeSizeInBits(VecElemTy) == 0);
+
+ // Validate that we can legally cast the extracted elements to the original
+ // types. This is critical for pointer types where we can't cast vectors to
+ // pointers.
+ for (const ChainElem &E : C) {
+ Type *OriginalTy = getLoadStoreType(E.Inst);
+ Type *OriginalScalarTy = OriginalTy->getScalarType();
+ unsigned OriginalSize = DL.getTypeSizeInBits(OriginalScalarTy);
+ unsigned NumElems = OriginalSize / VecElemSize;
+
+ // Check if the cast from VecElemTy (or vector of it) to OriginalTy would be
+ // valid. For loads, we extract elements and cast them. For stores, we cast
+ // values to VecElemTy.
+ if (IsLoadChain) {
+ Type *ExtractedTy = (NumElems == 1)
+ ? VecElemTy
+ : FixedVectorType::get(VecElemTy, NumElems);
+ // We can cast integer to pointer via inttoptr, but only scalars.
+ // We can bitcast between same-sized types.
+ // We can't cast vectors to pointers.
+ if (OriginalScalarTy->isPointerTy()) {
+ // Verify sizes match for inttoptr cast
+ if (DL.getTypeSizeInBits(ExtractedTy) !=
+ DL.getTypeSizeInBits(OriginalScalarTy)) {
+ {
+ dbgs() << "LSV: Cannot vectorize chain: size mismatch for inttoptr "
+ "cast from "
+ << *ExtractedTy << " to " << *OriginalScalarTy << "\n";
+ };
+ return false;
+ }
+ }
+ }
+ }
+
+ assert(ChainBytes % DL.getTypeStoreSize(VecElemTy) == 0);
// VecTy is a power of 2 and 1 byte at smallest, but VecElemTy may be smaller
// than 1 byte (e.g. VecTy == <32 x i1>).
unsigned NumElem = 8 * ChainBytes / DL.getTypeSizeInBits(VecElemTy);
@@ -1126,11 +1327,17 @@ bool Vectorizer::vectorizeChain(Chain &C) {
MaybeAlign(), DL, C[0].Inst, nullptr, &DT));
}
- // All elements of the chain must have the same scalar-type size.
#ifndef NDEBUG
- for (const ChainElem &E : C)
- assert(DL.getTypeStoreSize(getLoadStoreType(E.Inst)->getScalarType()) ==
- DL.getTypeStoreSize(VecElemTy));
+ for (const ChainElem &E : C) {
+ unsigned ElementSize =
+ DL.getTypeStoreSize(getLoadStoreType(E.Inst)->getScalarType());
+ unsigned VectorElementSize = DL.getTypeStoreSize(VecElemTy);
+ // Verify that the individual element size is a multiple of the vector
+ // element size.
+ assert(ElementSize % VectorElementSize == 0 &&
+ "Element size is not a multiple of vector element size, which would "
+ "break vectorization layout.");
+ }
#endif
Instruction *VecInst;
@@ -1164,6 +1371,8 @@ bool Vectorizer::vectorizeChain(Chain &C) {
for (const ChainElem &E : C) {
Instruction *I = E.Inst;
Value *V;
+ unsigned NumElems =
+ DL.getTypeSizeInBits(getLoadStoreType(I)) / VecElemSize;
Type *T = getLoadStoreType(I);
unsigned EOffset =
(E.OffsetFromLeader - C[0].OffsetFromLeader).getZExtValue();
@@ -1174,13 +1383,23 @@ bool Vectorizer::vectorizeChain(Chain &C) {
auto Mask = llvm::to_vector<8>(
llvm::seq<int>(VecIdx, VecIdx + VT->getNumElements()));
V = Builder.CreateShuffleVector(VecInst, Mask, I->getName());
+ } else if (NumElems > 1) {
+ auto Mask =
+ llvm::to_vector<8>(llvm::seq<int>(VecIdx, VecIdx + NumElems));
+ V = Builder.CreateShuffleVector(VecInst, Mask, I->getName());
} else {
V = Builder.CreateExtractElement(VecInst, Builder.getInt32(VecIdx),
I->getName());
}
- if (V->getType() != I->getType())
- V = Builder.CreateBitOrPointerCast(V, I->getType());
+
+ Type *ExpectedTy = getLoadStoreType(I);
+ if (V->getType() != ExpectedTy) {
+ LLVM_DEBUG(dbgs() << "LSV: Casting " << *V << " to " << *ExpectedTy
+ << '\n');
+ V = insertCast(V, ExpectedTy);
+ }
I->replaceAllUsesWith(V);
+ VecIdx += NumElems;
}
// Finally, we need to reorder the instrs in the BB so that the (transitive)
@@ -1211,26 +1430,32 @@ bool Vectorizer::vectorizeChain(Chain &C) {
// Build the vector to store.
Value *Vec = PoisonValue::get(VecTy);
- auto InsertElem = [&](Value *V, unsigned VecIdx) {
- if (V->getType() != VecElemTy)
- V = Builder.CreateBitOrPointerCast(V, VecElemTy);
- Vec = Builder.CreateInsertElement(Vec, V, Builder.getInt32(VecIdx));
- };
+ unsigned VecIdx = 0;
+
for (const ChainElem &E : C) {
auto *I = cast<StoreInst>(E.Inst);
- unsigned EOffset =
- (E.OffsetFromLeader - C[0].OffsetFromLeader).getZExtValue();
- unsigned VecIdx = 8 * EOffset / DL.getTypeSizeInBits(VecElemTy);
- if (FixedVectorType *VT =
- dyn_cast<FixedVectorType>(getLoadStoreType(I))) {
- for (int J = 0, JE = VT->getNumElements(); J < JE; ++J) {
- InsertElem(Builder.CreateExtractElement(I->getValueOperand(),
- Builder.getInt32(J)),
- VecIdx++);
- }
+ Value *Val = I->getValueOperand();
+ unsigned VecElemSize = DL.getTypeSizeInBits(VecElemTy);
+ unsigned NumElems = DL.getTypeSizeInBits(Val->getType()) / VecElemSize;
+
+ Value *CastVal = Val;
+ if (NumElems == 1) {
+ CastVal = Builder.CreateBitPreservingCastChain(DL, CastVal, VecElemTy);
+ Vec =
+ Builder.CreateInsertElement(Vec, CastVal, Builder.getInt32(VecIdx));
} else {
- InsertElem(I->getValueOperand(), VecIdx);
+ Type *SubVecTy = FixedVectorType::get(VecElemTy, NumElems);
+ if (CastVal->getType() != SubVecTy)
+ CastVal = Builder.CreateBitOrPointerCast(CastVal, SubVecTy);
+
+ for (unsigned ElmIdx = 0; ElmIdx < NumElems; ++ElmIdx) {
+ Value *Elt =
+ Builder.CreateExtractElement(CastVal, Builder.getInt32(ElmIdx));
+ Vec = Builder.CreateInsertElement(Vec, Elt,
+ Builder.getInt32(VecIdx + ElmIdx));
+ }
}
+ VecIdx += NumElems;
}
// If the chain originates from extra stores, we need to vectorize into a
@@ -1450,8 +1675,7 @@ std::optional<APInt> Vectorizer::getConstantOffsetComplexAddrs(
// Look through GEPs after checking they're the same except for the last
// index.
if (GEPA->getNumOperands() != GEPB->getNumOperands() ||
- GEPA->getPointerOperand() != GEPB->getPointerOperand() ||
- GEPA->getSourceElementType() != GEPB->getSourceElementType())
+ GEPA->getPointerOperand() != GEPB->getPointerOperand())
return std::nullopt;
gep_type_iterator GTIA = gep_type_begin(GEPA);
gep_type_iterator GTIB = gep_type_begin(GEPB);
@@ -1580,13 +1804,12 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
return;
// The reduced key has all elements of the ECClassKey except the underlying
- // object. Check that EqClassKey has 4 elements and define the reduced key.
- static_assert(std::tuple_size_v<EqClassKey> == 4,
+ // object. Check that EqClassKey has 3 elements and define the reduced key.
+ static_assert(std::tuple_size_v<EqClassKey> == 3,
"EqClassKey has changed - EqClassReducedKey needs changes too");
using EqClassReducedKey =
std::tuple<std::tuple_element_t<1, EqClassKey> /* AddrSpace */,
- std::tuple_element_t<2, EqClassKey> /* Element size */,
- std::tuple_element_t<3, EqClassKey> /* IsLoad; */>;
+ std::tuple_element_t<2, EqClassKey> /* IsLoad; */>;
using ECReducedKeyToUnderlyingObjectMap =
MapVector<EqClassReducedKey,
SmallPtrSet<std::tuple_element_t<0, EqClassKey>, 4>>;
@@ -1598,8 +1821,7 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
bool FoundPotentiallyOptimizableEC = false;
for (const auto &EC : EQClasses) {
const auto &Key = EC.first;
- EqClassReducedKey RedKey{std::get<1>(Key), std::get<2>(Key),
- std::get<3>(Key)};
+ EqClassReducedKey RedKey{std::get<1>(Key), std::get<2>(Key)};
auto &UOMap = RedKeyToUOMap[RedKey];
UOMap.insert(std::get<0>(Key));
if (UOMap.size() > 1)
@@ -1620,8 +1842,7 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
dbgs() << "LSV: mergeEquivalenceClasses: RedKeyToUOMap:\n";
for (const auto &RedKeyToUO : RedKeyToUOMap) {
dbgs() << " Reduced key: {" << std::get<0>(RedKeyToUO.first) << ", "
- << std::get<1>(RedKeyToUO.first) << ", "
- << static_cast<int>(std::get<2>(RedKeyToUO.first)) << "} --> "
+ << static_cast<int>(std::get<1>(RedKeyToUO.first)) << "} --> "
<< RedKeyToUO.second.size() << " underlying objects:\n";
for (auto UObject : RedKeyToUO.second)
dbgs() << " " << *UObject << '\n';
@@ -1661,10 +1882,9 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
if (UObject == UltimateTarget)
continue;
- EqClassKey KeyFrom{UObject, std::get<0>(RedKey), std::get<1>(RedKey),
- std::get<2>(RedKey)};
- EqClassKey KeyTo{UltimateTarget, std::get<0>(RedKey), std::get<1>(RedKey),
- std::get<2>(RedKey)};
+ EqClassKey KeyFrom{UObject, std::get<0>(RedKey), std::get<1>(RedKey)};
+ EqClassKey KeyTo{UltimateTarget, std::get<0>(RedKey),
+ std::get<1>(RedKey)};
// The entry for KeyFrom is guarantted to exist, unlike KeyTo. Thus,
// request the reference to the instructions vector for KeyTo first.
const auto &VecTo = EQClasses[KeyTo];
@@ -1756,7 +1976,6 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
continue;
Ret[{GetUnderlyingObject(Ptr), AS,
- DL.getTypeSizeInBits(getLoadStoreType(&I)->getScalarType()),
/*IsLoad=*/LI != nullptr}]
.emplace_back(&I);
}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
index d6b51039d5b44..fdea3800044b7 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
@@ -135,9 +135,10 @@ define amdgpu_kernel void @merge_load_i32_v2i16(ptr addrspace(1) nocapture %a) #
; CHECK-LABEL: define amdgpu_kernel void @merge_load_i32_v2i16(
; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[A_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i32 1
-; CHECK-NEXT: [[LD_0:%.*]] = load i32, ptr addrspace(1) [[A]], align 4
-; CHECK-NEXT: [[LD_1:%.*]] = load <2 x i16>, ptr addrspace(1) [[A_1]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[LD_01:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i16> [[LD_01]] to i32
+; CHECK-NEXT: [[LD_12:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list