[llvm-branch-commits] [llvm] [SandboxVec][LoadStoreVec] Support mixed type vectorization of loads (PR #224998)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Sep 20 22:37:19 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Anshil Gandhi (gandhi56)
<details>
<summary>Changes</summary>
Stacked on #<!-- -->224997, which is the base of this PR.
vectorizeLoads() bailed out unless every load in the chain had the same element type. Vectorize mixed-type chains too: the chain is loaded with a single vector load of its combined vector type, whose element type is an integer when the types differ, and each lane is reinterpreted as the type of the load it replaces. A load wider than one lane, e.g. a double in a `<N x i32>`, spans several lanes and is unpacked with a shuffle.
Lanes are counted in bits rather than elements, since with mixed types a lane no longer corresponds to one load. Chains whose loads do not cover a whole number of lanes, or that involve non-integral pointers, are left alone.
The new test covers int+float, ptr+int, a double spanning two lanes, a vector element mixed with a scalar, loads with no uses, and a non-integral pointer chain that is left alone.
Measured against the LoadStoreVectorizer lit tests (X86 and AMDGPU, load and store seeds), this takes the SandboxVectorizer from 101.6% to 103.4% of LSV's vectorized lanes and reduces the functions where it vectorizes nothing that LSV vectorizes from 46 to 32. The functions that change are AMDGPU/pointer-elements.ll (merge_load_* and merge_ptr_i32*), X86/preserve-order32.ll and preserve-order64.ll, which now match LSV's output, and X86/merge-tbaa.ll, whose load chain is now vectorized while its store chain still is not.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---
Full diff: https://github.com/llvm/llvm-project/pull/224998.diff
3 Files Affected:
- (modified) llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h (+4-2)
- (modified) llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp (+65-37)
- (added) llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_mixed_type_loads.ll (+140)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h
index 55b36d60d8e35..9cae0f990f4ea 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.h
@@ -57,8 +57,10 @@ class LLVM_ABI LoadStoreVec final : public RegionPass {
/// nullopt otherwise.
std::optional<Type *> canVectorize(BndlRef<Instruction *> Bndl);
- /// Builds a single vector load out of \p Loads. \returns the new load,
- /// or nullptr if \p Loads are not a vectorizable.
+ /// Builds a single vector load out of \p Loads. A chain of mixed types is
+ /// loaded as a vector of integers, which the caller casts back to the
+ /// original types. \returns the new load, or nullptr if \p Loads are not
+ /// vectorizable.
LoadInst *createVectorLoad(BndlRef<Instruction *> Loads);
/// Builds a ConstantVector with \p LaneTy elements from the constant store
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp
index f86cee65057ad..4bf8f2abff103 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/LoadStoreVec.cpp
@@ -40,6 +40,17 @@ std::optional<Type *> LoadStoreVec::canVectorize(BndlRef<Instruction *> Bndl) {
if (!Sched->trySchedule(Bndl))
return std::nullopt;
+ // Check contiguity
+ if (isa<StoreInst>(Bndl[0])) {
+ if (!VecUtils::areConsecutive<StoreInst, Instruction>(
+ Bndl, A->getScalarEvolution(), *DL))
+ return std::nullopt;
+ } else if (isa<LoadInst>(Bndl[0])) {
+ if (!VecUtils::areConsecutive<LoadInst, Instruction>(
+ Bndl, A->getScalarEvolution(), *DL))
+ return std::nullopt;
+ }
+
return VecUtils::getCombinedVectorTypeFor(Bndl, *DL);
}
@@ -67,21 +78,6 @@ bool LoadStoreVec::acceptOrRevert() {
return true;
}
-LoadInst *LoadStoreVec::createVectorLoad(BndlRef<Instruction *> Loads) {
- if (!VecUtils::areConsecutive<LoadInst, Instruction>(
- Loads, A->getScalarEvolution(), *DL))
- return nullptr;
- if (!canVectorize(Loads))
- return nullptr;
-
- Type *Ty = VecUtils::getCombinedVectorTypeFor(Loads, *DL);
- Value *LdPtr = cast<LoadInst>(Loads[0])->getPointerOperand();
- // TODO: Compute alignment.
- Align LdAlign(1);
- auto LdWhereIt = std::next(VecUtils::getLowest(Loads)->getIterator());
- return LoadInst::create(Ty, LdPtr, LdAlign, LdWhereIt, *Ctx, "VecIinitL");
-}
-
/// \returns an integer type with the same layout as \p Ty: iN for scalars,
/// <N x iM> for vectors.
static Type *getIntTypeFor(Type *Ty, Context &Ctx, const DataLayout &DL) {
@@ -93,6 +89,33 @@ static Type *getIntTypeFor(Type *Ty, Context &Ctx, const DataLayout &DL) {
return IntegerType::get(Ctx, Utils::getNumBits(Ty, DL));
}
+/// \returns true if each load in \p Loads can be unpacked from a single
+/// vector load of \p VecTy.
+static bool canUnpackFrom(BndlRef<Instruction *> Loads, FixedVectorType *VecTy,
+ const DataLayout &DL) {
+ unsigned LaneBits = Utils::getNumBits(VecTy->getElementType(), DL);
+ return all_of(Loads, [LaneBits, &DL](Instruction *I) {
+ // Each load has to cover a whole number of lanes.
+ if (Utils::getNumBits(I->getType(), DL) % LaneBits != 0)
+ return false;
+ // Non-integral pointers have no stable integer representation.
+ Type *ScalarTy = I->getType()->getScalarType();
+ return !ScalarTy->isPointerTy() ||
+ !DL.isNonIntegralAddressSpace(ScalarTy->getPointerAddressSpace());
+ });
+}
+
+LoadInst *LoadStoreVec::createVectorLoad(BndlRef<Instruction *> Loads) {
+ auto *Ty =
+ cast<FixedVectorType>(VecUtils::getCombinedVectorTypeFor(Loads, *DL));
+ if (!canUnpackFrom(Loads, Ty, *DL))
+ return nullptr;
+ Value *LdPtr = cast<LoadInst>(Loads[0])->getPointerOperand();
+ // TODO: Compute alignment.
+ Align LdAlign(1);
+ auto LdWhereIt = std::next(VecUtils::getLowest(Loads)->getIterator());
+ return LoadInst::create(Ty, LdPtr, LdAlign, LdWhereIt, *Ctx, "VecIinitL");
+}
/// Reinterprets the bits of \p V as \p DestTy, which must have the same size
/// in \p DL. Pointers only convert with ptrtoint / inttoptr, so those go
/// through an integer of matching layout, everything else through a bitcast.
@@ -224,9 +247,6 @@ Value *LoadStoreVec::createConstantVector(ArrayRef<Value *> Operands,
}
bool LoadStoreVec::vectorizeStores(BndlRef<Instruction *> Stores, Region &Rgn) {
- if (!VecUtils::areConsecutive<StoreInst, Instruction>(
- Stores, A->getScalarEvolution(), *DL))
- return false;
if (!canVectorize(Stores))
return false;
SmallVector<Value *, 4> Operands;
@@ -265,6 +285,8 @@ bool LoadStoreVec::vectorizeStores(BndlRef<Instruction *> Stores, Region &Rgn) {
Loads.reserve(Operands.size());
for (Value *Op : Operands)
Loads.push_back(cast<Instruction>(Op));
+ if (!canVectorize(Loads))
+ return false;
VecOp = createVectorLoad(Loads);
if (VecOp == nullptr) {
Ctx->accept();
@@ -298,18 +320,7 @@ bool LoadStoreVec::vectorizeStores(BndlRef<Instruction *> Stores, Region &Rgn) {
LoadInst *LoadStoreVec::vectorizeLoads(BndlRef<Instruction *> Loads,
Region &Rgn) {
- if (!VecUtils::areConsecutive<LoadInst, Instruction>(
- Loads, A->getScalarEvolution(), *DL))
- return nullptr;
- auto VecTy = canVectorize(Loads);
- if (!VecTy)
- return nullptr;
-
- // TODO: Support mixed-type top-level load chains.
- Type *VecElemTy = cast<FixedVectorType>(*VecTy)->getElementType();
- if (!all_of(Loads, [VecElemTy](Instruction *I) {
- return VecUtils::getElementType(I->getType()) == VecElemTy;
- }))
+ if (!canVectorize(Loads))
return nullptr;
saveIR(Rgn);
@@ -320,14 +331,31 @@ LoadInst *LoadStoreVec::vectorizeLoads(BndlRef<Instruction *> Loads,
return nullptr;
}
+ // Walk the chain in address order, tracking which lanes each load occupies.
+ // Lane counts are in units of the vector's element type, so a load wider
+ // than that element spans several lanes. For a chain of mixed types the
+ // element type is an integer, so each unpacked lane is cast back to the
+ // type of the load it replaces.
+ auto *ElmTy = cast<FixedVectorType>(VecLoad->getType())->getElementType();
+ unsigned LaneBits = Utils::getNumBits(ElmTy, *DL);
BasicBlock::iterator WhereIt = std::next(VecLoad->getIterator());
- for (auto [Lane, OrigV] : VecUtils::enumerateLanes(Loads)) {
- auto *OrigLoad = cast<LoadInst>(OrigV);
- if (OrigLoad->hasNUses(0))
- continue;
- Value *Unpacked =
- VecUtils::unpack(VecLoad, OrigLoad->getType(), Lane, WhereIt);
- OrigLoad->replaceAllUsesWith(Unpacked);
+ unsigned Lane = 0;
+ for (Instruction *I : Loads) {
+ auto *OrigLoad = cast<LoadInst>(I);
+ unsigned NumLanes = Utils::getNumBits(OrigLoad->getType(), *DL) / LaneBits;
+ if (!OrigLoad->hasNUses(0)) {
+ Type *ExtrTy =
+ NumLanes > 1 ? VecUtils::getWideType(ElmTy, NumLanes) : ElmTy;
+ Value *Unpacked = VecUtils::unpack(VecLoad, ExtrTy, Lane, WhereIt);
+ Unpacked =
+ reinterpretValue(Unpacked, OrigLoad->getType(), WhereIt, *Ctx, *DL);
+ if (Unpacked == nullptr) {
+ Ctx->revert();
+ return nullptr;
+ }
+ OrigLoad->replaceAllUsesWith(Unpacked);
+ }
+ Lane += NumLanes;
}
DeadInstrMorgue.collectPotentiallyDeadInstrs(Loads);
diff --git a/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_mixed_type_loads.ll b/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_mixed_type_loads.ll
new file mode 100644
index 0000000000000..221f1cc455330
--- /dev/null
+++ b/llvm/test/Transforms/SandboxVectorizer/Passes/LoadStoreVec/load_store_vec_mixed_type_loads.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=sandbox-vectorizer -sbvec-passes="seed-collection(loads,stores,enable-diff-types)<load-store-vec>" -sbvec-vec-reg-bits=1024 %s -S | FileCheck %s
+
+target datalayout = "e-p:64:64-ni:4"
+
+; A load chain of mixed types is loaded as a vector of integers and each lane
+; is cast back to the original type.
+
+; int + float: one lane each, unpacked with a bitcast.
+define void @mixed_int_float_loads(ptr %ptr) {
+; CHECK-LABEL: define void @mixed_int_float_loads(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 1, !sandboxvec [[META0:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i32> [[VECIINITL]], i32 0, !sandboxvec [[META0]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i32> [[VECIINITL]], i32 1, !sandboxvec [[META0]]
+; CHECK-NEXT: [[VCAST:%.*]] = bitcast i32 [[UNPACK1]] to float, !sandboxvec [[META0]]
+; CHECK-NEXT: call void @use_i32_f32(i32 [[UNPACK]], float [[VCAST]])
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i32, ptr %ptr, i32 0
+ %ptr1 = getelementptr i32, ptr %ptr, i32 1
+ %ld0 = load i32, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+ call void @use_i32_f32(i32 %ld0, float %ld1)
+ ret void
+}
+
+; ptr + int: the pointer lane needs an inttoptr.
+define void @mixed_ptr_int_loads(ptr %ptr) {
+; CHECK-LABEL: define void @mixed_ptr_int_loads(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i64, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i64>, ptr [[PTR0]], align 1, !sandboxvec [[META1:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = extractelement <2 x i64> [[VECIINITL]], i32 0, !sandboxvec [[META1]]
+; CHECK-NEXT: [[VCAST:%.*]] = inttoptr i64 [[UNPACK]] to ptr, !sandboxvec [[META1]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <2 x i64> [[VECIINITL]], i32 1, !sandboxvec [[META1]]
+; CHECK-NEXT: call void @use_ptr_i64(ptr [[VCAST]], i64 [[UNPACK1]])
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i64, ptr %ptr, i32 0
+ %ptr1 = getelementptr i64, ptr %ptr, i32 1
+ %ld0 = load ptr, ptr %ptr0
+ %ld1 = load i64, ptr %ptr1
+ call void @use_ptr_i64(ptr %ld0, i64 %ld1)
+ ret void
+}
+
+; A double spans two i32 lanes, so it is unpacked with a shuffle and a bitcast.
+define void @mixed_double_int_loads(ptr %ptr) {
+; CHECK-LABEL: define void @mixed_double_int_loads(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i32>, ptr [[PTR0]], align 1, !sandboxvec [[META2:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = shufflevector <4 x i32> [[VECIINITL]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>, !sandboxvec [[META2]]
+; CHECK-NEXT: [[VCAST:%.*]] = bitcast <2 x i32> [[UNPACK]] to double, !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = extractelement <4 x i32> [[VECIINITL]], i32 2, !sandboxvec [[META2]]
+; CHECK-NEXT: [[UNPACK2:%.*]] = extractelement <4 x i32> [[VECIINITL]], i32 3, !sandboxvec [[META2]]
+; CHECK-NEXT: call void @use_f64_i32_i32(double [[VCAST]], i32 [[UNPACK1]], i32 [[UNPACK2]])
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i8, ptr %ptr, i32 0
+ %ptr1 = getelementptr i8, ptr %ptr, i32 8
+ %ptr2 = getelementptr i8, ptr %ptr, i32 12
+ %ld0 = load double, ptr %ptr0
+ %ld1 = load i32, ptr %ptr1
+ %ld2 = load i32, ptr %ptr2
+ call void @use_f64_i32_i32(double %ld0, i32 %ld1, i32 %ld2)
+ ret void
+}
+
+; A vector element mixed with a scalar of another type.
+define void @mixed_vector_element_loads(ptr %ptr) {
+; CHECK-LABEL: define void @mixed_vector_element_loads(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <4 x i32>, ptr [[PTR0]], align 1, !sandboxvec [[META3:![0-9]+]]
+; CHECK-NEXT: [[UNPACK:%.*]] = shufflevector <4 x i32> [[VECIINITL]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>, !sandboxvec [[META3]]
+; CHECK-NEXT: [[VCAST:%.*]] = bitcast <2 x i32> [[UNPACK]] to <2 x float>, !sandboxvec [[META3]]
+; CHECK-NEXT: [[UNPACK1:%.*]] = shufflevector <4 x i32> [[VECIINITL]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>, !sandboxvec [[META3]]
+; CHECK-NEXT: [[VCAST2:%.*]] = bitcast <2 x i32> [[UNPACK1]] to i64, !sandboxvec [[META3]]
+; CHECK-NEXT: call void @use_v2f32_i64(<2 x float> [[VCAST]], i64 [[VCAST2]])
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i8, ptr %ptr, i32 0
+ %ptr1 = getelementptr i8, ptr %ptr, i32 8
+ %ld0 = load <2 x float>, ptr %ptr0
+ %ld1 = load i64, ptr %ptr1
+ call void @use_v2f32_i64(<2 x float> %ld0, i64 %ld1)
+ ret void
+}
+
+; Loads with no uses still merge, and emit no casts.
+define void @mixed_loads_no_uses(ptr %ptr) {
+; CHECK-LABEL: define void @mixed_loads_no_uses(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[VECIINITL:%.*]] = load <2 x i32>, ptr [[PTR0]], align 1, !sandboxvec [[META4:![0-9]+]]
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i32, ptr %ptr, i32 0
+ %ptr1 = getelementptr i32, ptr %ptr, i32 1
+ %ld0 = load i32, ptr %ptr0
+ %ld1 = load float, ptr %ptr1
+ ret void
+}
+
+; Non-integral pointers have no stable integer representation, so this chain is
+; left alone.
+define void @mixed_non_integral_ptr_loads(ptr %ptr) {
+; CHECK-LABEL: define void @mixed_non_integral_ptr_loads(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i64, ptr [[PTR]], i32 0
+; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i64, ptr [[PTR]], i32 1
+; CHECK-NEXT: [[LD0:%.*]] = load ptr addrspace(4), ptr [[PTR0]], align 8, !sandboxvec [[META5:![0-9]+]]
+; CHECK-NEXT: [[LD1:%.*]] = load i64, ptr [[PTR1]], align 4, !sandboxvec [[META5]]
+; CHECK-NEXT: call void @use_p4_i64(ptr addrspace(4) [[LD0]], i64 [[LD1]])
+; CHECK-NEXT: ret void
+;
+ %ptr0 = getelementptr i64, ptr %ptr, i32 0
+ %ptr1 = getelementptr i64, ptr %ptr, i32 1
+ %ld0 = load ptr addrspace(4), ptr %ptr0
+ %ld1 = load i64, ptr %ptr1
+ call void @use_p4_i64(ptr addrspace(4) %ld0, i64 %ld1)
+ ret void
+}
+
+declare void @use_i32_f32(i32, float)
+declare void @use_ptr_i64(ptr, i64)
+declare void @use_f64_i32_i32(double, i32, i32)
+declare void @use_v2f32_i64(<2 x float>, i64)
+declare void @use_p4_i64(ptr addrspace(4), i64)
+;.
+; CHECK: [[META0]] = distinct !{!"sandboxregion"}
+; CHECK: [[META1]] = distinct !{!"sandboxregion"}
+; CHECK: [[META2]] = distinct !{!"sandboxregion"}
+; CHECK: [[META3]] = distinct !{!"sandboxregion"}
+; CHECK: [[META4]] = distinct !{!"sandboxregion"}
+; CHECK: [[META5]] = distinct !{!"sandboxregion"}
+;.
``````````
</details>
https://github.com/llvm/llvm-project/pull/224998
More information about the llvm-branch-commits
mailing list