[llvm] [VectorCombine] Align shrinkLoadForShuffles to register boundaries (PR #209775)
Timur Golubovich via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 06:34:01 PDT 2026
https://github.com/timurgol007 updated https://github.com/llvm/llvm-project/pull/209775
>From ef09e91cc71fa32358234e99a311bce28fb9dd50 Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Tue, 14 Jul 2026 20:04:19 +0200
Subject: [PATCH 1/2] [VectorCombine] Align shrinkLoadForShuffles to register
boundaries
Round up the shrunk load element count to the next full vector
register boundary to avoid scalar remainders that prevent shuffle
lowering.
For example, on X86 AVX-512 a <16 x double> load with shuffle
indices 1-8 was shrunk to <9 x double>. This legalizes to
v8f64 + scalar f64, and the scalar operand in BUILD_VECTOR
prevents DAGCombiner from folding it into a VECTOR_SHUFFLE,
producing ~15 instructions instead of 2 (VMOVDQA64 + VALIGNQ).
---
.../Transforms/Vectorize/VectorCombine.cpp | 32 ++++++++++++++++++-
.../AArch64/hoist-load-from-vector-loop.ll | 8 ++---
.../PhaseOrdering/X86/vec-load-combine.ll | 8 ++---
.../VectorCombine/X86/load-widening.ll | 4 +--
.../VectorCombine/X86/shuffle-of-shuffles.ll | 24 ++++++++------
5 files changed, 56 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 8648125222e2a..6c49df7a1c9d3 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -6214,9 +6214,39 @@ bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
return OutputRange;
};
+ // Compute the new number of elements, aligning to vector register
+ // boundaries for fixed-width vectors to avoid non-power-of-2 remainders
+ // that legalize poorly.
+ auto GetAlignedNumElements = [&](unsigned MaxIdx,
+ FixedVectorType *LoadTy) -> unsigned {
+ unsigned RawNumElements = MaxIdx + 1u;
+ // Preserve the default behavior for scalable-vector targets.
+ if (TTI.supportsScalableVectors())
+ return RawNumElements;
+
+ Type *ElemTy = LoadTy->getElementType();
+ // Skip alignment for illegal element types.
+ if (!TTI.isTypeLegal(ElemTy))
+ return RawNumElements;
+
+ unsigned ElemBits = ElemTy->getPrimitiveSizeInBits();
+ if (ElemBits == 0)
+ return RawNumElements;
+
+ unsigned RegBits =
+ TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector);
+ unsigned ElemsPerReg = RegBits / ElemBits;
+ if (ElemsPerReg == 0)
+ return RawNumElements;
+
+ // Round up to the next full register boundary.
+ return alignTo(RawNumElements, ElemsPerReg);
+ };
+
// Get the range of vector elements used by shufflevector instructions.
if (std::optional<IndexRange> Indices = GetIndexRangeInShuffles()) {
- unsigned const NewNumElements = Indices->second + 1u;
+ unsigned const NewNumElements =
+ GetAlignedNumElements(Indices->second, OldLoadTy);
// If the range of vector elements is smaller than the full load, attempt
// to create a smaller load.
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
index b0c163737c206..53ea8530047f5 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-load-from-vector-loop.ll
@@ -41,10 +41,10 @@ define void @hoist_invariant_load(ptr %invariant_ptr, i64 %num_elements, ptr %ar
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 64
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr [32 x i8], ptr [[ARRAY]], i64 [[I2]]
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 96
-; CHECK-NEXT: [[TMP12:%.*]] = load <5 x double>, ptr [[TMP5]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <5 x double> [[TMP12]], <5 x double> poison, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT: [[TMP13:%.*]] = load <5 x double>, ptr [[TMP9]], align 8, !alias.scope [[META3]], !noalias [[META0]]
-; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <5 x double> [[TMP13]], <5 x double> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT: [[TMP12:%.*]] = load <6 x double>, ptr [[TMP5]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <6 x double> [[TMP12]], <6 x double> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT: [[TMP13:%.*]] = load <6 x double>, ptr [[TMP9]], align 8, !alias.scope [[META3]], !noalias [[META0]]
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = shufflevector <6 x double> [[TMP13]], <6 x double> poison, <2 x i32> <i32 0, i32 4>
; CHECK-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
; CHECK-NEXT: [[TMP17:%.*]] = fadd <2 x double> [[BROADCAST_SPLAT]], [[STRIDED_VEC5]]
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[TMP14]], i64 0
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll b/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
index 86bdd125e5e57..f6e8fcd5d1d8c 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
@@ -11,13 +11,13 @@ $getAt = comdat any
define dso_local noundef <4 x float> @ConvertVectors_ByRef(ptr noundef nonnull align 16 dereferenceable(16) %0) #0 {
; SSE-LABEL: @ConvertVectors_ByRef(
-; SSE-NEXT: [[TMP2:%.*]] = load <3 x float>, ptr [[TMP0:%.*]], align 16
-; SSE-NEXT: [[TMP3:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; SSE-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[TMP0:%.*]], align 16
+; SSE-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; SSE-NEXT: ret <4 x float> [[TMP3]]
;
; AVX-LABEL: @ConvertVectors_ByRef(
-; AVX-NEXT: [[TMP2:%.*]] = load <3 x float>, ptr [[TMP0:%.*]], align 16
-; AVX-NEXT: [[TMP3:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[TMP0:%.*]], align 16
+; AVX-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; AVX-NEXT: ret <4 x float> [[TMP3]]
;
%2 = alloca ptr, align 8
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-widening.ll b/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
index eacc40bfa9b53..30a089818074e 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
@@ -443,8 +443,8 @@ define <8 x float> @load_v2f32_v8f32_hwasan(ptr dereferenceable(32) %p) sanitize
define <4 x i32> @load_v2i32_v4i32_asan(ptr dereferenceable(16) %p) sanitize_address {
; CHECK-LABEL: @load_v2i32_v4i32_asan(
-; CHECK-NEXT: [[TMP1:%.*]] = load <1 x i32>, ptr [[P:%.*]], align 1
-; CHECK-NEXT: [[S:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[L:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
+; CHECK-NEXT: [[S:%.*]] = shufflevector <2 x i32> [[L]], <2 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: ret <4 x i32> [[S]]
;
%l = load <2 x i32>, ptr %p, align 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
index faa28bd7e5155..515a46c437f14 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
@@ -47,12 +47,21 @@ define <8 x i32> @concat_extract_subvectors_poison(<8 x i32> %x) {
; broadcast loads are free on AVX (and blends are much cheap than general 2-operand shuffles)
define <4 x double> @blend_broadcasts_v4f64(ptr %p0, ptr %p1) {
-; CHECK-LABEL: define <4 x double> @blend_broadcasts_v4f64(
-; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <1 x double>, ptr [[P0]], align 32
-; CHECK-NEXT: [[TMP2:%.*]] = load <1 x double>, ptr [[P1]], align 32
-; CHECK-NEXT: [[BLEND:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 1, i32 0>
-; CHECK-NEXT: ret <4 x double> [[BLEND]]
+; SSE-LABEL: define <4 x double> @blend_broadcasts_v4f64(
+; SSE-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P0]], align 32
+; SSE-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[P1]], align 32
+; SSE-NEXT: [[BLEND:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 2, i32 2, i32 0>
+; SSE-NEXT: ret <4 x double> [[BLEND]]
+;
+; AVX-LABEL: define <4 x double> @blend_broadcasts_v4f64(
+; AVX-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
+; AVX-NEXT: [[LD0:%.*]] = load <4 x double>, ptr [[P0]], align 32
+; AVX-NEXT: [[LD1:%.*]] = load <4 x double>, ptr [[P1]], align 32
+; AVX-NEXT: [[BCST0:%.*]] = shufflevector <4 x double> [[LD0]], <4 x double> undef, <4 x i32> zeroinitializer
+; AVX-NEXT: [[BCST1:%.*]] = shufflevector <4 x double> [[LD1]], <4 x double> undef, <4 x i32> zeroinitializer
+; AVX-NEXT: [[BLEND:%.*]] = shufflevector <4 x double> [[BCST0]], <4 x double> [[BCST1]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; AVX-NEXT: ret <4 x double> [[BLEND]]
;
%ld0 = load <4 x double>, ptr %p0, align 32
%ld1 = load <4 x double>, ptr %p1, align 32
@@ -118,6 +127,3 @@ define <4 x i32> @concat_inner_shuffles_all_poison_masks(<4 x i32> %x, <4 x i32>
ret <4 x i32> %r
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX: {{.*}}
-; SSE: {{.*}}
>From ca19675bc0c4fafcb5d1996fe0a8f5c2b2e949ed Mon Sep 17 00:00:00 2001
From: Timur Golubovich <timur.golubovich at intel.com>
Date: Fri, 17 Jul 2026 15:30:49 +0200
Subject: [PATCH 2/2] avoid shrinking loads below register width
---
.../Transforms/Vectorize/VectorCombine.cpp | 4 +++-
.../PhaseOrdering/X86/vec-load-combine.ll | 8 +++----
.../VectorCombine/X86/load-widening.ll | 23 ++++++++++++++++--
.../VectorCombine/X86/shuffle-of-shuffles.ll | 24 +++++++------------
4 files changed, 37 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 6c49df7a1c9d3..a617a72a26824 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -6236,7 +6236,9 @@ bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
unsigned RegBits =
TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector);
unsigned ElemsPerReg = RegBits / ElemBits;
- if (ElemsPerReg == 0)
+ // If the load already fits in a register, keep the exact size.
+ // Otherwise round up to the next full register boundary.
+ if (ElemsPerReg == 0 || RawNumElements <= ElemsPerReg)
return RawNumElements;
// Round up to the next full register boundary.
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll b/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
index f6e8fcd5d1d8c..86bdd125e5e57 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
@@ -11,13 +11,13 @@ $getAt = comdat any
define dso_local noundef <4 x float> @ConvertVectors_ByRef(ptr noundef nonnull align 16 dereferenceable(16) %0) #0 {
; SSE-LABEL: @ConvertVectors_ByRef(
-; SSE-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[TMP0:%.*]], align 16
-; SSE-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; SSE-NEXT: [[TMP2:%.*]] = load <3 x float>, ptr [[TMP0:%.*]], align 16
+; SSE-NEXT: [[TMP3:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; SSE-NEXT: ret <4 x float> [[TMP3]]
;
; AVX-LABEL: @ConvertVectors_ByRef(
-; AVX-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[TMP0:%.*]], align 16
-; AVX-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; AVX-NEXT: [[TMP2:%.*]] = load <3 x float>, ptr [[TMP0:%.*]], align 16
+; AVX-NEXT: [[TMP3:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; AVX-NEXT: ret <4 x float> [[TMP3]]
;
%2 = alloca ptr, align 8
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-widening.ll b/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
index 30a089818074e..d089edc32776f 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
@@ -252,6 +252,25 @@ define <2 x i4> @vec_with_2elts_128bits_i4(ptr align 16 dereferenceable(16) %p)
ret <2 x i4> %r
}
+; Shrinking this to <9 x double> would change the legalized type.
+
+define <8 x double> @load_v16f64(ptr %p) {
+; SSE-LABEL: @load_v16f64(
+; SSE-NEXT: [[TMP1:%.*]] = load <10 x double>, ptr [[P:%.*]], align 512
+; SSE-NEXT: [[S:%.*]] = shufflevector <10 x double> [[TMP1]], <10 x double> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; SSE-NEXT: ret <8 x double> [[S]]
+;
+; AVX-LABEL: @load_v16f64(
+; AVX-NEXT: [[TMP1:%.*]] = load <12 x double>, ptr [[P:%.*]], align 512
+; AVX-NEXT: [[S:%.*]] = shufflevector <12 x double> [[TMP1]], <12 x double> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; AVX-NEXT: ret <8 x double> [[S]]
+;
+ %l = load <16 x double>, ptr %p, align 512
+ %s = shufflevector <16 x double> %l, <16 x double> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+ ret <8 x double> %s
+}
+
+
; Load the 128-bit vector because there is no additional cost.
define <4 x float> @load_v1f32_v4f32(ptr dereferenceable(16) %p) {
@@ -443,8 +462,8 @@ define <8 x float> @load_v2f32_v8f32_hwasan(ptr dereferenceable(32) %p) sanitize
define <4 x i32> @load_v2i32_v4i32_asan(ptr dereferenceable(16) %p) sanitize_address {
; CHECK-LABEL: @load_v2i32_v4i32_asan(
-; CHECK-NEXT: [[L:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
-; CHECK-NEXT: [[S:%.*]] = shufflevector <2 x i32> [[L]], <2 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP1:%.*]] = load <1 x i32>, ptr [[P:%.*]], align 1
+; CHECK-NEXT: [[S:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: ret <4 x i32> [[S]]
;
%l = load <2 x i32>, ptr %p, align 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
index 515a46c437f14..faa28bd7e5155 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
@@ -47,21 +47,12 @@ define <8 x i32> @concat_extract_subvectors_poison(<8 x i32> %x) {
; broadcast loads are free on AVX (and blends are much cheap than general 2-operand shuffles)
define <4 x double> @blend_broadcasts_v4f64(ptr %p0, ptr %p1) {
-; SSE-LABEL: define <4 x double> @blend_broadcasts_v4f64(
-; SSE-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
-; SSE-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P0]], align 32
-; SSE-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[P1]], align 32
-; SSE-NEXT: [[BLEND:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 2, i32 2, i32 0>
-; SSE-NEXT: ret <4 x double> [[BLEND]]
-;
-; AVX-LABEL: define <4 x double> @blend_broadcasts_v4f64(
-; AVX-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
-; AVX-NEXT: [[LD0:%.*]] = load <4 x double>, ptr [[P0]], align 32
-; AVX-NEXT: [[LD1:%.*]] = load <4 x double>, ptr [[P1]], align 32
-; AVX-NEXT: [[BCST0:%.*]] = shufflevector <4 x double> [[LD0]], <4 x double> undef, <4 x i32> zeroinitializer
-; AVX-NEXT: [[BCST1:%.*]] = shufflevector <4 x double> [[LD1]], <4 x double> undef, <4 x i32> zeroinitializer
-; AVX-NEXT: [[BLEND:%.*]] = shufflevector <4 x double> [[BCST0]], <4 x double> [[BCST1]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-; AVX-NEXT: ret <4 x double> [[BLEND]]
+; CHECK-LABEL: define <4 x double> @blend_broadcasts_v4f64(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <1 x double>, ptr [[P0]], align 32
+; CHECK-NEXT: [[TMP2:%.*]] = load <1 x double>, ptr [[P1]], align 32
+; CHECK-NEXT: [[BLEND:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: ret <4 x double> [[BLEND]]
;
%ld0 = load <4 x double>, ptr %p0, align 32
%ld1 = load <4 x double>, ptr %p1, align 32
@@ -127,3 +118,6 @@ define <4 x i32> @concat_inner_shuffles_all_poison_masks(<4 x i32> %x, <4 x i32>
ret <4 x i32> %r
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
+; SSE: {{.*}}
More information about the llvm-commits
mailing list