[llvm] [SROA] Split fixed-vector loads into per-lane scalar loads when a smaller store overlaps (PR #199291)
Syadus Sefat via llvm-commits
llvm-commits at lists.llvm.org
Sat May 23 23:54:45 PDT 2026
https://github.com/mssefat updated https://github.com/llvm/llvm-project/pull/199291
>From 7bb8a6b6f64c3fee4680ff0663e42774636c8c0a Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Thu, 21 May 2026 13:51:52 -0500
Subject: [PATCH 1/3] [NFC][SROA] Pre-commit test for vector-load-split
---
.../test/Transforms/SROA/vector-load-split.ll | 523 ++++++++++++++++++
1 file changed, 523 insertions(+)
create mode 100644 llvm/test/Transforms/SROA/vector-load-split.ll
diff --git a/llvm/test/Transforms/SROA/vector-load-split.ll b/llvm/test/Transforms/SROA/vector-load-split.ll
new file mode 100644
index 0000000000000..4b53671737c6a
--- /dev/null
+++ b/llvm/test/Transforms/SROA/vector-load-split.ll
@@ -0,0 +1,523 @@
+; RUN: opt -passes='sroa,simplifycfg' -S < %s | FileCheck %s --check-prefixes=SROA
+; RUN: opt -passes='sroa,simplifycfg,instcombine' -S < %s | FileCheck %s --check-prefixes=COMBINED
+
+%struct.S = type { ptr, i64, i32, i32, i64, i64 }
+%desc = type { ptr, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 }
+
+define double @descriptor_copy_vector_load(ptr %data, ptr %dst) {
+; SROA-LABEL: define double @descriptor_copy_vector_load(
+; SROA-SAME: ptr [[DATA:%.*]], ptr [[DST:%.*]]) {
+; SROA-NEXT: [[ENTRY:.*:]]
+; SROA-NEXT: [[SRC_DESC:%.*]] = alloca [[DESC:%.*]], align 8, addrspace(5)
+; SROA-NEXT: [[TMP_DESC:%.*]] = alloca [[DESC]], align 8, addrspace(5)
+; SROA-NEXT: [[SRC_DESC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[SRC_DESC]] to ptr
+; SROA-NEXT: [[TMP_DESC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP_DESC]] to ptr
+; SROA-NEXT: store ptr [[DATA]], ptr addrspace(5) [[SRC_DESC]], align 8
+; SROA-NEXT: [[SRC_LB0:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC_DESC]], i32 24
+; SROA-NEXT: store i64 1, ptr addrspace(5) [[SRC_LB0]], align 8
+; SROA-NEXT: [[SRC_STRIDE0:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC_DESC]], i32 40
+; SROA-NEXT: store i64 8, ptr addrspace(5) [[SRC_STRIDE0]], align 8
+; SROA-NEXT: [[ARGS_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x ptr> {{.*}}, ptr [[SRC_DESC_CAST]], i32 0
+; SROA-NEXT: [[ARGS_SROA_0_8_VEC_INSERT:%.*]] = insertelement <2 x ptr> [[ARGS_SROA_0_0_VEC_INSERT]], ptr [[TMP_DESC_CAST]], i32 1
+; SROA-NEXT: [[TMP_PTR:%.*]] = extractelement <2 x ptr> [[ARGS_SROA_0_8_VEC_INSERT]], i32 1
+; SROA-NEXT: [[SRC_PTR:%.*]] = extractelement <2 x ptr> [[ARGS_SROA_0_8_VEC_INSERT]], i32 0
+; SROA-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(96) [[TMP_PTR]], ptr noundef nonnull align 8 dereferenceable(96) [[SRC_PTR]], i32 96, i1 false)
+; SROA-NEXT: [[TMP_BASE:%.*]] = load ptr, ptr [[TMP_PTR]], align 8
+; SROA-NEXT: [[TMP_LB0_P:%.*]] = getelementptr inbounds i8, ptr [[TMP_PTR]], i64 24
+; SROA-NEXT: [[TMP_LB0:%.*]] = load i64, ptr [[TMP_LB0_P]], align 8
+; SROA-NEXT: [[TMP_STRIDE0_P:%.*]] = getelementptr inbounds i8, ptr [[TMP_PTR]], i64 40
+; SROA-NEXT: [[TMP_STRIDE0:%.*]] = load i64, ptr [[TMP_STRIDE0_P]], align 8
+; SROA-NEXT: [[IDX:%.*]] = sub nsw i64 4, [[TMP_LB0]]
+; SROA-NEXT: [[OFFSET:%.*]] = mul nuw nsw i64 [[TMP_STRIDE0]], [[IDX]]
+; SROA-NEXT: [[ELT_PTR:%.*]] = getelementptr i8, ptr [[TMP_BASE]], i64 [[OFFSET]]
+; SROA-NEXT: [[RETVAL:%.*]] = load double, ptr [[ELT_PTR]], align 8
+; SROA-NEXT: store double 0.000000e+00, ptr [[DST]], align 8
+; SROA-NEXT: ret double [[RETVAL]]
+;
+; COMBINED-LABEL: define double @descriptor_copy_vector_load(
+; COMBINED-SAME: ptr [[DATA:%.*]], ptr [[DST:%.*]]) {
+; COMBINED-NEXT: [[ENTRY:.*:]]
+; COMBINED-NEXT: [[SRC_DESC:%.*]] = alloca [[DESC:%.*]], align 8, addrspace(5)
+; COMBINED-NEXT: [[TMP_DESC:%.*]] = alloca [[DESC]], align 8, addrspace(5)
+; COMBINED-NEXT: [[SRC_DESC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[SRC_DESC]] to ptr
+; COMBINED-NEXT: [[TMP_DESC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP_DESC]] to ptr
+; COMBINED-NEXT: store ptr [[DATA]], ptr addrspace(5) [[SRC_DESC]], align 8
+; COMBINED-NEXT: [[SRC_LB0:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SRC_DESC]], i64 24
+; COMBINED-NEXT: store i64 1, ptr addrspace(5) [[SRC_LB0]], align 8
+; COMBINED-NEXT: [[SRC_STRIDE0:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SRC_DESC]], i64 40
+; COMBINED-NEXT: store i64 8, ptr addrspace(5) [[SRC_STRIDE0]], align 8
+; COMBINED-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(96) [[TMP_DESC_CAST]], ptr noundef nonnull align 8 dereferenceable(96) [[SRC_DESC_CAST]], i32 96, i1 false)
+; COMBINED-NEXT: [[TMP_BASE:%.*]] = load ptr, ptr [[TMP_DESC_CAST]], align 8
+; COMBINED-NEXT: [[TMP_LB0_P:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP_DESC_CAST]], i64 24
+; COMBINED-NEXT: [[TMP_LB0:%.*]] = load i64, ptr [[TMP_LB0_P]], align 8
+; COMBINED-NEXT: [[TMP_STRIDE0_P:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP_DESC_CAST]], i64 40
+; COMBINED-NEXT: [[TMP_STRIDE0:%.*]] = load i64, ptr [[TMP_STRIDE0_P]], align 8
+; COMBINED-NEXT: [[IDX:%.*]] = sub nsw i64 4, [[TMP_LB0]]
+; COMBINED-NEXT: [[OFFSET:%.*]] = mul nuw nsw i64 [[TMP_STRIDE0]], [[IDX]]
+; COMBINED-NEXT: [[ELT_PTR:%.*]] = getelementptr i8, ptr [[TMP_BASE]], i64 [[OFFSET]]
+; COMBINED-NEXT: [[RETVAL:%.*]] = load double, ptr [[ELT_PTR]], align 8
+; COMBINED-NEXT: store double 0.000000e+00, ptr [[DST]], align 8
+; COMBINED-NEXT: ret double [[RETVAL]]
+;
+entry:
+ %src.desc = alloca %desc, align 8, addrspace(5)
+ %tmp.desc = alloca %desc, align 8, addrspace(5)
+ %args = alloca { ptr, ptr }, align 8, addrspace(5)
+ %src.desc.cast = addrspacecast ptr addrspace(5) %src.desc to ptr
+ %tmp.desc.cast = addrspacecast ptr addrspace(5) %tmp.desc to ptr
+
+ store ptr %data, ptr addrspace(5) %src.desc, align 8
+ %src.lb0 = getelementptr inbounds i8, ptr addrspace(5) %src.desc, i32 24
+ store i64 1, ptr addrspace(5) %src.lb0, align 8
+ %src.stride0 = getelementptr inbounds i8, ptr addrspace(5) %src.desc, i32 40
+ store i64 8, ptr addrspace(5) %src.stride0, align 8
+
+ store ptr %src.desc.cast, ptr addrspace(5) %args, align 8
+ %arg1 = getelementptr inbounds i8, ptr addrspace(5) %args, i32 8
+ store ptr %tmp.desc.cast, ptr addrspace(5) %arg1, align 8
+
+ %pair = load <2 x ptr>, ptr addrspace(5) %args, align 8
+ %tmp.ptr = extractelement <2 x ptr> %pair, i32 1
+ %src.ptr = extractelement <2 x ptr> %pair, i32 0
+ call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(96) %tmp.ptr, ptr noundef nonnull align 8 dereferenceable(96) %src.ptr, i32 96, i1 false)
+
+ %tmp.base = load ptr, ptr %tmp.ptr, align 8
+ %tmp.lb0.p = getelementptr inbounds i8, ptr %tmp.ptr, i64 24
+ %tmp.lb0 = load i64, ptr %tmp.lb0.p, align 8
+ %tmp.stride0.p = getelementptr inbounds i8, ptr %tmp.ptr, i64 40
+ %tmp.stride0 = load i64, ptr %tmp.stride0.p, align 8
+ %idx = sub nsw i64 4, %tmp.lb0
+ %offset = mul nuw nsw i64 %tmp.stride0, %idx
+ %elt.ptr = getelementptr i8, ptr %tmp.base, i64 %offset
+ %retval = load double, ptr %elt.ptr, align 8
+
+ store double 0.000000e+00, ptr %dst, align 8
+ ret double %retval
+}
+
+define double @descriptor_copy_scalar_load(ptr %data, ptr %dst) {
+; SROA-LABEL: define double @descriptor_copy_scalar_load(
+; SROA-SAME: ptr [[DATA:%.*]], ptr [[DST:%.*]]) {
+; SROA-NEXT: [[ENTRY:.*:]]
+; SROA-NEXT: [[SRC_DESC_SROA_2:%.*]] = alloca { i64, i64 }, align 8, addrspace(5)
+; SROA-NEXT: [[SRC_DESC_SROA_8:%.*]] = alloca { i64, i64, i64, i64, i64, i64 }, align 8, addrspace(5)
+; SROA-NEXT: [[TMP_DESC_SROA_2:%.*]] = alloca { i64, i64 }, align 8, addrspace(5)
+; SROA-NEXT: [[TMP_DESC_SROA_4:%.*]] = alloca { i64, i64, i64, i64, i64, i64 }, align 8, addrspace(5)
+; SROA-NEXT: [[TMP_DESC_SROA_2_0_TMP_DESC_CAST_SROA_CAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP_DESC_SROA_2]] to ptr
+; SROA-NEXT: [[SRC_DESC_SROA_2_8_SRC_DESC_CAST_SROA_CAST:%.*]] = addrspacecast ptr addrspace(5) [[SRC_DESC_SROA_2]] to ptr
+; SROA-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[TMP_DESC_SROA_2_0_TMP_DESC_CAST_SROA_CAST]], ptr align 8 [[SRC_DESC_SROA_2_8_SRC_DESC_CAST_SROA_CAST]], i32 16, i1 false)
+; SROA-NEXT: [[TMP_DESC_SROA_4_0_TMP_DESC_CAST_SROA_CAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP_DESC_SROA_4]] to ptr
+; SROA-NEXT: [[SRC_DESC_SROA_8_48_SRC_DESC_CAST_SROA_CAST:%.*]] = addrspacecast ptr addrspace(5) [[SRC_DESC_SROA_8]] to ptr
+; SROA-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[TMP_DESC_SROA_4_0_TMP_DESC_CAST_SROA_CAST]], ptr align 8 [[SRC_DESC_SROA_8_48_SRC_DESC_CAST_SROA_CAST]], i32 48, i1 false)
+; SROA-NEXT: [[IDX:%.*]] = sub nsw i64 4, 1
+; SROA-NEXT: [[OFFSET:%.*]] = mul nuw nsw i64 8, [[IDX]]
+; SROA-NEXT: [[ELT_PTR:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[OFFSET]]
+; SROA-NEXT: [[RETVAL:%.*]] = load double, ptr [[ELT_PTR]], align 8
+; SROA-NEXT: store double 0.000000e+00, ptr [[DST]], align 8
+; SROA-NEXT: ret double [[RETVAL]]
+;
+; COMBINED-LABEL: define double @descriptor_copy_scalar_load(
+; COMBINED-SAME: ptr [[DATA:%.*]], ptr [[DST:%.*]]) {
+; COMBINED-NEXT: [[ENTRY:.*:]]
+; COMBINED-NEXT: [[ELT_PTR:%.*]] = getelementptr i8, ptr [[DATA]], i64 24
+; COMBINED-NEXT: [[RETVAL:%.*]] = load double, ptr [[ELT_PTR]], align 8
+; COMBINED-NEXT: store double 0.000000e+00, ptr [[DST]], align 8
+; COMBINED-NEXT: ret double [[RETVAL]]
+;
+entry:
+ %src.desc = alloca %desc, align 8, addrspace(5)
+ %tmp.desc = alloca %desc, align 8, addrspace(5)
+ %args = alloca { ptr, ptr }, align 8, addrspace(5)
+ %src.desc.cast = addrspacecast ptr addrspace(5) %src.desc to ptr
+ %tmp.desc.cast = addrspacecast ptr addrspace(5) %tmp.desc to ptr
+
+ store ptr %data, ptr addrspace(5) %src.desc, align 8
+ %src.lb0 = getelementptr inbounds i8, ptr addrspace(5) %src.desc, i32 24
+ store i64 1, ptr addrspace(5) %src.lb0, align 8
+ %src.stride0 = getelementptr inbounds i8, ptr addrspace(5) %src.desc, i32 40
+ store i64 8, ptr addrspace(5) %src.stride0, align 8
+
+ store ptr %src.desc.cast, ptr addrspace(5) %args, align 8
+ %arg1 = getelementptr inbounds i8, ptr addrspace(5) %args, i32 8
+ store ptr %tmp.desc.cast, ptr addrspace(5) %arg1, align 8
+
+ %src.ptr = load ptr, ptr addrspace(5) %args, align 8
+ %tmp.ptr = load ptr, ptr addrspace(5) %arg1, align 8
+ call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(96) %tmp.ptr, ptr noundef nonnull align 8 dereferenceable(96) %src.ptr, i32 96, i1 false)
+
+ %tmp.base = load ptr, ptr %tmp.ptr, align 8
+ %tmp.lb0.p = getelementptr inbounds i8, ptr %tmp.ptr, i64 24
+ %tmp.lb0 = load i64, ptr %tmp.lb0.p, align 8
+ %tmp.stride0.p = getelementptr inbounds i8, ptr %tmp.ptr, i64 40
+ %tmp.stride0 = load i64, ptr %tmp.stride0.p, align 8
+ %idx = sub nsw i64 4, %tmp.lb0
+ %offset = mul nuw nsw i64 %tmp.stride0, %idx
+ %elt.ptr = getelementptr i8, ptr %tmp.base, i64 %offset
+ %retval = load double, ptr %elt.ptr, align 8
+
+ store double 0.000000e+00, ptr %dst, align 8
+ ret double %retval
+}
+
+define void @split_all_lanes_select_same_alloca(float %v0, float %v1, float %v2, float %v3, i1 %cond, ptr addrspace(1) %out) {
+; SROA-LABEL: define void @split_all_lanes_select_same_alloca(
+; SROA-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; SROA-NEXT: [[ENTRY:.*:]]
+; SROA-NEXT: [[SLOT_SROA_0:%.*]] = alloca <4 x float>, align 16, addrspace(5)
+; SROA-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT_SROA_0]], align 16
+; SROA-NEXT: [[SLOT_SROA_0_4_GEP_SROA_IDX8:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
+; SROA-NEXT: [[SLOT_SROA_0_4_GEP1_SROA_IDX9:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
+; SROA-NEXT: [[PTR_SROA_SEL:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP_SROA_IDX8]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP1_SROA_IDX9]]
+; SROA-NEXT: store float [[V1]], ptr addrspace(5) [[PTR_SROA_SEL]], align 4
+; SROA-NEXT: [[SLOT_SROA_0_8_GEP2_SROA_IDX10:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
+; SROA-NEXT: [[SLOT_SROA_0_8_GEP3_SROA_IDX11:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
+; SROA-NEXT: [[PTR_SROA_SEL4:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP2_SROA_IDX10]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP3_SROA_IDX11]]
+; SROA-NEXT: store float [[V2]], ptr addrspace(5) [[PTR_SROA_SEL4]], align 4
+; SROA-NEXT: [[SLOT_SROA_0_12_GEP5_SROA_IDX12:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
+; SROA-NEXT: [[SLOT_SROA_0_12_GEP6_SROA_IDX13:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
+; SROA-NEXT: [[PTR_SROA_SEL7:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP5_SROA_IDX12]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP6_SROA_IDX13]]
+; SROA-NEXT: store float [[V3]], ptr addrspace(5) [[PTR_SROA_SEL7]], align 4
+; SROA-NEXT: [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT_SROA_0]], align 16
+; SROA-NEXT: [[E0:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 0
+; SROA-NEXT: [[E1:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 1
+; SROA-NEXT: [[E2:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 2
+; SROA-NEXT: [[E3:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 3
+; SROA-NEXT: [[SUM01:%.*]] = fadd float [[E0]], [[E1]]
+; SROA-NEXT: [[SUM23:%.*]] = fadd float [[E2]], [[E3]]
+; SROA-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
+; SROA-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
+; SROA-NEXT: ret void
+;
+; COMBINED-LABEL: define void @split_all_lanes_select_same_alloca(
+; COMBINED-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; COMBINED-NEXT: [[ENTRY:.*:]]
+; COMBINED-NEXT: [[SLOT_SROA_0:%.*]] = alloca <4 x float>, align 16, addrspace(5)
+; COMBINED-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT_SROA_0]], align 16
+; COMBINED-NEXT: [[PTR_SROA_SEL:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
+; COMBINED-NEXT: store float [[V1]], ptr addrspace(5) [[PTR_SROA_SEL]], align 4
+; COMBINED-NEXT: [[PTR_SROA_SEL4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
+; COMBINED-NEXT: store float [[V2]], ptr addrspace(5) [[PTR_SROA_SEL4]], align 4
+; COMBINED-NEXT: [[PTR_SROA_SEL7:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
+; COMBINED-NEXT: store float [[V3]], ptr addrspace(5) [[PTR_SROA_SEL7]], align 4
+; COMBINED-NEXT: [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT_SROA_0]], align 16
+; COMBINED-NEXT: [[E0:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 0
+; COMBINED-NEXT: [[E1:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 1
+; COMBINED-NEXT: [[E2:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 2
+; COMBINED-NEXT: [[E3:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 3
+; COMBINED-NEXT: [[SUM01:%.*]] = fadd float [[E0]], [[E1]]
+; COMBINED-NEXT: [[SUM23:%.*]] = fadd float [[E2]], [[E3]]
+; COMBINED-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
+; COMBINED-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
+; COMBINED-NEXT: ret void
+;
+entry:
+ %slot = alloca [4 x float], align 16, addrspace(5)
+
+ %ptr = select i1 %cond, ptr addrspace(5) %slot, ptr addrspace(5) %slot
+ store float %v0, ptr addrspace(5) %ptr, align 16
+ %p1 = getelementptr inbounds float, ptr addrspace(5) %ptr, i32 1
+ store float %v1, ptr addrspace(5) %p1, align 4
+ %p2 = getelementptr inbounds float, ptr addrspace(5) %ptr, i32 2
+ store float %v2, ptr addrspace(5) %p2, align 4
+ %p3 = getelementptr inbounds float, ptr addrspace(5) %ptr, i32 3
+ store float %v3, ptr addrspace(5) %p3, align 4
+
+ %vec = load <4 x float>, ptr addrspace(5) %slot, align 16
+ %e0 = extractelement <4 x float> %vec, i32 0
+ %e1 = extractelement <4 x float> %vec, i32 1
+ %e2 = extractelement <4 x float> %vec, i32 2
+ %e3 = extractelement <4 x float> %vec, i32 3
+
+ %sum01 = fadd float %e0, %e1
+ %sum23 = fadd float %e2, %e3
+ %sum = fadd float %sum01, %sum23
+ store float %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @split_all_lanes_phi_same_alloca(float %v0, float %v1, float %v2, float %v3, i1 %cond, ptr addrspace(1) %out) {
+; SROA-LABEL: define void @split_all_lanes_phi_same_alloca(
+; SROA-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; SROA-NEXT: [[ENTRY:.*:]]
+; SROA-NEXT: [[SLOT_SROA_0:%.*]] = alloca <4 x float>, align 16, addrspace(5)
+; SROA-NEXT: [[SLOT_SROA_0_4_GEP_SROA_IDX8:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
+; SROA-NEXT: [[SLOT_SROA_0_4_GEP1_SROA_IDX9:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
+; SROA-NEXT: [[SLOT_SROA_0_8_GEP3_SROA_IDX10:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
+; SROA-NEXT: [[SLOT_SROA_0_8_GEP4_SROA_IDX11:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
+; SROA-NEXT: [[SLOT_SROA_0_12_GEP6_SROA_IDX12:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
+; SROA-NEXT: [[SLOT_SROA_0_12_GEP7_SROA_IDX13:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
+; SROA-NEXT: [[SLOT_SROA_0_4_GEP_SROA_IDX8_SLOT_SROA_0_4_GEP1_SROA_IDX9:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP_SROA_IDX8]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP1_SROA_IDX9]]
+; SROA-NEXT: [[SLOT_SROA_0_8_GEP3_SROA_IDX10_SLOT_SROA_0_8_GEP4_SROA_IDX11:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP3_SROA_IDX10]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP4_SROA_IDX11]]
+; SROA-NEXT: [[SLOT_SROA_0_12_GEP6_SROA_IDX12_SLOT_SROA_0_12_GEP7_SROA_IDX13:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP6_SROA_IDX12]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP7_SROA_IDX13]]
+; SROA-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT_SROA_0]], align 16
+; SROA-NEXT: store float [[V1]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP_SROA_IDX8_SLOT_SROA_0_4_GEP1_SROA_IDX9]], align 4
+; SROA-NEXT: store float [[V2]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP3_SROA_IDX10_SLOT_SROA_0_8_GEP4_SROA_IDX11]], align 4
+; SROA-NEXT: store float [[V3]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP6_SROA_IDX12_SLOT_SROA_0_12_GEP7_SROA_IDX13]], align 4
+; SROA-NEXT: [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT_SROA_0]], align 16
+; SROA-NEXT: [[E0:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 0
+; SROA-NEXT: [[E1:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 1
+; SROA-NEXT: [[E2:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 2
+; SROA-NEXT: [[E3:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 3
+; SROA-NEXT: [[SUM01:%.*]] = fadd float [[E0]], [[E1]]
+; SROA-NEXT: [[SUM23:%.*]] = fadd float [[E2]], [[E3]]
+; SROA-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
+; SROA-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
+; SROA-NEXT: ret void
+;
+; COMBINED-LABEL: define void @split_all_lanes_phi_same_alloca(
+; COMBINED-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; COMBINED-NEXT: [[ENTRY:.*:]]
+; COMBINED-NEXT: [[SLOT_SROA_0:%.*]] = alloca <4 x float>, align 16, addrspace(5)
+; COMBINED-NEXT: [[SLOT_SROA_0_4_GEP_SROA_IDX8_SLOT_SROA_0_4_GEP1_SROA_IDX9:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
+; COMBINED-NEXT: [[SLOT_SROA_0_8_GEP3_SROA_IDX10_SLOT_SROA_0_8_GEP4_SROA_IDX11:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
+; COMBINED-NEXT: [[SLOT_SROA_0_12_GEP6_SROA_IDX12_SLOT_SROA_0_12_GEP7_SROA_IDX13:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
+; COMBINED-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT_SROA_0]], align 16
+; COMBINED-NEXT: store float [[V1]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP_SROA_IDX8_SLOT_SROA_0_4_GEP1_SROA_IDX9]], align 4
+; COMBINED-NEXT: store float [[V2]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP3_SROA_IDX10_SLOT_SROA_0_8_GEP4_SROA_IDX11]], align 4
+; COMBINED-NEXT: store float [[V3]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP6_SROA_IDX12_SLOT_SROA_0_12_GEP7_SROA_IDX13]], align 4
+; COMBINED-NEXT: [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT_SROA_0]], align 16
+; COMBINED-NEXT: [[E0:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 0
+; COMBINED-NEXT: [[E1:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 1
+; COMBINED-NEXT: [[E2:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 2
+; COMBINED-NEXT: [[E3:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 3
+; COMBINED-NEXT: [[SUM01:%.*]] = fadd float [[E0]], [[E1]]
+; COMBINED-NEXT: [[SUM23:%.*]] = fadd float [[E2]], [[E3]]
+; COMBINED-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
+; COMBINED-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
+; COMBINED-NEXT: ret void
+;
+entry:
+ %slot = alloca [4 x float], align 16, addrspace(5)
+ br i1 %cond, label %left, label %right
+
+left:
+ br label %merge
+
+right:
+ br label %merge
+
+merge:
+ %ptr = phi ptr addrspace(5) [ %slot, %left ], [ %slot, %right ]
+ store float %v0, ptr addrspace(5) %ptr, align 16
+ %p1 = getelementptr inbounds float, ptr addrspace(5) %ptr, i32 1
+ store float %v1, ptr addrspace(5) %p1, align 4
+ %p2 = getelementptr inbounds float, ptr addrspace(5) %ptr, i32 2
+ store float %v2, ptr addrspace(5) %p2, align 4
+ %p3 = getelementptr inbounds float, ptr addrspace(5) %ptr, i32 3
+ store float %v3, ptr addrspace(5) %p3, align 4
+
+ %vec = load <4 x float>, ptr addrspace(5) %slot, align 16
+ %e0 = extractelement <4 x float> %vec, i32 0
+ %e1 = extractelement <4 x float> %vec, i32 1
+ %e2 = extractelement <4 x float> %vec, i32 2
+ %e3 = extractelement <4 x float> %vec, i32 3
+
+ %sum01 = fadd float %e0, %e1
+ %sum23 = fadd float %e2, %e3
+ %sum = fadd float %sum01, %sum23
+ store float %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define float @no_split_pointer_store() {
+; SROA-LABEL: define float @no_split_pointer_store() {
+; SROA-NEXT: [[ENTRY:.*:]]
+; SROA-NEXT: [[SLOT:%.*]] = alloca [4 x float], align 16, addrspace(5)
+; SROA-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, ptr addrspace(5) [[SLOT]], i32 0
+; SROA-NEXT: [[OTHER:%.*]] = getelementptr inbounds float, ptr addrspace(5) [[SLOT]], i32 1
+; SROA-NEXT: store ptr addrspace(5) [[OTHER]], ptr addrspace(5) [[GEP0]], align 8
+; SROA-NEXT: [[VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT]], align 16
+; SROA-NEXT: [[E0:%.*]] = extractelement <4 x float> [[VEC]], i32 0
+; SROA-NEXT: ret float [[E0]]
+;
+; COMBINED-LABEL: define float @no_split_pointer_store() {
+; COMBINED-NEXT: [[ENTRY:.*:]]
+; COMBINED-NEXT: [[SLOT:%.*]] = alloca [4 x float], align 16, addrspace(5)
+; COMBINED-NEXT: [[OTHER:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT]], i64 4
+; COMBINED-NEXT: store ptr addrspace(5) [[OTHER]], ptr addrspace(5) [[SLOT]], align 8
+; COMBINED-NEXT: [[VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT]], align 16
+; COMBINED-NEXT: [[E0:%.*]] = extractelement <4 x float> [[VEC]], i64 0
+; COMBINED-NEXT: ret float [[E0]]
+;
+entry:
+ %slot = alloca [4 x float], align 16, addrspace(5)
+ %gep0 = getelementptr inbounds float, ptr addrspace(5) %slot, i32 0
+ %other = getelementptr inbounds float, ptr addrspace(5) %slot, i32 1
+ store ptr addrspace(5) %other, ptr addrspace(5) %gep0
+ %vec = load <4 x float>, ptr addrspace(5) %slot, align 16
+ %e0 = extractelement <4 x float> %vec, i32 0
+ ret float %e0
+}
+
+define float @no_split_escaped_alloca() {
+; SROA-LABEL: define float @no_split_escaped_alloca() {
+; SROA-NEXT: [[ENTRY:.*:]]
+; SROA-NEXT: [[SLOT:%.*]] = alloca <4 x float>, align 16
+; SROA-NEXT: call void @escape(ptr [[SLOT]])
+; SROA-NEXT: store <2 x float> <float 1.000000e+00, float 2.000000e+00>, ptr [[SLOT]], align 16
+; SROA-NEXT: [[VEC:%.*]] = load <4 x float>, ptr [[SLOT]], align 16
+; SROA-NEXT: [[E0:%.*]] = extractelement <4 x float> [[VEC]], i32 0
+; SROA-NEXT: ret float [[E0]]
+;
+; COMBINED-LABEL: define float @no_split_escaped_alloca() {
+; COMBINED-NEXT: [[ENTRY:.*:]]
+; COMBINED-NEXT: [[SLOT:%.*]] = alloca <4 x float>, align 16
+; COMBINED-NEXT: call void @escape(ptr nonnull [[SLOT]])
+; COMBINED-NEXT: store <2 x float> <float 1.000000e+00, float 2.000000e+00>, ptr [[SLOT]], align 16
+; COMBINED-NEXT: [[VEC:%.*]] = load <4 x float>, ptr [[SLOT]], align 16
+; COMBINED-NEXT: [[E0:%.*]] = extractelement <4 x float> [[VEC]], i64 0
+; COMBINED-NEXT: ret float [[E0]]
+;
+entry:
+ %slot = alloca <4 x float>, align 16
+ call void @escape(ptr %slot)
+ store <2 x float> <float 1.0, float 2.0>, ptr %slot, align 16
+ %vec = load <4 x float>, ptr %slot, align 16
+ %e0 = extractelement <4 x float> %vec, i32 0
+ ret float %e0
+}
+
+define void @no_split_phi_diff_allocas( float %v0, float %v1, float %v2, float %v3, i1 %cond, ptr addrspace(1) %out) {
+; SROA-LABEL: define void @no_split_phi_diff_allocas(
+; SROA-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; SROA-NEXT: [[ENTRY:.*:]]
+; SROA-NEXT: [[SLOT1:%.*]] = alloca [4 x float], align 16, addrspace(5)
+; SROA-NEXT: [[SLOT2:%.*]] = alloca [4 x float], align 16, addrspace(5)
+; SROA-NEXT: br i1 [[COND]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; SROA: [[LEFT]]:
+; SROA-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT1]], align 16
+; SROA-NEXT: [[SLOT1_4_S1P1_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT1]], i64 4
+; SROA-NEXT: store float [[V1]], ptr addrspace(5) [[SLOT1_4_S1P1_SROA_IDX]], align 4
+; SROA-NEXT: [[SLOT1_8_S1P2_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT1]], i64 8
+; SROA-NEXT: store float [[V2]], ptr addrspace(5) [[SLOT1_8_S1P2_SROA_IDX]], align 8
+; SROA-NEXT: [[SLOT1_12_S1P3_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT1]], i64 12
+; SROA-NEXT: store float [[V3]], ptr addrspace(5) [[SLOT1_12_S1P3_SROA_IDX]], align 4
+; SROA-NEXT: br label %[[MERGE:.*]]
+; SROA: [[RIGHT]]:
+; SROA-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT2]], align 16
+; SROA-NEXT: [[SLOT2_4_S2P1_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT2]], i64 4
+; SROA-NEXT: store float [[V1]], ptr addrspace(5) [[SLOT2_4_S2P1_SROA_IDX]], align 4
+; SROA-NEXT: [[SLOT2_8_S2P2_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT2]], i64 8
+; SROA-NEXT: store float [[V2]], ptr addrspace(5) [[SLOT2_8_S2P2_SROA_IDX]], align 8
+; SROA-NEXT: [[SLOT2_12_S2P3_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT2]], i64 12
+; SROA-NEXT: store float [[V3]], ptr addrspace(5) [[SLOT2_12_S2P3_SROA_IDX]], align 4
+; SROA-NEXT: br label %[[MERGE]]
+; SROA: [[MERGE]]:
+; SROA-NEXT: [[PTR:%.*]] = phi ptr addrspace(5) [ [[SLOT1]], %[[LEFT]] ], [ [[SLOT2]], %[[RIGHT]] ]
+; SROA-NEXT: [[VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[PTR]], align 16
+; SROA-NEXT: [[F0:%.*]] = extractelement <4 x float> [[VEC]], i32 0
+; SROA-NEXT: [[F1:%.*]] = extractelement <4 x float> [[VEC]], i32 1
+; SROA-NEXT: [[F2:%.*]] = extractelement <4 x float> [[VEC]], i32 2
+; SROA-NEXT: [[F3:%.*]] = extractelement <4 x float> [[VEC]], i32 3
+; SROA-NEXT: [[SUM01:%.*]] = fadd float [[F0]], [[F1]]
+; SROA-NEXT: [[SUM23:%.*]] = fadd float [[F2]], [[F3]]
+; SROA-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
+; SROA-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
+; SROA-NEXT: ret void
+;
+; COMBINED-LABEL: define void @no_split_phi_diff_allocas(
+; COMBINED-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; COMBINED-NEXT: [[ENTRY:.*:]]
+; COMBINED-NEXT: [[SLOT1:%.*]] = alloca [4 x float], align 16, addrspace(5)
+; COMBINED-NEXT: [[SLOT2:%.*]] = alloca [4 x float], align 16, addrspace(5)
+; COMBINED-NEXT: br i1 [[COND]], label %[[LEFT:.*]], label %[[RIGHT:.*]]
+; COMBINED: [[LEFT]]:
+; COMBINED-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT1]], align 16
+; COMBINED-NEXT: [[SLOT1_4_S1P1_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT1]], i64 4
+; COMBINED-NEXT: store float [[V1]], ptr addrspace(5) [[SLOT1_4_S1P1_SROA_IDX]], align 4
+; COMBINED-NEXT: [[SLOT1_8_S1P2_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT1]], i64 8
+; COMBINED-NEXT: store float [[V2]], ptr addrspace(5) [[SLOT1_8_S1P2_SROA_IDX]], align 8
+; COMBINED-NEXT: [[SLOT1_12_S1P3_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT1]], i64 12
+; COMBINED-NEXT: store float [[V3]], ptr addrspace(5) [[SLOT1_12_S1P3_SROA_IDX]], align 4
+; COMBINED-NEXT: br label %[[MERGE:.*]]
+; COMBINED: [[RIGHT]]:
+; COMBINED-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT2]], align 16
+; COMBINED-NEXT: [[SLOT2_4_S2P1_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT2]], i64 4
+; COMBINED-NEXT: store float [[V1]], ptr addrspace(5) [[SLOT2_4_S2P1_SROA_IDX]], align 4
+; COMBINED-NEXT: [[SLOT2_8_S2P2_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT2]], i64 8
+; COMBINED-NEXT: store float [[V2]], ptr addrspace(5) [[SLOT2_8_S2P2_SROA_IDX]], align 8
+; COMBINED-NEXT: [[SLOT2_12_S2P3_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT2]], i64 12
+; COMBINED-NEXT: store float [[V3]], ptr addrspace(5) [[SLOT2_12_S2P3_SROA_IDX]], align 4
+; COMBINED-NEXT: br label %[[MERGE]]
+; COMBINED: [[MERGE]]:
+; COMBINED-NEXT: [[PTR:%.*]] = phi ptr addrspace(5) [ [[SLOT1]], %[[LEFT]] ], [ [[SLOT2]], %[[RIGHT]] ]
+; COMBINED-NEXT: [[VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[PTR]], align 16
+; COMBINED-NEXT: [[F0:%.*]] = extractelement <4 x float> [[VEC]], i64 0
+; COMBINED-NEXT: [[F1:%.*]] = extractelement <4 x float> [[VEC]], i64 1
+; COMBINED-NEXT: [[F2:%.*]] = extractelement <4 x float> [[VEC]], i64 2
+; COMBINED-NEXT: [[F3:%.*]] = extractelement <4 x float> [[VEC]], i64 3
+; COMBINED-NEXT: [[SUM01:%.*]] = fadd float [[F0]], [[F1]]
+; COMBINED-NEXT: [[SUM23:%.*]] = fadd float [[F2]], [[F3]]
+; COMBINED-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
+; COMBINED-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
+; COMBINED-NEXT: ret void
+;
+entry:
+ %slot1 = alloca [4 x float], align 16, addrspace(5)
+ %slot2 = alloca [4 x float], align 16, addrspace(5)
+ br i1 %cond, label %left, label %right
+
+left:
+ store float %v0, ptr addrspace(5) %slot1, align 16
+ %s1p1 = getelementptr inbounds float, ptr addrspace(5) %slot1, i32 1
+ store float %v1, ptr addrspace(5) %s1p1, align 4
+ %s1p2 = getelementptr inbounds float, ptr addrspace(5) %slot1, i32 2
+ store float %v2, ptr addrspace(5) %s1p2, align 4
+ %s1p3 = getelementptr inbounds float, ptr addrspace(5) %slot1, i32 3
+ store float %v3, ptr addrspace(5) %s1p3, align 4
+ br label %merge
+
+right:
+ store float %v0, ptr addrspace(5) %slot2, align 16
+ %s2p1 = getelementptr inbounds float, ptr addrspace(5) %slot2, i32 1
+ store float %v1, ptr addrspace(5) %s2p1, align 4
+ %s2p2 = getelementptr inbounds float, ptr addrspace(5) %slot2, i32 2
+ store float %v2, ptr addrspace(5) %s2p2, align 8
+ %s2p3 = getelementptr inbounds float, ptr addrspace(5) %slot2, i32 3
+ store float %v3, ptr addrspace(5) %s2p3, align 4
+ br label %merge
+
+merge:
+ %ptr = phi ptr addrspace(5) [ %slot1, %left ], [ %slot2, %right ]
+ %vec = load <4 x float>, ptr addrspace(5) %ptr, align 16
+ %f0 = extractelement <4 x float> %vec, i32 0
+ %f1 = extractelement <4 x float> %vec, i32 1
+ %f2 = extractelement <4 x float> %vec, i32 2
+ %f3 = extractelement <4 x float> %vec, i32 3
+
+ %sum01 = fadd float %f0, %f1
+ %sum23 = fadd float %f2, %f3
+ %sum = fadd float %sum01, %sum23
+ store float %sum, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define float @no_split_scalable_vector_store(<vscale x 4 x float> %sv) {
+; SROA-LABEL: define float @no_split_scalable_vector_store(
+; SROA-SAME: <vscale x 4 x float> [[SV:%.*]]) {
+; SROA-NEXT: [[ENTRY:.*:]]
+; SROA-NEXT: [[SLOT:%.*]] = alloca <4 x float>, align 16
+; SROA-NEXT: store <vscale x 4 x float> [[SV]], ptr [[SLOT]], align 16
+; SROA-NEXT: [[VEC:%.*]] = load <4 x float>, ptr [[SLOT]], align 16
+; SROA-NEXT: [[E0:%.*]] = extractelement <4 x float> [[VEC]], i32 0
+; SROA-NEXT: ret float [[E0]]
+;
+; COMBINED-LABEL: define float @no_split_scalable_vector_store(
+; COMBINED-SAME: <vscale x 4 x float> [[SV:%.*]]) {
+; COMBINED-NEXT: [[ENTRY:.*:]]
+; COMBINED-NEXT: [[SLOT:%.*]] = alloca <4 x float>, align 16
+; COMBINED-NEXT: store <vscale x 4 x float> [[SV]], ptr [[SLOT]], align 16
+; COMBINED-NEXT: [[VEC:%.*]] = load <4 x float>, ptr [[SLOT]], align 16
+; COMBINED-NEXT: [[E0:%.*]] = extractelement <4 x float> [[VEC]], i64 0
+; COMBINED-NEXT: ret float [[E0]]
+;
+entry:
+ %slot = alloca <4 x float>, align 16
+ store <vscale x 4 x float> %sv, ptr %slot, align 16
+ %vec = load <4 x float>, ptr %slot, align 16
+ %e0 = extractelement <4 x float> %vec, i32 0
+ ret float %e0
+}
+
+declare void @escape(ptr)
+declare void @llvm.memcpy.p0.p0.i32(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i32, i1 immarg)
>From d316142cdc36cadff8ff81849f6e35cb75a20c77 Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Fri, 22 May 2026 16:46:38 -0500
Subject: [PATCH 2/3] [SROA] Split fixed-vector loads into per-lane scalar
loads when a smaller store overlaps
---
llvm/lib/Transforms/Scalar/SROA.cpp | 289 +++++++++++++++++-
.../test/Transforms/SROA/vector-load-split.ll | 131 ++------
2 files changed, 309 insertions(+), 111 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 83e40edb64541..824eaf4d48600 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -79,6 +79,7 @@
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TypeSize.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -4436,6 +4437,282 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
}
};
+/// Splits fixed-vector alloca loads into per-lane scalar loads when a smaller
+/// overlapping store exists for the same alloca.
+class VecLoadLaneSplitter : public InstVisitor<VecLoadLaneSplitter, bool> {
+
+ friend class InstVisitor<VecLoadLaneSplitter, bool>;
+
+ const DataLayout &DL;
+ SmallPtrSet<LoadInst *, 4> CandidateLoadSet;
+ SmallPtrSet<StoreInst *, 4> CandidateStoreSet;
+
+ SmallVector<Use *, 4> Queue;
+ SmallPtrSet<Use *, 4> Visited;
+ IRBuilderTy &IRB;
+ Use *U = nullptr;
+
+ bool isLoadLaneExtractOnly(LoadInst &LI) {
+ if (!LI.isSimple() || LI.use_empty())
+ return false;
+
+ FixedVectorType *VTy = dyn_cast<FixedVectorType>(LI.getType());
+ if (!VTy)
+ return false;
+
+ Type *EltTy = VTy->getElementType();
+ if (!DL.typeSizeEqualsStoreSize(EltTy))
+ return false;
+
+ unsigned NumElts = VTy->getNumElements();
+ for (Use &Us : LI.uses()) {
+ auto *EE = dyn_cast<ExtractElementInst>(Us.getUser());
+ if (!EE)
+ return false;
+ auto *CI = dyn_cast<ConstantInt>(EE->getIndexOperand());
+ if (!CI || CI->getValue().uge(NumElts))
+ return false;
+ }
+ return true;
+ }
+
+ bool isStoreCandidate(StoreInst &SI) {
+ if (!SI.isSimple())
+ return false;
+ Type *StoreTy = SI.getValueOperand()->getType();
+ return !isa<ScalableVectorType>(StoreTy);
+ }
+
+ bool isScalarOrSubVectorStore(StoreInst &SI, const Slice &StoreSlice,
+ const Slice &LoadSlice) {
+ if (!isStoreCandidate(SI))
+ return false;
+
+ Type *StoreTy = SI.getValueOperand()->getType();
+ bool IsContained = StoreSlice.beginOffset() >= LoadSlice.beginOffset() &&
+ StoreSlice.endOffset() <= LoadSlice.endOffset();
+ if (!isa<FixedVectorType>(StoreTy))
+ return IsContained;
+ uint64_t StoreSize = StoreSlice.endOffset() - StoreSlice.beginOffset();
+ uint64_t LoadSize = LoadSlice.endOffset() - LoadSlice.beginOffset();
+ return IsContained && StoreSize < LoadSize;
+ }
+
+ // Return true if slice S's use is the address operand of the instruction I.
+ bool isAddressOperandSlice(const Slice &S, Instruction &I) {
+ if (S.getUse()->getUser() != &I)
+ return false;
+ if (auto *LI = dyn_cast<LoadInst>(&I))
+ return S.getUse() == &LI->getOperandUse(LI->getPointerOperandIndex());
+ if (auto *SI = dyn_cast<StoreInst>(&I))
+ return S.getUse() == &SI->getOperandUse(SI->getPointerOperandIndex());
+ return false;
+ }
+
+ void enqueueUsers(Instruction &I) {
+ for (Use &U : I.uses())
+ if (Visited.insert(&U).second)
+ Queue.push_back(&U);
+ }
+
+ bool visitInstruction(Instruction &I) { return false; }
+
+ bool visitBitCastInst(BitCastInst &BC) {
+ enqueueUsers(BC);
+ return false;
+ }
+
+ bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
+ enqueueUsers(ASC);
+ return false;
+ }
+
+ bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
+ enqueueUsers(GEPI);
+ return false;
+ }
+
+ bool visitSelectInst(SelectInst &SI) {
+ enqueueUsers(SI);
+ return false;
+ }
+
+ bool visitPHINode(PHINode &PN) {
+ enqueueUsers(PN);
+ return false;
+ }
+
+ bool visitStoreInst(StoreInst &SI) {
+ if (!SI.isSimple() || SI.getPointerOperand() != *U)
+ return false;
+ if (isStoreCandidate(SI))
+ CandidateStoreSet.insert(&SI);
+ return false;
+ }
+
+ bool visitLoadInst(LoadInst &LI) {
+ if (!LI.isSimple() || LI.getPointerOperand() != *U)
+ return false;
+
+ if (isLoadLaneExtractOnly(LI))
+ CandidateLoadSet.insert(&LI);
+ return false;
+ }
+
+ bool trySplitVectorLoad(LoadInst &LI) {
+
+ assert(LI.isSimple() && "Load is not simple");
+ // Erase the load if it has no users.
+ if (LI.use_empty()) {
+ LI.eraseFromParent();
+ return true;
+ }
+
+ FixedVectorType *VTy = cast<FixedVectorType>(LI.getType());
+ Type *EltTy = VTy->getElementType();
+
+ // Reject elements where the store size is not equal to the element size.
+ if (!DL.typeSizeEqualsStoreSize(EltTy))
+ return false;
+
+ unsigned NumElts = VTy->getNumElements();
+ LLVM_DEBUG(dbgs() << "Consider load: " << LI << " NumElts=" << NumElts
+ << " EltTy=" << *EltTy << "\n");
+
+ // Collect the lane users.
+ TypeSize EltSize = DL.getTypeStoreSize(EltTy);
+ SmallVector<SmallVector<ExtractElementInst *, 4>, 4> LaneUsers(NumElts);
+ for (Use &Us : LI.uses()) {
+ auto *EE = dyn_cast<ExtractElementInst>(Us.getUser());
+ if (!EE)
+ return false;
+ auto *CI = dyn_cast<ConstantInt>(EE->getIndexOperand());
+ if (!CI || CI->getValue().uge(NumElts))
+ return false;
+ LaneUsers[CI->getZExtValue()].push_back(EE);
+ }
+
+ // Emit one scalar load per lane.
+ IRB.SetInsertPoint(&LI);
+ Value *BasePtr = LI.getPointerOperand();
+ unsigned IdxBits = DL.getIndexSizeInBits(LI.getPointerAddressSpace());
+ uint64_t Stride = EltSize.getFixedValue();
+ SmallVector<ExtractElementInst *, 4> ToErase;
+ for (auto [Lane, Users] : enumerate(LaneUsers)) {
+ if (Users.empty())
+ continue;
+ uint64_t Offset = Lane * Stride;
+ LLVM_DEBUG(dbgs() << "Lane " << Lane << " Offset=" << Offset
+ << " NumUsers=" << Users.size() << "\n");
+ Value *LanePtr = getAdjustedPtr(IRB, DL, BasePtr, APInt(IdxBits, Offset),
+ LI.getPointerOperandType(),
+ BasePtr->getName() + ".lane");
+ auto *NewLI = IRB.CreateAlignedLoad(EltTy, LanePtr,
+ getAdjustedAlignment(&LI, Offset),
+ LI.getName() + ".lane");
+ copyMetadataForLoad(*NewLI, LI);
+ // Adjust AA metadata for the lane byte offset.
+ if (AAMDNodes AATags = LI.getAAMetadata())
+ NewLI->setAAMetadata(
+ AATags.adjustForAccess(Offset, NewLI->getType(), DL));
+
+ for (ExtractElementInst *EE : Users) {
+ EE->replaceAllUsesWith(NewLI);
+ ToErase.push_back(EE);
+ }
+ }
+
+ for (ExtractElementInst *EE : ToErase)
+ EE->eraseFromParent();
+ LI.eraseFromParent();
+ return true;
+ }
+
+public:
+ explicit VecLoadLaneSplitter(const DataLayout &DL, IRBuilderTy &IRB)
+ : DL(DL), IRB(IRB) {}
+
+ bool rewrite(AllocaInst &AI, AllocaSlices &AS) {
+
+ CandidateLoadSet.clear();
+ CandidateStoreSet.clear();
+ Queue.clear();
+ Visited.clear();
+ U = nullptr;
+
+ // Reject escaped alloca slices. Let runOnAlloca route the escapes.
+ if (AS.isEscaped() || AS.isEscapedReadOnly())
+ return false;
+
+ // Walk all uses and classify.
+ enqueueUsers(AI);
+ while (!Queue.empty()) {
+ U = Queue.pop_back_val();
+ visit(*cast<Instruction>(U->getUser())); // dispatches to visitXxx above
+ }
+
+ LLVM_DEBUG(dbgs() << "CandidateLoads=" << CandidateLoadSet.size()
+ << " CandidateStoreSet=" << CandidateStoreSet.size()
+ << "\n");
+
+ if (CandidateLoadSet.empty() || CandidateStoreSet.empty())
+ return false;
+
+ auto SlicesOverlap = [](const Slice &S1, const Slice &S2) {
+ return S1.beginOffset() < S2.endOffset() &&
+ S2.beginOffset() < S1.endOffset();
+ };
+
+ SmallSetVector<LoadInst *, 4> SplitEligibleLoads;
+
+ // For each partition, find eligible loads and stores.
+ for (Partition &P : AS.partitions()) {
+ SmallVector<Slice *, 4> StoreSlices;
+ for (Slice &S : P) {
+ auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser());
+ if (SI && CandidateStoreSet.count(SI) && isAddressOperandSlice(S, *SI))
+ StoreSlices.push_back(&S);
+ }
+ if (StoreSlices.empty()) {
+ LLVM_DEBUG(dbgs() << "Skip partition: no store slices\n");
+ continue;
+ }
+
+ for (Slice &S : P) {
+ auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser());
+ if (!LI || !CandidateLoadSet.count(LI) ||
+ !isAddressOperandSlice(S, *LI))
+ continue;
+
+ // Check if vector loads overlap with any store slices.
+ if (any_of(StoreSlices, [&](const Slice *StoreSlice) {
+ auto *SI = cast<StoreInst>(StoreSlice->getUse()->getUser());
+ return SlicesOverlap(S, *StoreSlice) &&
+ isScalarOrSubVectorStore(*SI, *StoreSlice, S);
+ })) {
+ if (SplitEligibleLoads.insert(LI)) {
+ LLVM_DEBUG(dbgs() << "Eligible load: " << *LI << "\n");
+ }
+ } else {
+ LLVM_DEBUG(dbgs() << "Skip load: no overlapping store slices\n");
+ }
+ }
+ }
+
+ if (SplitEligibleLoads.empty()) {
+ LLVM_DEBUG(dbgs() << "No eligible loads to split\n");
+ return false;
+ }
+
+ // Split each candidate load.
+ bool Changed = false;
+ for (LoadInst *LI : SplitEligibleLoads)
+ Changed |= trySplitVectorLoad(*LI);
+
+ return Changed;
+ }
+};
+
} // end anonymous namespace
/// Strip aggregate type wrapping.
@@ -5819,10 +6096,20 @@ SROA::runOnAlloca(AllocaInst &AI) {
IRBuilderTy IRB(&AI);
AggLoadStoreRewriter AggRewriter(DL, IRB);
Changed |= AggRewriter.rewrite(AI);
-
// Build the slices using a recursive instruction-visiting builder.
AllocaSlices AS(DL, AI);
LLVM_DEBUG(AS.print(dbgs()));
+ VecLoadLaneSplitter VecLoadSplitter(DL, IRB);
+
+ // VecLoadSplitter may change IR (erase vector-loads/extractelements)
+ // so AS may have dangling Use pointers. Re-queue for rebuilding the AS
+ // on next iteration.
+ if (VecLoadSplitter.rewrite(AI, AS)) {
+ Changed = true;
+ Worklist.insert(&AI);
+ return {Changed, CFGChanged};
+ }
+
if (AS.isEscaped())
return {Changed, CFGChanged};
diff --git a/llvm/test/Transforms/SROA/vector-load-split.ll b/llvm/test/Transforms/SROA/vector-load-split.ll
index 4b53671737c6a..bb53704b54cdf 100644
--- a/llvm/test/Transforms/SROA/vector-load-split.ll
+++ b/llvm/test/Transforms/SROA/vector-load-split.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes='sroa,simplifycfg' -S < %s | FileCheck %s --check-prefixes=SROA
; RUN: opt -passes='sroa,simplifycfg,instcombine' -S < %s | FileCheck %s --check-prefixes=COMBINED
@@ -8,28 +9,19 @@ define double @descriptor_copy_vector_load(ptr %data, ptr %dst) {
; SROA-LABEL: define double @descriptor_copy_vector_load(
; SROA-SAME: ptr [[DATA:%.*]], ptr [[DST:%.*]]) {
; SROA-NEXT: [[ENTRY:.*:]]
-; SROA-NEXT: [[SRC_DESC:%.*]] = alloca [[DESC:%.*]], align 8, addrspace(5)
-; SROA-NEXT: [[TMP_DESC:%.*]] = alloca [[DESC]], align 8, addrspace(5)
+; SROA-NEXT: [[TMP_DESC:%.*]] = alloca { i64, i64 }, align 8, addrspace(5)
+; SROA-NEXT: [[SRC_DESC_SROA_8:%.*]] = alloca { i64, i64, i64, i64, i64, i64 }, align 8, addrspace(5)
+; SROA-NEXT: [[SRC_DESC:%.*]] = alloca { i64, i64 }, align 8, addrspace(5)
+; SROA-NEXT: [[TMP_DESC_SROA_4:%.*]] = alloca { i64, i64, i64, i64, i64, i64 }, align 8, addrspace(5)
; SROA-NEXT: [[SRC_DESC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[SRC_DESC]] to ptr
; SROA-NEXT: [[TMP_DESC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP_DESC]] to ptr
-; SROA-NEXT: store ptr [[DATA]], ptr addrspace(5) [[SRC_DESC]], align 8
-; SROA-NEXT: [[SRC_LB0:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC_DESC]], i32 24
-; SROA-NEXT: store i64 1, ptr addrspace(5) [[SRC_LB0]], align 8
-; SROA-NEXT: [[SRC_STRIDE0:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC_DESC]], i32 40
-; SROA-NEXT: store i64 8, ptr addrspace(5) [[SRC_STRIDE0]], align 8
-; SROA-NEXT: [[ARGS_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x ptr> {{.*}}, ptr [[SRC_DESC_CAST]], i32 0
-; SROA-NEXT: [[ARGS_SROA_0_8_VEC_INSERT:%.*]] = insertelement <2 x ptr> [[ARGS_SROA_0_0_VEC_INSERT]], ptr [[TMP_DESC_CAST]], i32 1
-; SROA-NEXT: [[TMP_PTR:%.*]] = extractelement <2 x ptr> [[ARGS_SROA_0_8_VEC_INSERT]], i32 1
-; SROA-NEXT: [[SRC_PTR:%.*]] = extractelement <2 x ptr> [[ARGS_SROA_0_8_VEC_INSERT]], i32 0
-; SROA-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(96) [[TMP_PTR]], ptr noundef nonnull align 8 dereferenceable(96) [[SRC_PTR]], i32 96, i1 false)
-; SROA-NEXT: [[TMP_BASE:%.*]] = load ptr, ptr [[TMP_PTR]], align 8
-; SROA-NEXT: [[TMP_LB0_P:%.*]] = getelementptr inbounds i8, ptr [[TMP_PTR]], i64 24
-; SROA-NEXT: [[TMP_LB0:%.*]] = load i64, ptr [[TMP_LB0_P]], align 8
-; SROA-NEXT: [[TMP_STRIDE0_P:%.*]] = getelementptr inbounds i8, ptr [[TMP_PTR]], i64 40
-; SROA-NEXT: [[TMP_STRIDE0:%.*]] = load i64, ptr [[TMP_STRIDE0_P]], align 8
-; SROA-NEXT: [[IDX:%.*]] = sub nsw i64 4, [[TMP_LB0]]
-; SROA-NEXT: [[OFFSET:%.*]] = mul nuw nsw i64 [[TMP_STRIDE0]], [[IDX]]
-; SROA-NEXT: [[ELT_PTR:%.*]] = getelementptr i8, ptr [[TMP_BASE]], i64 [[OFFSET]]
+; SROA-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[SRC_DESC_CAST]], ptr align 8 [[TMP_DESC_CAST]], i32 16, i1 false)
+; SROA-NEXT: [[TMP_DESC_SROA_4_0_TMP_DESC_CAST_SROA_CAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP_DESC_SROA_4]] to ptr
+; SROA-NEXT: [[SRC_DESC_SROA_8_48_SRC_DESC_CAST_SROA_CAST:%.*]] = addrspacecast ptr addrspace(5) [[SRC_DESC_SROA_8]] to ptr
+; SROA-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 8 [[TMP_DESC_SROA_4_0_TMP_DESC_CAST_SROA_CAST]], ptr align 8 [[SRC_DESC_SROA_8_48_SRC_DESC_CAST_SROA_CAST]], i32 48, i1 false)
+; SROA-NEXT: [[IDX:%.*]] = sub nsw i64 4, 1
+; SROA-NEXT: [[OFFSET:%.*]] = mul nuw nsw i64 8, [[IDX]]
+; SROA-NEXT: [[ELT_PTR:%.*]] = getelementptr i8, ptr [[DATA]], i64 [[OFFSET]]
; SROA-NEXT: [[RETVAL:%.*]] = load double, ptr [[ELT_PTR]], align 8
; SROA-NEXT: store double 0.000000e+00, ptr [[DST]], align 8
; SROA-NEXT: ret double [[RETVAL]]
@@ -37,24 +29,7 @@ define double @descriptor_copy_vector_load(ptr %data, ptr %dst) {
; COMBINED-LABEL: define double @descriptor_copy_vector_load(
; COMBINED-SAME: ptr [[DATA:%.*]], ptr [[DST:%.*]]) {
; COMBINED-NEXT: [[ENTRY:.*:]]
-; COMBINED-NEXT: [[SRC_DESC:%.*]] = alloca [[DESC:%.*]], align 8, addrspace(5)
-; COMBINED-NEXT: [[TMP_DESC:%.*]] = alloca [[DESC]], align 8, addrspace(5)
-; COMBINED-NEXT: [[SRC_DESC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[SRC_DESC]] to ptr
-; COMBINED-NEXT: [[TMP_DESC_CAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP_DESC]] to ptr
-; COMBINED-NEXT: store ptr [[DATA]], ptr addrspace(5) [[SRC_DESC]], align 8
-; COMBINED-NEXT: [[SRC_LB0:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SRC_DESC]], i64 24
-; COMBINED-NEXT: store i64 1, ptr addrspace(5) [[SRC_LB0]], align 8
-; COMBINED-NEXT: [[SRC_STRIDE0:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SRC_DESC]], i64 40
-; COMBINED-NEXT: store i64 8, ptr addrspace(5) [[SRC_STRIDE0]], align 8
-; COMBINED-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 8 dereferenceable(96) [[TMP_DESC_CAST]], ptr noundef nonnull align 8 dereferenceable(96) [[SRC_DESC_CAST]], i32 96, i1 false)
-; COMBINED-NEXT: [[TMP_BASE:%.*]] = load ptr, ptr [[TMP_DESC_CAST]], align 8
-; COMBINED-NEXT: [[TMP_LB0_P:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP_DESC_CAST]], i64 24
-; COMBINED-NEXT: [[TMP_LB0:%.*]] = load i64, ptr [[TMP_LB0_P]], align 8
-; COMBINED-NEXT: [[TMP_STRIDE0_P:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP_DESC_CAST]], i64 40
-; COMBINED-NEXT: [[TMP_STRIDE0:%.*]] = load i64, ptr [[TMP_STRIDE0_P]], align 8
-; COMBINED-NEXT: [[IDX:%.*]] = sub nsw i64 4, [[TMP_LB0]]
-; COMBINED-NEXT: [[OFFSET:%.*]] = mul nuw nsw i64 [[TMP_STRIDE0]], [[IDX]]
-; COMBINED-NEXT: [[ELT_PTR:%.*]] = getelementptr i8, ptr [[TMP_BASE]], i64 [[OFFSET]]
+; COMBINED-NEXT: [[ELT_PTR:%.*]] = getelementptr i8, ptr [[DATA]], i64 24
; COMBINED-NEXT: [[RETVAL:%.*]] = load double, ptr [[ELT_PTR]], align 8
; COMBINED-NEXT: store double 0.000000e+00, ptr [[DST]], align 8
; COMBINED-NEXT: ret double [[RETVAL]]
@@ -163,27 +138,8 @@ define void @split_all_lanes_select_same_alloca(float %v0, float %v1, float %v2,
; SROA-LABEL: define void @split_all_lanes_select_same_alloca(
; SROA-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; SROA-NEXT: [[ENTRY:.*:]]
-; SROA-NEXT: [[SLOT_SROA_0:%.*]] = alloca <4 x float>, align 16, addrspace(5)
-; SROA-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT_SROA_0]], align 16
-; SROA-NEXT: [[SLOT_SROA_0_4_GEP_SROA_IDX8:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
-; SROA-NEXT: [[SLOT_SROA_0_4_GEP1_SROA_IDX9:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
-; SROA-NEXT: [[PTR_SROA_SEL:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP_SROA_IDX8]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP1_SROA_IDX9]]
-; SROA-NEXT: store float [[V1]], ptr addrspace(5) [[PTR_SROA_SEL]], align 4
-; SROA-NEXT: [[SLOT_SROA_0_8_GEP2_SROA_IDX10:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
-; SROA-NEXT: [[SLOT_SROA_0_8_GEP3_SROA_IDX11:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
-; SROA-NEXT: [[PTR_SROA_SEL4:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP2_SROA_IDX10]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP3_SROA_IDX11]]
-; SROA-NEXT: store float [[V2]], ptr addrspace(5) [[PTR_SROA_SEL4]], align 4
-; SROA-NEXT: [[SLOT_SROA_0_12_GEP5_SROA_IDX12:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
-; SROA-NEXT: [[SLOT_SROA_0_12_GEP6_SROA_IDX13:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
-; SROA-NEXT: [[PTR_SROA_SEL7:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP5_SROA_IDX12]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP6_SROA_IDX13]]
-; SROA-NEXT: store float [[V3]], ptr addrspace(5) [[PTR_SROA_SEL7]], align 4
-; SROA-NEXT: [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT_SROA_0]], align 16
-; SROA-NEXT: [[E0:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 0
-; SROA-NEXT: [[E1:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 1
-; SROA-NEXT: [[E2:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 2
-; SROA-NEXT: [[E3:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 3
-; SROA-NEXT: [[SUM01:%.*]] = fadd float [[E0]], [[E1]]
-; SROA-NEXT: [[SUM23:%.*]] = fadd float [[E2]], [[E3]]
+; SROA-NEXT: [[SUM01:%.*]] = fadd float [[V0]], [[V1]]
+; SROA-NEXT: [[SUM23:%.*]] = fadd float [[V2]], [[V3]]
; SROA-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
; SROA-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
; SROA-NEXT: ret void
@@ -191,21 +147,8 @@ define void @split_all_lanes_select_same_alloca(float %v0, float %v1, float %v2,
; COMBINED-LABEL: define void @split_all_lanes_select_same_alloca(
; COMBINED-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; COMBINED-NEXT: [[ENTRY:.*:]]
-; COMBINED-NEXT: [[SLOT_SROA_0:%.*]] = alloca <4 x float>, align 16, addrspace(5)
-; COMBINED-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT_SROA_0]], align 16
-; COMBINED-NEXT: [[PTR_SROA_SEL:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
-; COMBINED-NEXT: store float [[V1]], ptr addrspace(5) [[PTR_SROA_SEL]], align 4
-; COMBINED-NEXT: [[PTR_SROA_SEL4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
-; COMBINED-NEXT: store float [[V2]], ptr addrspace(5) [[PTR_SROA_SEL4]], align 4
-; COMBINED-NEXT: [[PTR_SROA_SEL7:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
-; COMBINED-NEXT: store float [[V3]], ptr addrspace(5) [[PTR_SROA_SEL7]], align 4
-; COMBINED-NEXT: [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT_SROA_0]], align 16
-; COMBINED-NEXT: [[E0:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 0
-; COMBINED-NEXT: [[E1:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 1
-; COMBINED-NEXT: [[E2:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 2
-; COMBINED-NEXT: [[E3:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 3
-; COMBINED-NEXT: [[SUM01:%.*]] = fadd float [[E0]], [[E1]]
-; COMBINED-NEXT: [[SUM23:%.*]] = fadd float [[E2]], [[E3]]
+; COMBINED-NEXT: [[SUM01:%.*]] = fadd float [[V0]], [[V1]]
+; COMBINED-NEXT: [[SUM23:%.*]] = fadd float [[V2]], [[V3]]
; COMBINED-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
; COMBINED-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
; COMBINED-NEXT: ret void
@@ -239,27 +182,8 @@ define void @split_all_lanes_phi_same_alloca(float %v0, float %v1, float %v2, fl
; SROA-LABEL: define void @split_all_lanes_phi_same_alloca(
; SROA-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; SROA-NEXT: [[ENTRY:.*:]]
-; SROA-NEXT: [[SLOT_SROA_0:%.*]] = alloca <4 x float>, align 16, addrspace(5)
-; SROA-NEXT: [[SLOT_SROA_0_4_GEP_SROA_IDX8:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
-; SROA-NEXT: [[SLOT_SROA_0_4_GEP1_SROA_IDX9:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
-; SROA-NEXT: [[SLOT_SROA_0_8_GEP3_SROA_IDX10:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
-; SROA-NEXT: [[SLOT_SROA_0_8_GEP4_SROA_IDX11:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
-; SROA-NEXT: [[SLOT_SROA_0_12_GEP6_SROA_IDX12:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
-; SROA-NEXT: [[SLOT_SROA_0_12_GEP7_SROA_IDX13:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
-; SROA-NEXT: [[SLOT_SROA_0_4_GEP_SROA_IDX8_SLOT_SROA_0_4_GEP1_SROA_IDX9:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP_SROA_IDX8]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP1_SROA_IDX9]]
-; SROA-NEXT: [[SLOT_SROA_0_8_GEP3_SROA_IDX10_SLOT_SROA_0_8_GEP4_SROA_IDX11:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP3_SROA_IDX10]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP4_SROA_IDX11]]
-; SROA-NEXT: [[SLOT_SROA_0_12_GEP6_SROA_IDX12_SLOT_SROA_0_12_GEP7_SROA_IDX13:%.*]] = select i1 [[COND]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP6_SROA_IDX12]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP7_SROA_IDX13]]
-; SROA-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT_SROA_0]], align 16
-; SROA-NEXT: store float [[V1]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP_SROA_IDX8_SLOT_SROA_0_4_GEP1_SROA_IDX9]], align 4
-; SROA-NEXT: store float [[V2]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP3_SROA_IDX10_SLOT_SROA_0_8_GEP4_SROA_IDX11]], align 4
-; SROA-NEXT: store float [[V3]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP6_SROA_IDX12_SLOT_SROA_0_12_GEP7_SROA_IDX13]], align 4
-; SROA-NEXT: [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT_SROA_0]], align 16
-; SROA-NEXT: [[E0:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 0
-; SROA-NEXT: [[E1:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 1
-; SROA-NEXT: [[E2:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 2
-; SROA-NEXT: [[E3:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i32 3
-; SROA-NEXT: [[SUM01:%.*]] = fadd float [[E0]], [[E1]]
-; SROA-NEXT: [[SUM23:%.*]] = fadd float [[E2]], [[E3]]
+; SROA-NEXT: [[SUM01:%.*]] = fadd float [[V0]], [[V1]]
+; SROA-NEXT: [[SUM23:%.*]] = fadd float [[V2]], [[V3]]
; SROA-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
; SROA-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
; SROA-NEXT: ret void
@@ -267,21 +191,8 @@ define void @split_all_lanes_phi_same_alloca(float %v0, float %v1, float %v2, fl
; COMBINED-LABEL: define void @split_all_lanes_phi_same_alloca(
; COMBINED-SAME: float [[V0:%.*]], float [[V1:%.*]], float [[V2:%.*]], float [[V3:%.*]], i1 [[COND:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; COMBINED-NEXT: [[ENTRY:.*:]]
-; COMBINED-NEXT: [[SLOT_SROA_0:%.*]] = alloca <4 x float>, align 16, addrspace(5)
-; COMBINED-NEXT: [[SLOT_SROA_0_4_GEP_SROA_IDX8_SLOT_SROA_0_4_GEP1_SROA_IDX9:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 4
-; COMBINED-NEXT: [[SLOT_SROA_0_8_GEP3_SROA_IDX10_SLOT_SROA_0_8_GEP4_SROA_IDX11:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 8
-; COMBINED-NEXT: [[SLOT_SROA_0_12_GEP6_SROA_IDX12_SLOT_SROA_0_12_GEP7_SROA_IDX13:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) [[SLOT_SROA_0]], i64 12
-; COMBINED-NEXT: store float [[V0]], ptr addrspace(5) [[SLOT_SROA_0]], align 16
-; COMBINED-NEXT: store float [[V1]], ptr addrspace(5) [[SLOT_SROA_0_4_GEP_SROA_IDX8_SLOT_SROA_0_4_GEP1_SROA_IDX9]], align 4
-; COMBINED-NEXT: store float [[V2]], ptr addrspace(5) [[SLOT_SROA_0_8_GEP3_SROA_IDX10_SLOT_SROA_0_8_GEP4_SROA_IDX11]], align 4
-; COMBINED-NEXT: store float [[V3]], ptr addrspace(5) [[SLOT_SROA_0_12_GEP6_SROA_IDX12_SLOT_SROA_0_12_GEP7_SROA_IDX13]], align 4
-; COMBINED-NEXT: [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC:%.*]] = load <4 x float>, ptr addrspace(5) [[SLOT_SROA_0]], align 16
-; COMBINED-NEXT: [[E0:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 0
-; COMBINED-NEXT: [[E1:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 1
-; COMBINED-NEXT: [[E2:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 2
-; COMBINED-NEXT: [[E3:%.*]] = extractelement <4 x float> [[SLOT_SROA_0_0_SLOT_SROA_0_0_VEC]], i64 3
-; COMBINED-NEXT: [[SUM01:%.*]] = fadd float [[E0]], [[E1]]
-; COMBINED-NEXT: [[SUM23:%.*]] = fadd float [[E2]], [[E3]]
+; COMBINED-NEXT: [[SUM01:%.*]] = fadd float [[V0]], [[V1]]
+; COMBINED-NEXT: [[SUM23:%.*]] = fadd float [[V2]], [[V3]]
; COMBINED-NEXT: [[SUM:%.*]] = fadd float [[SUM01]], [[SUM23]]
; COMBINED-NEXT: store float [[SUM]], ptr addrspace(1) [[OUT]], align 4
; COMBINED-NEXT: ret void
>From 9b98b5b058c748b593ac76e94ba8011e7f7ed92c Mon Sep 17 00:00:00 2001
From: mssefat <syadus.sefat at gmail.com>
Date: Sat, 23 May 2026 22:13:11 -0500
Subject: [PATCH 3/3] Fixup strictly smaller store
---
llvm/lib/Transforms/Scalar/SROA.cpp | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 824eaf4d48600..ba76ac29a2b06 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -4488,14 +4488,14 @@ class VecLoadLaneSplitter : public InstVisitor<VecLoadLaneSplitter, bool> {
if (!isStoreCandidate(SI))
return false;
- Type *StoreTy = SI.getValueOperand()->getType();
bool IsContained = StoreSlice.beginOffset() >= LoadSlice.beginOffset() &&
StoreSlice.endOffset() <= LoadSlice.endOffset();
- if (!isa<FixedVectorType>(StoreTy))
- return IsContained;
+ if (!IsContained)
+ return false;
+
uint64_t StoreSize = StoreSlice.endOffset() - StoreSlice.beginOffset();
uint64_t LoadSize = LoadSlice.endOffset() - LoadSlice.beginOffset();
- return IsContained && StoreSize < LoadSize;
+ return StoreSize < LoadSize;
}
// Return true if slice S's use is the address operand of the instruction I.
More information about the llvm-commits
mailing list