[llvm] [SROA] Adjust struct layout requirements for struct to vector canonicalization (PR #201967)
Yonah Goldberg via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 25 08:15:31 PDT 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/201967
>From f3bc07a1980947e62f94a08806c636c0eed9c3de Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 5 Jun 2026 23:12:35 +0000
Subject: [PATCH 1/7] padding fix
---
llvm/lib/Transforms/Scalar/SROA.cpp | 19 +++++++++++++++----
llvm/test/Transforms/SROA/struct-to-vector.ll | 16 ++++++++++++++++
2 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 811dd373eca94..2c39118446357 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5090,9 +5090,10 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
/// Try to canonicalize a homogeneous struct partition to a vector type.
///
-/// We can do this if all the elements of the struct are the same and tightly
-/// packed. This can sometimes eliminate allocas because structs cannot get
-/// promoted to LLVM values, but vectors can.
+/// We can do this if all the elements of the struct are the same and the
+/// corresponding vector has the same byte-level layout. This can sometimes
+/// eliminate allocas because structs cannot get promoted to LLVM values, but
+/// vectors can.
///
/// We only apply this transformation when all users of the alloca are memory
/// intrinsics. Otherwise, if there is a load or store of some other type to the
@@ -5125,9 +5126,19 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
!IsIntegralPointerTy)
return nullptr;
+ // Ensure the struct is tightly packed so that the bit-layout is the same as
+ // the corresponding vector. For example, this prevents a miscompile for
+ // { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
+ // tightly packed elements and trailing padding. So if you loaded an { i5, i5 }
+ // memory layout as <i5, i5> you wouldn't correctly load each element.
+ if (DL.getTypeSizeInBits(EltTy) != DL.getTypeAllocSizeInBits(EltTy))
+ return nullptr;
+
auto *VTy = FixedVectorType::get(EltTy, NumElts);
TypeSize StructSize = DL.getStructLayout(STy)->getSizeInBytes();
- TypeSize VectorSize = DL.getTypeAllocSize(VTy);
+ TypeSize VectorSize = DL.getTypeStoreSize(VTy);
+ // After ruling out per-element padding, make sure a vector load/store
+ // covers the same number of bytes as the struct layout.
if (StructSize != VectorSize)
return nullptr;
diff --git a/llvm/test/Transforms/SROA/struct-to-vector.ll b/llvm/test/Transforms/SROA/struct-to-vector.ll
index ad7d3df129d6c..aeb7421a31f01 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector.ll
@@ -387,3 +387,19 @@ cond.end.ptr:
call void @llvm.lifetime.end.p0(ptr %temp)
ret void
}
+
+%struct.i5x2 = type { i5, i5 }
+define void @struct_i5x2_memcpy_into_alloca(ptr %c) {
+entry:
+ %e = alloca %struct.i5x2, align 1
+ call void @llvm.memcpy.p0.p0.i32(ptr align 1 %e, ptr align 1 %c, i32 2, i1 true)
+ ret void
+}
+
+%struct.i32x3 = type { i32, i32, i32 }
+define void @struct_i32x3_memcpy_into_alloca(ptr %c) {
+entry:
+ %e = alloca %struct.i32x3, align 4
+ call void @llvm.memcpy.p0.p0.i32(ptr align 4 %e, ptr align 4 %c, i32 12, i1 true)
+ ret void
+}
>From 8e95b431b556e0afe7d33571a714cd0693d56b0d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 5 Jun 2026 23:18:33 +0000
Subject: [PATCH 2/7] fix
---
llvm/test/Transforms/SROA/struct-to-vector.ll | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/llvm/test/Transforms/SROA/struct-to-vector.ll b/llvm/test/Transforms/SROA/struct-to-vector.ll
index aeb7421a31f01..b7fb3cc97d66a 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector.ll
@@ -390,6 +390,13 @@ cond.end.ptr:
%struct.i5x2 = type { i5, i5 }
define void @struct_i5x2_memcpy_into_alloca(ptr %c) {
+; CHECK-LABEL: define void @struct_i5x2_memcpy_into_alloca(
+; CHECK-SAME: ptr [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[E:%.*]] = alloca [[STRUCT_I5X2:%.*]], align 1
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[E]], ptr align 1 [[C]], i32 2, i1 true)
+; CHECK-NEXT: ret void
+;
entry:
%e = alloca %struct.i5x2, align 1
call void @llvm.memcpy.p0.p0.i32(ptr align 1 %e, ptr align 1 %c, i32 2, i1 true)
@@ -398,6 +405,14 @@ entry:
%struct.i32x3 = type { i32, i32, i32 }
define void @struct_i32x3_memcpy_into_alloca(ptr %c) {
+; CHECK-LABEL: define void @struct_i32x3_memcpy_into_alloca(
+; CHECK-SAME: ptr [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[E_SROA_0:%.*]] = alloca <3 x i32>, align 16
+; CHECK-NEXT: [[E_SROA_0_0_COPYLOAD:%.*]] = load volatile <3 x i32>, ptr [[C]], align 4
+; CHECK-NEXT: store volatile <3 x i32> [[E_SROA_0_0_COPYLOAD]], ptr [[E_SROA_0]], align 16
+; CHECK-NEXT: ret void
+;
entry:
%e = alloca %struct.i32x3, align 4
call void @llvm.memcpy.p0.p0.i32(ptr align 4 %e, ptr align 4 %c, i32 12, i1 true)
>From c88688a0cf4e6699aab6d01a07849b40fc15a30d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 5 Jun 2026 23:20:02 +0000
Subject: [PATCH 3/7] format
---
llvm/lib/Transforms/Scalar/SROA.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 2c39118446357..29b6935c86857 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5129,8 +5129,8 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
// Ensure the struct is tightly packed so that the bit-layout is the same as
// the corresponding vector. For example, this prevents a miscompile for
// { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
- // tightly packed elements and trailing padding. So if you loaded an { i5, i5 }
- // memory layout as <i5, i5> you wouldn't correctly load each element.
+ // tightly packed elements and trailing padding. So if you loaded an
+ // { i5, i5 } memory layout as <i5, i5> you wouldn't correctly load each element.
if (DL.getTypeSizeInBits(EltTy) != DL.getTypeAllocSizeInBits(EltTy))
return nullptr;
>From 73337a5fb752b3434794bc83fba9423cc3f99803 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Fri, 5 Jun 2026 23:20:16 +0000
Subject: [PATCH 4/7] format
---
llvm/lib/Transforms/Scalar/SROA.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 29b6935c86857..caaba9e82315a 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5130,7 +5130,8 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
// the corresponding vector. For example, this prevents a miscompile for
// { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
// tightly packed elements and trailing padding. So if you loaded an
- // { i5, i5 } memory layout as <i5, i5> you wouldn't correctly load each element.
+ // { i5, i5 } memory layout as <i5, i5> you wouldn't correctly load each
+ // element.
if (DL.getTypeSizeInBits(EltTy) != DL.getTypeAllocSizeInBits(EltTy))
return nullptr;
>From 9f6f2e6196d357bbcaa1d117d8ecd651181fa16f Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Sat, 6 Jun 2026 00:34:44 +0000
Subject: [PATCH 5/7] fix fp-store-only-tail test
---
.../struct-to-vector-fp-store-only-tail.ll | 37 +++++++------------
1 file changed, 14 insertions(+), 23 deletions(-)
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
index d1b7dcab99612..08674a9295d07 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=sroa -S %s | FileCheck %s --check-prefixes=NO-CANON
-; RUN: opt -passes='sroa<aggregate-to-vector>' -S %s | FileCheck %s --check-prefixes=CANON
+; RUN: opt -passes='memcpyopt,sroa<aggregate-to-vector>' -S %s | FileCheck %s --check-prefixes=GOOD
+; RUN: opt -passes='sroa<aggregate-to-vector>,memcpyopt' -S %s | FileCheck %s --check-prefixes=BAD
%class.aiMatrix4x4t = type { float, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }
@@ -17,28 +17,19 @@ declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr no
; uninitialized memory becomes a placeholder value. If you instead delay struct to vector canonicalization and allow memcpyopt
; to run, you'll get a 4-byte store of 0.
define ptr @store_only_fp_tail() {
-; NO-CANON-LABEL: define ptr @store_only_fp_tail() {
-; NO-CANON-NEXT: [[DOTSROA_3:%.*]] = alloca { float, float, float }, align 8
-; NO-CANON-NEXT: [[DOTSROA_4:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; NO-CANON-NEXT: [[DOTSROA_0_SROA_1:%.*]] = alloca { float, float, float }, align 8
-; NO-CANON-NEXT: [[DOTSROA_2:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_3]], ptr align 8 [[DOTSROA_0_SROA_1]], i64 12, i1 false)
-; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_4]], ptr align 8 [[DOTSROA_2]], i64 44, i1 false)
-; NO-CANON-NEXT: store float 0.000000e+00, ptr null, align 1
-; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 4), ptr align 8 [[DOTSROA_3]], i64 12, i1 false)
-; NO-CANON-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
-; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 [[DOTSROA_4]], i64 44, i1 false)
-; NO-CANON-NEXT: ret ptr null
+; GOOD-LABEL: define ptr @store_only_fp_tail() {
+; GOOD-NEXT: store float 0.000000e+00, ptr null, align 1
+; GOOD-NEXT: store <3 x float> undef, ptr getelementptr inbounds (i8, ptr null, i64 4), align 1
+; GOOD-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
+; GOOD-NEXT: store <11 x float> undef, ptr getelementptr inbounds (i8, ptr null, i64 20), align 1
+; GOOD-NEXT: ret ptr null
;
-; CANON-LABEL: define ptr @store_only_fp_tail() {
-; CANON-NEXT: [[DOTSROA_4:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; CANON-NEXT: [[DOTSROA_2:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_4]], ptr align 8 [[DOTSROA_2]], i64 44, i1 false)
-; CANON-NEXT: [[DST_SROA_0_0_VEC_INSERT:%.*]] = insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 0
-; CANON-NEXT: store <4 x float> [[DST_SROA_0_0_VEC_INSERT]], ptr null, align 1
-; CANON-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
-; CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 [[DOTSROA_4]], i64 44, i1 false)
-; CANON-NEXT: ret ptr null
+; BAD-LABEL: define ptr @store_only_fp_tail() {
+; BAD-NEXT: [[DST_SROA_0_0_VEC_INSERT:%.*]] = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+; BAD-NEXT: store <4 x float> [[DST_SROA_0_0_VEC_INSERT]], ptr null, align 1
+; BAD-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
+; BAD-NEXT: store <11 x float> undef, ptr getelementptr inbounds (i8, ptr null, i64 20), align 1
+; BAD-NEXT: ret ptr null
;
%dst = alloca %class.aiMatrix4x4t, align 4
%src = alloca %class.aiMatrix4x4t, align 4
>From 1e4219a6fd339800a4cba8de81fda59d0553a336 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Tue, 23 Jun 2026 20:14:30 +0000
Subject: [PATCH 6/7] [SROA] Keep fp-store-only-tail test shape
---
.../struct-to-vector-fp-store-only-tail.ll | 34 +++++++++++--------
1 file changed, 20 insertions(+), 14 deletions(-)
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
index 08674a9295d07..aa5e7b10a089b 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='memcpyopt,sroa<aggregate-to-vector>' -S %s | FileCheck %s --check-prefixes=GOOD
-; RUN: opt -passes='sroa<aggregate-to-vector>,memcpyopt' -S %s | FileCheck %s --check-prefixes=BAD
+; RUN: opt -passes=sroa -S %s | FileCheck %s --check-prefixes=NO-CANON
+; RUN: opt -passes='sroa<aggregate-to-vector>' -S %s | FileCheck %s --check-prefixes=CANON
%class.aiMatrix4x4t = type { float, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }
@@ -17,19 +17,25 @@ declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr no
; uninitialized memory becomes a placeholder value. If you instead delay struct to vector canonicalization and allow memcpyopt
; to run, you'll get a 4-byte store of 0.
define ptr @store_only_fp_tail() {
-; GOOD-LABEL: define ptr @store_only_fp_tail() {
-; GOOD-NEXT: store float 0.000000e+00, ptr null, align 1
-; GOOD-NEXT: store <3 x float> undef, ptr getelementptr inbounds (i8, ptr null, i64 4), align 1
-; GOOD-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
-; GOOD-NEXT: store <11 x float> undef, ptr getelementptr inbounds (i8, ptr null, i64 20), align 1
-; GOOD-NEXT: ret ptr null
+; NO-CANON-LABEL: define ptr @store_only_fp_tail() {
+; NO-CANON-NEXT: [[DOTSROA_3:%.*]] = alloca { float, float, float }, align 8
+; NO-CANON-NEXT: [[DOTSROA_4:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
+; NO-CANON-NEXT: [[DOTSROA_0_SROA_1:%.*]] = alloca { float, float, float }, align 8
+; NO-CANON-NEXT: [[DOTSROA_2:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
+; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_3]], ptr align 8 [[DOTSROA_0_SROA_1]], i64 12, i1 false)
+; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_4]], ptr align 8 [[DOTSROA_2]], i64 44, i1 false)
+; NO-CANON-NEXT: store float 0.000000e+00, ptr null, align 1
+; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 4), ptr align 8 [[DOTSROA_3]], i64 12, i1 false)
+; NO-CANON-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
+; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 [[DOTSROA_4]], i64 44, i1 false)
+; NO-CANON-NEXT: ret ptr null
;
-; BAD-LABEL: define ptr @store_only_fp_tail() {
-; BAD-NEXT: [[DST_SROA_0_0_VEC_INSERT:%.*]] = insertelement <4 x float> undef, float 0.000000e+00, i32 0
-; BAD-NEXT: store <4 x float> [[DST_SROA_0_0_VEC_INSERT]], ptr null, align 1
-; BAD-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
-; BAD-NEXT: store <11 x float> undef, ptr getelementptr inbounds (i8, ptr null, i64 20), align 1
-; BAD-NEXT: ret ptr null
+; CANON-LABEL: define ptr @store_only_fp_tail() {
+; CANON-NEXT: [[DST_SROA_0_0_VEC_INSERT:%.*]] = insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 0
+; CANON-NEXT: store <4 x float> [[DST_SROA_0_0_VEC_INSERT]], ptr null, align 1
+; CANON-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
+; CANON-NEXT: store <11 x float> {{.*}}, ptr getelementptr inbounds (i8, ptr null, i64 20), align 1
+; CANON-NEXT: ret ptr null
;
%dst = alloca %class.aiMatrix4x4t, align 4
%src = alloca %class.aiMatrix4x4t, align 4
>From d40ed4794af401acfb0557c59c9c1a2e7549a963 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 25 Jun 2026 15:15:11 +0000
Subject: [PATCH 7/7] shorten comment
---
llvm/lib/Transforms/Scalar/SROA.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index caaba9e82315a..c3936b49027fb 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5129,9 +5129,7 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
// Ensure the struct is tightly packed so that the bit-layout is the same as
// the corresponding vector. For example, this prevents a miscompile for
// { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
- // tightly packed elements and trailing padding. So if you loaded an
- // { i5, i5 } memory layout as <i5, i5> you wouldn't correctly load each
- // element.
+ // tightly packed elements and trailing padding.
if (DL.getTypeSizeInBits(EltTy) != DL.getTypeAllocSizeInBits(EltTy))
return nullptr;
More information about the llvm-commits
mailing list