[clang] [llvm] [SROA] Canonicalize homogeneous structs to fixed vectors (opt-in, after memcpyopt) (PR #165159)
Yaxun Liu via cfe-commits
cfe-commits at lists.llvm.org
Mon Jun 1 08:06:57 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/165159
>From 0f423e5c5d6a66d5487bcc0357b18712de9662d2 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 14 Apr 2026 09:48:54 -0400
Subject: [PATCH 01/13] [SROA] Canonicalize homogeneous structs to vectors with
guarded fallback
When SROA's getTypePartition yields a homogeneous struct (no padding, no
pointers, no i1 fields), canonicalize it to a fixed vector (e.g.
{ i64, i64 } -> <2 x i64>) so allocas can promote through vector load/store
patterns such as std::function swap with three memcpys.
Introduce shouldCanonicalizeHomogeneousStructToVector for the struct-to-vector
fallback after normal promotion paths fail: require a non-splittable
whole-partition use, reject non-splittable sub-element loads, and for i64
homogeneous partitions recover splittable MemIntrinsic transfers only for
interior subaggregates or full-record integer aggregates of at least 32 bytes
(avoiding the broad 16-byte memcpy bucket that reopened llvm-opt-benchmark
regressions).
Add LLVM_DEBUG tracing for partition-type decisions, extend SROA lit coverage,
update DebugInfo SROA FileChecks for struct-to-vector codegen, and regenerate
NVPTX lower-byval-args.ll expectations (@memcpy_to_param) using llc from
llvm-dev Docker.
---
clang/test/CodeGenOpenCL/nullptr.cl | 4 +-
llvm/lib/Transforms/Scalar/SROA.cpp | 254 ++++++++++-
.../assignment-tracking/sroa/user-memcpy.ll | 28 +-
.../DebugInfo/Generic/sroa-alloca-offset.ll | 18 +-
llvm/test/DebugInfo/X86/sroasplit-4.ll | 16 +-
.../struct-to-vector-fp-store-only-tail.ll | 39 ++
.../struct-to-vector-mapok-extra-alloca.ll | 35 ++
.../SROA/struct-to-vector-subpartition.ll | 76 ++++
llvm/test/Transforms/SROA/struct-to-vector.ll | 396 ++++++++++++++++++
llvm/test/Transforms/SROA/tbaa-struct3.ll | 9 +-
10 files changed, 829 insertions(+), 46 deletions(-)
create mode 100644 llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
create mode 100644 llvm/test/Transforms/SROA/struct-to-vector-mapok-extra-alloca.ll
create mode 100644 llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
create mode 100644 llvm/test/Transforms/SROA/struct-to-vector.ll
diff --git a/clang/test/CodeGenOpenCL/nullptr.cl b/clang/test/CodeGenOpenCL/nullptr.cl
index 976e12c0bef47..f45df110ec243 100644
--- a/clang/test/CodeGenOpenCL/nullptr.cl
+++ b/clang/test/CodeGenOpenCL/nullptr.cl
@@ -597,10 +597,10 @@ typedef struct {
} StructTy3;
// CHECK-LABEL: test_memset_private
-// SPIR64: call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) %ptr, i8 0, i64 32, i1 false)
+// SPIR64: store <4 x i64> zeroinitializer, ptr %ptr, align 8
// SPIR64: [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr %ptr, i64 32
// SPIR64: store ptr addrspacecast (ptr addrspace(4) null to ptr), ptr [[GEP]], align 8
-// AMDGCN: call void @llvm.memset.p5.i64(ptr addrspace(5) noundef align 8 {{.*}}, i8 0, i64 32, i1 false)
+// AMDGCN: store <4 x i64> zeroinitializer, ptr addrspace(5) %ptr, align 8
// AMDGCN: [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) %ptr, i32 32
// AMDGCN: store ptr addrspace(5) addrspacecast (ptr null to ptr addrspace(5)), ptr addrspace(5) [[GEP]]
// AMDGCN: [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) {{.*}}, i32 36
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 83e40edb64541..5991db4b992f1 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5086,6 +5086,162 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
return true;
}
+/// Try to canonicalize a homogeneous, tightly-packed struct to a vector type.
+///
+/// For structs where all elements have the same type and are tightly packed
+/// (no padding), we can represent them as a fixed vector which enables better
+/// optimization (e.g., vector selects instead of memcpy).
+///
+/// \param STy The struct type to try to canonicalize.
+/// \param DL The DataLayout for size/alignment queries.
+/// \returns The equivalent vector type, or nullptr if not applicable.
+static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
+ const DataLayout &DL) {
+ unsigned NumElts = STy->getNumElements();
+ if (NumElts != 2 && NumElts != 4)
+ return nullptr;
+
+ // All elements must be the same type.
+ Type *EltTy = STy->getElementType(0);
+ for (unsigned I = 1; I < NumElts; ++I)
+ if (STy->getElementType(I) != EltTy)
+ return nullptr;
+
+ // Element type must be valid for vectors.
+ if (!VectorType::isValidElementType(EltTy))
+ return nullptr;
+
+ // Only allow integer types >= 8 bits or floating point.
+ if (auto *IT = dyn_cast<IntegerType>(EltTy)) {
+ if (IT->getBitWidth() < 8)
+ return nullptr;
+ } else if (!EltTy->isFloatingPointTy()) {
+ return nullptr;
+ }
+
+ // Element size must be fixed and non-zero.
+ TypeSize EltTS = DL.getTypeAllocSize(EltTy);
+ if (!EltTS.isFixed())
+ return nullptr;
+ uint64_t EltSize = EltTS.getFixedValue();
+ if (EltSize < 1)
+ return nullptr;
+
+ const StructLayout *SL = DL.getStructLayout(STy);
+ uint64_t StructSize = SL->getSizeInBytes();
+ if (StructSize == 0)
+ return nullptr;
+
+ // Must be tightly packed: size == NumElts * EltSize.
+ if (StructSize != NumElts * EltSize)
+ return nullptr;
+
+ // Verify each element is at the expected offset (no padding).
+ for (unsigned I = 0; I < NumElts; ++I)
+ if (SL->getElementOffset(I) != I * EltSize)
+ return nullptr;
+
+ return FixedVectorType::get(EltTy, NumElts);
+}
+
+/// Decide whether it is profitable to canonicalize a homogeneous struct
+/// partition to a vector after the usual promotion choices have already failed.
+///
+/// This helper is intentionally conservative: we only canonicalize when the
+/// partition has a non-splittable whole-partition use and does not have any
+/// non-splittable sub-element loads.
+///
+/// Default allow case:
+/// - a real whole-partition use, because that is the clearest signal that a
+/// vector type can carry a whole value profitably.
+///
+/// Default reject case:
+/// - sub-element loads, because they usually turn the vector back into lane
+/// extraction traffic.
+///
+/// We also recover a narrow class of memcpy-only integer cases, even though
+/// they are classified as splittable rather than whole-partition uses:
+/// - interior i64 partitions within a larger fixed-size allocation
+/// - original full-record integer aggregates of size >= 32 bytes
+///
+/// Rationale:
+/// - Some memcpy-only integer cases are real wins and stay in whole-value form
+/// after canonicalization.
+/// - But recovering all full-record memcpy-only cases is too broad: an
+/// original full-record 16-byte {i64, i64} helper can look locally
+/// profitable while still making later code generation worse.
+/// - So the fallback stays narrow and uses the current partition's offsets plus
+/// allocation size to distinguish the safer recovered cases from the broader
+/// risky bucket.
+///
+/// Intuition:
+/// - Good: whole-value traffic can benefit from a vector type.
+/// %tmp = load { i64, i64 }, ptr %src
+/// store { i64, i64 } %tmp, ptr %dst
+/// Canonicalizing to <2 x i64> exposes a single whole-value load/store.
+///
+/// - Bad: field-by-field reads become extractelement traffic.
+/// %x = load i32, ptr %p
+/// %y = load i32, ptr (gep %p, 4)
+/// Canonicalizing { i32, i32 } to <2 x i32> only adds lane extraction.
+///
+/// - Bad: a store-only FP tail can seed later SLP divergence without a clear
+/// SROA win.
+/// store float %a, ptr %p0
+/// store float %b, ptr %p1
+/// store float %c, ptr %p2
+/// store float %d, ptr %p3
+/// Canonicalizing this to a temporary <4 x float> store was enough to change
+/// later vectorization without a clear benefit.
+static bool shouldCanonicalizeHomogeneousStructToVector(Partition &P,
+ const DataLayout &DL,
+ AllocaInst &AI,
+ bool IsI64Candidate) {
+ bool HasWholePartitionUse = false;
+ bool HasSubElementLoad = false;
+ bool HasRecoverableSplittableTransfer = false;
+ std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL);
+ bool IsInteriorSubaggregate = AllocSize && AllocSize->isFixed() &&
+ P.beginOffset() != 0 &&
+ P.endOffset() < AllocSize->getFixedValue();
+ bool IsOriginalFullRecord = P.beginOffset() == 0 && AllocSize &&
+ AllocSize->isFixed() &&
+ AllocSize->getFixedValue() == P.size();
+
+ for (const Slice &S : P) {
+ if (S.isDead())
+ continue;
+
+ auto *U = S.getUse();
+ if (!U)
+ continue;
+
+ if (S.isSplittable()) {
+ if (IsI64Candidate && IsInteriorSubaggregate &&
+ S.beginOffset() == P.beginOffset() &&
+ S.endOffset() == P.endOffset() && isa<MemIntrinsic>(U->getUser()))
+ HasRecoverableSplittableTransfer = true;
+ if (IsI64Candidate && IsOriginalFullRecord && P.size() >= 32 &&
+ S.beginOffset() == P.beginOffset() &&
+ S.endOffset() == P.endOffset() && isa<MemIntrinsic>(U->getUser()))
+ HasRecoverableSplittableTransfer = true;
+ continue;
+ }
+
+ uint64_t SliceSize = S.endOffset() - S.beginOffset();
+ if (SliceSize < P.size()) {
+ if (isa<LoadInst>(U->getUser()))
+ HasSubElementLoad = true;
+ continue;
+ }
+
+ HasWholePartitionUse = true;
+ }
+
+ return (HasWholePartitionUse || HasRecoverableSplittableTransfer) &&
+ !HasSubElementLoad;
+}
+
/// Select a partition type for an alloca partition.
///
/// Try to compute a friendly type for this partition of the alloca. This
@@ -5100,6 +5256,26 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
static std::tuple<Type *, bool, VectorType *>
selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
LLVMContext &C) {
+ auto LogChoice = [&](StringRef Path, Type *ChosenTy, VectorType *ChosenVecTy,
+ bool ChosenIntWidening) {
+ LLVM_DEBUG({
+ dbgs() << "selectPartitionType path=" << Path
+ << " func=" << AI.getFunction()->getName() << " alloca=";
+ if (AI.hasName())
+ dbgs() << AI.getName();
+ else
+ dbgs() << "<unnamed>";
+ dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
+ << ") size=" << P.size();
+ if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
+ dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
+ if (ChosenTy)
+ dbgs() << " chosen=" << *ChosenTy;
+ if (ChosenVecTy)
+ dbgs() << " vec=" << *ChosenVecTy;
+ dbgs() << " intwiden=" << ChosenIntWidening << "\n";
+ });
+ };
// First check if the partition is viable for vector promotion.
//
// We prefer vector promotion over integer widening promotion when:
@@ -5116,8 +5292,10 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
// promotion. If the vector has one element, let the below code select
// whether we promote with the vector or scalar.
if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
- VecTy->getElementCount().getFixedValue() > 1)
+ VecTy->getElementCount().getFixedValue() > 1) {
+ LogChoice("direct-fp-vecty", VecTy, VecTy, false);
return {VecTy, false, VecTy};
+ }
// Check if there is a common type that all slices of the partition use that
// spans the partition.
@@ -5129,10 +5307,13 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
// We prefer vector promotion here because if vector promotion is viable
// and there is a common type used, then it implies the second listed
// condition for preferring vector promotion is true.
- if (VecTy)
+ if (VecTy) {
+ LogChoice("common-type-vecty", VecTy, VecTy, false);
return {VecTy, false, VecTy};
- return {CommonUseTy, isIntegerWideningViable(P, CommonUseTy, DL),
- nullptr};
+ }
+ bool IntWiden = isIntegerWideningViable(P, CommonUseTy, DL);
+ LogChoice("common-type", CommonUseTy, nullptr, IntWiden);
+ return {CommonUseTy, IntWiden, nullptr};
}
}
@@ -5148,32 +5329,83 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
DL.isLegalInteger(P.size() * 8))
TypePartitionTy = Type::getIntNTy(C, P.size() * 8);
// There was no common type used, so we prefer integer widening promotion.
- if (isIntegerWideningViable(P, TypePartitionTy, DL))
+ if (isIntegerWideningViable(P, TypePartitionTy, DL)) {
+ LogChoice("type-partition-intwiden", TypePartitionTy, nullptr, true);
return {TypePartitionTy, true, nullptr};
- if (VecTy)
+ }
+ if (VecTy) {
+ LogChoice("type-partition-vecty", VecTy, VecTy, false);
return {VecTy, false, VecTy};
+ }
// If we couldn't promote with TypePartitionTy, try with the largest
// integer type used.
if (LargestIntTy &&
DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size() &&
- isIntegerWideningViable(P, LargestIntTy, DL))
+ isIntegerWideningViable(P, LargestIntTy, DL)) {
+ LogChoice("largest-int-intwiden", LargestIntTy, nullptr, true);
return {LargestIntTy, true, nullptr};
+ }
+
+ // Try homogeneous struct to vector canonicalization.
+ //
+ // This is intentionally more conservative than tryCanonicalize...
+ // alone: after the normal promotion paths above fail, we only want the
+ // struct-to-vector fallback when it can expose a real whole-value use,
+ // not when it would merely create vector lanes that later passes have to
+ // unpack again.
+ if (auto *STy = dyn_cast<StructType>(TypePartitionTy))
+ if (auto *VTy = tryCanonicalizeStructToVector(STy, DL)) {
+ // Recover only a narrow integer memcpy-only subset here. Broader
+ // fallback recovery was too aggressive:
+ // - full-record 16-byte { i64, i64 } cases reopened regressions
+ // - FP and lane-oriented cases were more likely to trigger bad later
+ // codegen
+ //
+ // So this path is restricted to 64-bit integer lanes, and only for
+ // interior partitions or original full records of size >= 32 bytes.
+ bool AllowStructFallback = shouldCanonicalizeHomogeneousStructToVector(
+ P, DL, AI, VTy->getElementType()->isIntegerTy(64));
+ LLVM_DEBUG({
+ dbgs() << "selectPartitionType struct-fallback-candidate"
+ << " func=" << AI.getFunction()->getName() << " alloca=";
+ if (AI.hasName())
+ dbgs() << AI.getName();
+ else
+ dbgs() << "<unnamed>";
+ dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
+ << ") size=" << P.size() << " type-partition=" << *STy
+ << " candidate-vec=" << *VTy
+ << " allow=" << AllowStructFallback << "\n";
+ });
+ if (AllowStructFallback) {
+ LogChoice("struct-fallback-vecty", VTy, nullptr, false);
+ return {VTy, false, nullptr};
+ }
+ }
// Fallback to TypePartitionTy and we probably won't promote.
+ LogChoice("type-partition-fallback", TypePartitionTy, nullptr, false);
return {TypePartitionTy, false, nullptr};
}
// Select the largest integer type used if it spans the partition.
if (LargestIntTy &&
- DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size())
+ DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size()) {
+ LogChoice("largest-int-fallback", LargestIntTy, nullptr, false);
return {LargestIntTy, false, nullptr};
+ }
// Select a legal integer type if it spans the partition.
- if (DL.isLegalInteger(P.size() * 8))
- return {Type::getIntNTy(C, P.size() * 8), false, nullptr};
+ if (DL.isLegalInteger(P.size() * 8)) {
+ Type *IntTy = Type::getIntNTy(C, P.size() * 8);
+ LogChoice("legal-int-fallback", IntTy, nullptr, false);
+ return {IntTy, false, nullptr};
+ }
// Fallback to an i8 array.
- return {ArrayType::get(Type::getInt8Ty(C), P.size()), false, nullptr};
+ Type *ArrayTy = ArrayType::get(Type::getInt8Ty(C), P.size());
+ LogChoice("byte-array-fallback", ArrayTy, nullptr, false);
+ return {ArrayTy, false, nullptr};
}
/// Rewrite an alloca partition's users.
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
index ded78f4ff83f4..5997cc1b9d041 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
@@ -21,31 +21,29 @@
;; Allocas have been promoted - the linked dbg.assigns have been removed.
;; | V3i point = {0, 0, 0};
-; CHECK-NEXT: #dbg_value(i64 0, ![[point:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64),
-; CHECK-NEXT: #dbg_value(i64 0, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64),
+;; Homogeneous i64 struct is split; point.x/point.y are described as separate i64 fragments.
+; CHECK-NEXT: #dbg_value(i64 0, ![[point:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 0, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
;; point.z = 5000;
-; CHECK-NEXT: #dbg_value(i64 5000, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64),
+; CHECK-NEXT: #dbg_value(i64 5000, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
;; | V3i other = {10, 9, 8};
;; other is global const:
;; local.other.x = global.other.x
;; local.other.y = global.other.y
;; local.other.z = global.other.z
-; CHECK-NEXT: %other.sroa.0.0.copyload = load i64, ptr @__const._Z3funv.other
-; CHECK-NEXT: %other.sroa.2.0.copyload = load i64, ptr getelementptr inbounds (i8, ptr @__const._Z3funv.other, i64 8)
-; CHECK-NEXT: %other.sroa.3.0.copyload = load i64, ptr getelementptr inbounds (i8, ptr @__const._Z3funv.other, i64 16)
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[other:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64),
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 64, 64),
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.3.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 128, 64),
+; CHECK-NEXT: %other.sroa.0.0.copyload = load i64, ptr @__const._Z3funv.other, align 8, !dbg !{{[0-9]+}}
+; CHECK-NEXT: %other.sroa.2.0.copyload = load i64, ptr getelementptr inbounds (i8, ptr @__const._Z3funv.other, i64 8), align 8, !dbg !{{[0-9]+}}
+; CHECK-NEXT: %other.sroa.3.0.copyload = load i64, ptr getelementptr inbounds (i8, ptr @__const._Z3funv.other, i64 16), align 8, !dbg !{{[0-9]+}}
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[other:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.3.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
;; | std::memcpy(&point.y, &other.x, sizeof(long) * 2);
-;; other is now 3 scalars:
-;; point.y = other.x
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64),
-;;
-;; point.z = other.y
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64),
+;; memcpy is folded into scalar dbg_value updates for point.y/point.z.
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
; CHECK: ![[point]] = !DILocalVariable(name: "point",
; CHECK: ![[other]] = !DILocalVariable(name: "other",
diff --git a/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll b/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
index 6718711f83e04..40c57730b1557 100644
--- a/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
+++ b/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
@@ -140,16 +140,18 @@ entry:
;; 16 bit variable f (!62): value vgf (lower bits)
;; 16 bit variable g (!63): value vgf (upper bits)
;;
-;; 16 bit variable h (!64): deref dead_64_128
-; COMMON-NEXT: %[[dead_64_128:.*]] = alloca %struct.two
-; COMMON-NEXT: #dbg_declare(ptr %[[dead_64_128]], ![[h:[0-9]+]], !DIExpression(),
-; COMMON-NEXT: %[[ve:.*]] = load i32, ptr @gf
+;; 16 bit variable h (!64): inner aggregate slice kept on stack; #dbg_declare on that alloca.
+; COMMON-NEXT: %[[h_alloca:.*]] = alloca %struct.two, align 8
+; COMMON-NEXT: #dbg_declare(ptr %[[h_alloca]], ![[h:[0-9]+]], !DIExpression(), !{{[0-9]+}})
+; COMMON-NEXT: %[[ve:.*]] = load i32, ptr @gf, align 4{{.*}}
;; FIXME: mem2reg bug - offset is incorrect - see comment above.
-; COMMON-NEXT: #dbg_value(i32 %[[ve]], ![[e:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2),
-; COMMON-NEXT: %[[vfg:.*]] = load i32, ptr getelementptr inbounds (i8, ptr @gf, i64 4)
-; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[f:[0-9]+]], !DIExpression(),
+; COMMON-NEXT: #dbg_value(i32 %[[ve]], ![[e:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2), !{{[0-9]+}})
+; COMMON-NEXT: %[[vfg:.*]] = load i32, ptr getelementptr inbounds (i8, ptr @gf, i64 4), align 4{{.*}}
+; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[f:[0-9]+]], !DIExpression(), !{{[0-9]+}})
;; FIXME: mem2reg bug - offset is incorrect - see comment above.
-; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[g:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2),
+; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[g:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2), !{{[0-9]+}})
+; COMMON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[h_alloca]], ptr align 4 getelementptr inbounds (i8, ptr @gf, i64 8), i64 8, i1 false){{.*}}
+; COMMON-NEXT: ret i32 %[[vfg]]{{.*}}
define dso_local noundef i32 @_Z4fun3v() #0 !dbg !55 {
entry:
%0 = alloca %struct.four, align 4
diff --git a/llvm/test/DebugInfo/X86/sroasplit-4.ll b/llvm/test/DebugInfo/X86/sroasplit-4.ll
index d5ce348e9896e..f483d60ad7e5b 100644
--- a/llvm/test/DebugInfo/X86/sroasplit-4.ll
+++ b/llvm/test/DebugInfo/X86/sroasplit-4.ll
@@ -1,12 +1,16 @@
; RUN: opt -passes='sroa' < %s -S -o - | FileCheck %s
;
; Test that recursively splitting an alloca updates the debug info correctly.
-; CHECK: %[[T:.*]] = load i64, ptr @t, align 8
-; CHECK: #dbg_value(i64 %[[T]], ![[Y:.*]], !DIExpression(DW_OP_LLVM_fragment, 0, 64),
-; CHECK: %[[T1:.*]] = load i64, ptr @t, align 8
-; CHECK: #dbg_value(i64 %[[T1]], ![[Y]], !DIExpression(DW_OP_LLVM_fragment, 64, 64),
-; CHECK: #dbg_value(i64 %[[T]], ![[R:.*]], !DIExpression(DW_OP_LLVM_fragment, 192, 64),
-; CHECK: #dbg_value(i64 %[[T1]], ![[R]], !DIExpression(DW_OP_LLVM_fragment, 256, 64),
+; CHECK-LABEL: if.end:
+; CHECK-NEXT: %[[T0:.*]] = load i64, ptr @t, align 8{{.*}}
+; CHECK-NEXT: #dbg_value(i64 %[[T0]], ![[Y:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64), !{{[0-9]+}})
+; CHECK-NEXT: %[[T1:.*]] = load i64, ptr @t, align 8{{.*}}
+; CHECK-NEXT: #dbg_value(i64 %[[T1]], ![[Y]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i32 0, ![[R:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 32), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 0, ![[R]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 0, ![[R]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 %[[T0]], ![[R]], !DIExpression(DW_OP_LLVM_fragment, 192, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 %[[T1]], ![[R]], !DIExpression(DW_OP_LLVM_fragment, 256, 64), !{{[0-9]+}})
;
; struct p {
; __SIZE_TYPE__ s;
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
new file mode 100644
index 0000000000000..f223c2e8efa70
--- /dev/null
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -0,0 +1,39 @@
+; RUN: opt -passes=sroa -S %s | FileCheck %s
+; NOTE: Do not autogenerate. This regression test checks a specific store-only
+; homogeneous float slice pattern that current SROA can over-vectorize.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+%class.aiMatrix4x4t = type { float, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }
+
+; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
+
+; This reduced case exercises a homogeneous FP tail slice that is only stored.
+; The fixed behavior keeps the scalar/memcpy shape; the buggy behavior deletes
+; the 3-float temporary and replaces it with a non-const vector store
+; (`store <4 x float> %...`) seeded by FP struct-to-vector canonicalization.
+;
+; CHECK-LABEL: define ptr @store_only_fp_tail()
+; CHECK: %.sroa.3 = alloca { float, float, float }, align 8
+; CHECK: %.sroa.4 = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
+; CHECK: %.sroa.0.sroa.1 = alloca { float, float, float }, align 8
+; CHECK: %.sroa.2 = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %.sroa.3, ptr align 8 %.sroa.0.sroa.1, i64 12, i1 false)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %.sroa.4, ptr align 8 %.sroa.2, i64 44, i1 false)
+; CHECK: store float 0.000000e+00, ptr null, align 1
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 4), ptr align 8 %.sroa.3, i64 12, i1 false)
+; CHECK: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 %.sroa.4, i64 44, i1 false)
+; CHECK-NOT: store <4 x float> %
+define ptr @store_only_fp_tail() {
+ %1 = alloca %class.aiMatrix4x4t, align 4
+ %2 = alloca %class.aiMatrix4x4t, align 4
+ %3 = getelementptr i8, ptr %2, i64 16
+ store float 0.000000e+00, ptr %3, align 4
+ call void @llvm.memcpy.p0.p0.i64(ptr %1, ptr %2, i64 64, i1 false)
+ store float 0.000000e+00, ptr %1, align 4
+ call void @llvm.memcpy.p0.p0.i64(ptr null, ptr %1, i64 64, i1 false)
+ ret ptr null
+}
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-mapok-extra-alloca.ll b/llvm/test/Transforms/SROA/struct-to-vector-mapok-extra-alloca.ll
new file mode 100644
index 0000000000000..f9ce644156b31
--- /dev/null
+++ b/llvm/test/Transforms/SROA/struct-to-vector-mapok-extra-alloca.ll
@@ -0,0 +1,35 @@
+; RUN: opt -passes='default<O3>' -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: define void @"_ZN102_$LT$futures_util..stream..try_stream..MapOk$LT$St$C$F$GT$$u20$as$u20$futures_core..stream..Stream$GT$9poll_next17h555df33481d9c33cE"
+; CHECK: [[TMP:%.*]] = alloca [11 x i64], align 8
+; CHECK: call void @llvm.lifetime.start.p0(ptr nonnull [[TMP]])
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(88) [[TMP]], ptr noundef nonnull align 1 dereferenceable(88) %0, i64 88, i1 false)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(88) %0, ptr noundef nonnull align 8 dereferenceable(88) [[TMP]], i64 88, i1 false)
+; CHECK: call void @llvm.lifetime.end.p0(ptr nonnull [[TMP]])
+define void @"_ZN102_$LT$futures_util..stream..try_stream..MapOk$LT$St$C$F$GT$$u20$as$u20$futures_core..stream..Stream$GT$9poll_next17h555df33481d9c33cE"(ptr %0) {
+ call void @"_ZN101_$LT$futures_util..stream..stream..map..Map$LT$St$C$F$GT$$u20$as$u20$futures_core..stream..Stream$GT$9poll_next17h5aa844c062b2077eE"(ptr %0)
+ ret void
+}
+
+; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg) #0
+
+define void @"_ZN101_$LT$futures_util..stream..stream..map..Map$LT$St$C$F$GT$$u20$as$u20$futures_core..stream..Stream$GT$9poll_next17h5aa844c062b2077eE"(ptr %0) {
+ %.sroa.5 = alloca [11 x i64], align 8
+ %2 = load i64, ptr %0, align 8
+ %3 = icmp eq i64 %2, 0
+ br i1 %3, label %"_ZN122_$LT$futures_util..fns..MapOkFn$LT$F$GT$$u20$as$u20$futures_util..fns..FnMut1$LT$core..result..Result$LT$T$C$E$GT$$GT$$GT$8call_mut17h252763b5559d12fbE.exit", label %4
+
+4: ; preds = %1
+ call void @llvm.memcpy.p0.p0.i64(ptr %.sroa.5, ptr %0, i64 88, i1 false)
+ br label %"_ZN122_$LT$futures_util..fns..MapOkFn$LT$F$GT$$u20$as$u20$futures_util..fns..FnMut1$LT$core..result..Result$LT$T$C$E$GT$$GT$$GT$8call_mut17h252763b5559d12fbE.exit"
+
+"_ZN122_$LT$futures_util..fns..MapOkFn$LT$F$GT$$u20$as$u20$futures_util..fns..FnMut1$LT$core..result..Result$LT$T$C$E$GT$$GT$$GT$8call_mut17h252763b5559d12fbE.exit": ; preds = %4, %1
+ call void @llvm.memcpy.p0.p0.i64(ptr %0, ptr %.sroa.5, i64 88, i1 false)
+ ret void
+}
+
+attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
new file mode 100644
index 0000000000000..fafded012ae4a
--- /dev/null
+++ b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
@@ -0,0 +1,76 @@
+; RUN: opt -passes=sroa -S %s | FileCheck %s
+; NOTE: Do not autogenerate. This test intentionally uses targeted CHECK
+; patterns for clarity.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; When SROA splits { ptr, i64, i64, i64 } into [0,8), [8,16), [16,32),
+; the [16,32) partition type from getTypePartition is { i64, i64 }.
+; The current whole-use heuristic intentionally does NOT canonicalize this
+; sub-partition to <2 x i64>, because the [16,32) slice is only touched by
+; splittable memcpy traffic and has no non-splittable whole-partition use.
+; Keeping it scalar avoids creating a vectorized temporary that later passes
+; may not be able to promote away.
+
+; CHECK-LABEL: define void @test_subpartition_type(
+; CHECK: %a.sroa.6 = alloca { i64, i64 }, align 8
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a.sroa.6, ptr align 8 %a.sroa.6.0.src.sroa_idx, i64 16, i1 false)
+; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a.sroa.6.0.dst.sroa_idx, ptr align 8 %a.sroa.6, i64 16, i1 false)
+define void @test_subpartition_type(ptr %src, ptr %dst) {
+entry:
+ %a = alloca { ptr, i64, i64, i64 }, align 8
+ call void @llvm.lifetime.start.p0(i64 32, ptr %a)
+
+ ; Copy all 32 bytes from src into %a (splittable)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a, ptr align 8 %src, i64 32, i1 false)
+
+ ; Load ptr at [0,8) -- forces partition boundary at 8
+ %p = load ptr, ptr %a, align 8
+
+ ; Load i64 at [8,16) -- forces partition boundary at 16
+ %gep.a.8 = getelementptr inbounds i8, ptr %a, i64 8
+ %v1 = load i64, ptr %gep.a.8, align 8
+
+ ; Only splittable memcpy uses touch [16,32), so SROA creates a single
+ ; [16,32) partition. getTypePartition returns { i64, i64 } for this, but
+ ; the whole-use heuristic keeps it in scalar/memcpy form.
+
+ ; Copy all 32 bytes from %a to dst (splittable)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %a, i64 32, i1 false)
+
+ call void @llvm.lifetime.end.p0(i64 32, ptr %a)
+ ret void
+}
+
+; Element-wise { double, double } access through a phi.
+; The phi between two allocas prevents SROA slice analysis
+; ("A pointer to this alloca escaped"), so the allocas survive.
+
+; CHECK-LABEL: define void @test_elementwise_phi(
+; CHECK-NOT: <2 x double>
+define void @test_elementwise_phi(ptr %src0, ptr %src1, i1 %cond, ptr %dst) {
+entry:
+ %a = alloca { double, double }, align 8
+ %b = alloca { double, double }, align 8
+ %a.1 = getelementptr inbounds i8, ptr %a, i64 8
+ %b.1 = getelementptr inbounds i8, ptr %b, i64 8
+ %v0 = load double, ptr %src0, align 8
+ %v1 = load double, ptr %src1, align 8
+ store double %v0, ptr %a, align 8
+ store double %v1, ptr %a.1, align 8
+ store double 0.0, ptr %b, align 8
+ store double 0.0, ptr %b.1, align 8
+ br i1 %cond, label %if.then, label %if.else
+
+if.then:
+ br label %merge
+
+if.else:
+ br label %merge
+
+merge:
+ %sel = phi ptr [ %a, %if.then ], [ %b, %if.else ]
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %sel, i64 16, i1 false)
+ ret void
+}
diff --git a/llvm/test/Transforms/SROA/struct-to-vector.ll b/llvm/test/Transforms/SROA/struct-to-vector.ll
new file mode 100644
index 0000000000000..58945334b7961
--- /dev/null
+++ b/llvm/test/Transforms/SROA/struct-to-vector.ll
@@ -0,0 +1,396 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='sroa,gvn,instcombine,simplifycfg' -S %s | FileCheck %s
+%struct.myint4 = type { i32, i32, i32, i32 }
+
+define dso_local void @foo_flat(ptr noundef %x, i64 %y.coerce0, i64 %y.coerce1, i32 noundef %cond) {
+; CHECK-LABEL: define dso_local void @foo_flat(
+; CHECK-SAME: ptr noundef [[X:%.*]], i64 [[Y_COERCE0:%.*]], i64 [[Y_COERCE1:%.*]], i32 noundef [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[COND]], 0
+; CHECK-NEXT: [[DOTY_COERCE0:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE0]]
+; CHECK-NEXT: [[DOTY_COERCE1:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE1]]
+; CHECK-NEXT: store i64 [[DOTY_COERCE0]], ptr [[X]], align 16
+; CHECK-NEXT: [[X_REPACK7:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
+; CHECK-NEXT: store i64 [[DOTY_COERCE1]], ptr [[X_REPACK7]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %y = alloca %struct.myint4, align 16
+ %x.addr = alloca ptr, align 8
+ %cond.addr = alloca i32, align 4
+ %temp = alloca %struct.myint4, align 16
+ %zero = alloca %struct.myint4, align 16
+ %data = alloca %struct.myint4, align 16
+ %0 = getelementptr inbounds nuw { i64, i64 }, ptr %y, i32 0, i32 0
+ store i64 %y.coerce0, ptr %0, align 16
+ %1 = getelementptr inbounds nuw { i64, i64 }, ptr %y, i32 0, i32 1
+ store i64 %y.coerce1, ptr %1, align 8
+ store ptr %x, ptr %x.addr, align 8
+ store i32 %cond, ptr %cond.addr, align 4
+ call void @llvm.lifetime.start.p0(ptr %temp)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 16 %temp, ptr align 16 %y, i64 16, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %zero)
+ call void @llvm.memset.p0.i64(ptr align 16 %zero, i8 0, i64 16, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %data)
+ %2 = load i32, ptr %cond.addr, align 4
+ %tobool = icmp ne i32 %2, 0
+ br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:
+ br label %cond.end
+
+cond.false:
+ br label %cond.end
+
+cond.end:
+ %cond1 = phi ptr [ %temp, %cond.true ], [ %zero, %cond.false ]
+ %whole = load { i64, i64 }, ptr %cond1, align 16
+ store { i64, i64 } %whole, ptr %data, align 16
+ %3 = load ptr, ptr %x.addr, align 8
+ %whole2 = load { i64, i64 }, ptr %data, align 16
+ store { i64, i64 } %whole2, ptr %3, align 16
+ call void @llvm.lifetime.end.p0(ptr %data)
+ call void @llvm.lifetime.end.p0(ptr %zero)
+ call void @llvm.lifetime.end.p0(ptr %temp)
+ ret void
+}
+%struct.myint4_base_n = type { i32, i32, i32, i32 }
+%struct.myint4_nested = type { %struct.myint4_base_n }
+
+define dso_local void @foo_nested(ptr noundef %x, i64 %y.coerce0, i64 %y.coerce1, i32 noundef %cond) {
+; CHECK-LABEL: define dso_local void @foo_nested(
+; CHECK-SAME: ptr noundef [[X:%.*]], i64 [[Y_COERCE0:%.*]], i64 [[Y_COERCE1:%.*]], i32 noundef [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[COND]], 0
+; CHECK-NEXT: [[DOTY_COERCE0:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE0]]
+; CHECK-NEXT: [[DOTY_COERCE1:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE1]]
+; CHECK-NEXT: store i64 [[DOTY_COERCE0]], ptr [[X]], align 16
+; CHECK-NEXT: [[X_REPACK7:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
+; CHECK-NEXT: store i64 [[DOTY_COERCE1]], ptr [[X_REPACK7]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %y = alloca %struct.myint4_nested, align 16
+ %x.addr = alloca ptr, align 8
+ %cond.addr = alloca i32, align 4
+ %temp = alloca %struct.myint4_nested, align 16
+ %zero = alloca %struct.myint4_nested, align 16
+ %data = alloca %struct.myint4_nested, align 16
+ %0 = getelementptr inbounds nuw { i64, i64 }, ptr %y, i32 0, i32 0
+ store i64 %y.coerce0, ptr %0, align 16
+ %1 = getelementptr inbounds nuw { i64, i64 }, ptr %y, i32 0, i32 1
+ store i64 %y.coerce1, ptr %1, align 8
+ store ptr %x, ptr %x.addr, align 8
+ store i32 %cond, ptr %cond.addr, align 4
+ call void @llvm.lifetime.start.p0(ptr %temp)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 16 %temp, ptr align 16 %y, i64 16, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %zero)
+ call void @llvm.memset.p0.i64(ptr align 16 %zero, i8 0, i64 16, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %data)
+ %2 = load i32, ptr %cond.addr, align 4
+ %tobool = icmp ne i32 %2, 0
+ br i1 %tobool, label %cond.true, label %cond.false
+
+cond.true:
+ br label %cond.end
+
+cond.false:
+ br label %cond.end
+
+cond.end:
+ %cond1 = phi ptr [ %temp, %cond.true ], [ %zero, %cond.false ]
+ %whole = load { i64, i64 }, ptr %cond1, align 16
+ store { i64, i64 } %whole, ptr %data, align 16
+ %3 = load ptr, ptr %x.addr, align 8
+ %whole2 = load { i64, i64 }, ptr %data, align 16
+ store { i64, i64 } %whole2, ptr %3, align 16
+ call void @llvm.lifetime.end.p0(ptr %data)
+ call void @llvm.lifetime.end.p0(ptr %zero)
+ call void @llvm.lifetime.end.p0(ptr %temp)
+ ret void
+}
+
+%struct.padded = type { i32, i8, i32, i8 }
+define dso_local void @foo_padded(ptr noundef %x, i32 %a0, i8 %a1,
+; CHECK-LABEL: define dso_local void @foo_padded(
+; CHECK-SAME: ptr noundef [[X:%.*]], i32 [[A0:%.*]], i8 [[A1:%.*]], i32 [[A2:%.*]], i8 [[A3:%.*]], i32 noundef [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TEMP:%.*]] = alloca [[STRUCT_PADDED:%.*]], align 4
+; CHECK-NEXT: [[ZERO:%.*]] = alloca [[STRUCT_PADDED]], align 4
+; CHECK-NEXT: [[DATA:%.*]] = alloca [[STRUCT_PADDED]], align 4
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[TEMP]])
+; CHECK-NEXT: store i32 [[A0]], ptr [[TEMP]], align 4
+; CHECK-NEXT: [[Y_SROA_2_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 4
+; CHECK-NEXT: store i8 [[A1]], ptr [[Y_SROA_2_0_TEMP_SROA_IDX]], align 4
+; CHECK-NEXT: [[Y_SROA_31_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 8
+; CHECK-NEXT: store i32 [[A2]], ptr [[Y_SROA_31_0_TEMP_SROA_IDX]], align 4
+; CHECK-NEXT: [[Y_SROA_4_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 12
+; CHECK-NEXT: store i8 [[A3]], ptr [[Y_SROA_4_0_TEMP_SROA_IDX]], align 4
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 4 dereferenceable(16) [[ZERO]], i8 0, i64 16, i1 false)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[DATA]])
+; CHECK-NEXT: [[TOBOOL_PAD_NOT:%.*]] = icmp eq i32 [[COND]], 0
+; CHECK-NEXT: [[ZERO_TEMP:%.*]] = select i1 [[TOBOOL_PAD_NOT]], ptr [[ZERO]], ptr [[TEMP]]
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(16) [[DATA]], ptr noundef nonnull align 4 dereferenceable(16) [[ZERO_TEMP]], i64 16, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(16) [[X]], ptr noundef nonnull align 4 dereferenceable(16) [[DATA]], i64 16, i1 false)
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[DATA]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[TEMP]])
+; CHECK-NEXT: ret void
+;
+ i32 %a2, i8 %a3,
+ i32 noundef %cond) {
+entry:
+ %y = alloca %struct.padded, align 4
+ %x.addr = alloca ptr, align 8
+ %cond.addr = alloca i32, align 4
+ %temp = alloca %struct.padded, align 4
+ %zero = alloca %struct.padded, align 4
+ %data = alloca %struct.padded, align 4
+ %y_i32_0 = getelementptr inbounds %struct.padded, ptr %y, i32 0, i32 0
+ store i32 %a0, ptr %y_i32_0, align 4
+ %y_i8_1 = getelementptr inbounds %struct.padded, ptr %y, i32 0, i32 1
+ store i8 %a1, ptr %y_i8_1, align 1
+ %y_i32_2 = getelementptr inbounds %struct.padded, ptr %y, i32 0, i32 2
+ store i32 %a2, ptr %y_i32_2, align 4
+ %y_i8_3 = getelementptr inbounds %struct.padded, ptr %y, i32 0, i32 3
+ store i8 %a3, ptr %y_i8_3, align 1
+ store ptr %x, ptr %x.addr, align 8
+ store i32 %cond, ptr %cond.addr, align 4
+ call void @llvm.lifetime.start.p0(ptr %temp)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 4 %temp, ptr align 4 %y,
+ i64 16, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %zero)
+ call void @llvm.memset.p0.i64(ptr align 4 %zero, i8 0, i64 16, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %data)
+ %c.pad = load i32, ptr %cond.addr, align 4
+ %tobool.pad = icmp ne i32 %c.pad, 0
+ br i1 %tobool.pad, label %cond.true.pad, label %cond.false.pad
+
+cond.true.pad:
+ br label %cond.end.pad
+
+cond.false.pad:
+ br label %cond.end.pad
+
+cond.end.pad:
+ %cond1.pad = phi ptr [ %temp, %cond.true.pad ], [ %zero, %cond.false.pad ]
+ call void @llvm.memcpy.p0.p0.i64(ptr align 4 %data, ptr align 4 %cond1.pad,
+ i64 16, i1 false)
+ %xv.pad = load ptr, ptr %x.addr, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr align 4 %xv.pad, ptr align 4 %data,
+ i64 16, i1 false)
+ call void @llvm.lifetime.end.p0(ptr %data)
+ call void @llvm.lifetime.end.p0(ptr %zero)
+ call void @llvm.lifetime.end.p0(ptr %temp)
+ ret void
+}
+
+%struct.nonhomo = type { i32, i64, i32, i64 }
+define dso_local void @foo_nonhomo(ptr noundef %x, i32 %a0, i64 %a1,
+; CHECK-LABEL: define dso_local void @foo_nonhomo(
+; CHECK-SAME: ptr noundef [[X:%.*]], i32 [[A0:%.*]], i64 [[A1:%.*]], i32 [[A2:%.*]], i64 [[A3:%.*]], i32 noundef [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TEMP:%.*]] = alloca [[STRUCT_NONHOMO:%.*]], align 8
+; CHECK-NEXT: [[ZERO:%.*]] = alloca [[STRUCT_NONHOMO]], align 8
+; CHECK-NEXT: [[DATA:%.*]] = alloca [[STRUCT_NONHOMO]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[TEMP]])
+; CHECK-NEXT: store i32 [[A0]], ptr [[TEMP]], align 8
+; CHECK-NEXT: [[Y_SROA_2_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 4
+; CHECK-NEXT: store i64 [[A1]], ptr [[Y_SROA_2_0_TEMP_SROA_IDX]], align 4
+; CHECK-NEXT: [[Y_SROA_3_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 12
+; CHECK-NEXT: store i32 [[A2]], ptr [[Y_SROA_3_0_TEMP_SROA_IDX]], align 4
+; CHECK-NEXT: [[Y_SROA_4_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 16
+; CHECK-NEXT: store i64 [[A3]], ptr [[Y_SROA_4_0_TEMP_SROA_IDX]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[ZERO]], i8 0, i64 32, i1 false)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[DATA]])
+; CHECK-NEXT: [[TOBOOL_NH_NOT:%.*]] = icmp eq i32 [[COND]], 0
+; CHECK-NEXT: [[ZERO_TEMP:%.*]] = select i1 [[TOBOOL_NH_NOT]], ptr [[ZERO]], ptr [[TEMP]]
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[DATA]], ptr noundef nonnull align 8 dereferenceable(32) [[ZERO_TEMP]], i64 32, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[X]], ptr noundef nonnull align 8 dereferenceable(32) [[DATA]], i64 32, i1 false)
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[DATA]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[TEMP]])
+; CHECK-NEXT: ret void
+;
+ i32 %a2, i64 %a3,
+ i32 noundef %cond) {
+entry:
+ %y = alloca %struct.nonhomo, align 8
+ %x.addr = alloca ptr, align 8
+ %cond.addr = alloca i32, align 4
+ %temp = alloca %struct.nonhomo, align 8
+ %zero = alloca %struct.nonhomo, align 8
+ %data = alloca %struct.nonhomo, align 8
+ %y_i32_0n = getelementptr inbounds %struct.nonhomo, ptr %y, i32 0, i32 0
+ store i32 %a0, ptr %y_i32_0n, align 4
+ %y_i64_1n = getelementptr inbounds %struct.nonhomo, ptr %y, i32 0, i32 1
+ store i64 %a1, ptr %y_i64_1n, align 8
+ %y_i32_2n = getelementptr inbounds %struct.nonhomo, ptr %y, i32 0, i32 2
+ store i32 %a2, ptr %y_i32_2n, align 4
+ %y_i64_3n = getelementptr inbounds %struct.nonhomo, ptr %y, i32 0, i32 3
+ store i64 %a3, ptr %y_i64_3n, align 8
+ store ptr %x, ptr %x.addr, align 8
+ store i32 %cond, ptr %cond.addr, align 4
+ call void @llvm.lifetime.start.p0(ptr %temp)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %temp, ptr align 8 %y,
+ i64 32, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %zero)
+ call void @llvm.memset.p0.i64(ptr align 8 %zero, i8 0, i64 32, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %data)
+ %c.nh = load i32, ptr %cond.addr, align 4
+ %tobool.nh = icmp ne i32 %c.nh, 0
+ br i1 %tobool.nh, label %cond.true.nh, label %cond.false.nh
+
+cond.true.nh:
+ br label %cond.end.nh
+
+cond.false.nh:
+ br label %cond.end.nh
+
+cond.end.nh:
+ %cond1.nh = phi ptr [ %temp, %cond.true.nh ], [ %zero, %cond.false.nh ]
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %data, ptr align 8 %cond1.nh,
+ i64 32, i1 false)
+ %xv.nh = load ptr, ptr %x.addr, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %xv.nh, ptr align 8 %data,
+ i64 32, i1 false)
+ call void @llvm.lifetime.end.p0(ptr %data)
+ call void @llvm.lifetime.end.p0(ptr %zero)
+ call void @llvm.lifetime.end.p0(ptr %temp)
+ ret void
+}
+
+%struct.i1x4 = type { i1, i1, i1, i1 }
+define dso_local void @foo_i1(ptr noundef %x, i64 %dummy0, i64 %dummy1,
+; CHECK-LABEL: define dso_local void @foo_i1(
+; CHECK-SAME: ptr noundef [[X:%.*]], i64 [[DUMMY0:%.*]], i64 [[DUMMY1:%.*]], i32 noundef [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TEMP:%.*]] = alloca [[STRUCT_I1X4:%.*]], align 1
+; CHECK-NEXT: [[ZERO:%.*]] = alloca [[STRUCT_I1X4]], align 1
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[TEMP]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: store i32 0, ptr [[ZERO]], align 1
+; CHECK-NEXT: [[TOBOOL_I1_NOT:%.*]] = icmp eq i32 [[COND]], 0
+; CHECK-NEXT: [[ZERO_TEMP:%.*]] = select i1 [[TOBOOL_I1_NOT]], ptr [[ZERO]], ptr [[TEMP]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ZERO_TEMP]], align 1
+; CHECK-NEXT: store i32 [[TMP0]], ptr [[X]], align 1
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[TEMP]])
+; CHECK-NEXT: ret void
+;
+ i32 noundef %cond) {
+entry:
+ %y = alloca %struct.i1x4, align 1
+ %x.addr = alloca ptr, align 8
+ %cond.addr = alloca i32, align 4
+ %temp = alloca %struct.i1x4, align 1
+ %zero = alloca %struct.i1x4, align 1
+ %data = alloca %struct.i1x4, align 1
+ store ptr %x, ptr %x.addr, align 8
+ store i32 %cond, ptr %cond.addr, align 4
+ call void @llvm.lifetime.start.p0(ptr %temp)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %temp, ptr align 1 %y,
+ i64 4, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %zero)
+ call void @llvm.memset.p0.i64(ptr align 1 %zero, i8 0, i64 4, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %data)
+ %c.i1 = load i32, ptr %cond.addr, align 4
+ %tobool.i1 = icmp ne i32 %c.i1, 0
+ br i1 %tobool.i1, label %cond.true.i1, label %cond.false.i1
+
+cond.true.i1:
+ br label %cond.end.i1
+
+cond.false.i1:
+ br label %cond.end.i1
+
+cond.end.i1:
+ %cond1.i1 = phi ptr [ %temp, %cond.true.i1 ], [ %zero, %cond.false.i1 ]
+ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %data, ptr align 1 %cond1.i1,
+ i64 4, i1 false)
+ %xv.i1 = load ptr, ptr %x.addr, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %xv.i1, ptr align 1 %data,
+ i64 4, i1 false)
+ call void @llvm.lifetime.end.p0(ptr %data)
+ call void @llvm.lifetime.end.p0(ptr %zero)
+ call void @llvm.lifetime.end.p0(ptr %temp)
+ ret void
+}
+
+%struct.ptr4 = type { ptr, ptr, ptr, ptr }
+define dso_local void @foo_ptr(ptr noundef %x, ptr %p0, ptr %p1,
+; CHECK-LABEL: define dso_local void @foo_ptr(
+; CHECK-SAME: ptr noundef [[X:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], i32 noundef [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TEMP:%.*]] = alloca [[STRUCT_PTR4:%.*]], align 8
+; CHECK-NEXT: [[ZERO:%.*]] = alloca [[STRUCT_PTR4]], align 8
+; CHECK-NEXT: [[DATA:%.*]] = alloca [[STRUCT_PTR4]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[TEMP]])
+; CHECK-NEXT: store ptr [[P0]], ptr [[TEMP]], align 8
+; CHECK-NEXT: [[Y_SROA_2_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 8
+; CHECK-NEXT: store ptr [[P1]], ptr [[Y_SROA_2_0_TEMP_SROA_IDX]], align 8
+; CHECK-NEXT: [[Y_SROA_3_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 16
+; CHECK-NEXT: store ptr [[P2]], ptr [[Y_SROA_3_0_TEMP_SROA_IDX]], align 8
+; CHECK-NEXT: [[Y_SROA_4_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 24
+; CHECK-NEXT: store ptr [[P3]], ptr [[Y_SROA_4_0_TEMP_SROA_IDX]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[ZERO]], i8 0, i64 32, i1 false)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[DATA]])
+; CHECK-NEXT: [[TOBOOL_PTR_NOT:%.*]] = icmp eq i32 [[COND]], 0
+; CHECK-NEXT: [[ZERO_TEMP:%.*]] = select i1 [[TOBOOL_PTR_NOT]], ptr [[ZERO]], ptr [[TEMP]]
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[DATA]], ptr noundef nonnull align 8 dereferenceable(32) [[ZERO_TEMP]], i64 32, i1 false)
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[X]], ptr noundef nonnull align 8 dereferenceable(32) [[DATA]], i64 32, i1 false)
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[DATA]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[TEMP]])
+; CHECK-NEXT: ret void
+;
+ ptr %p2, ptr %p3,
+ i32 noundef %cond) {
+entry:
+ %y = alloca %struct.ptr4, align 8
+ %x.addr = alloca ptr, align 8
+ %cond.addr = alloca i32, align 4
+ %temp = alloca %struct.ptr4, align 8
+ %zero = alloca %struct.ptr4, align 8
+ %data = alloca %struct.ptr4, align 8
+ %y_p0 = getelementptr inbounds %struct.ptr4, ptr %y, i32 0, i32 0
+ store ptr %p0, ptr %y_p0, align 8
+ %y_p1 = getelementptr inbounds %struct.ptr4, ptr %y, i32 0, i32 1
+ store ptr %p1, ptr %y_p1, align 8
+ %y_p2 = getelementptr inbounds %struct.ptr4, ptr %y, i32 0, i32 2
+ store ptr %p2, ptr %y_p2, align 8
+ %y_p3 = getelementptr inbounds %struct.ptr4, ptr %y, i32 0, i32 3
+ store ptr %p3, ptr %y_p3, align 8
+ store ptr %x, ptr %x.addr, align 8
+ store i32 %cond, ptr %cond.addr, align 4
+ call void @llvm.lifetime.start.p0(ptr %temp)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %temp, ptr align 8 %y,
+ i64 32, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %zero)
+ call void @llvm.memset.p0.i64(ptr align 8 %zero, i8 0, i64 32, i1 false)
+ call void @llvm.lifetime.start.p0(ptr %data)
+ %c.ptr = load i32, ptr %cond.addr, align 4
+ %tobool.ptr = icmp ne i32 %c.ptr, 0
+ br i1 %tobool.ptr, label %cond.true.ptr, label %cond.false.ptr
+
+cond.true.ptr:
+ br label %cond.end.ptr
+
+cond.false.ptr:
+ br label %cond.end.ptr
+
+cond.end.ptr:
+ %cond1.ptr = phi ptr [ %temp, %cond.true.ptr ], [ %zero, %cond.false.ptr ]
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %data, ptr align 8 %cond1.ptr,
+ i64 32, i1 false)
+ %xv.ptr = load ptr, ptr %x.addr, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %xv.ptr, ptr align 8 %data,
+ i64 32, i1 false)
+ call void @llvm.lifetime.end.p0(ptr %data)
+ call void @llvm.lifetime.end.p0(ptr %zero)
+ call void @llvm.lifetime.end.p0(ptr %temp)
+ ret void
+}
diff --git a/llvm/test/Transforms/SROA/tbaa-struct3.ll b/llvm/test/Transforms/SROA/tbaa-struct3.ll
index 6a0cacc7016f7..97e82db27c378 100644
--- a/llvm/test/Transforms/SROA/tbaa-struct3.ll
+++ b/llvm/test/Transforms/SROA/tbaa-struct3.ll
@@ -73,12 +73,13 @@ define void @load_store_transfer_split_struct_tbaa_2_i31(ptr dereferenceable(24)
; CHECK-LABEL: define void @load_store_transfer_split_struct_tbaa_2_i31(
; CHECK-SAME: ptr dereferenceable(24) [[RES:%.*]], i31 [[A:%.*]], i31 [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP:%.*]] = alloca { i31, i31 }, align 4
-; CHECK-NEXT: store i31 [[A]], ptr [[TMP]], align 4
+; CHECK-NEXT: [[TMP:%.*]] = alloca <2 x i31>, align 8
+; CHECK-NEXT: store i31 [[A]], ptr [[TMP]], align 8
; CHECK-NEXT: [[TMP_4_TMP_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 4
; CHECK-NEXT: store i31 [[B]], ptr [[TMP_4_TMP_4_SROA_IDX]], align 4
-; CHECK-NEXT: [[TMP_0_L1:%.*]] = load i62, ptr [[TMP]], align 4, !tbaa.struct [[TBAA_STRUCT4:![0-9]+]]
-; CHECK-NEXT: store i62 [[TMP_0_L1]], ptr [[RES]], align 4, !tbaa.struct [[TBAA_STRUCT4]]
+; CHECK-NEXT: [[TMP_SROA_0_0_TMP_SROA_0_0_L1:%.*]] = load <2 x i31>, ptr [[TMP]], align 8, !tbaa.struct [[TBAA_STRUCT4:![0-9]+]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i31> [[TMP_SROA_0_0_TMP_SROA_0_0_L1]] to i62
+; CHECK-NEXT: store i62 [[TMP0]], ptr [[RES]], align 4, !tbaa.struct [[TBAA_STRUCT4]]
; CHECK-NEXT: ret void
;
entry:
>From 4cd8c3344d5eece1b303d96474a005524aad0d44 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Fri, 24 Apr 2026 11:20:05 -0400
Subject: [PATCH 02/13] [SROA] Simplify struct-to-vector fallback helper
Fold the fallback decision into tryCanonicalizeStructToVector and drop
redundant structural checks and debug noise so the current conservative
policy is easier to read and review.
---
llvm/lib/Transforms/Scalar/SROA.cpp | 177 ++++--------------
.../struct-to-vector-mapok-extra-alloca.ll | 35 ----
2 files changed, 37 insertions(+), 175 deletions(-)
delete mode 100644 llvm/test/Transforms/SROA/struct-to-vector-mapok-extra-alloca.ll
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 5991db4b992f1..853a857269db8 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5086,32 +5086,24 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
return true;
}
-/// Try to canonicalize a homogeneous, tightly-packed struct to a vector type.
+/// Try to canonicalize a homogeneous struct partition to a vector type.
///
-/// For structs where all elements have the same type and are tightly packed
-/// (no padding), we can represent them as a fixed vector which enables better
-/// optimization (e.g., vector selects instead of memcpy).
-///
-/// \param STy The struct type to try to canonicalize.
-/// \param DL The DataLayout for size/alignment queries.
-/// \returns The equivalent vector type, or nullptr if not applicable.
+/// This is only used as a fallback after the usual promotion choices fail, so
+/// it stays intentionally conservative: besides requiring a tightly-packed
+/// homogeneous struct shape, it rejects sub-element loads and only recovers a
+/// narrow class of memcpy-only i64 cases.
static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
- const DataLayout &DL) {
+ Partition &P,
+ const DataLayout &DL,
+ AllocaInst &AI) {
unsigned NumElts = STy->getNumElements();
if (NumElts != 2 && NumElts != 4)
return nullptr;
- // All elements must be the same type.
Type *EltTy = STy->getElementType(0);
- for (unsigned I = 1; I < NumElts; ++I)
- if (STy->getElementType(I) != EltTy)
- return nullptr;
-
- // Element type must be valid for vectors.
- if (!VectorType::isValidElementType(EltTy))
+ if (!llvm::all_equal(STy->elements()))
return nullptr;
- // Only allow integer types >= 8 bits or floating point.
if (auto *IT = dyn_cast<IntegerType>(EltTy)) {
if (IT->getBitWidth() < 8)
return nullptr;
@@ -5119,87 +5111,19 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
return nullptr;
}
- // Element size must be fixed and non-zero.
TypeSize EltTS = DL.getTypeAllocSize(EltTy);
if (!EltTS.isFixed())
return nullptr;
uint64_t EltSize = EltTS.getFixedValue();
- if (EltSize < 1)
- return nullptr;
-
- const StructLayout *SL = DL.getStructLayout(STy);
- uint64_t StructSize = SL->getSizeInBytes();
- if (StructSize == 0)
- return nullptr;
- // Must be tightly packed: size == NumElts * EltSize.
- if (StructSize != NumElts * EltSize)
+ if (DL.getStructLayout(STy)->getSizeInBytes() != NumElts * EltSize)
return nullptr;
- // Verify each element is at the expected offset (no padding).
- for (unsigned I = 0; I < NumElts; ++I)
- if (SL->getElementOffset(I) != I * EltSize)
- return nullptr;
-
- return FixedVectorType::get(EltTy, NumElts);
-}
-
-/// Decide whether it is profitable to canonicalize a homogeneous struct
-/// partition to a vector after the usual promotion choices have already failed.
-///
-/// This helper is intentionally conservative: we only canonicalize when the
-/// partition has a non-splittable whole-partition use and does not have any
-/// non-splittable sub-element loads.
-///
-/// Default allow case:
-/// - a real whole-partition use, because that is the clearest signal that a
-/// vector type can carry a whole value profitably.
-///
-/// Default reject case:
-/// - sub-element loads, because they usually turn the vector back into lane
-/// extraction traffic.
-///
-/// We also recover a narrow class of memcpy-only integer cases, even though
-/// they are classified as splittable rather than whole-partition uses:
-/// - interior i64 partitions within a larger fixed-size allocation
-/// - original full-record integer aggregates of size >= 32 bytes
-///
-/// Rationale:
-/// - Some memcpy-only integer cases are real wins and stay in whole-value form
-/// after canonicalization.
-/// - But recovering all full-record memcpy-only cases is too broad: an
-/// original full-record 16-byte {i64, i64} helper can look locally
-/// profitable while still making later code generation worse.
-/// - So the fallback stays narrow and uses the current partition's offsets plus
-/// allocation size to distinguish the safer recovered cases from the broader
-/// risky bucket.
-///
-/// Intuition:
-/// - Good: whole-value traffic can benefit from a vector type.
-/// %tmp = load { i64, i64 }, ptr %src
-/// store { i64, i64 } %tmp, ptr %dst
-/// Canonicalizing to <2 x i64> exposes a single whole-value load/store.
-///
-/// - Bad: field-by-field reads become extractelement traffic.
-/// %x = load i32, ptr %p
-/// %y = load i32, ptr (gep %p, 4)
-/// Canonicalizing { i32, i32 } to <2 x i32> only adds lane extraction.
-///
-/// - Bad: a store-only FP tail can seed later SLP divergence without a clear
-/// SROA win.
-/// store float %a, ptr %p0
-/// store float %b, ptr %p1
-/// store float %c, ptr %p2
-/// store float %d, ptr %p3
-/// Canonicalizing this to a temporary <4 x float> store was enough to change
-/// later vectorization without a clear benefit.
-static bool shouldCanonicalizeHomogeneousStructToVector(Partition &P,
- const DataLayout &DL,
- AllocaInst &AI,
- bool IsI64Candidate) {
+ auto *VTy = FixedVectorType::get(EltTy, NumElts);
bool HasWholePartitionUse = false;
bool HasSubElementLoad = false;
bool HasRecoverableSplittableTransfer = false;
+ bool IsI64Candidate = VTy->getElementType()->isIntegerTy(64);
std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL);
bool IsInteriorSubaggregate = AllocSize && AllocSize->isFixed() &&
P.beginOffset() != 0 &&
@@ -5238,8 +5162,10 @@ static bool shouldCanonicalizeHomogeneousStructToVector(Partition &P,
HasWholePartitionUse = true;
}
- return (HasWholePartitionUse || HasRecoverableSplittableTransfer) &&
- !HasSubElementLoad;
+ if ((HasWholePartitionUse || HasRecoverableSplittableTransfer) &&
+ !HasSubElementLoad)
+ return VTy;
+ return nullptr;
}
/// Select a partition type for an alloca partition.
@@ -5256,8 +5182,9 @@ static bool shouldCanonicalizeHomogeneousStructToVector(Partition &P,
static std::tuple<Type *, bool, VectorType *>
selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
LLVMContext &C) {
- auto LogChoice = [&](StringRef Path, Type *ChosenTy, VectorType *ChosenVecTy,
- bool ChosenIntWidening) {
+ auto LogSelection = [&](StringRef Path, Type *SelectedTy,
+ VectorType *SelectedVecTy,
+ bool SelectedIntWidening) {
LLVM_DEBUG({
dbgs() << "selectPartitionType path=" << Path
<< " func=" << AI.getFunction()->getName() << " alloca=";
@@ -5269,11 +5196,11 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
<< ") size=" << P.size();
if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
- if (ChosenTy)
- dbgs() << " chosen=" << *ChosenTy;
- if (ChosenVecTy)
- dbgs() << " vec=" << *ChosenVecTy;
- dbgs() << " intwiden=" << ChosenIntWidening << "\n";
+ if (SelectedTy)
+ dbgs() << " chosen=" << *SelectedTy;
+ if (SelectedVecTy)
+ dbgs() << " vec=" << *SelectedVecTy;
+ dbgs() << " intwiden=" << SelectedIntWidening << "\n";
});
};
// First check if the partition is viable for vector promotion.
@@ -5293,7 +5220,7 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
// whether we promote with the vector or scalar.
if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
VecTy->getElementCount().getFixedValue() > 1) {
- LogChoice("direct-fp-vecty", VecTy, VecTy, false);
+ LogSelection("direct-fp-vecty", VecTy, VecTy, false);
return {VecTy, false, VecTy};
}
@@ -5308,11 +5235,11 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
// and there is a common type used, then it implies the second listed
// condition for preferring vector promotion is true.
if (VecTy) {
- LogChoice("common-type-vecty", VecTy, VecTy, false);
+ LogSelection("common-type-vecty", VecTy, VecTy, false);
return {VecTy, false, VecTy};
}
bool IntWiden = isIntegerWideningViable(P, CommonUseTy, DL);
- LogChoice("common-type", CommonUseTy, nullptr, IntWiden);
+ LogSelection("common-type", CommonUseTy, nullptr, IntWiden);
return {CommonUseTy, IntWiden, nullptr};
}
}
@@ -5330,11 +5257,11 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
TypePartitionTy = Type::getIntNTy(C, P.size() * 8);
// There was no common type used, so we prefer integer widening promotion.
if (isIntegerWideningViable(P, TypePartitionTy, DL)) {
- LogChoice("type-partition-intwiden", TypePartitionTy, nullptr, true);
+ LogSelection("type-partition-int-widen", TypePartitionTy, nullptr, true);
return {TypePartitionTy, true, nullptr};
}
if (VecTy) {
- LogChoice("type-partition-vecty", VecTy, VecTy, false);
+ LogSelection("type-partition-vecty", VecTy, VecTy, false);
return {VecTy, false, VecTy};
}
// If we couldn't promote with TypePartitionTy, try with the largest
@@ -5342,69 +5269,39 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
if (LargestIntTy &&
DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size() &&
isIntegerWideningViable(P, LargestIntTy, DL)) {
- LogChoice("largest-int-intwiden", LargestIntTy, nullptr, true);
+ LogSelection("largest-int-int-widen", LargestIntTy, nullptr, true);
return {LargestIntTy, true, nullptr};
}
// Try homogeneous struct to vector canonicalization.
- //
- // This is intentionally more conservative than tryCanonicalize...
- // alone: after the normal promotion paths above fail, we only want the
- // struct-to-vector fallback when it can expose a real whole-value use,
- // not when it would merely create vector lanes that later passes have to
- // unpack again.
if (auto *STy = dyn_cast<StructType>(TypePartitionTy))
- if (auto *VTy = tryCanonicalizeStructToVector(STy, DL)) {
- // Recover only a narrow integer memcpy-only subset here. Broader
- // fallback recovery was too aggressive:
- // - full-record 16-byte { i64, i64 } cases reopened regressions
- // - FP and lane-oriented cases were more likely to trigger bad later
- // codegen
- //
- // So this path is restricted to 64-bit integer lanes, and only for
- // interior partitions or original full records of size >= 32 bytes.
- bool AllowStructFallback = shouldCanonicalizeHomogeneousStructToVector(
- P, DL, AI, VTy->getElementType()->isIntegerTy(64));
- LLVM_DEBUG({
- dbgs() << "selectPartitionType struct-fallback-candidate"
- << " func=" << AI.getFunction()->getName() << " alloca=";
- if (AI.hasName())
- dbgs() << AI.getName();
- else
- dbgs() << "<unnamed>";
- dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
- << ") size=" << P.size() << " type-partition=" << *STy
- << " candidate-vec=" << *VTy
- << " allow=" << AllowStructFallback << "\n";
- });
- if (AllowStructFallback) {
- LogChoice("struct-fallback-vecty", VTy, nullptr, false);
- return {VTy, false, nullptr};
- }
+ if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL, AI)) {
+ LogSelection("struct-fallback-vecty", VTy, nullptr, false);
+ return {VTy, false, nullptr};
}
// Fallback to TypePartitionTy and we probably won't promote.
- LogChoice("type-partition-fallback", TypePartitionTy, nullptr, false);
+ LogSelection("type-partition-fallback", TypePartitionTy, nullptr, false);
return {TypePartitionTy, false, nullptr};
}
// Select the largest integer type used if it spans the partition.
if (LargestIntTy &&
DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size()) {
- LogChoice("largest-int-fallback", LargestIntTy, nullptr, false);
+ LogSelection("largest-int-fallback", LargestIntTy, nullptr, false);
return {LargestIntTy, false, nullptr};
}
// Select a legal integer type if it spans the partition.
if (DL.isLegalInteger(P.size() * 8)) {
Type *IntTy = Type::getIntNTy(C, P.size() * 8);
- LogChoice("legal-int-fallback", IntTy, nullptr, false);
+ LogSelection("legal-int-fallback", IntTy, nullptr, false);
return {IntTy, false, nullptr};
}
// Fallback to an i8 array.
Type *ArrayTy = ArrayType::get(Type::getInt8Ty(C), P.size());
- LogChoice("byte-array-fallback", ArrayTy, nullptr, false);
+ LogSelection("byte-array-fallback", ArrayTy, nullptr, false);
return {ArrayTy, false, nullptr};
}
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-mapok-extra-alloca.ll b/llvm/test/Transforms/SROA/struct-to-vector-mapok-extra-alloca.ll
deleted file mode 100644
index f9ce644156b31..0000000000000
--- a/llvm/test/Transforms/SROA/struct-to-vector-mapok-extra-alloca.ll
+++ /dev/null
@@ -1,35 +0,0 @@
-; RUN: opt -passes='default<O3>' -S %s | FileCheck %s
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK-LABEL: define void @"_ZN102_$LT$futures_util..stream..try_stream..MapOk$LT$St$C$F$GT$$u20$as$u20$futures_core..stream..Stream$GT$9poll_next17h555df33481d9c33cE"
-; CHECK: [[TMP:%.*]] = alloca [11 x i64], align 8
-; CHECK: call void @llvm.lifetime.start.p0(ptr nonnull [[TMP]])
-; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(88) [[TMP]], ptr noundef nonnull align 1 dereferenceable(88) %0, i64 88, i1 false)
-; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 1 dereferenceable(88) %0, ptr noundef nonnull align 8 dereferenceable(88) [[TMP]], i64 88, i1 false)
-; CHECK: call void @llvm.lifetime.end.p0(ptr nonnull [[TMP]])
-define void @"_ZN102_$LT$futures_util..stream..try_stream..MapOk$LT$St$C$F$GT$$u20$as$u20$futures_core..stream..Stream$GT$9poll_next17h555df33481d9c33cE"(ptr %0) {
- call void @"_ZN101_$LT$futures_util..stream..stream..map..Map$LT$St$C$F$GT$$u20$as$u20$futures_core..stream..Stream$GT$9poll_next17h5aa844c062b2077eE"(ptr %0)
- ret void
-}
-
-; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg) #0
-
-define void @"_ZN101_$LT$futures_util..stream..stream..map..Map$LT$St$C$F$GT$$u20$as$u20$futures_core..stream..Stream$GT$9poll_next17h5aa844c062b2077eE"(ptr %0) {
- %.sroa.5 = alloca [11 x i64], align 8
- %2 = load i64, ptr %0, align 8
- %3 = icmp eq i64 %2, 0
- br i1 %3, label %"_ZN122_$LT$futures_util..fns..MapOkFn$LT$F$GT$$u20$as$u20$futures_util..fns..FnMut1$LT$core..result..Result$LT$T$C$E$GT$$GT$$GT$8call_mut17h252763b5559d12fbE.exit", label %4
-
-4: ; preds = %1
- call void @llvm.memcpy.p0.p0.i64(ptr %.sroa.5, ptr %0, i64 88, i1 false)
- br label %"_ZN122_$LT$futures_util..fns..MapOkFn$LT$F$GT$$u20$as$u20$futures_util..fns..FnMut1$LT$core..result..Result$LT$T$C$E$GT$$GT$$GT$8call_mut17h252763b5559d12fbE.exit"
-
-"_ZN122_$LT$futures_util..fns..MapOkFn$LT$F$GT$$u20$as$u20$futures_util..fns..FnMut1$LT$core..result..Result$LT$T$C$E$GT$$GT$$GT$8call_mut17h252763b5559d12fbE.exit": ; preds = %4, %1
- call void @llvm.memcpy.p0.p0.i64(ptr %0, ptr %.sroa.5, i64 88, i1 false)
- ret void
-}
-
-attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
>From b65170b56e4e1133d65b00af382768f9f8501698 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 29 Apr 2026 19:12:20 -0400
Subject: [PATCH 03/13] [SROA] Fix struct-to-vector helper formatting
Wrap the LogSelection lambda parameters as requested by clang-format.
---
llvm/lib/Transforms/Scalar/SROA.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 853a857269db8..30f217f7063b2 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5183,8 +5183,7 @@ static std::tuple<Type *, bool, VectorType *>
selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
LLVMContext &C) {
auto LogSelection = [&](StringRef Path, Type *SelectedTy,
- VectorType *SelectedVecTy,
- bool SelectedIntWidening) {
+ VectorType *SelectedVecTy, bool SelectedIntWidening) {
LLVM_DEBUG({
dbgs() << "selectPartitionType path=" << Path
<< " func=" << AI.getFunction()->getName() << " alloca=";
>From 9e6b1c52cc285919a2f19db0882991f5345fde61 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Mon, 27 Apr 2026 09:40:55 -0400
Subject: [PATCH 04/13] [SROA] Simplify struct-to-vector fallback policy
Keep the fallback rule focused on simple memory-use shapes so downstream passes can handle case-specific codegen fallout.
---
llvm/lib/Transforms/Scalar/SROA.cpp | 55 ++++++-------------
.../struct-to-vector-fp-store-only-tail.ll | 17 ++----
.../SROA/struct-to-vector-subpartition.ll | 16 ++----
llvm/test/Transforms/SROA/struct-to-vector.ll | 4 +-
llvm/test/Transforms/SROA/tbaa-struct3.ll | 9 ++-
5 files changed, 35 insertions(+), 66 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 30f217f7063b2..d1df263a39e38 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -5088,14 +5088,12 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
/// Try to canonicalize a homogeneous struct partition to a vector type.
///
-/// This is only used as a fallback after the usual promotion choices fail, so
-/// it stays intentionally conservative: besides requiring a tightly-packed
-/// homogeneous struct shape, it rejects sub-element loads and only recovers a
-/// narrow class of memcpy-only i64 cases.
+/// This is only used as a fallback after the usual promotion choices fail.
+/// Keep the policy simple: canonicalize when every remaining use of the
+/// partition is either a mem intrinsic or a whole-partition load/store.
static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
Partition &P,
- const DataLayout &DL,
- AllocaInst &AI) {
+ const DataLayout &DL) {
unsigned NumElts = STy->getNumElements();
if (NumElts != 2 && NumElts != 4)
return nullptr;
@@ -5120,17 +5118,9 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
return nullptr;
auto *VTy = FixedVectorType::get(EltTy, NumElts);
- bool HasWholePartitionUse = false;
- bool HasSubElementLoad = false;
- bool HasRecoverableSplittableTransfer = false;
- bool IsI64Candidate = VTy->getElementType()->isIntegerTy(64);
- std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL);
- bool IsInteriorSubaggregate = AllocSize && AllocSize->isFixed() &&
- P.beginOffset() != 0 &&
- P.endOffset() < AllocSize->getFixedValue();
- bool IsOriginalFullRecord = P.beginOffset() == 0 && AllocSize &&
- AllocSize->isFixed() &&
- AllocSize->getFixedValue() == P.size();
+ bool SawUse = false;
+ bool AllMemIntrinsics = true;
+ bool AllWholePartitionLoadStore = true;
for (const Slice &S : P) {
if (S.isDead())
@@ -5140,30 +5130,19 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
if (!U)
continue;
- if (S.isSplittable()) {
- if (IsI64Candidate && IsInteriorSubaggregate &&
- S.beginOffset() == P.beginOffset() &&
- S.endOffset() == P.endOffset() && isa<MemIntrinsic>(U->getUser()))
- HasRecoverableSplittableTransfer = true;
- if (IsI64Candidate && IsOriginalFullRecord && P.size() >= 32 &&
- S.beginOffset() == P.beginOffset() &&
- S.endOffset() == P.endOffset() && isa<MemIntrinsic>(U->getUser()))
- HasRecoverableSplittableTransfer = true;
+ User *Usr = U->getUser();
+ if (isa<LifetimeIntrinsic>(Usr) || isa<DbgInfoIntrinsic>(Usr))
continue;
- }
-
- uint64_t SliceSize = S.endOffset() - S.beginOffset();
- if (SliceSize < P.size()) {
- if (isa<LoadInst>(U->getUser()))
- HasSubElementLoad = true;
- continue;
- }
+ SawUse = true;
+ bool CoversWholePartition =
+ S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset();
- HasWholePartitionUse = true;
+ AllMemIntrinsics &= isa<MemIntrinsic>(Usr);
+ AllWholePartitionLoadStore &=
+ CoversWholePartition && (isa<LoadInst>(Usr) || isa<StoreInst>(Usr));
}
- if ((HasWholePartitionUse || HasRecoverableSplittableTransfer) &&
- !HasSubElementLoad)
+ if (SawUse && (AllMemIntrinsics || AllWholePartitionLoadStore))
return VTy;
return nullptr;
}
@@ -5274,7 +5253,7 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
// Try homogeneous struct to vector canonicalization.
if (auto *STy = dyn_cast<StructType>(TypePartitionTy))
- if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL, AI)) {
+ if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
LogSelection("struct-fallback-vecty", VTy, nullptr, false);
return {VTy, false, nullptr};
}
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
index f223c2e8efa70..5556ae5c01ad2 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -1,6 +1,6 @@
; RUN: opt -passes=sroa -S %s | FileCheck %s
-; NOTE: Do not autogenerate. This regression test checks a specific store-only
-; homogeneous float slice pattern that current SROA can over-vectorize.
+; NOTE: Do not autogenerate. This regression test checks the direct SROA shape
+; for a store-only homogeneous float slice under the simplified fallback rule.
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
target triple = "x86_64-pc-linux-gnu"
@@ -11,22 +11,17 @@ target triple = "x86_64-pc-linux-gnu"
declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
; This reduced case exercises a homogeneous FP tail slice that is only stored.
-; The fixed behavior keeps the scalar/memcpy shape; the buggy behavior deletes
-; the 3-float temporary and replaces it with a non-const vector store
-; (`store <4 x float> %...`) seeded by FP struct-to-vector canonicalization.
+; With the simplified fallback rule, SROA canonicalizes the first 16 bytes to a
+; vector store and leaves the remaining 44 bytes as memcpy traffic.
;
; CHECK-LABEL: define ptr @store_only_fp_tail()
-; CHECK: %.sroa.3 = alloca { float, float, float }, align 8
; CHECK: %.sroa.4 = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; CHECK: %.sroa.0.sroa.1 = alloca { float, float, float }, align 8
; CHECK: %.sroa.2 = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %.sroa.3, ptr align 8 %.sroa.0.sroa.1, i64 12, i1 false)
; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %.sroa.4, ptr align 8 %.sroa.2, i64 44, i1 false)
-; CHECK: store float 0.000000e+00, ptr null, align 1
-; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 4), ptr align 8 %.sroa.3, i64 12, i1 false)
+; CHECK: %.sroa.01.0.vec.insert = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+; CHECK: store <4 x float> %.sroa.01.0.vec.insert, ptr null, align 1
; CHECK: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 %.sroa.4, i64 44, i1 false)
-; CHECK-NOT: store <4 x float> %
define ptr @store_only_fp_tail() {
%1 = alloca %class.aiMatrix4x4t, align 4
%2 = alloca %class.aiMatrix4x4t, align 4
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
index fafded012ae4a..2d3066a6f2a57 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
@@ -7,16 +7,12 @@ target triple = "x86_64-unknown-linux-gnu"
; When SROA splits { ptr, i64, i64, i64 } into [0,8), [8,16), [16,32),
; the [16,32) partition type from getTypePartition is { i64, i64 }.
-; The current whole-use heuristic intentionally does NOT canonicalize this
-; sub-partition to <2 x i64>, because the [16,32) slice is only touched by
-; splittable memcpy traffic and has no non-splittable whole-partition use.
-; Keeping it scalar avoids creating a vectorized temporary that later passes
-; may not be able to promote away.
+; With the simplified fallback rule, that partition canonicalizes to <2 x i64>
+; because its remaining users are all mem intrinsics.
; CHECK-LABEL: define void @test_subpartition_type(
-; CHECK: %a.sroa.6 = alloca { i64, i64 }, align 8
-; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a.sroa.6, ptr align 8 %a.sroa.6.0.src.sroa_idx, i64 16, i1 false)
-; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a.sroa.6.0.dst.sroa_idx, ptr align 8 %a.sroa.6, i64 16, i1 false)
+; CHECK: %a.sroa.6.sroa.0.0.copyload = load <2 x i64>, ptr %a.sroa.6.0.src.sroa_idx, align 8
+; CHECK: store <2 x i64> %a.sroa.6.sroa.0.0.copyload, ptr %a.sroa.6.0.dst.sroa_idx, align 8
define void @test_subpartition_type(ptr %src, ptr %dst) {
entry:
%a = alloca { ptr, i64, i64, i64 }, align 8
@@ -33,8 +29,8 @@ entry:
%v1 = load i64, ptr %gep.a.8, align 8
; Only splittable memcpy uses touch [16,32), so SROA creates a single
- ; [16,32) partition. getTypePartition returns { i64, i64 } for this, but
- ; the whole-use heuristic keeps it in scalar/memcpy form.
+ ; [16,32) partition. getTypePartition returns { i64, i64 } for this, and
+ ; the simplified fallback rule canonicalizes it to <2 x i64>.
; Copy all 32 bytes from %a to dst (splittable)
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %a, i64 32, i1 false)
diff --git a/llvm/test/Transforms/SROA/struct-to-vector.ll b/llvm/test/Transforms/SROA/struct-to-vector.ll
index 58945334b7961..09fb50809592d 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector.ll
@@ -7,8 +7,8 @@ define dso_local void @foo_flat(ptr noundef %x, i64 %y.coerce0, i64 %y.coerce1,
; CHECK-SAME: ptr noundef [[X:%.*]], i64 [[Y_COERCE0:%.*]], i64 [[Y_COERCE1:%.*]], i32 noundef [[COND:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[COND]], 0
-; CHECK-NEXT: [[DOTY_COERCE0:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE0]]
; CHECK-NEXT: [[DOTY_COERCE1:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE1]]
+; CHECK-NEXT: [[DOTY_COERCE0:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE0]]
; CHECK-NEXT: store i64 [[DOTY_COERCE0]], ptr [[X]], align 16
; CHECK-NEXT: [[X_REPACK7:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
; CHECK-NEXT: store i64 [[DOTY_COERCE1]], ptr [[X_REPACK7]], align 8
@@ -62,8 +62,8 @@ define dso_local void @foo_nested(ptr noundef %x, i64 %y.coerce0, i64 %y.coerce1
; CHECK-SAME: ptr noundef [[X:%.*]], i64 [[Y_COERCE0:%.*]], i64 [[Y_COERCE1:%.*]], i32 noundef [[COND:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[COND]], 0
-; CHECK-NEXT: [[DOTY_COERCE0:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE0]]
; CHECK-NEXT: [[DOTY_COERCE1:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE1]]
+; CHECK-NEXT: [[DOTY_COERCE0:%.*]] = select i1 [[TOBOOL_NOT]], i64 0, i64 [[Y_COERCE0]]
; CHECK-NEXT: store i64 [[DOTY_COERCE0]], ptr [[X]], align 16
; CHECK-NEXT: [[X_REPACK7:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 8
; CHECK-NEXT: store i64 [[DOTY_COERCE1]], ptr [[X_REPACK7]], align 8
diff --git a/llvm/test/Transforms/SROA/tbaa-struct3.ll b/llvm/test/Transforms/SROA/tbaa-struct3.ll
index 97e82db27c378..6a0cacc7016f7 100644
--- a/llvm/test/Transforms/SROA/tbaa-struct3.ll
+++ b/llvm/test/Transforms/SROA/tbaa-struct3.ll
@@ -73,13 +73,12 @@ define void @load_store_transfer_split_struct_tbaa_2_i31(ptr dereferenceable(24)
; CHECK-LABEL: define void @load_store_transfer_split_struct_tbaa_2_i31(
; CHECK-SAME: ptr dereferenceable(24) [[RES:%.*]], i31 [[A:%.*]], i31 [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP:%.*]] = alloca <2 x i31>, align 8
-; CHECK-NEXT: store i31 [[A]], ptr [[TMP]], align 8
+; CHECK-NEXT: [[TMP:%.*]] = alloca { i31, i31 }, align 4
+; CHECK-NEXT: store i31 [[A]], ptr [[TMP]], align 4
; CHECK-NEXT: [[TMP_4_TMP_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 4
; CHECK-NEXT: store i31 [[B]], ptr [[TMP_4_TMP_4_SROA_IDX]], align 4
-; CHECK-NEXT: [[TMP_SROA_0_0_TMP_SROA_0_0_L1:%.*]] = load <2 x i31>, ptr [[TMP]], align 8, !tbaa.struct [[TBAA_STRUCT4:![0-9]+]]
-; CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x i31> [[TMP_SROA_0_0_TMP_SROA_0_0_L1]] to i62
-; CHECK-NEXT: store i62 [[TMP0]], ptr [[RES]], align 4, !tbaa.struct [[TBAA_STRUCT4]]
+; CHECK-NEXT: [[TMP_0_L1:%.*]] = load i62, ptr [[TMP]], align 4, !tbaa.struct [[TBAA_STRUCT4:![0-9]+]]
+; CHECK-NEXT: store i62 [[TMP_0_L1]], ptr [[RES]], align 4, !tbaa.struct [[TBAA_STRUCT4]]
; CHECK-NEXT: ret void
;
entry:
>From 49db25bec91ae5565e4fc49714d9b61dcd36fc17 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 5 May 2026 21:52:30 -0400
Subject: [PATCH 05/13] [SROA] Fix test updates for simplified fallback
Avoid adding a literal undef CHECK and update the NVPTX byval PTX output after the simplified fallback changes the memcpy_to_param store shape.
---
llvm/test/CodeGen/NVPTX/lower-byval-args.ll | 77 +++++++------------
.../struct-to-vector-fp-store-only-tail.ll | 2 +-
2 files changed, 27 insertions(+), 52 deletions(-)
diff --git a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
index 2b99a2af52719..a3144c5768431 100644
--- a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
@@ -455,64 +455,39 @@ define dso_local ptx_kernel void @memcpy_to_param(ptr nocapture noundef readonly
; PTX-NEXT: .local .align 8 .b8 __local_depot9[8];
; PTX-NEXT: .reg .b64 %SP;
; PTX-NEXT: .reg .b64 %SPL;
-; PTX-NEXT: .reg .b32 %r<3>;
-; PTX-NEXT: .reg .b64 %rd<47>;
+; PTX-NEXT: .reg .b32 %r<23>;
+; PTX-NEXT: .reg .b64 %rd<4>;
; PTX-EMPTY:
; PTX-NEXT: // %bb.0: // %entry
; PTX-NEXT: mov.b64 %SPL, __local_depot9;
; PTX-NEXT: cvta.local.u64 %SP, %SPL;
; PTX-NEXT: ld.param.b64 %rd1, [memcpy_to_param_param_0];
-; PTX-NEXT: add.u64 %rd2, %SPL, 0;
+; PTX-NEXT: cvta.to.global.u64 %rd2, %rd1;
; PTX-NEXT: ld.param.b32 %r1, [memcpy_to_param_param_1+4];
-; PTX-NEXT: st.local.b32 [%rd2+4], %r1;
; PTX-NEXT: ld.param.b32 %r2, [memcpy_to_param_param_1];
-; PTX-NEXT: st.local.b32 [%rd2], %r2;
-; PTX-NEXT: ld.volatile.b8 %rd3, [%rd1];
-; PTX-NEXT: ld.volatile.b8 %rd4, [%rd1+1];
-; PTX-NEXT: shl.b64 %rd5, %rd4, 8;
-; PTX-NEXT: or.b64 %rd6, %rd5, %rd3;
-; PTX-NEXT: ld.volatile.b8 %rd7, [%rd1+2];
-; PTX-NEXT: shl.b64 %rd8, %rd7, 16;
-; PTX-NEXT: ld.volatile.b8 %rd9, [%rd1+3];
-; PTX-NEXT: shl.b64 %rd10, %rd9, 24;
-; PTX-NEXT: or.b64 %rd11, %rd10, %rd8;
-; PTX-NEXT: or.b64 %rd12, %rd11, %rd6;
-; PTX-NEXT: ld.volatile.b8 %rd13, [%rd1+4];
-; PTX-NEXT: ld.volatile.b8 %rd14, [%rd1+5];
-; PTX-NEXT: shl.b64 %rd15, %rd14, 8;
-; PTX-NEXT: or.b64 %rd16, %rd15, %rd13;
-; PTX-NEXT: ld.volatile.b8 %rd17, [%rd1+6];
-; PTX-NEXT: shl.b64 %rd18, %rd17, 16;
-; PTX-NEXT: ld.volatile.b8 %rd19, [%rd1+7];
-; PTX-NEXT: shl.b64 %rd20, %rd19, 24;
-; PTX-NEXT: or.b64 %rd21, %rd20, %rd18;
-; PTX-NEXT: or.b64 %rd22, %rd21, %rd16;
-; PTX-NEXT: shl.b64 %rd23, %rd22, 32;
-; PTX-NEXT: or.b64 %rd24, %rd23, %rd12;
-; PTX-NEXT: st.volatile.b64 [%SP], %rd24;
-; PTX-NEXT: ld.volatile.b8 %rd25, [%rd1+8];
-; PTX-NEXT: ld.volatile.b8 %rd26, [%rd1+9];
-; PTX-NEXT: shl.b64 %rd27, %rd26, 8;
-; PTX-NEXT: or.b64 %rd28, %rd27, %rd25;
-; PTX-NEXT: ld.volatile.b8 %rd29, [%rd1+10];
-; PTX-NEXT: shl.b64 %rd30, %rd29, 16;
-; PTX-NEXT: ld.volatile.b8 %rd31, [%rd1+11];
-; PTX-NEXT: shl.b64 %rd32, %rd31, 24;
-; PTX-NEXT: or.b64 %rd33, %rd32, %rd30;
-; PTX-NEXT: or.b64 %rd34, %rd33, %rd28;
-; PTX-NEXT: ld.volatile.b8 %rd35, [%rd1+12];
-; PTX-NEXT: ld.volatile.b8 %rd36, [%rd1+13];
-; PTX-NEXT: shl.b64 %rd37, %rd36, 8;
-; PTX-NEXT: or.b64 %rd38, %rd37, %rd35;
-; PTX-NEXT: ld.volatile.b8 %rd39, [%rd1+14];
-; PTX-NEXT: shl.b64 %rd40, %rd39, 16;
-; PTX-NEXT: ld.volatile.b8 %rd41, [%rd1+15];
-; PTX-NEXT: shl.b64 %rd42, %rd41, 24;
-; PTX-NEXT: or.b64 %rd43, %rd42, %rd40;
-; PTX-NEXT: or.b64 %rd44, %rd43, %rd38;
-; PTX-NEXT: shl.b64 %rd45, %rd44, 32;
-; PTX-NEXT: or.b64 %rd46, %rd45, %rd34;
-; PTX-NEXT: st.volatile.b64 [%SP+8], %rd46;
+; PTX-NEXT: st.v2.b32 [%SP], {%r2, %r1};
+; PTX-NEXT: ld.volatile.global.b8 %r3, [%rd2+4];
+; PTX-NEXT: ld.volatile.global.b8 %r4, [%rd2+5];
+; PTX-NEXT: shl.b32 %r5, %r4, 8;
+; PTX-NEXT: or.b32 %r6, %r5, %r3;
+; PTX-NEXT: ld.volatile.global.b8 %r7, [%rd2+6];
+; PTX-NEXT: shl.b32 %r8, %r7, 16;
+; PTX-NEXT: ld.volatile.global.b8 %r9, [%rd2+7];
+; PTX-NEXT: shl.b32 %r10, %r9, 24;
+; PTX-NEXT: or.b32 %r11, %r10, %r8;
+; PTX-NEXT: or.b32 %r12, %r11, %r6;
+; PTX-NEXT: ld.volatile.global.b8 %r13, [%rd2];
+; PTX-NEXT: ld.volatile.global.b8 %r14, [%rd2+1];
+; PTX-NEXT: shl.b32 %r15, %r14, 8;
+; PTX-NEXT: or.b32 %r16, %r15, %r13;
+; PTX-NEXT: ld.volatile.global.b8 %r17, [%rd2+2];
+; PTX-NEXT: shl.b32 %r18, %r17, 16;
+; PTX-NEXT: ld.volatile.global.b8 %r19, [%rd2+3];
+; PTX-NEXT: shl.b32 %r20, %r19, 24;
+; PTX-NEXT: or.b32 %r21, %r20, %r18;
+; PTX-NEXT: or.b32 %r22, %r21, %r16;
+; PTX-NEXT: add.u64 %rd3, %SPL, 0;
+; PTX-NEXT: st.local.v2.b32 [%rd3], {%r22, %r12};
; PTX-NEXT: ret;
entry:
tail call void @llvm.memcpy.p0.p0.i64(ptr %s, ptr %in, i64 16, i1 true)
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
index 5556ae5c01ad2..04f1d0d1af914 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -18,7 +18,7 @@ declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr no
; CHECK: %.sroa.4 = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
; CHECK: %.sroa.2 = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %.sroa.4, ptr align 8 %.sroa.2, i64 44, i1 false)
-; CHECK: %.sroa.01.0.vec.insert = insertelement <4 x float> undef, float 0.000000e+00, i32 0
+; CHECK: %.sroa.01.0.vec.insert = insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 0
; CHECK: store <4 x float> %.sroa.01.0.vec.insert, ptr null, align 1
; CHECK: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 %.sroa.4, i64 44, i1 false)
>From 5b059645cc8815ff0dbbc86d4aeacb6919e78342 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 6 May 2026 17:50:23 -0400
Subject: [PATCH 06/13] [SROA] Update DebugInfo tests for struct-to-vector
sub-partition canonicalization
2-element homogeneous struct sub-partitions (e.g. the [x,y] slice of a 3-element
struct, or a struct.two / struct.p field) are now canonicalized to fixed vectors
by tryCanonicalizeStructToVector. Update three DebugInfo tests whose CHECK lines
expected scalar i64/i32 fragments but now see <2 x i64> / <2 x i32> debug values.
---
.../Generic/assignment-tracking/sroa/user-memcpy.ll | 8 ++++----
llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll | 8 ++++----
llvm/test/DebugInfo/X86/sroasplit-4.ll | 9 +++++----
3 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
index 5997cc1b9d041..adbcbc61b9dda 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
@@ -21,9 +21,8 @@
;; Allocas have been promoted - the linked dbg.assigns have been removed.
;; | V3i point = {0, 0, 0};
-;; Homogeneous i64 struct is split; point.x/point.y are described as separate i64 fragments.
-; CHECK-NEXT: #dbg_value(i64 0, ![[point:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i64 0, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
+;; Homogeneous i64 sub-partition [x,y] is canonicalized to <2 x i64>.
+; CHECK-NEXT: #dbg_value(<2 x i64> zeroinitializer, ![[point:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), !{{[0-9]+}})
;; point.z = 5000;
; CHECK-NEXT: #dbg_value(i64 5000, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
@@ -41,7 +40,8 @@
; CHECK-NEXT: #dbg_value(i64 %other.sroa.3.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
;; | std::memcpy(&point.y, &other.x, sizeof(long) * 2);
-;; memcpy is folded into scalar dbg_value updates for point.y/point.z.
+;; memcpy updates scalar point.y/point.z lanes; the vector slot for [x,y] is kept live.
+; CHECK-NEXT: %{{.*}} = insertelement <2 x i64> zeroinitializer, i64 %other.sroa.0.0.copyload, i32 1{{.*}}
; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
diff --git a/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll b/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
index 40c57730b1557..59f2a1bdb1dc4 100644
--- a/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
+++ b/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
@@ -140,9 +140,8 @@ entry:
;; 16 bit variable f (!62): value vgf (lower bits)
;; 16 bit variable g (!63): value vgf (upper bits)
;;
-;; 16 bit variable h (!64): inner aggregate slice kept on stack; #dbg_declare on that alloca.
-; COMMON-NEXT: %[[h_alloca:.*]] = alloca %struct.two, align 8
-; COMMON-NEXT: #dbg_declare(ptr %[[h_alloca]], ![[h:[0-9]+]], !DIExpression(), !{{[0-9]+}})
+;; 16 bit variable h (!64): struct.two = {i32,i32} is a 2-element homogeneous struct,
+;; canonicalized to <2 x i32> by SROA; no alloca or memcpy survives.
; COMMON-NEXT: %[[ve:.*]] = load i32, ptr @gf, align 4{{.*}}
;; FIXME: mem2reg bug - offset is incorrect - see comment above.
; COMMON-NEXT: #dbg_value(i32 %[[ve]], ![[e:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2), !{{[0-9]+}})
@@ -150,7 +149,8 @@ entry:
; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[f:[0-9]+]], !DIExpression(), !{{[0-9]+}})
;; FIXME: mem2reg bug - offset is incorrect - see comment above.
; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[g:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2), !{{[0-9]+}})
-; COMMON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[h_alloca]], ptr align 4 getelementptr inbounds (i8, ptr @gf, i64 8), i64 8, i1 false){{.*}}
+; COMMON-NEXT: %[[vh:.*]] = load <2 x i32>, ptr getelementptr inbounds (i8, ptr @gf, i64 8), align 4{{.*}}
+; COMMON-NEXT: #dbg_value(<2 x i32> %[[vh]], ![[h:[0-9]+]], !DIExpression(), !{{[0-9]+}})
; COMMON-NEXT: ret i32 %[[vfg]]{{.*}}
define dso_local noundef i32 @_Z4fun3v() #0 !dbg !55 {
entry:
diff --git a/llvm/test/DebugInfo/X86/sroasplit-4.ll b/llvm/test/DebugInfo/X86/sroasplit-4.ll
index f483d60ad7e5b..16fe331eeae9b 100644
--- a/llvm/test/DebugInfo/X86/sroasplit-4.ll
+++ b/llvm/test/DebugInfo/X86/sroasplit-4.ll
@@ -3,14 +3,15 @@
; Test that recursively splitting an alloca updates the debug info correctly.
; CHECK-LABEL: if.end:
; CHECK-NEXT: %[[T0:.*]] = load i64, ptr @t, align 8{{.*}}
-; CHECK-NEXT: #dbg_value(i64 %[[T0]], ![[Y:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64), !{{[0-9]+}})
+; CHECK-NEXT: %[[Y0:.*]] = insertelement <2 x i64> undef, i64 %[[T0]], i32 0{{.*}}
+; CHECK-NEXT: #dbg_value(<2 x i64> %[[Y0]], ![[Y:[0-9]+]], !DIExpression(), !{{[0-9]+}})
; CHECK-NEXT: %[[T1:.*]] = load i64, ptr @t, align 8{{.*}}
-; CHECK-NEXT: #dbg_value(i64 %[[T1]], ![[Y]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
+; CHECK-NEXT: %[[Y1:.*]] = insertelement <2 x i64> %[[Y0]], i64 %[[T1]], i32 1{{.*}}
+; CHECK-NEXT: #dbg_value(<2 x i64> %[[Y1]], ![[Y]], !DIExpression(), !{{[0-9]+}})
; CHECK-NEXT: #dbg_value(i32 0, ![[R:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 32), !{{[0-9]+}})
; CHECK-NEXT: #dbg_value(i64 0, ![[R]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
; CHECK-NEXT: #dbg_value(i64 0, ![[R]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i64 %[[T0]], ![[R]], !DIExpression(DW_OP_LLVM_fragment, 192, 64), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i64 %[[T1]], ![[R]], !DIExpression(DW_OP_LLVM_fragment, 256, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(<2 x i64> %[[Y1]], ![[R]], !DIExpression(DW_OP_LLVM_fragment, 192, 128), !{{[0-9]+}})
;
; struct p {
; __SIZE_TYPE__ s;
>From c597271316c1f288ed24a9282465e0acc4bd01c3 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 6 May 2026 18:05:55 -0400
Subject: [PATCH 07/13] [SROA] Avoid undef in sroasplit-4.ll CHECK pattern
Use {{.*}} instead of literal undef to satisfy the undef deprecator.
---
llvm/test/DebugInfo/X86/sroasplit-4.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/DebugInfo/X86/sroasplit-4.ll b/llvm/test/DebugInfo/X86/sroasplit-4.ll
index 16fe331eeae9b..1003db59a08ae 100644
--- a/llvm/test/DebugInfo/X86/sroasplit-4.ll
+++ b/llvm/test/DebugInfo/X86/sroasplit-4.ll
@@ -3,7 +3,7 @@
; Test that recursively splitting an alloca updates the debug info correctly.
; CHECK-LABEL: if.end:
; CHECK-NEXT: %[[T0:.*]] = load i64, ptr @t, align 8{{.*}}
-; CHECK-NEXT: %[[Y0:.*]] = insertelement <2 x i64> undef, i64 %[[T0]], i32 0{{.*}}
+; CHECK-NEXT: %[[Y0:.*]] = insertelement <2 x i64> {{.*}}, i64 %[[T0]], i32 0{{.*}}
; CHECK-NEXT: #dbg_value(<2 x i64> %[[Y0]], ![[Y:[0-9]+]], !DIExpression(), !{{[0-9]+}})
; CHECK-NEXT: %[[T1:.*]] = load i64, ptr @t, align 8{{.*}}
; CHECK-NEXT: %[[Y1:.*]] = insertelement <2 x i64> %[[Y0]], i64 %[[T1]], i32 1{{.*}}
>From 731613e5fce23ed1ac092a3ed073baa557317947 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Thu, 14 May 2026 00:07:14 -0400
Subject: [PATCH 08/13] [SROA] Gate struct-to-vector canonicalization on a
per-pass flag
Add a `canonicalize-struct-to-vector` option to `SROAOptions`, off by default.
Only the late SROA passes in `addVectorPasses` and the two NVPTX legacy-PM
SROAs enable it, so canonicalization runs after `MemCpyOptPass`. Running it
earlier can hide memcpy chains from memcpyopt or emit wide stores with undef
suffix lanes (see the two new SROA tests). With firing gated, the helper is
simplified to require memory-intrinsic-only users, and the element-shape rule
accepts any homogeneous element count, any integer width, any FP, and integral
pointers. Opt in via `opt -passes='sroa<canonicalize-struct-to-vector>'`.
---
llvm/include/llvm/Transforms/Scalar.h | 3 +-
llvm/include/llvm/Transforms/Scalar/SROA.h | 17 ++-
llvm/lib/Passes/PassBuilder.cpp | 40 ++++--
llvm/lib/Passes/PassBuilderPipelines.cpp | 22 +++-
llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 6 +-
llvm/lib/Transforms/Scalar/SROA.cpp | 115 ++++++++++--------
llvm/test/DebugInfo/X86/sroasplit-4.ll | 17 +--
.../SROA/struct-to-vector-before-memcpyopt.ll | 65 ++++++++++
.../struct-to-vector-fp-store-only-tail.ll | 52 +++++---
.../SROA/struct-to-vector-subpartition.ll | 9 +-
llvm/test/Transforms/SROA/struct-to-vector.ll | 15 +--
11 files changed, 246 insertions(+), 115 deletions(-)
create mode 100644 llvm/test/Transforms/SROA/struct-to-vector-before-memcpyopt.ll
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index e2a236458dd79..29de5a26d3503 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -44,7 +44,8 @@ LLVM_ABI FunctionPass *createDeadStoreEliminationPass();
//
// SROA - Replace aggregates or pieces of aggregates with scalar SSA values.
//
-LLVM_ABI FunctionPass *createSROAPass(bool PreserveCFG = true);
+LLVM_ABI FunctionPass *createSROAPass(bool PreserveCFG = true,
+ bool CanonicalizeStructToVector = false);
//===----------------------------------------------------------------------===//
//
diff --git a/llvm/include/llvm/Transforms/Scalar/SROA.h b/llvm/include/llvm/Transforms/Scalar/SROA.h
index 745e5aee64165..77703661b7f61 100644
--- a/llvm/include/llvm/Transforms/Scalar/SROA.h
+++ b/llvm/include/llvm/Transforms/Scalar/SROA.h
@@ -21,15 +21,26 @@ namespace llvm {
class Function;
-enum class SROAOptions : bool { ModifyCFG, PreserveCFG };
+struct SROAOptions {
+ enum CFGOption { ModifyCFG, PreserveCFG };
+
+ CFGOption CFG;
+ bool CanonicalizeStructToVector;
+
+ SROAOptions(CFGOption CFG = PreserveCFG,
+ bool CanonicalizeStructToVector = false)
+ : CFG(CFG), CanonicalizeStructToVector(CanonicalizeStructToVector) {}
+};
class SROAPass : public OptionalPassInfoMixin<SROAPass> {
- const SROAOptions PreserveCFG;
+ const SROAOptions Options;
public:
/// If \p PreserveCFG is set, then the pass is not allowed to modify CFG
/// in any way, even if it would update CFG analyses.
- LLVM_ABI SROAPass(SROAOptions PreserveCFG);
+ /// If \p CanonicalizeStructToVector is set, then the pass will try to convert
+ /// allocas of homogeneous structs into vector allocas.
+ LLVM_ABI SROAPass(SROAOptions Options);
/// Run the pass over the function.
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 0ada0f7e766dc..f16bfbbc19cc3 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1433,16 +1433,36 @@ Expected<ScalarizerPassOptions> parseScalarizerOptions(StringRef Params) {
}
Expected<SROAOptions> parseSROAOptions(StringRef Params) {
- if (Params.empty() || Params == "modify-cfg")
- return SROAOptions::ModifyCFG;
- if (Params == "preserve-cfg")
- return SROAOptions::PreserveCFG;
- return make_error<StringError>(
- formatv("invalid SROA pass parameter '{}' (either preserve-cfg or "
- "modify-cfg can be specified)",
- Params)
- .str(),
- inconvertibleErrorCode());
+ SROAOptions Result(SROAOptions::ModifyCFG);
+ bool SawCFGOption = false;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ if (ParamName == "modify-cfg") {
+ if (SawCFGOption)
+ return make_error<StringError>("multiple SROA CFG options specified",
+ inconvertibleErrorCode());
+ Result.CFG = SROAOptions::ModifyCFG;
+ SawCFGOption = true;
+ } else if (ParamName == "preserve-cfg") {
+ if (SawCFGOption)
+ return make_error<StringError>("multiple SROA CFG options specified",
+ inconvertibleErrorCode());
+ Result.CFG = SROAOptions::PreserveCFG;
+ SawCFGOption = true;
+ } else if (ParamName == "canonicalize-struct-to-vector") {
+ Result.CanonicalizeStructToVector = true;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid SROA pass parameter '{}' (expected preserve-cfg, "
+ "modify-cfg, or canonicalize-struct-to-vector)",
+ ParamName)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
}
Expected<StackLifetime::LivenessType>
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index a78a5afe05ae9..0dcadae70536e 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1372,7 +1372,16 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// NOTE: we are very late in the pipeline, and we don't have any LICM
// or SimplifyCFG passes scheduled after us, that would cleanup
// the CFG mess this may created if allowed to modify CFG, so forbid that.
- FPM.addPass(SROAPass(SROAOptions::PreserveCFG));
+
+ // We also turn on struct to vector canonicalization here, which allows
+ // converting allocas of homogeneous structs into vector allocas when the
+ // allocas' users are all memory intrinsics. This allows promotion in some
+ // cases because structs cannot promote to SSA values, but vectors can. We
+ // only turn this on after memcpyopt runs because this might hinder
+ // memcpyopt's optimizations if done before. Look at the documentation for
+ // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
+ FPM.addPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
+ /*CanonicalizeStructToVector=*/true)));
}
if (!isFullLTOPostLink(LTOPhase)) {
@@ -1464,7 +1473,16 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// NOTE: we are very late in the pipeline, and we don't have any LICM
// or SimplifyCFG passes scheduled after us, that would cleanup
// the CFG mess this may created if allowed to modify CFG, so forbid that.
- FPM.addPass(SROAPass(SROAOptions::PreserveCFG));
+
+ // We also turn on struct to vector canonicalization here, which allows
+ // converting allocas of homogeneous structs into vector allocas when the
+ // allocas' users are all memory intrinsics. This allows promotion in some
+ // cases because structs cannot promote to SSA values, but vectors can. We
+ // only turn this on after memcpyopt runs because this might hinder
+ // memcpyopt's optimizations if done before. Look at the documentation for
+ // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
+ FPM.addPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
+ /*CanonicalizeStructToVector=*/true)));
}
FPM.addPass(InferAlignmentPass());
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 9351c8dde60d4..6770c94b57626 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -300,7 +300,8 @@ void NVPTXPassConfig::addEarlyCSEOrGVNPass() {
void NVPTXPassConfig::addAddressSpaceInferencePasses() {
// NVPTXLowerArgs emits alloca for byval parameters which can often
// be eliminated by SROA.
- addPass(createSROAPass());
+ addPass(createSROAPass(/*PreserveCFG=*/true,
+ /*CanonicalizeStructToVector=*/true));
addPass(createNVPTXLowerAllocaPass());
// TODO: Consider running InferAddressSpaces during opt, earlier in the
// compilation flow.
@@ -391,7 +392,8 @@ void NVPTXPassConfig::addIRPasses() {
addEarlyCSEOrGVNPass();
if (!DisableLoadStoreVectorizer)
addPass(createLoadStoreVectorizerPass());
- addPass(createSROAPass());
+ addPass(createSROAPass(/*PreserveCFG=*/true,
+ /*CanonicalizeStructToVector=*/true));
addPass(createNVPTXTagInvariantLoadsPass());
if (!DisableNVPTXIRPeephole)
addPass(createNVPTXIRPeepholePass());
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index d1df263a39e38..c10cd8eaf385f 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -178,6 +178,7 @@ class SROA {
DomTreeUpdater *const DTU;
AssumptionCache *const AC;
const bool PreserveCFG;
+ const bool CanonicalizeStructToVector;
/// Worklist of alloca instructions to simplify.
///
@@ -240,9 +241,10 @@ class SROA {
public:
SROA(LLVMContext *C, DomTreeUpdater *DTU, AssumptionCache *AC,
- SROAOptions PreserveCFG_)
+ SROAOptions Options)
: C(C), DTU(DTU), AC(AC),
- PreserveCFG(PreserveCFG_ == SROAOptions::PreserveCFG) {}
+ PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
+ CanonicalizeStructToVector(Options.CanonicalizeStructToVector) {}
/// Main run method used by both the SROAPass and by the legacy pass.
std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
@@ -5088,44 +5090,50 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
/// Try to canonicalize a homogeneous struct partition to a vector type.
///
-/// This is only used as a fallback after the usual promotion choices fail.
-/// Keep the policy simple: canonicalize when every remaining use of the
-/// partition is either a mem intrinsic or a whole-partition load/store.
+/// We can do this if all the elements of the struct are the same and tightly
+/// packed. This can sometimes eliminate allocas because structs cannot get
+/// promoted to LLVM values, but vectors can.
+///
+/// We only apply this transformation when all users of the alloca are memory
+/// intrinsics. Otherwise, if there is a load or store of some other type to the
+/// partition, SROA would select that type.
+///
+/// Applying this transformation too early may hinder memcpyopt, which may
+/// generate better code when eliminating allocas. For example, see
+/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
+/// this before memcpyopt can initialize previously uninitialized memory when
+/// the alloca gets promoted to an SSA value. For another example, see
+/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
+/// this before memcpyopt can result in promoting an alloca so that we load a
+/// tempory value instead of copying the tempory value into memory, whereas
+/// memcpyopt eliminates the tempory altogether.
+///
+/// As such, we only apply this transformation after memcpyopt has run. We gate
+/// this transformation by the "CanonicalizeStructToVector" pass option.
static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
Partition &P,
const DataLayout &DL) {
unsigned NumElts = STy->getNumElements();
- if (NumElts != 2 && NumElts != 4)
- return nullptr;
Type *EltTy = STy->getElementType(0);
if (!llvm::all_equal(STy->elements()))
return nullptr;
- if (auto *IT = dyn_cast<IntegerType>(EltTy)) {
- if (IT->getBitWidth() < 8)
- return nullptr;
- } else if (!EltTy->isFloatingPointTy()) {
- return nullptr;
- }
-
- TypeSize EltTS = DL.getTypeAllocSize(EltTy);
- if (!EltTS.isFixed())
- return nullptr;
- uint64_t EltSize = EltTS.getFixedValue();
-
- if (DL.getStructLayout(STy)->getSizeInBytes() != NumElts * EltSize)
+ bool IsIntegralPointerTy =
+ EltTy->isPointerTy() && !DL.isNonIntegralPointerType(EltTy);
+ if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
+ !IsIntegralPointerTy)
return nullptr;
auto *VTy = FixedVectorType::get(EltTy, NumElts);
- bool SawUse = false;
- bool AllMemIntrinsics = true;
- bool AllWholePartitionLoadStore = true;
+ TypeSize StructSize = DL.getStructLayout(STy)->getSizeInBytes();
+ TypeSize VectorSize = DL.getTypeAllocSize(VTy);
+ if (StructSize != VectorSize)
+ return nullptr;
for (const Slice &S : P) {
if (S.isDead())
continue;
-
auto *U = S.getUse();
if (!U)
continue;
@@ -5133,18 +5141,12 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
User *Usr = U->getUser();
if (isa<LifetimeIntrinsic>(Usr) || isa<DbgInfoIntrinsic>(Usr))
continue;
- SawUse = true;
- bool CoversWholePartition =
- S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset();
- AllMemIntrinsics &= isa<MemIntrinsic>(Usr);
- AllWholePartitionLoadStore &=
- CoversWholePartition && (isa<LoadInst>(Usr) || isa<StoreInst>(Usr));
+ if (!isa<MemIntrinsic>(Usr))
+ return nullptr;
}
- if (SawUse && (AllMemIntrinsics || AllWholePartitionLoadStore))
- return VTy;
- return nullptr;
+ return VTy;
}
/// Select a partition type for an alloca partition.
@@ -5160,7 +5162,7 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
/// nullptr.
static std::tuple<Type *, bool, VectorType *>
selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
- LLVMContext &C) {
+ LLVMContext &C, bool CanonicalizeStructToVector) {
auto LogSelection = [&](StringRef Path, Type *SelectedTy,
VectorType *SelectedVecTy, bool SelectedIntWidening) {
LLVM_DEBUG({
@@ -5251,12 +5253,16 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
return {LargestIntTy, true, nullptr};
}
- // Try homogeneous struct to vector canonicalization.
- if (auto *STy = dyn_cast<StructType>(TypePartitionTy))
- if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
- LogSelection("struct-fallback-vecty", VTy, nullptr, false);
- return {VTy, false, nullptr};
+ // Try homogeneous struct to vector canonicalization when requested. Running
+ // this too early can hide memcpy chains from MemCpyOpt.
+ if (CanonicalizeStructToVector) {
+ if (auto *STy = dyn_cast<StructType>(TypePartitionTy)) {
+ if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
+ LogSelection("struct-fallback-vecty", VTy, nullptr, false);
+ return {VTy, false, nullptr};
+ }
}
+ }
// Fallback to TypePartitionTy and we probably won't promote.
LogSelection("type-partition-fallback", TypePartitionTy, nullptr, false);
@@ -5298,7 +5304,7 @@ SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
const DataLayout &DL = AI.getDataLayout();
// Select the type for the new alloca that spans the partition.
auto [PartitionTy, IsIntegerWideningViable, VecTy] =
- selectPartitionType(P, DL, AI, *C);
+ selectPartitionType(P, DL, AI, *C, CanonicalizeStructToVector);
// Check for the case where we're going to rewrite to a new alloca of the
// exact same type as the original, and with the same access offsets. In that
@@ -6107,7 +6113,7 @@ PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) {
AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(F);
DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
auto [Changed, CFGChanged] =
- SROA(&F.getContext(), &DTU, &AC, PreserveCFG).runSROA(F);
+ SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
if (!Changed)
return PreservedAnalyses::all();
PreservedAnalyses PA;
@@ -6121,23 +6127,27 @@ void SROAPass::printPipeline(
raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
static_cast<PassInfoMixin<SROAPass> *>(this)->printPipeline(
OS, MapClassName2PassName);
- OS << (PreserveCFG == SROAOptions::PreserveCFG ? "<preserve-cfg>"
- : "<modify-cfg>");
+ OS << '<'
+ << (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
+ : "modify-cfg");
+ if (Options.CanonicalizeStructToVector)
+ OS << ";canonicalize-struct-to-vector";
+ OS << '>';
}
-SROAPass::SROAPass(SROAOptions PreserveCFG) : PreserveCFG(PreserveCFG) {}
+SROAPass::SROAPass(SROAOptions Options) : Options(Options) {}
namespace {
/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
class SROALegacyPass : public FunctionPass {
- SROAOptions PreserveCFG;
+ SROAOptions Options;
public:
static char ID;
- SROALegacyPass(SROAOptions PreserveCFG = SROAOptions::PreserveCFG)
- : FunctionPass(ID), PreserveCFG(PreserveCFG) {
+ SROALegacyPass(SROAOptions Options = SROAOptions::PreserveCFG)
+ : FunctionPass(ID), Options(Options) {
initializeSROALegacyPassPass(*PassRegistry::getPassRegistry());
}
@@ -6149,8 +6159,7 @@ class SROALegacyPass : public FunctionPass {
AssumptionCache &AC =
getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
- auto [Changed, _] =
- SROA(&F.getContext(), &DTU, &AC, PreserveCFG).runSROA(F);
+ auto [Changed, _] = SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
return Changed;
}
@@ -6168,9 +6177,11 @@ class SROALegacyPass : public FunctionPass {
char SROALegacyPass::ID = 0;
-FunctionPass *llvm::createSROAPass(bool PreserveCFG) {
- return new SROALegacyPass(PreserveCFG ? SROAOptions::PreserveCFG
- : SROAOptions::ModifyCFG);
+FunctionPass *llvm::createSROAPass(bool PreserveCFG,
+ bool CanonicalizeStructToVector) {
+ return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
+ : SROAOptions::ModifyCFG,
+ CanonicalizeStructToVector));
}
INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
diff --git a/llvm/test/DebugInfo/X86/sroasplit-4.ll b/llvm/test/DebugInfo/X86/sroasplit-4.ll
index 1003db59a08ae..d5ce348e9896e 100644
--- a/llvm/test/DebugInfo/X86/sroasplit-4.ll
+++ b/llvm/test/DebugInfo/X86/sroasplit-4.ll
@@ -1,17 +1,12 @@
; RUN: opt -passes='sroa' < %s -S -o - | FileCheck %s
;
; Test that recursively splitting an alloca updates the debug info correctly.
-; CHECK-LABEL: if.end:
-; CHECK-NEXT: %[[T0:.*]] = load i64, ptr @t, align 8{{.*}}
-; CHECK-NEXT: %[[Y0:.*]] = insertelement <2 x i64> {{.*}}, i64 %[[T0]], i32 0{{.*}}
-; CHECK-NEXT: #dbg_value(<2 x i64> %[[Y0]], ![[Y:[0-9]+]], !DIExpression(), !{{[0-9]+}})
-; CHECK-NEXT: %[[T1:.*]] = load i64, ptr @t, align 8{{.*}}
-; CHECK-NEXT: %[[Y1:.*]] = insertelement <2 x i64> %[[Y0]], i64 %[[T1]], i32 1{{.*}}
-; CHECK-NEXT: #dbg_value(<2 x i64> %[[Y1]], ![[Y]], !DIExpression(), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i32 0, ![[R:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 32), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i64 0, ![[R]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i64 0, ![[R]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(<2 x i64> %[[Y1]], ![[R]], !DIExpression(DW_OP_LLVM_fragment, 192, 128), !{{[0-9]+}})
+; CHECK: %[[T:.*]] = load i64, ptr @t, align 8
+; CHECK: #dbg_value(i64 %[[T]], ![[Y:.*]], !DIExpression(DW_OP_LLVM_fragment, 0, 64),
+; CHECK: %[[T1:.*]] = load i64, ptr @t, align 8
+; CHECK: #dbg_value(i64 %[[T1]], ![[Y]], !DIExpression(DW_OP_LLVM_fragment, 64, 64),
+; CHECK: #dbg_value(i64 %[[T]], ![[R:.*]], !DIExpression(DW_OP_LLVM_fragment, 192, 64),
+; CHECK: #dbg_value(i64 %[[T1]], ![[R]], !DIExpression(DW_OP_LLVM_fragment, 256, 64),
;
; struct p {
; __SIZE_TYPE__ s;
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-before-memcpyopt.ll b/llvm/test/Transforms/SROA/struct-to-vector-before-memcpyopt.ll
new file mode 100644
index 0000000000000..2a694e8899b6a
--- /dev/null
+++ b/llvm/test/Transforms/SROA/struct-to-vector-before-memcpyopt.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='sroa,memcpyopt,dse,instcombine,sroa<canonicalize-struct-to-vector>' -S %s | FileCheck %s --check-prefix=DELAYED
+; RUN: opt -passes='sroa<canonicalize-struct-to-vector>,memcpyopt,dse,instcombine,sroa<canonicalize-struct-to-vector>' -S %s | FileCheck %s --check-prefix=EARLY
+
+%pair = type { i64, i64 }
+%quad = type { i64, i64, i64, i64 }
+
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
+declare void @llvm.memset.p0.i64(ptr writeonly captures(none), i8, i64, i1 immarg)
+
+; This test case has the following pattern:
+;
+; memcpy tmp, obj, 16
+; memset obj + 16, 0, 16
+;
+; ----- SWAP(other, tmp) -----
+;
+; memcpy swap.tmp, tmp, 16
+; memcpy tmp, other, 16
+; memcpy other, swap.tmp, 16
+;
+; It swaps the first 16-bytes of other and tmp, but the first 16-bytes of tmp are actually the same as the first 16-bytes
+; of obj. This comes from real code from DuckDB, where the swap function is inlined. If struct-to-vector canonicalization runs before
+; memcpyopt, then swap.tmp gets promoted to an SSA value and we are stuck with saving tmp to swap.tmp.
+; But if we delay canonicalization until after memcpyopt,
+; then memcpyopt realizes that tmp and obj share the same first 16-bytes, and so we don't need to save tmp to swap.tmp. So it can rewrite to:
+;
+; memset obj + 16, 0, 16
+; memcpy tmp, other, 16
+; memcpy other, obj, 16
+;
+; This is a win because it avoids the extra step of saving tmp to swap.tmp.
+define void @move_then_swap(ptr %dst, ptr %src, ptr %other) {
+; DELAYED-LABEL: define void @move_then_swap(
+; DELAYED-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[OTHER:%.*]]) {
+; DELAYED-NEXT: [[ENTRY:.*:]]
+; DELAYED-NEXT: [[TMP_SROA_0_0_COPYLOAD:%.*]] = load <2 x i64>, ptr [[OTHER]], align 8
+; DELAYED-NEXT: call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) [[OTHER]], ptr noundef nonnull align 8 dereferenceable(16) [[SRC]], i64 16, i1 false)
+; DELAYED-NEXT: store <2 x i64> [[TMP_SROA_0_0_COPYLOAD]], ptr [[DST]], align 8
+; DELAYED-NEXT: ret void
+;
+; EARLY-LABEL: define void @move_then_swap(
+; EARLY-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[OTHER:%.*]]) {
+; EARLY-NEXT: [[ENTRY:.*:]]
+; EARLY-NEXT: [[OBJ_SROA_0_0_COPYLOAD:%.*]] = load <4 x i64>, ptr [[SRC]], align 8
+; EARLY-NEXT: [[OBJ_SROA_0_0_VEC_EXTRACT:%.*]] = shufflevector <4 x i64> [[OBJ_SROA_0_0_COPYLOAD]], <4 x i64> poison, <2 x i32> <i32 0, i32 1>
+; EARLY-NEXT: [[TMP_SROA_0_0_COPYLOAD:%.*]] = load <2 x i64>, ptr [[OTHER]], align 8
+; EARLY-NEXT: store <2 x i64> [[OBJ_SROA_0_0_VEC_EXTRACT]], ptr [[OTHER]], align 8
+; EARLY-NEXT: store <2 x i64> [[TMP_SROA_0_0_COPYLOAD]], ptr [[DST]], align 8
+; EARLY-NEXT: ret void
+;
+entry:
+ %tmp = alloca %pair, align 8
+ %obj = alloca %quad, align 8
+ %swap.tmp = alloca %pair, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %obj, ptr align 8 %src, i64 32, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %obj, i64 16, i1 false)
+ %obj.tail = getelementptr inbounds i8, ptr %obj, i64 16
+ call void @llvm.memset.p0.i64(ptr align 8 %obj.tail, i8 0, i64 16, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %swap.tmp, ptr align 8 %tmp, i64 16, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %other, i64 16, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %other, ptr align 8 %swap.tmp, i64 16, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp, i64 16, i1 false)
+ ret void
+}
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
index 04f1d0d1af914..5920f01e37188 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -1,28 +1,46 @@
-; RUN: opt -passes=sroa -S %s | FileCheck %s
-; NOTE: Do not autogenerate. This regression test checks the direct SROA shape
-; for a store-only homogeneous float slice under the simplified fallback rule.
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-pc-linux-gnu"
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=sroa -S %s | FileCheck %s --check-prefixes=NO-CANON
+; RUN: opt -passes='sroa<canonicalize-struct-to-vector>' -S %s | FileCheck %s --check-prefixes=CANON
%class.aiMatrix4x4t = type { float, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }
; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
-; This reduced case exercises a homogeneous FP tail slice that is only stored.
-; With the simplified fallback rule, SROA canonicalizes the first 16 bytes to a
-; vector store and leaves the remaining 44 bytes as memcpy traffic.
+; This test case shows an example where we have:
+;
+; 1. memcpy %1, %2
+; 2. memcpy null, %1
;
-; CHECK-LABEL: define ptr @store_only_fp_tail()
-; CHECK: %.sroa.4 = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; CHECK: %.sroa.2 = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
-; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %.sroa.4, ptr align 8 %.sroa.2, i64 44, i1 false)
-; CHECK: %.sroa.01.0.vec.insert = insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 0
-; CHECK: store <4 x float> %.sroa.01.0.vec.insert, ptr null, align 1
-; CHECK: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
-; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 %.sroa.4, i64 44, i1 false)
+; but %2 is only partially initialized. Note that null is just a placeholder for any pointer to simplify the test case.
+; If SROA is too agressive and canonicalizes %2 to a vector partition and promotes it to an SSA value, then you end up with a 16-byte store
+; whose first lane is 0 and whose other three lanes carry no defined value. That is a wider store than necessary because the
+; uninitialized memory becomes a placeholder value. If you instead delay struct to vector canonicalization and allow memcpyopt
+; to run, you'll get a 4-byte store of 0.
define ptr @store_only_fp_tail() {
+; NO-CANON-LABEL: define ptr @store_only_fp_tail() {
+; NO-CANON-NEXT: [[DOTSROA_3:%.*]] = alloca { float, float, float }, align 8
+; NO-CANON-NEXT: [[DOTSROA_4:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
+; NO-CANON-NEXT: [[DOTSROA_0_SROA_1:%.*]] = alloca { float, float, float }, align 8
+; NO-CANON-NEXT: [[DOTSROA_2:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
+; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_3]], ptr align 8 [[DOTSROA_0_SROA_1]], i64 12, i1 false)
+; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_4]], ptr align 8 [[DOTSROA_2]], i64 44, i1 false)
+; NO-CANON-NEXT: store float 0.000000e+00, ptr null, align 1
+; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 4), ptr align 8 [[DOTSROA_3]], i64 12, i1 false)
+; NO-CANON-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
+; NO-CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 [[DOTSROA_4]], i64 44, i1 false)
+; NO-CANON-NEXT: ret ptr null
+;
+; CANON-LABEL: define ptr @store_only_fp_tail() {
+; CANON-NEXT: [[DOTSROA_4:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
+; CANON-NEXT: [[DOTSROA_2:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
+; CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_4]], ptr align 8 [[DOTSROA_2]], i64 44, i1 false)
+; CANON-NEXT: [[DOTSROA_01_0_VEC_INSERT:%.*]] = insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 0
+; CANON-NEXT: store <4 x float> [[DOTSROA_01_0_VEC_INSERT]], ptr null, align 1
+; CANON-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
+; CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 [[DOTSROA_4]], i64 44, i1 false)
+; CANON-NEXT: ret ptr null
+;
%1 = alloca %class.aiMatrix4x4t, align 4
%2 = alloca %class.aiMatrix4x4t, align 4
%3 = getelementptr i8, ptr %2, i64 16
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
index 2d3066a6f2a57..b731796c0a40e 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
@@ -1,18 +1,15 @@
-; RUN: opt -passes=sroa -S %s | FileCheck %s
+; RUN: opt -passes='sroa<canonicalize-struct-to-vector>' -S %s | FileCheck %s
; NOTE: Do not autogenerate. This test intentionally uses targeted CHECK
; patterns for clarity.
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
; When SROA splits { ptr, i64, i64, i64 } into [0,8), [8,16), [16,32),
; the [16,32) partition type from getTypePartition is { i64, i64 }.
; With the simplified fallback rule, that partition canonicalizes to <2 x i64>
; because its remaining users are all mem intrinsics.
; CHECK-LABEL: define void @test_subpartition_type(
-; CHECK: %a.sroa.6.sroa.0.0.copyload = load <2 x i64>, ptr %a.sroa.6.0.src.sroa_idx, align 8
-; CHECK: store <2 x i64> %a.sroa.6.sroa.0.0.copyload, ptr %a.sroa.6.0.dst.sroa_idx, align 8
+; CHECK: %a.sroa.6.0.copyload = load <2 x i64>, ptr %a.sroa.6.0.src.sroa_idx, align 8
+; CHECK: store <2 x i64> %a.sroa.6.0.copyload, ptr %a.sroa.6.0.dst.sroa_idx, align 8
define void @test_subpartition_type(ptr %src, ptr %dst) {
entry:
%a = alloca { ptr, i64, i64, i64 }, align 8
diff --git a/llvm/test/Transforms/SROA/struct-to-vector.ll b/llvm/test/Transforms/SROA/struct-to-vector.ll
index 09fb50809592d..f95ceaead87bf 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='sroa,gvn,instcombine,simplifycfg' -S %s | FileCheck %s
+; RUN: opt -passes='sroa<canonicalize-struct-to-vector>,gvn,instcombine,simplifycfg' -S %s | FileCheck %s
%struct.myint4 = type { i32, i32, i32, i32 }
define dso_local void @foo_flat(ptr noundef %x, i64 %y.coerce0, i64 %y.coerce1, i32 noundef %cond) {
@@ -325,8 +325,6 @@ define dso_local void @foo_ptr(ptr noundef %x, ptr %p0, ptr %p1,
; CHECK-SAME: ptr noundef [[X:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]], i32 noundef [[COND:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TEMP:%.*]] = alloca [[STRUCT_PTR4:%.*]], align 8
-; CHECK-NEXT: [[ZERO:%.*]] = alloca [[STRUCT_PTR4]], align 8
-; CHECK-NEXT: [[DATA:%.*]] = alloca [[STRUCT_PTR4]], align 8
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[TEMP]])
; CHECK-NEXT: store ptr [[P0]], ptr [[TEMP]], align 8
; CHECK-NEXT: [[Y_SROA_2_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 8
@@ -335,15 +333,10 @@ define dso_local void @foo_ptr(ptr noundef %x, ptr %p0, ptr %p1,
; CHECK-NEXT: store ptr [[P2]], ptr [[Y_SROA_3_0_TEMP_SROA_IDX]], align 8
; CHECK-NEXT: [[Y_SROA_4_0_TEMP_SROA_IDX:%.*]] = getelementptr inbounds nuw i8, ptr [[TEMP]], i64 24
; CHECK-NEXT: store ptr [[P3]], ptr [[Y_SROA_4_0_TEMP_SROA_IDX]], align 8
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[ZERO]])
-; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[ZERO]], i8 0, i64 32, i1 false)
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull [[DATA]])
; CHECK-NEXT: [[TOBOOL_PTR_NOT:%.*]] = icmp eq i32 [[COND]], 0
-; CHECK-NEXT: [[ZERO_TEMP:%.*]] = select i1 [[TOBOOL_PTR_NOT]], ptr [[ZERO]], ptr [[TEMP]]
-; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[DATA]], ptr noundef nonnull align 8 dereferenceable(32) [[ZERO_TEMP]], i64 32, i1 false)
-; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[X]], ptr noundef nonnull align 8 dereferenceable(32) [[DATA]], i64 32, i1 false)
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[DATA]])
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[ZERO]])
+; CHECK-NEXT: [[DATA_SROA_0_0_COPYLOAD_PRE:%.*]] = load <4 x ptr>, ptr [[TEMP]], align 8
+; CHECK-NEXT: [[DATA_SROA_0_0_COPYLOAD:%.*]] = select i1 [[TOBOOL_PTR_NOT]], <4 x ptr> splat (ptr null), <4 x ptr> [[DATA_SROA_0_0_COPYLOAD_PRE]]
+; CHECK-NEXT: store <4 x ptr> [[DATA_SROA_0_0_COPYLOAD]], ptr [[X]], align 8
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[TEMP]])
; CHECK-NEXT: ret void
;
>From 83bbd33297e57103d95ec98e0ab2a677e087ee3e Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Thu, 14 May 2026 00:07:16 -0400
Subject: [PATCH 09/13] [SROA] Revert three tests to upstream form
The default `sroa` pass no longer canonicalizes homogeneous struct partitions
to fixed vectors, so restore the scalar CHECKs in `sroa-alloca-offset.ll`,
`user-memcpy.ll`, and `nullptr.cl` to match what the default pipeline now
produces. NVPTX still opts in, so `lower-byval-args.ll` keeps its
vector-shaped CHECKs.
---
clang/test/CodeGenOpenCL/nullptr.cl | 4 +--
.../assignment-tracking/sroa/user-memcpy.ll | 28 ++++++++++---------
.../DebugInfo/Generic/sroa-alloca-offset.ll | 18 ++++++------
3 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/clang/test/CodeGenOpenCL/nullptr.cl b/clang/test/CodeGenOpenCL/nullptr.cl
index f45df110ec243..976e12c0bef47 100644
--- a/clang/test/CodeGenOpenCL/nullptr.cl
+++ b/clang/test/CodeGenOpenCL/nullptr.cl
@@ -597,10 +597,10 @@ typedef struct {
} StructTy3;
// CHECK-LABEL: test_memset_private
-// SPIR64: store <4 x i64> zeroinitializer, ptr %ptr, align 8
+// SPIR64: call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) %ptr, i8 0, i64 32, i1 false)
// SPIR64: [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr %ptr, i64 32
// SPIR64: store ptr addrspacecast (ptr addrspace(4) null to ptr), ptr [[GEP]], align 8
-// AMDGCN: store <4 x i64> zeroinitializer, ptr addrspace(5) %ptr, align 8
+// AMDGCN: call void @llvm.memset.p5.i64(ptr addrspace(5) noundef align 8 {{.*}}, i8 0, i64 32, i1 false)
// AMDGCN: [[GEP:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) %ptr, i32 32
// AMDGCN: store ptr addrspace(5) addrspacecast (ptr null to ptr addrspace(5)), ptr addrspace(5) [[GEP]]
// AMDGCN: [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(5) {{.*}}, i32 36
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
index adbcbc61b9dda..ded78f4ff83f4 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
@@ -21,29 +21,31 @@
;; Allocas have been promoted - the linked dbg.assigns have been removed.
;; | V3i point = {0, 0, 0};
-;; Homogeneous i64 sub-partition [x,y] is canonicalized to <2 x i64>.
-; CHECK-NEXT: #dbg_value(<2 x i64> zeroinitializer, ![[point:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 0, ![[point:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64),
+; CHECK-NEXT: #dbg_value(i64 0, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64),
;; point.z = 5000;
-; CHECK-NEXT: #dbg_value(i64 5000, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
+; CHECK-NEXT: #dbg_value(i64 5000, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64),
;; | V3i other = {10, 9, 8};
;; other is global const:
;; local.other.x = global.other.x
;; local.other.y = global.other.y
;; local.other.z = global.other.z
-; CHECK-NEXT: %other.sroa.0.0.copyload = load i64, ptr @__const._Z3funv.other, align 8, !dbg !{{[0-9]+}}
-; CHECK-NEXT: %other.sroa.2.0.copyload = load i64, ptr getelementptr inbounds (i8, ptr @__const._Z3funv.other, i64 8), align 8, !dbg !{{[0-9]+}}
-; CHECK-NEXT: %other.sroa.3.0.copyload = load i64, ptr getelementptr inbounds (i8, ptr @__const._Z3funv.other, i64 16), align 8, !dbg !{{[0-9]+}}
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[other:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.3.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
+; CHECK-NEXT: %other.sroa.0.0.copyload = load i64, ptr @__const._Z3funv.other
+; CHECK-NEXT: %other.sroa.2.0.copyload = load i64, ptr getelementptr inbounds (i8, ptr @__const._Z3funv.other, i64 8)
+; CHECK-NEXT: %other.sroa.3.0.copyload = load i64, ptr getelementptr inbounds (i8, ptr @__const._Z3funv.other, i64 16)
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[other:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64),
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 64, 64),
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.3.0.copyload, ![[other]], !DIExpression(DW_OP_LLVM_fragment, 128, 64),
;; | std::memcpy(&point.y, &other.x, sizeof(long) * 2);
-;; memcpy updates scalar point.y/point.z lanes; the vector slot for [x,y] is kept live.
-; CHECK-NEXT: %{{.*}} = insertelement <2 x i64> zeroinitializer, i64 %other.sroa.0.0.copyload, i32 1{{.*}}
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64), !{{[0-9]+}})
-; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64), !{{[0-9]+}})
+;; other is now 3 scalars:
+;; point.y = other.x
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.0.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 64, 64),
+;;
+;; point.z = other.y
+; CHECK-NEXT: #dbg_value(i64 %other.sroa.2.0.copyload, ![[point]], !DIExpression(DW_OP_LLVM_fragment, 128, 64),
; CHECK: ![[point]] = !DILocalVariable(name: "point",
; CHECK: ![[other]] = !DILocalVariable(name: "other",
diff --git a/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll b/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
index 59f2a1bdb1dc4..6718711f83e04 100644
--- a/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
+++ b/llvm/test/DebugInfo/Generic/sroa-alloca-offset.ll
@@ -140,18 +140,16 @@ entry:
;; 16 bit variable f (!62): value vgf (lower bits)
;; 16 bit variable g (!63): value vgf (upper bits)
;;
-;; 16 bit variable h (!64): struct.two = {i32,i32} is a 2-element homogeneous struct,
-;; canonicalized to <2 x i32> by SROA; no alloca or memcpy survives.
-; COMMON-NEXT: %[[ve:.*]] = load i32, ptr @gf, align 4{{.*}}
+;; 16 bit variable h (!64): deref dead_64_128
+; COMMON-NEXT: %[[dead_64_128:.*]] = alloca %struct.two
+; COMMON-NEXT: #dbg_declare(ptr %[[dead_64_128]], ![[h:[0-9]+]], !DIExpression(),
+; COMMON-NEXT: %[[ve:.*]] = load i32, ptr @gf
;; FIXME: mem2reg bug - offset is incorrect - see comment above.
-; COMMON-NEXT: #dbg_value(i32 %[[ve]], ![[e:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2), !{{[0-9]+}})
-; COMMON-NEXT: %[[vfg:.*]] = load i32, ptr getelementptr inbounds (i8, ptr @gf, i64 4), align 4{{.*}}
-; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[f:[0-9]+]], !DIExpression(), !{{[0-9]+}})
+; COMMON-NEXT: #dbg_value(i32 %[[ve]], ![[e:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2),
+; COMMON-NEXT: %[[vfg:.*]] = load i32, ptr getelementptr inbounds (i8, ptr @gf, i64 4)
+; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[f:[0-9]+]], !DIExpression(),
;; FIXME: mem2reg bug - offset is incorrect - see comment above.
-; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[g:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2), !{{[0-9]+}})
-; COMMON-NEXT: %[[vh:.*]] = load <2 x i32>, ptr getelementptr inbounds (i8, ptr @gf, i64 8), align 4{{.*}}
-; COMMON-NEXT: #dbg_value(<2 x i32> %[[vh]], ![[h:[0-9]+]], !DIExpression(), !{{[0-9]+}})
-; COMMON-NEXT: ret i32 %[[vfg]]{{.*}}
+; COMMON-NEXT: #dbg_value(i32 %[[vfg]], ![[g:[0-9]+]], !DIExpression(DW_OP_plus_uconst, 2),
define dso_local noundef i32 @_Z4fun3v() #0 !dbg !55 {
entry:
%0 = alloca %struct.four, align 4
>From 1f29fdbb872f943fa4211179944924ece74b5743 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 26 May 2026 12:02:59 -0400
Subject: [PATCH 10/13] [SROA] Rename CanonicalizeStructToVector option to
StructToVector
Per @arsenm review feedback on PR #165159: drop the redundant
"Canonicalize" verb from the pass-option name. Renames the SROAOptions
field, the pipeline-parser string ("canonicalize-struct-to-vector" ->
"struct-to-vector"), the createSROAPass parameter, all call sites, and
the existing RUN-line strings in two SROA tests.
---
llvm/include/llvm/Transforms/Scalar.h | 2 +-
llvm/include/llvm/Transforms/Scalar/SROA.h | 9 ++++----
llvm/lib/Passes/PassBuilder.cpp | 6 +++---
llvm/lib/Passes/PassBuilderPipelines.cpp | 4 ++--
llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 4 ++--
llvm/lib/Transforms/Scalar/SROA.cpp | 21 +++++++++----------
.../SROA/struct-to-vector-subpartition.ll | 2 +-
llvm/test/Transforms/SROA/struct-to-vector.ll | 2 +-
8 files changed, 24 insertions(+), 26 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index 29de5a26d3503..b4d807614d665 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -45,7 +45,7 @@ LLVM_ABI FunctionPass *createDeadStoreEliminationPass();
// SROA - Replace aggregates or pieces of aggregates with scalar SSA values.
//
LLVM_ABI FunctionPass *createSROAPass(bool PreserveCFG = true,
- bool CanonicalizeStructToVector = false);
+ bool StructToVector = false);
//===----------------------------------------------------------------------===//
//
diff --git a/llvm/include/llvm/Transforms/Scalar/SROA.h b/llvm/include/llvm/Transforms/Scalar/SROA.h
index 77703661b7f61..96a20ee44c593 100644
--- a/llvm/include/llvm/Transforms/Scalar/SROA.h
+++ b/llvm/include/llvm/Transforms/Scalar/SROA.h
@@ -25,11 +25,10 @@ struct SROAOptions {
enum CFGOption { ModifyCFG, PreserveCFG };
CFGOption CFG;
- bool CanonicalizeStructToVector;
+ bool StructToVector;
- SROAOptions(CFGOption CFG = PreserveCFG,
- bool CanonicalizeStructToVector = false)
- : CFG(CFG), CanonicalizeStructToVector(CanonicalizeStructToVector) {}
+ SROAOptions(CFGOption CFG = PreserveCFG, bool StructToVector = false)
+ : CFG(CFG), StructToVector(StructToVector) {}
};
class SROAPass : public OptionalPassInfoMixin<SROAPass> {
@@ -38,7 +37,7 @@ class SROAPass : public OptionalPassInfoMixin<SROAPass> {
public:
/// If \p PreserveCFG is set, then the pass is not allowed to modify CFG
/// in any way, even if it would update CFG analyses.
- /// If \p CanonicalizeStructToVector is set, then the pass will try to convert
+ /// If \p StructToVector is set, then the pass will try to convert
/// allocas of homogeneous structs into vector allocas.
LLVM_ABI SROAPass(SROAOptions Options);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index f16bfbbc19cc3..6dbc3b2d512c2 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1451,12 +1451,12 @@ Expected<SROAOptions> parseSROAOptions(StringRef Params) {
inconvertibleErrorCode());
Result.CFG = SROAOptions::PreserveCFG;
SawCFGOption = true;
- } else if (ParamName == "canonicalize-struct-to-vector") {
- Result.CanonicalizeStructToVector = true;
+ } else if (ParamName == "struct-to-vector") {
+ Result.StructToVector = true;
} else {
return make_error<StringError>(
formatv("invalid SROA pass parameter '{}' (expected preserve-cfg, "
- "modify-cfg, or canonicalize-struct-to-vector)",
+ "modify-cfg, or struct-to-vector)",
ParamName)
.str(),
inconvertibleErrorCode());
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 0dcadae70536e..572a61a4a24b5 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1381,7 +1381,7 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// memcpyopt's optimizations if done before. Look at the documentation for
// `tryCanonicalizeStructToVector` in SROA.cpp to see why.
FPM.addPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
- /*CanonicalizeStructToVector=*/true)));
+ /*StructToVector=*/true)));
}
if (!isFullLTOPostLink(LTOPhase)) {
@@ -1482,7 +1482,7 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// memcpyopt's optimizations if done before. Look at the documentation for
// `tryCanonicalizeStructToVector` in SROA.cpp to see why.
FPM.addPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
- /*CanonicalizeStructToVector=*/true)));
+ /*StructToVector=*/true)));
}
FPM.addPass(InferAlignmentPass());
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 6770c94b57626..c2fd2b560f729 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -301,7 +301,7 @@ void NVPTXPassConfig::addAddressSpaceInferencePasses() {
// NVPTXLowerArgs emits alloca for byval parameters which can often
// be eliminated by SROA.
addPass(createSROAPass(/*PreserveCFG=*/true,
- /*CanonicalizeStructToVector=*/true));
+ /*StructToVector=*/true));
addPass(createNVPTXLowerAllocaPass());
// TODO: Consider running InferAddressSpaces during opt, earlier in the
// compilation flow.
@@ -393,7 +393,7 @@ void NVPTXPassConfig::addIRPasses() {
if (!DisableLoadStoreVectorizer)
addPass(createLoadStoreVectorizerPass());
addPass(createSROAPass(/*PreserveCFG=*/true,
- /*CanonicalizeStructToVector=*/true));
+ /*StructToVector=*/true));
addPass(createNVPTXTagInvariantLoadsPass());
if (!DisableNVPTXIRPeephole)
addPass(createNVPTXIRPeepholePass());
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index c10cd8eaf385f..abcedc12a09f1 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -178,7 +178,7 @@ class SROA {
DomTreeUpdater *const DTU;
AssumptionCache *const AC;
const bool PreserveCFG;
- const bool CanonicalizeStructToVector;
+ const bool StructToVector;
/// Worklist of alloca instructions to simplify.
///
@@ -244,7 +244,7 @@ class SROA {
SROAOptions Options)
: C(C), DTU(DTU), AC(AC),
PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
- CanonicalizeStructToVector(Options.CanonicalizeStructToVector) {}
+ StructToVector(Options.StructToVector) {}
/// Main run method used by both the SROAPass and by the legacy pass.
std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
@@ -5109,7 +5109,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
/// memcpyopt eliminates the tempory altogether.
///
/// As such, we only apply this transformation after memcpyopt has run. We gate
-/// this transformation by the "CanonicalizeStructToVector" pass option.
+/// this transformation by the "StructToVector" pass option.
static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
Partition &P,
const DataLayout &DL) {
@@ -5162,7 +5162,7 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
/// nullptr.
static std::tuple<Type *, bool, VectorType *>
selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
- LLVMContext &C, bool CanonicalizeStructToVector) {
+ LLVMContext &C, bool StructToVector) {
auto LogSelection = [&](StringRef Path, Type *SelectedTy,
VectorType *SelectedVecTy, bool SelectedIntWidening) {
LLVM_DEBUG({
@@ -5255,7 +5255,7 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
// Try homogeneous struct to vector canonicalization when requested. Running
// this too early can hide memcpy chains from MemCpyOpt.
- if (CanonicalizeStructToVector) {
+ if (StructToVector) {
if (auto *STy = dyn_cast<StructType>(TypePartitionTy)) {
if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
LogSelection("struct-fallback-vecty", VTy, nullptr, false);
@@ -5304,7 +5304,7 @@ SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
const DataLayout &DL = AI.getDataLayout();
// Select the type for the new alloca that spans the partition.
auto [PartitionTy, IsIntegerWideningViable, VecTy] =
- selectPartitionType(P, DL, AI, *C, CanonicalizeStructToVector);
+ selectPartitionType(P, DL, AI, *C, StructToVector);
// Check for the case where we're going to rewrite to a new alloca of the
// exact same type as the original, and with the same access offsets. In that
@@ -6130,8 +6130,8 @@ void SROAPass::printPipeline(
OS << '<'
<< (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
: "modify-cfg");
- if (Options.CanonicalizeStructToVector)
- OS << ";canonicalize-struct-to-vector";
+ if (Options.StructToVector)
+ OS << ";struct-to-vector";
OS << '>';
}
@@ -6177,11 +6177,10 @@ class SROALegacyPass : public FunctionPass {
char SROALegacyPass::ID = 0;
-FunctionPass *llvm::createSROAPass(bool PreserveCFG,
- bool CanonicalizeStructToVector) {
+FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool StructToVector) {
return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
: SROAOptions::ModifyCFG,
- CanonicalizeStructToVector));
+ StructToVector));
}
INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
index b731796c0a40e..6962fd7291465 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes='sroa<canonicalize-struct-to-vector>' -S %s | FileCheck %s
+; RUN: opt -passes='sroa<struct-to-vector>' -S %s | FileCheck %s
; NOTE: Do not autogenerate. This test intentionally uses targeted CHECK
; patterns for clarity.
diff --git a/llvm/test/Transforms/SROA/struct-to-vector.ll b/llvm/test/Transforms/SROA/struct-to-vector.ll
index f95ceaead87bf..9ce21ad689d11 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='sroa<canonicalize-struct-to-vector>,gvn,instcombine,simplifycfg' -S %s | FileCheck %s
+; RUN: opt -passes='sroa<struct-to-vector>,gvn,instcombine,simplifycfg' -S %s | FileCheck %s
%struct.myint4 = type { i32, i32, i32, i32 }
define dso_local void @foo_flat(ptr noundef %x, i64 %y.coerce0, i64 %y.coerce1, i32 noundef %cond) {
>From 5f1c1317a8e84400aaf5db9b35a88c00a09ebc5e Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 26 May 2026 12:03:08 -0400
Subject: [PATCH 11/13] [SROA] Clean up struct-to-vector-fp-store-only-tail.ll
Per @arsenm review feedback on PR #165159:
- Rename numeric SSA values (%1, %2, %3) to named values (%dst, %src,
%src.tail) so the test reads more clearly and won't churn on future
edits. Regenerated CHECK lines via update_test_checks.py.
- Drop the auto-emitted "Function Attrs:" comment line above the
llvm.memcpy declaration; the attrs already appear on the declare
itself.
---
.../struct-to-vector-fp-store-only-tail.ll | 29 +++++++++----------
1 file changed, 14 insertions(+), 15 deletions(-)
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
index 5920f01e37188..dd403c55ca64d 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -1,19 +1,18 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=sroa -S %s | FileCheck %s --check-prefixes=NO-CANON
-; RUN: opt -passes='sroa<canonicalize-struct-to-vector>' -S %s | FileCheck %s --check-prefixes=CANON
+; RUN: opt -passes='sroa<struct-to-vector>' -S %s | FileCheck %s --check-prefixes=CANON
%class.aiMatrix4x4t = type { float, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }
-; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite)
declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
; This test case shows an example where we have:
;
-; 1. memcpy %1, %2
-; 2. memcpy null, %1
+; 1. memcpy %dst, %src
+; 2. memcpy null, %dst
;
-; but %2 is only partially initialized. Note that null is just a placeholder for any pointer to simplify the test case.
-; If SROA is too agressive and canonicalizes %2 to a vector partition and promotes it to an SSA value, then you end up with a 16-byte store
+; but %src is only partially initialized. Note that null is just a placeholder for any pointer to simplify the test case.
+; If SROA is too agressive and canonicalizes %src to a vector partition and promotes it to an SSA value, then you end up with a 16-byte store
; whose first lane is 0 and whose other three lanes carry no defined value. That is a wider store than necessary because the
; uninitialized memory becomes a placeholder value. If you instead delay struct to vector canonicalization and allow memcpyopt
; to run, you'll get a 4-byte store of 0.
@@ -35,18 +34,18 @@ define ptr @store_only_fp_tail() {
; CANON-NEXT: [[DOTSROA_4:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
; CANON-NEXT: [[DOTSROA_2:%.*]] = alloca { float, float, float, float, float, float, float, float, float, float, float }, align 8
; CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DOTSROA_4]], ptr align 8 [[DOTSROA_2]], i64 44, i1 false)
-; CANON-NEXT: [[DOTSROA_01_0_VEC_INSERT:%.*]] = insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 0
-; CANON-NEXT: store <4 x float> [[DOTSROA_01_0_VEC_INSERT]], ptr null, align 1
+; CANON-NEXT: [[DST_SROA_0_0_VEC_INSERT:%.*]] = insertelement <4 x float> {{.*}}, float 0.000000e+00, i32 0
+; CANON-NEXT: store <4 x float> [[DST_SROA_0_0_VEC_INSERT]], ptr null, align 1
; CANON-NEXT: store float 0.000000e+00, ptr getelementptr inbounds (i8, ptr null, i64 16), align 1
; CANON-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 getelementptr inbounds (i8, ptr null, i64 20), ptr align 8 [[DOTSROA_4]], i64 44, i1 false)
; CANON-NEXT: ret ptr null
;
- %1 = alloca %class.aiMatrix4x4t, align 4
- %2 = alloca %class.aiMatrix4x4t, align 4
- %3 = getelementptr i8, ptr %2, i64 16
- store float 0.000000e+00, ptr %3, align 4
- call void @llvm.memcpy.p0.p0.i64(ptr %1, ptr %2, i64 64, i1 false)
- store float 0.000000e+00, ptr %1, align 4
- call void @llvm.memcpy.p0.p0.i64(ptr null, ptr %1, i64 64, i1 false)
+ %dst = alloca %class.aiMatrix4x4t, align 4
+ %src = alloca %class.aiMatrix4x4t, align 4
+ %src.tail = getelementptr i8, ptr %src, i64 16
+ store float 0.000000e+00, ptr %src.tail, align 4
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 64, i1 false)
+ store float 0.000000e+00, ptr %dst, align 4
+ call void @llvm.memcpy.p0.p0.i64(ptr null, ptr %dst, i64 64, i1 false)
ret ptr null
}
>From 0f4a98cdfeda62c07e9b914890e9b89deaac624b Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 26 May 2026 12:03:16 -0400
Subject: [PATCH 12/13] [SROA] Move struct-to-vector-before-memcpyopt.ll to
test/PhaseOrdering
Per @arsenm review feedback on PR #165159: the test exists to verify
that struct-to-vector canonicalization runs only after memcpyopt in the
default pipeline, which is a pipeline-configuration concern rather than
a SROA-pass unit test. Move it under test/PhaseOrdering/ and switch the
single RUN line to "opt -passes='default<O3>'" so it exercises the real
pipeline directly instead of a synthetic pass list.
---
.../struct-to-vector-before-memcpyopt.ll | 51 +++++++++++++++
.../SROA/struct-to-vector-before-memcpyopt.ll | 65 -------------------
2 files changed, 51 insertions(+), 65 deletions(-)
create mode 100644 llvm/test/Transforms/PhaseOrdering/struct-to-vector-before-memcpyopt.ll
delete mode 100644 llvm/test/Transforms/SROA/struct-to-vector-before-memcpyopt.ll
diff --git a/llvm/test/Transforms/PhaseOrdering/struct-to-vector-before-memcpyopt.ll b/llvm/test/Transforms/PhaseOrdering/struct-to-vector-before-memcpyopt.ll
new file mode 100644
index 0000000000000..e98c9074e4f82
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/struct-to-vector-before-memcpyopt.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes='default<O3>' %s | FileCheck %s
+
+%pair = type { i64, i64 }
+%quad = type { i64, i64, i64, i64 }
+
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
+declare void @llvm.memset.p0.i64(ptr writeonly captures(none), i8, i64, i1 immarg)
+
+; This test verifies that the default O3 pipeline canonicalizes struct allocas
+; to vectors only after memcpyopt has run. The input pattern is:
+;
+; memcpy tmp, obj, 16
+; memset obj + 16, 0, 16
+;
+; ----- SWAP(other, tmp) -----
+;
+; memcpy swap.tmp, tmp, 16
+; memcpy tmp, other, 16
+; memcpy other, swap.tmp, 16
+;
+; It swaps the first 16-bytes of other and tmp, but the first 16-bytes of tmp
+; are the same as the first 16-bytes of obj. This comes from real code from
+; DuckDB, where the swap function is inlined. If struct-to-vector canonicalization
+; runs before memcpyopt, swap.tmp gets promoted to an SSA value and we are stuck
+; saving tmp to swap.tmp. Delaying canonicalization until after memcpyopt lets
+; memcpyopt notice that tmp and obj share the same first 16-bytes, so swap.tmp
+; is no longer needed and the IR collapses to a single load/memmove/store.
+define void @move_then_swap(ptr %dst, ptr %src, ptr %other) {
+; CHECK-LABEL: define void @move_then_swap(
+; CHECK-SAME: ptr writeonly captures(none) initializes((0, 16)) [[DST:%.*]], ptr readonly captures(none) [[SRC:%.*]], ptr captures(none) [[OTHER:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP_SROA_0_0_COPYLOAD:%.*]] = load <2 x i64>, ptr [[OTHER]], align 8
+; CHECK-NEXT: tail call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) [[OTHER]], ptr noundef nonnull align 8 dereferenceable(16) [[SRC]], i64 16, i1 false)
+; CHECK-NEXT: store <2 x i64> [[TMP_SROA_0_0_COPYLOAD]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %tmp = alloca %pair, align 8
+ %obj = alloca %quad, align 8
+ %swap.tmp = alloca %pair, align 8
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %obj, ptr align 8 %src, i64 32, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %obj, i64 16, i1 false)
+ %obj.tail = getelementptr inbounds i8, ptr %obj, i64 16
+ call void @llvm.memset.p0.i64(ptr align 8 %obj.tail, i8 0, i64 16, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %swap.tmp, ptr align 8 %tmp, i64 16, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %other, i64 16, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %other, ptr align 8 %swap.tmp, i64 16, i1 false)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp, i64 16, i1 false)
+ ret void
+}
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-before-memcpyopt.ll b/llvm/test/Transforms/SROA/struct-to-vector-before-memcpyopt.ll
deleted file mode 100644
index 2a694e8899b6a..0000000000000
--- a/llvm/test/Transforms/SROA/struct-to-vector-before-memcpyopt.ll
+++ /dev/null
@@ -1,65 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='sroa,memcpyopt,dse,instcombine,sroa<canonicalize-struct-to-vector>' -S %s | FileCheck %s --check-prefix=DELAYED
-; RUN: opt -passes='sroa<canonicalize-struct-to-vector>,memcpyopt,dse,instcombine,sroa<canonicalize-struct-to-vector>' -S %s | FileCheck %s --check-prefix=EARLY
-
-%pair = type { i64, i64 }
-%quad = type { i64, i64, i64, i64 }
-
-declare void @llvm.memcpy.p0.p0.i64(ptr noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg)
-declare void @llvm.memset.p0.i64(ptr writeonly captures(none), i8, i64, i1 immarg)
-
-; This test case has the following pattern:
-;
-; memcpy tmp, obj, 16
-; memset obj + 16, 0, 16
-;
-; ----- SWAP(other, tmp) -----
-;
-; memcpy swap.tmp, tmp, 16
-; memcpy tmp, other, 16
-; memcpy other, swap.tmp, 16
-;
-; It swaps the first 16-bytes of other and tmp, but the first 16-bytes of tmp are actually the same as the first 16-bytes
-; of obj. This comes from real code from DuckDB, where the swap function is inlined. If struct-to-vector canonicalization runs before
-; memcpyopt, then swap.tmp gets promoted to an SSA value and we are stuck with saving tmp to swap.tmp.
-; But if we delay canonicalization until after memcpyopt,
-; then memcpyopt realizes that tmp and obj share the same first 16-bytes, and so we don't need to save tmp to swap.tmp. So it can rewrite to:
-;
-; memset obj + 16, 0, 16
-; memcpy tmp, other, 16
-; memcpy other, obj, 16
-;
-; This is a win because it avoids the extra step of saving tmp to swap.tmp.
-define void @move_then_swap(ptr %dst, ptr %src, ptr %other) {
-; DELAYED-LABEL: define void @move_then_swap(
-; DELAYED-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[OTHER:%.*]]) {
-; DELAYED-NEXT: [[ENTRY:.*:]]
-; DELAYED-NEXT: [[TMP_SROA_0_0_COPYLOAD:%.*]] = load <2 x i64>, ptr [[OTHER]], align 8
-; DELAYED-NEXT: call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) [[OTHER]], ptr noundef nonnull align 8 dereferenceable(16) [[SRC]], i64 16, i1 false)
-; DELAYED-NEXT: store <2 x i64> [[TMP_SROA_0_0_COPYLOAD]], ptr [[DST]], align 8
-; DELAYED-NEXT: ret void
-;
-; EARLY-LABEL: define void @move_then_swap(
-; EARLY-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], ptr [[OTHER:%.*]]) {
-; EARLY-NEXT: [[ENTRY:.*:]]
-; EARLY-NEXT: [[OBJ_SROA_0_0_COPYLOAD:%.*]] = load <4 x i64>, ptr [[SRC]], align 8
-; EARLY-NEXT: [[OBJ_SROA_0_0_VEC_EXTRACT:%.*]] = shufflevector <4 x i64> [[OBJ_SROA_0_0_COPYLOAD]], <4 x i64> poison, <2 x i32> <i32 0, i32 1>
-; EARLY-NEXT: [[TMP_SROA_0_0_COPYLOAD:%.*]] = load <2 x i64>, ptr [[OTHER]], align 8
-; EARLY-NEXT: store <2 x i64> [[OBJ_SROA_0_0_VEC_EXTRACT]], ptr [[OTHER]], align 8
-; EARLY-NEXT: store <2 x i64> [[TMP_SROA_0_0_COPYLOAD]], ptr [[DST]], align 8
-; EARLY-NEXT: ret void
-;
-entry:
- %tmp = alloca %pair, align 8
- %obj = alloca %quad, align 8
- %swap.tmp = alloca %pair, align 8
- call void @llvm.memcpy.p0.p0.i64(ptr align 8 %obj, ptr align 8 %src, i64 32, i1 false)
- call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %obj, i64 16, i1 false)
- %obj.tail = getelementptr inbounds i8, ptr %obj, i64 16
- call void @llvm.memset.p0.i64(ptr align 8 %obj.tail, i8 0, i64 16, i1 false)
- call void @llvm.memcpy.p0.p0.i64(ptr align 8 %swap.tmp, ptr align 8 %tmp, i64 16, i1 false)
- call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %other, i64 16, i1 false)
- call void @llvm.memcpy.p0.p0.i64(ptr align 8 %other, ptr align 8 %swap.tmp, i64 16, i1 false)
- call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp, i64 16, i1 false)
- ret void
-}
>From 867e3035056d0a23908e9ae2bb6ad2ea3bd642bf Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 26 May 2026 16:15:07 -0400
Subject: [PATCH 13/13] [SROA] Rename StructToVector option to
AggregateToVector
Per @YonahGoldberg follow-up on PR #165159: the transformation is
expected to extend to array allocas in a follow-up (motivated by
issue #164308 for the Julia frontend), so the per-pass option should
not be tied to "struct". Renames the SROAOptions field, the
pipeline-parser string ("struct-to-vector" -> "aggregate-to-vector"),
the createSROAPass parameter, all call sites, and the existing RUN-line
strings in the three SROA tests.
---
llvm/include/llvm/Transforms/Scalar.h | 2 +-
llvm/include/llvm/Transforms/Scalar/SROA.h | 8 +++----
llvm/lib/Passes/PassBuilder.cpp | 6 ++---
llvm/lib/Passes/PassBuilderPipelines.cpp | 4 ++--
llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp | 4 ++--
llvm/lib/Transforms/Scalar/SROA.cpp | 24 +++++++++----------
.../struct-to-vector-fp-store-only-tail.ll | 2 +-
.../SROA/struct-to-vector-subpartition.ll | 2 +-
llvm/test/Transforms/SROA/struct-to-vector.ll | 2 +-
9 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index b4d807614d665..f57a86c772bb1 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -45,7 +45,7 @@ LLVM_ABI FunctionPass *createDeadStoreEliminationPass();
// SROA - Replace aggregates or pieces of aggregates with scalar SSA values.
//
LLVM_ABI FunctionPass *createSROAPass(bool PreserveCFG = true,
- bool StructToVector = false);
+ bool AggregateToVector = false);
//===----------------------------------------------------------------------===//
//
diff --git a/llvm/include/llvm/Transforms/Scalar/SROA.h b/llvm/include/llvm/Transforms/Scalar/SROA.h
index 96a20ee44c593..3c0e99f6dee19 100644
--- a/llvm/include/llvm/Transforms/Scalar/SROA.h
+++ b/llvm/include/llvm/Transforms/Scalar/SROA.h
@@ -25,10 +25,10 @@ struct SROAOptions {
enum CFGOption { ModifyCFG, PreserveCFG };
CFGOption CFG;
- bool StructToVector;
+ bool AggregateToVector;
- SROAOptions(CFGOption CFG = PreserveCFG, bool StructToVector = false)
- : CFG(CFG), StructToVector(StructToVector) {}
+ SROAOptions(CFGOption CFG = PreserveCFG, bool AggregateToVector = false)
+ : CFG(CFG), AggregateToVector(AggregateToVector) {}
};
class SROAPass : public OptionalPassInfoMixin<SROAPass> {
@@ -37,7 +37,7 @@ class SROAPass : public OptionalPassInfoMixin<SROAPass> {
public:
/// If \p PreserveCFG is set, then the pass is not allowed to modify CFG
/// in any way, even if it would update CFG analyses.
- /// If \p StructToVector is set, then the pass will try to convert
+ /// If \p AggregateToVector is set, then the pass will try to convert
/// allocas of homogeneous structs into vector allocas.
LLVM_ABI SROAPass(SROAOptions Options);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 6dbc3b2d512c2..8a0a407c569f4 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1451,12 +1451,12 @@ Expected<SROAOptions> parseSROAOptions(StringRef Params) {
inconvertibleErrorCode());
Result.CFG = SROAOptions::PreserveCFG;
SawCFGOption = true;
- } else if (ParamName == "struct-to-vector") {
- Result.StructToVector = true;
+ } else if (ParamName == "aggregate-to-vector") {
+ Result.AggregateToVector = true;
} else {
return make_error<StringError>(
formatv("invalid SROA pass parameter '{}' (expected preserve-cfg, "
- "modify-cfg, or struct-to-vector)",
+ "modify-cfg, or aggregate-to-vector)",
ParamName)
.str(),
inconvertibleErrorCode());
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 572a61a4a24b5..fc8cf9f472c12 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1381,7 +1381,7 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// memcpyopt's optimizations if done before. Look at the documentation for
// `tryCanonicalizeStructToVector` in SROA.cpp to see why.
FPM.addPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
- /*StructToVector=*/true)));
+ /*AggregateToVector=*/true)));
}
if (!isFullLTOPostLink(LTOPhase)) {
@@ -1482,7 +1482,7 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// memcpyopt's optimizations if done before. Look at the documentation for
// `tryCanonicalizeStructToVector` in SROA.cpp to see why.
FPM.addPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
- /*StructToVector=*/true)));
+ /*AggregateToVector=*/true)));
}
FPM.addPass(InferAlignmentPass());
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index c2fd2b560f729..636258459b1a2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -301,7 +301,7 @@ void NVPTXPassConfig::addAddressSpaceInferencePasses() {
// NVPTXLowerArgs emits alloca for byval parameters which can often
// be eliminated by SROA.
addPass(createSROAPass(/*PreserveCFG=*/true,
- /*StructToVector=*/true));
+ /*AggregateToVector=*/true));
addPass(createNVPTXLowerAllocaPass());
// TODO: Consider running InferAddressSpaces during opt, earlier in the
// compilation flow.
@@ -393,7 +393,7 @@ void NVPTXPassConfig::addIRPasses() {
if (!DisableLoadStoreVectorizer)
addPass(createLoadStoreVectorizerPass());
addPass(createSROAPass(/*PreserveCFG=*/true,
- /*StructToVector=*/true));
+ /*AggregateToVector=*/true));
addPass(createNVPTXTagInvariantLoadsPass());
if (!DisableNVPTXIRPeephole)
addPass(createNVPTXIRPeepholePass());
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index abcedc12a09f1..811dd373eca94 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -178,7 +178,7 @@ class SROA {
DomTreeUpdater *const DTU;
AssumptionCache *const AC;
const bool PreserveCFG;
- const bool StructToVector;
+ const bool AggregateToVector;
/// Worklist of alloca instructions to simplify.
///
@@ -244,7 +244,7 @@ class SROA {
SROAOptions Options)
: C(C), DTU(DTU), AC(AC),
PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
- StructToVector(Options.StructToVector) {}
+ AggregateToVector(Options.AggregateToVector) {}
/// Main run method used by both the SROAPass and by the legacy pass.
std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
@@ -5105,11 +5105,11 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
/// the alloca gets promoted to an SSA value. For another example, see
/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
/// this before memcpyopt can result in promoting an alloca so that we load a
-/// tempory value instead of copying the tempory value into memory, whereas
-/// memcpyopt eliminates the tempory altogether.
+/// temporary value instead of copying the temporary value into memory, whereas
+/// memcpyopt eliminates the temporary altogether.
///
/// As such, we only apply this transformation after memcpyopt has run. We gate
-/// this transformation by the "StructToVector" pass option.
+/// this transformation by the "AggregateToVector" pass option.
static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
Partition &P,
const DataLayout &DL) {
@@ -5162,7 +5162,7 @@ static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
/// nullptr.
static std::tuple<Type *, bool, VectorType *>
selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
- LLVMContext &C, bool StructToVector) {
+ LLVMContext &C, bool AggregateToVector) {
auto LogSelection = [&](StringRef Path, Type *SelectedTy,
VectorType *SelectedVecTy, bool SelectedIntWidening) {
LLVM_DEBUG({
@@ -5255,7 +5255,7 @@ selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
// Try homogeneous struct to vector canonicalization when requested. Running
// this too early can hide memcpy chains from MemCpyOpt.
- if (StructToVector) {
+ if (AggregateToVector) {
if (auto *STy = dyn_cast<StructType>(TypePartitionTy)) {
if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
LogSelection("struct-fallback-vecty", VTy, nullptr, false);
@@ -5304,7 +5304,7 @@ SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
const DataLayout &DL = AI.getDataLayout();
// Select the type for the new alloca that spans the partition.
auto [PartitionTy, IsIntegerWideningViable, VecTy] =
- selectPartitionType(P, DL, AI, *C, StructToVector);
+ selectPartitionType(P, DL, AI, *C, AggregateToVector);
// Check for the case where we're going to rewrite to a new alloca of the
// exact same type as the original, and with the same access offsets. In that
@@ -6130,8 +6130,8 @@ void SROAPass::printPipeline(
OS << '<'
<< (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
: "modify-cfg");
- if (Options.StructToVector)
- OS << ";struct-to-vector";
+ if (Options.AggregateToVector)
+ OS << ";aggregate-to-vector";
OS << '>';
}
@@ -6177,10 +6177,10 @@ class SROALegacyPass : public FunctionPass {
char SROALegacyPass::ID = 0;
-FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool StructToVector) {
+FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool AggregateToVector) {
return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
: SROAOptions::ModifyCFG,
- StructToVector));
+ AggregateToVector));
}
INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
index dd403c55ca64d..d1b7dcab99612 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-fp-store-only-tail.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=sroa -S %s | FileCheck %s --check-prefixes=NO-CANON
-; RUN: opt -passes='sroa<struct-to-vector>' -S %s | FileCheck %s --check-prefixes=CANON
+; RUN: opt -passes='sroa<aggregate-to-vector>' -S %s | FileCheck %s --check-prefixes=CANON
%class.aiMatrix4x4t = type { float, float, float, float, float, float, float, float, float, float, float, float, float, float, float, float }
diff --git a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
index 6962fd7291465..9edb8492aa460 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector-subpartition.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes='sroa<struct-to-vector>' -S %s | FileCheck %s
+; RUN: opt -passes='sroa<aggregate-to-vector>' -S %s | FileCheck %s
; NOTE: Do not autogenerate. This test intentionally uses targeted CHECK
; patterns for clarity.
diff --git a/llvm/test/Transforms/SROA/struct-to-vector.ll b/llvm/test/Transforms/SROA/struct-to-vector.ll
index 9ce21ad689d11..ad7d3df129d6c 100644
--- a/llvm/test/Transforms/SROA/struct-to-vector.ll
+++ b/llvm/test/Transforms/SROA/struct-to-vector.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes='sroa<struct-to-vector>,gvn,instcombine,simplifycfg' -S %s | FileCheck %s
+; RUN: opt -passes='sroa<aggregate-to-vector>,gvn,instcombine,simplifycfg' -S %s | FileCheck %s
%struct.myint4 = type { i32, i32, i32, i32 }
define dso_local void @foo_flat(ptr noundef %x, i64 %y.coerce0, i64 %y.coerce1, i32 noundef %cond) {
More information about the cfe-commits
mailing list