[llvm] [SROA] Pre-split overlapping move slices within a partition (PR #210061)
John Brawn via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 08:50:36 PDT 2026
https://github.com/john-brawn-arm updated https://github.com/llvm/llvm-project/pull/210061
>From 202c141ae4abc7c60e4e06b2aaf3721ea9e39265 Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn at arm.com>
Date: Thu, 9 Jul 2026 11:26:54 +0100
Subject: [PATCH 1/3] [SROA] Pre-split overlapping move slices within a
partition
When we have an overlapping load and store within an alloca, e.g. due
to a memmove that has been transformed into a load+store by
InstCombine, then currently SROA can't do anything with it. By
pre-splitting such loads and stores we eliminate any overlap between
slices and thus SROA can optimize them.
---
llvm/lib/Transforms/Scalar/SROA.cpp | 73 ++-
.../Transforms/SROA/load-store-overlap.ll | 474 ++++++++++++++++++
2 files changed, 545 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/SROA/load-store-overlap.ll
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index bae6e766c5ee3..e9a4eb4dc7d21 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -4805,6 +4805,43 @@ static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
return SubTy;
}
+/// Try to find a slice in P that overlaps with S, and which forms a load/store
+/// pair with S (i.e. the two slices are used to perform a memmove-like copy
+/// within P).
+static Slice *findOverlappingCopySlice(Slice &S, Partition &P) {
+ // Single byte slices can't overlap anything
+ if (S.endOffset() - S.beginOffset() == 1)
+ return nullptr;
+ // The source/destination of this slice needs to be a memory instruction
+ // whose slice overlaps this one.
+ Instruction *I = cast<Instruction>(S.getUse()->getUser());
+ Instruction *J = nullptr;
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ if (!LI->hasOneUser())
+ return nullptr;
+ J = dyn_cast<Instruction>(*LI->user_begin());
+ if (!J)
+ return nullptr;
+ } else if (auto *SI = dyn_cast<StoreInst>(I)) {
+ J = dyn_cast<Instruction>(SI->getValueOperand());
+ if (!J || !J->hasOneUser())
+ return nullptr;
+ } else {
+ return nullptr;
+ }
+ // Check if there's a slice that corresponds to J that overlaps this slice
+ for (Slice &JS : P) {
+ if (JS.getUse()->getUser() != J)
+ continue;
+ if (S.beginOffset() > JS.beginOffset() && S.beginOffset() < JS.endOffset())
+ return &JS;
+ if (JS.beginOffset() > S.beginOffset() && JS.beginOffset() < S.endOffset())
+ return &JS;
+ return nullptr;
+ }
+ return nullptr;
+}
+
/// Pre-split loads and stores to simplify rewriting.
///
/// We want to break up the splittable load+store pairs as much as
@@ -4867,7 +4904,9 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
for (auto &P : AS.partitions()) {
for (Slice &S : P) {
Instruction *I = cast<Instruction>(S.getUse()->getUser());
- if (!S.isSplittable() || S.endOffset() <= P.endOffset()) {
+ bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
+ Slice *CopyOverlap = findOverlappingCopySlice(S, P);
+ if (!S.isSplittable() || !(ExtendsPastPartitionEnd || CopyOverlap)) {
// If this is a load we have to track that it can't participate in any
// pre-splitting. If this is a store of a load we have to track that
// that load also can't participate in any pre-splitting.
@@ -4923,7 +4962,37 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
assert(Offsets.Splits.empty() &&
"Should not have splits the first time we see an instruction!");
Offsets.S = &S;
- Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
+ if (CopyOverlap) {
+ // S is being moved to CopyOverlap, which overlaps with S, so we split S
+ // into three part, which:
+ // * Starts outside the overlap and is copied into the overlap
+ // * Either starts inside the overlap and is copied inside the overlap,
+ // or starts outside and is copied outside, depending on if the
+ // overlap or non-overlap area is larger, and which will be empty if
+ // they are equal.
+ // * Starts inside the overlap and is copied outside the overlap
+ // This should result in the first and last parts being promoted to
+ // scalars, which may result in the middle part now being an overlapping
+ // copy which will then be presplit in the same way in the next
+ // iteration of the SROA loop.
+ uint64_t OverlapStart =
+ std::max(S.beginOffset(), CopyOverlap->beginOffset());
+ uint64_t OverlapEnd = std::min(S.endOffset(), CopyOverlap->endOffset());
+ uint64_t OverlapSize = OverlapEnd - OverlapStart;
+ uint64_t NonOverlapSize =
+ (S.endOffset() - S.beginOffset()) - OverlapSize;
+ if (OverlapSize < NonOverlapSize) {
+ Offsets.Splits.push_back(OverlapSize);
+ Offsets.Splits.push_back(NonOverlapSize);
+ } else if (OverlapSize > NonOverlapSize) {
+ Offsets.Splits.push_back(NonOverlapSize);
+ Offsets.Splits.push_back(OverlapSize);
+ } else {
+ Offsets.Splits.push_back(OverlapSize);
+ }
+ } else {
+ Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
+ }
}
// Now scan the already split slices, and add a split for any of them which
diff --git a/llvm/test/Transforms/SROA/load-store-overlap.ll b/llvm/test/Transforms/SROA/load-store-overlap.ll
new file mode 100644
index 0000000000000..d58275ad33e25
--- /dev/null
+++ b/llvm/test/Transforms/SROA/load-store-overlap.ll
@@ -0,0 +1,474 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=sroa < %s | FileCheck %s
+
+; In these tests we have memmove-like loads and stores within an alloca, where
+; the load slice overlaps with the store slice.
+
+define void @move_up(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 0, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 1, ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 2, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.1, align 1
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @move_down(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: store i8 1, ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 2, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 4, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ %move = load i32, ptr %arr.1, align 1
+ store i32 %move, ptr %arr, align 1
+ store i8 %arg, ptr %arr.4, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @move_up_small_overlap(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up_small_overlap(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR_SROA_4_1_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_4_1_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_4_1_INSERT_MASK]], 1
+; CHECK-NEXT: [[ARR_SROA_4_2_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_4_1_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_4_2_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_4_2_INSERT_MASK]], 512
+; CHECK-NEXT: [[ARR_SROA_13_4_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_13_4_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_13_4_INSERT_MASK]], 4
+; CHECK-NEXT: [[ARR_SROA_13_5_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_13_4_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_13_5_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_13_5_INSERT_MASK]], 1280
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[DEST_5:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 5
+; CHECK-NEXT: [[DEST_6:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 6
+; CHECK-NEXT: [[ARR_SROA_4_1_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_4_2_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_4_2_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_4_2_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: [[ARR_SROA_13_4_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_13_5_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_4_2_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_13_5_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_13_5_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_4_1_EXTRACT_TRUNC]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_4_2_EXTRACT_TRUNC]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 0, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_13_4_EXTRACT_TRUNC]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_13_5_EXTRACT_TRUNC]], ptr [[DEST_5]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_6]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [8 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ %arr.5 = getelementptr inbounds nuw i8, ptr %arr, i64 5
+ %arr.6 = getelementptr inbounds nuw i8, ptr %arr, i64 6
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ store i8 5, ptr %arr.5, align 1
+ store i8 6, ptr %arr.6, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.3, align 1
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %dest.5 = getelementptr inbounds nuw i8, ptr %dest, i64 5
+ %dest.6 = getelementptr inbounds nuw i8, ptr %dest, i64 6
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ %val.5 = load i8, ptr %arr.5, align 1
+ %val.6 = load i8, ptr %arr.6, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ store i8 %val.5, ptr %dest.5, align 1
+ store i8 %val.6, ptr %dest.6, align 1
+ ret void
+}
+
+define void @move_down_small_overlap(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down_small_overlap(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR_SROA_3_1_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_3_1_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_3_1_INSERT_MASK]], 1
+; CHECK-NEXT: [[ARR_SROA_3_2_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_3_1_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_3_2_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_3_2_INSERT_MASK]], 512
+; CHECK-NEXT: [[ARR_SROA_12_4_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_12_4_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_12_4_INSERT_MASK]], 4
+; CHECK-NEXT: [[ARR_SROA_12_5_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_12_4_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_12_5_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_12_5_INSERT_MASK]], 1280
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[DEST_5:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 5
+; CHECK-NEXT: [[DEST_6:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 6
+; CHECK-NEXT: [[ARR_SROA_3_1_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_3_2_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_12_5_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_3_2_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_3_2_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: [[ARR_SROA_12_4_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_12_5_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_12_5_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_12_5_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: store i8 3, ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_3_1_EXTRACT_TRUNC]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_3_2_EXTRACT_TRUNC]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 6, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_12_4_EXTRACT_TRUNC]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_12_5_EXTRACT_TRUNC]], ptr [[DEST_5]], align 1
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST_6]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [8 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ %arr.5 = getelementptr inbounds nuw i8, ptr %arr, i64 5
+ %arr.6 = getelementptr inbounds nuw i8, ptr %arr, i64 6
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ store i8 5, ptr %arr.5, align 1
+ store i8 6, ptr %arr.6, align 1
+ %move = load i32, ptr %arr.3, align 1
+ store i32 %move, ptr %arr, align 1
+ store i8 %arg, ptr %arr.6, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %dest.5 = getelementptr inbounds nuw i8, ptr %dest, i64 5
+ %dest.6 = getelementptr inbounds nuw i8, ptr %dest, i64 6
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ %val.5 = load i8, ptr %arr.5, align 1
+ %val.6 = load i8, ptr %arr.6, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ store i8 %val.5, ptr %dest.5, align 1
+ store i8 %val.6, ptr %dest.6, align 1
+ ret void
+}
+
+; The size of the overlap area equals the size of the non-overlap area
+define void @move_up_equal_overlap_nooverlap(i16 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up_equal_overlap_nooverlap(
+; CHECK-SAME: i16 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 2
+; CHECK-NEXT: store i16 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i16 0, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i16 1, ptr [[DEST_2]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [3 x i16], align 1
+ %arr.1 = getelementptr inbounds nuw i16, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i16, ptr %arr, i64 2
+ store i16 0, ptr %arr, align 1
+ store i16 1, ptr %arr.1, align 1
+ store i16 2, ptr %arr.2, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.1, align 1
+ store i16 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i16, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i16, ptr %dest, i64 2
+ %val.0 = load i16, ptr %arr, align 1
+ %val.1 = load i16, ptr %arr.1, align 1
+ %val.2 = load i16, ptr %arr.2, align 1
+ store i16 %val.0, ptr %dest, align 1
+ store i16 %val.1, ptr %dest.1, align 1
+ store i16 %val.2, ptr %dest.2, align 1
+ ret void
+}
+
+define void @move_down_equal_overlap_nooverlap(i16 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down_equal_overlap_nooverlap(
+; CHECK-SAME: i16 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 2
+; CHECK-NEXT: store i16 1, ptr [[DEST]], align 1
+; CHECK-NEXT: store i16 2, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i16 [[ARG]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [3 x i16], align 1
+ %arr.1 = getelementptr inbounds nuw i16, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i16, ptr %arr, i64 2
+ store i16 0, ptr %arr, align 1
+ store i16 1, ptr %arr.1, align 1
+ store i16 2, ptr %arr.2, align 1
+ %move = load i32, ptr %arr.1, align 1
+ store i32 %move, ptr %arr, align 1
+ store i16 %arg, ptr %arr.2, align 1
+ %dest.1 = getelementptr inbounds nuw i16, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i16, ptr %dest, i64 2
+ %val.0 = load i16, ptr %arr, align 1
+ %val.1 = load i16, ptr %arr.1, align 1
+ %val.2 = load i16, ptr %arr.2, align 1
+ store i16 %val.0, ptr %dest, align 1
+ store i16 %val.1, ptr %dest.1, align 1
+ store i16 %val.2, ptr %dest.2, align 1
+ ret void
+}
+
+define void @move_inside_loop(ptr %src, ptr %dest, i64 %n) {
+; CHECK-LABEL: define void @move_inside_loop(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_3_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_7_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_5_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_7_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_7_0]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_5_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_3_0]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[VAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_DEST:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 [[IV]]
+; CHECK-NEXT: [[VAL]] = load i8, ptr [[GEP_DEST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %arr = alloca [9 x i8], align 1
+ store i64 0, ptr %arr, align 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %move = load i64, ptr %arr, align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ store i64 %move, ptr %arr.1, align 1
+ %gep.dest = getelementptr inbounds nuw i8, ptr %dest, i64 %iv
+ %val = load i8, ptr %gep.dest, align 1
+ store i8 %val, ptr %arr, align 1
+ %iv.next = add i64 %iv, 1
+ %cmp = icmp eq i64 %iv.next, %n
+ br i1 %cmp, label %loop, label %end
+
+end:
+ ret void
+}
+
+; FIXME: SROA can't split memmove calls, so we can't optimize this
+define void @memmove_up(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @memmove_up(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT: store i8 0, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX1:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: store i8 1, ptr [[ARR_1_ARR_1_SROA_IDX1]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: store i8 2, ptr [[ARR_2_ARR_2_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: store i8 3, ptr [[ARR_3_ARR_3_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 4, ptr [[ARR_4_ARR_4_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i32(ptr align 1 [[ARR_1_ARR_1_SROA_IDX]], ptr align 1 [[ARR]], i32 4, i1 false)
+; CHECK-NEXT: store i8 [[ARG]], ptr [[ARR]], align 1
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[ARR_0_VAL_0:%.*]] = load i8, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX2:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: [[ARR_1_VAL_1:%.*]] = load i8, ptr [[ARR_1_ARR_1_SROA_IDX2]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX3:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: [[ARR_2_VAL_2:%.*]] = load i8, ptr [[ARR_2_ARR_2_SROA_IDX3]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX4:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: [[ARR_3_VAL_3:%.*]] = load i8, ptr [[ARR_3_ARR_3_SROA_IDX4]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX5:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: [[ARR_4_VAL_4:%.*]] = load i8, ptr [[ARR_4_ARR_4_SROA_IDX5]], align 1
+; CHECK-NEXT: store i8 [[ARR_0_VAL_0]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_1_VAL_1]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_2_VAL_2]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 [[ARR_3_VAL_3]], ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_4_VAL_4]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ call void @llvm.memmove.p0.p0.i32(ptr %arr.1, ptr %arr, i32 4, i1 false)
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @memmove_down(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @memmove_down(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT: store i8 0, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX1:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: store i8 1, ptr [[ARR_1_ARR_1_SROA_IDX1]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: store i8 2, ptr [[ARR_2_ARR_2_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: store i8 3, ptr [[ARR_3_ARR_3_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 4, ptr [[ARR_4_ARR_4_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i32(ptr align 1 [[ARR]], ptr align 1 [[ARR_1_ARR_1_SROA_IDX]], i32 4, i1 false)
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX5:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 [[ARG]], ptr [[ARR_4_ARR_4_SROA_IDX5]], align 1
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[ARR_0_VAL_0:%.*]] = load i8, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX2:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: [[ARR_1_VAL_1:%.*]] = load i8, ptr [[ARR_1_ARR_1_SROA_IDX2]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX3:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: [[ARR_2_VAL_2:%.*]] = load i8, ptr [[ARR_2_ARR_2_SROA_IDX3]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX4:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: [[ARR_3_VAL_3:%.*]] = load i8, ptr [[ARR_3_ARR_3_SROA_IDX4]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX6:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: [[ARR_4_VAL_4:%.*]] = load i8, ptr [[ARR_4_ARR_4_SROA_IDX6]], align 1
+; CHECK-NEXT: store i8 [[ARR_0_VAL_0]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_1_VAL_1]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_2_VAL_2]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 [[ARR_3_VAL_3]], ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_4_VAL_4]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ call void @llvm.memmove.p0.p0.i32(ptr %arr, ptr %arr.1, i32 4, i1 false)
+ store i8 %arg, ptr %arr.4, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
>From aff060f428bd50f60ccdcb44ce42feba8a76afa5 Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn at arm.com>
Date: Tue, 1 Sep 2026 16:38:55 +0100
Subject: [PATCH 2/3] Adjust based on review comments
---
llvm/lib/Transforms/Scalar/SROA.cpp | 51 +++++++++++++++++------------
1 file changed, 30 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index e9a4eb4dc7d21..95d71300c3835 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -129,6 +129,7 @@ namespace {
class AllocaSliceRewriter;
class AllocaSlices;
class Partition;
+class Slice;
class SelectHandSpeculativity {
unsigned char Storage = 0; // None are speculatable by default.
@@ -153,6 +154,8 @@ using UnspeculatableStore = StoreInst *;
using RewriteableMemOp =
std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
using RewriteableMemOps = SmallVector<RewriteableMemOp, 2>;
+using InstructionSliceMap =
+ SmallDenseMap<Instruction *, SmallPtrSet<Slice *, 8>, 8>;
/// An optimization pass providing Scalar Replacement of Aggregates.
///
@@ -4805,11 +4808,14 @@ static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
return SubTy;
}
-/// Try to find a slice in P that overlaps with S, and which forms a load/store
-/// pair with S (i.e. the two slices are used to perform a memmove-like copy
-/// within P).
-static Slice *findOverlappingCopySlice(Slice &S, Partition &P) {
- // Single byte slices can't overlap anything
+/// Try to find a slice in the map that partially overlaps with S, i.e. some but
+/// not all of its range is contained within the range of S, and which forms a
+/// load/store pair with S (i.e. the two slices are used to perform a
+/// memmove-like copy). SliceMap is expected to contain the slices for a single
+/// Partition.
+static Slice *findOverlappingCopySlice(Slice &S,
+ InstructionSliceMap &SliceMap) {
+ // Single byte slices can't partially overlap anything
if (S.endOffset() - S.beginOffset() == 1)
return nullptr;
// The source/destination of this slice needs to be a memory instruction
@@ -4830,14 +4836,13 @@ static Slice *findOverlappingCopySlice(Slice &S, Partition &P) {
return nullptr;
}
// Check if there's a slice that corresponds to J that overlaps this slice
- for (Slice &JS : P) {
- if (JS.getUse()->getUser() != J)
- continue;
- if (S.beginOffset() > JS.beginOffset() && S.beginOffset() < JS.endOffset())
- return &JS;
- if (JS.beginOffset() > S.beginOffset() && JS.beginOffset() < S.endOffset())
- return &JS;
- return nullptr;
+ for (Slice *JS : SliceMap[J]) {
+ if (S.beginOffset() > JS->beginOffset() &&
+ S.beginOffset() < JS->endOffset())
+ return JS;
+ if (JS->beginOffset() > S.beginOffset() &&
+ JS->beginOffset() < S.endOffset())
+ return JS;
}
return nullptr;
}
@@ -4902,10 +4907,14 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
for (auto &P : AS.partitions()) {
+ InstructionSliceMap SliceMap;
+ for (Slice &S : P) {
+ SliceMap[cast<Instruction>(S.getUse()->getUser())].insert(&S);
+ }
for (Slice &S : P) {
Instruction *I = cast<Instruction>(S.getUse()->getUser());
bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
- Slice *CopyOverlap = findOverlappingCopySlice(S, P);
+ Slice *CopyOverlap = findOverlappingCopySlice(S, SliceMap);
if (!S.isSplittable() || !(ExtendsPastPartitionEnd || CopyOverlap)) {
// If this is a load we have to track that it can't participate in any
// pre-splitting. If this is a store of a load we have to track that
@@ -4964,13 +4973,13 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
Offsets.S = &S;
if (CopyOverlap) {
// S is being moved to CopyOverlap, which overlaps with S, so we split S
- // into three part, which:
- // * Starts outside the overlap and is copied into the overlap
- // * Either starts inside the overlap and is copied inside the overlap,
- // or starts outside and is copied outside, depending on if the
- // overlap or non-overlap area is larger, and which will be empty if
- // they are equal.
- // * Starts inside the overlap and is copied outside the overlap
+ // into three parts:
+ // * Initially outside the overlap and copied into the overlap.
+ // * Either initially inside the overlap and remains inside it after
+ // the copy, or initially outside and remains outside, depending on
+ // if the overlap or non-overlap area is larger, and which will be
+ // empty if they are equal.
+ // * Initially inside the overlap and copied outside the overlap.
// This should result in the first and last parts being promoted to
// scalars, which may result in the middle part now being an overlapping
// copy which will then be presplit in the same way in the next
>From 56f2d7ed1c46ede5b8c23b7302041215dea37570 Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn at arm.com>
Date: Mon, 7 Sep 2026 13:37:59 +0100
Subject: [PATCH 3/3] Update based on comments
Repeatedly split instead of letting it be handled by the next iteration of SROA.
Don't need dyn_cast for user of instruction.
Adjust splittable condition.
---
llvm/lib/Transforms/Scalar/SROA.cpp | 32 ++++++++++++-------
.../Transforms/SROA/load-store-overlap.ll | 14 ++++----
2 files changed, 28 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 95d71300c3835..f565a7831c92f 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -4825,9 +4825,7 @@ static Slice *findOverlappingCopySlice(Slice &S,
if (auto *LI = dyn_cast<LoadInst>(I)) {
if (!LI->hasOneUser())
return nullptr;
- J = dyn_cast<Instruction>(*LI->user_begin());
- if (!J)
- return nullptr;
+ J = cast<Instruction>(*LI->user_begin());
} else if (auto *SI = dyn_cast<StoreInst>(I)) {
J = dyn_cast<Instruction>(SI->getValueOperand());
if (!J || !J->hasOneUser())
@@ -4915,7 +4913,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
Instruction *I = cast<Instruction>(S.getUse()->getUser());
bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
Slice *CopyOverlap = findOverlappingCopySlice(S, SliceMap);
- if (!S.isSplittable() || !(ExtendsPastPartitionEnd || CopyOverlap)) {
+ if (!S.isSplittable() || (!ExtendsPastPartitionEnd && !CopyOverlap)) {
// If this is a load we have to track that it can't participate in any
// pre-splitting. If this is a store of a load we have to track that
// that load also can't participate in any pre-splitting.
@@ -4981,22 +4979,34 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
// empty if they are equal.
// * Initially inside the overlap and copied outside the overlap.
// This should result in the first and last parts being promoted to
- // scalars, which may result in the middle part now being an overlapping
- // copy which will then be presplit in the same way in the next
- // iteration of the SROA loop.
+ // scalars. If the middle part is an overlapping copy then we repeat
+ // this process until it isn't.
uint64_t OverlapStart =
std::max(S.beginOffset(), CopyOverlap->beginOffset());
uint64_t OverlapEnd = std::min(S.endOffset(), CopyOverlap->endOffset());
uint64_t OverlapSize = OverlapEnd - OverlapStart;
- uint64_t NonOverlapSize =
- (S.endOffset() - S.beginOffset()) - OverlapSize;
+ uint64_t SliceSize = S.endOffset() - S.beginOffset();
+ uint64_t NonOverlapSize = SliceSize - OverlapSize;
if (OverlapSize < NonOverlapSize) {
+ // When the overlap area is smaller the middle part is initially
+ // and remains outside the overlap, so splitting once is enough.
Offsets.Splits.push_back(OverlapSize);
Offsets.Splits.push_back(NonOverlapSize);
} else if (OverlapSize > NonOverlapSize) {
- Offsets.Splits.push_back(NonOverlapSize);
- Offsets.Splits.push_back(OverlapSize);
+ // When the overlap area is larger the middle part is initially and
+ // remains inside the overlap, so we repeatedly split it.
+ for (uint64_t Split = NonOverlapSize; Split <= SliceSize / 2;
+ Split += NonOverlapSize) {
+ Offsets.Splits.push_back(Split);
+ if (Split != SliceSize - Split) {
+ Offsets.Splits.push_back(SliceSize - Split);
+ }
+ }
+ // Sort the splits as they need to be in ascending order.
+ llvm::sort(Offsets.Splits);
} else {
+ // Here the overlap and non-overlap size are the same, so the middle
+ // part is empty and the slice is split exactly in the center.
Offsets.Splits.push_back(OverlapSize);
}
} else {
diff --git a/llvm/test/Transforms/SROA/load-store-overlap.ll b/llvm/test/Transforms/SROA/load-store-overlap.ll
index d58275ad33e25..87340fd772e6a 100644
--- a/llvm/test/Transforms/SROA/load-store-overlap.ll
+++ b/llvm/test/Transforms/SROA/load-store-overlap.ll
@@ -303,13 +303,13 @@ define void @move_inside_loop(ptr %src, ptr %dest, i64 %n) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_3_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_7_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_5_0:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ARR_SROA_3_SROA_7_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_7_0]], %[[LOOP]] ]
-; CHECK-NEXT: [[ARR_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_0_0:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_5_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_3_0]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_15_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_13_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_13_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_11_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_11_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_9_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_9_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_7_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_7_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_5_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_5_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_0_0:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[ARR_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[VAL:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[GEP_DEST:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 [[IV]]
More information about the llvm-commits
mailing list