[llvm] [SROA] Pre-split overlapping move slices within a partition (PR #210061)
John Brawn via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 06:40:12 PDT 2026
https://github.com/john-brawn-arm created https://github.com/llvm/llvm-project/pull/210061
When we have an overlapping load and store within an alloca, e.g. due to a memmove that has been transformed into a load+store by InstCombine, then currently SROA can't do anything with it. By pre-splitting such loads and stores we eliminate any overlap between slices and thus SROA can optimize them.
>From 202c141ae4abc7c60e4e06b2aaf3721ea9e39265 Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn at arm.com>
Date: Thu, 9 Jul 2026 11:26:54 +0100
Subject: [PATCH] [SROA] Pre-split overlapping move slices within a partition
When we have an overlapping load and store within an alloca, e.g. due
to a memmove that has been transformed into a load+store by
InstCombine, then currently SROA can't do anything with it. By
pre-splitting such loads and stores we eliminate any overlap between
slices and thus SROA can optimize them.
---
llvm/lib/Transforms/Scalar/SROA.cpp | 73 ++-
.../Transforms/SROA/load-store-overlap.ll | 474 ++++++++++++++++++
2 files changed, 545 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/SROA/load-store-overlap.ll
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index bae6e766c5ee3..e9a4eb4dc7d21 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -4805,6 +4805,43 @@ static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
return SubTy;
}
+/// Try to find a slice in P that overlaps with S, and which forms a load/store
+/// pair with S (i.e. the two slices are used to perform a memmove-like copy
+/// within P).
+static Slice *findOverlappingCopySlice(Slice &S, Partition &P) {
+ // Single byte slices can't overlap anything
+ if (S.endOffset() - S.beginOffset() == 1)
+ return nullptr;
+ // The source/destination of this slice needs to be a memory instruction
+ // whose slice overlaps this one.
+ Instruction *I = cast<Instruction>(S.getUse()->getUser());
+ Instruction *J = nullptr;
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ if (!LI->hasOneUser())
+ return nullptr;
+ J = dyn_cast<Instruction>(*LI->user_begin());
+ if (!J)
+ return nullptr;
+ } else if (auto *SI = dyn_cast<StoreInst>(I)) {
+ J = dyn_cast<Instruction>(SI->getValueOperand());
+ if (!J || !J->hasOneUser())
+ return nullptr;
+ } else {
+ return nullptr;
+ }
+ // Check if there's a slice that corresponds to J that overlaps this slice
+ for (Slice &JS : P) {
+ if (JS.getUse()->getUser() != J)
+ continue;
+ if (S.beginOffset() > JS.beginOffset() && S.beginOffset() < JS.endOffset())
+ return &JS;
+ if (JS.beginOffset() > S.beginOffset() && JS.beginOffset() < S.endOffset())
+ return &JS;
+ return nullptr;
+ }
+ return nullptr;
+}
+
/// Pre-split loads and stores to simplify rewriting.
///
/// We want to break up the splittable load+store pairs as much as
@@ -4867,7 +4904,9 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
for (auto &P : AS.partitions()) {
for (Slice &S : P) {
Instruction *I = cast<Instruction>(S.getUse()->getUser());
- if (!S.isSplittable() || S.endOffset() <= P.endOffset()) {
+ bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
+ Slice *CopyOverlap = findOverlappingCopySlice(S, P);
+ if (!S.isSplittable() || !(ExtendsPastPartitionEnd || CopyOverlap)) {
// If this is a load we have to track that it can't participate in any
// pre-splitting. If this is a store of a load we have to track that
// that load also can't participate in any pre-splitting.
@@ -4923,7 +4962,37 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
assert(Offsets.Splits.empty() &&
"Should not have splits the first time we see an instruction!");
Offsets.S = &S;
- Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
+ if (CopyOverlap) {
+ // S is being moved to CopyOverlap, which overlaps with S, so we split S
+ // into three part, which:
+ // * Starts outside the overlap and is copied into the overlap
+ // * Either starts inside the overlap and is copied inside the overlap,
+ // or starts outside and is copied outside, depending on if the
+ // overlap or non-overlap area is larger, and which will be empty if
+ // they are equal.
+ // * Starts inside the overlap and is copied outside the overlap
+ // This should result in the first and last parts being promoted to
+ // scalars, which may result in the middle part now being an overlapping
+ // copy which will then be presplit in the same way in the next
+ // iteration of the SROA loop.
+ uint64_t OverlapStart =
+ std::max(S.beginOffset(), CopyOverlap->beginOffset());
+ uint64_t OverlapEnd = std::min(S.endOffset(), CopyOverlap->endOffset());
+ uint64_t OverlapSize = OverlapEnd - OverlapStart;
+ uint64_t NonOverlapSize =
+ (S.endOffset() - S.beginOffset()) - OverlapSize;
+ if (OverlapSize < NonOverlapSize) {
+ Offsets.Splits.push_back(OverlapSize);
+ Offsets.Splits.push_back(NonOverlapSize);
+ } else if (OverlapSize > NonOverlapSize) {
+ Offsets.Splits.push_back(NonOverlapSize);
+ Offsets.Splits.push_back(OverlapSize);
+ } else {
+ Offsets.Splits.push_back(OverlapSize);
+ }
+ } else {
+ Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
+ }
}
// Now scan the already split slices, and add a split for any of them which
diff --git a/llvm/test/Transforms/SROA/load-store-overlap.ll b/llvm/test/Transforms/SROA/load-store-overlap.ll
new file mode 100644
index 0000000000000..d58275ad33e25
--- /dev/null
+++ b/llvm/test/Transforms/SROA/load-store-overlap.ll
@@ -0,0 +1,474 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=sroa < %s | FileCheck %s
+
+; In these tests we have memmove-like loads and stores within an alloca, where
+; the load slice overlaps with the store slice.
+
+define void @move_up(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 0, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 1, ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 2, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.1, align 1
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @move_down(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: store i8 1, ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 2, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 4, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ %move = load i32, ptr %arr.1, align 1
+ store i32 %move, ptr %arr, align 1
+ store i8 %arg, ptr %arr.4, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @move_up_small_overlap(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up_small_overlap(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR_SROA_4_1_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_4_1_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_4_1_INSERT_MASK]], 1
+; CHECK-NEXT: [[ARR_SROA_4_2_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_4_1_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_4_2_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_4_2_INSERT_MASK]], 512
+; CHECK-NEXT: [[ARR_SROA_13_4_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_13_4_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_13_4_INSERT_MASK]], 4
+; CHECK-NEXT: [[ARR_SROA_13_5_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_13_4_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_13_5_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_13_5_INSERT_MASK]], 1280
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[DEST_5:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 5
+; CHECK-NEXT: [[DEST_6:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 6
+; CHECK-NEXT: [[ARR_SROA_4_1_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_4_2_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_4_2_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_4_2_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: [[ARR_SROA_13_4_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_13_5_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_4_2_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_13_5_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_13_5_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_4_1_EXTRACT_TRUNC]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_4_2_EXTRACT_TRUNC]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 0, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_13_4_EXTRACT_TRUNC]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_13_5_EXTRACT_TRUNC]], ptr [[DEST_5]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_6]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [8 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ %arr.5 = getelementptr inbounds nuw i8, ptr %arr, i64 5
+ %arr.6 = getelementptr inbounds nuw i8, ptr %arr, i64 6
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ store i8 5, ptr %arr.5, align 1
+ store i8 6, ptr %arr.6, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.3, align 1
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %dest.5 = getelementptr inbounds nuw i8, ptr %dest, i64 5
+ %dest.6 = getelementptr inbounds nuw i8, ptr %dest, i64 6
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ %val.5 = load i8, ptr %arr.5, align 1
+ %val.6 = load i8, ptr %arr.6, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ store i8 %val.5, ptr %dest.5, align 1
+ store i8 %val.6, ptr %dest.6, align 1
+ ret void
+}
+
+define void @move_down_small_overlap(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down_small_overlap(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR_SROA_3_1_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_3_1_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_3_1_INSERT_MASK]], 1
+; CHECK-NEXT: [[ARR_SROA_3_2_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_3_1_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_3_2_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_3_2_INSERT_MASK]], 512
+; CHECK-NEXT: [[ARR_SROA_12_4_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_12_4_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_12_4_INSERT_MASK]], 4
+; CHECK-NEXT: [[ARR_SROA_12_5_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_12_4_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_12_5_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_12_5_INSERT_MASK]], 1280
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[DEST_5:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 5
+; CHECK-NEXT: [[DEST_6:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 6
+; CHECK-NEXT: [[ARR_SROA_3_1_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_3_2_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_12_5_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_3_2_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_3_2_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: [[ARR_SROA_12_4_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_12_5_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_12_5_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_12_5_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: store i8 3, ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_3_1_EXTRACT_TRUNC]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_3_2_EXTRACT_TRUNC]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 6, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_12_4_EXTRACT_TRUNC]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_12_5_EXTRACT_TRUNC]], ptr [[DEST_5]], align 1
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST_6]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [8 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ %arr.5 = getelementptr inbounds nuw i8, ptr %arr, i64 5
+ %arr.6 = getelementptr inbounds nuw i8, ptr %arr, i64 6
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ store i8 5, ptr %arr.5, align 1
+ store i8 6, ptr %arr.6, align 1
+ %move = load i32, ptr %arr.3, align 1
+ store i32 %move, ptr %arr, align 1
+ store i8 %arg, ptr %arr.6, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %dest.5 = getelementptr inbounds nuw i8, ptr %dest, i64 5
+ %dest.6 = getelementptr inbounds nuw i8, ptr %dest, i64 6
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ %val.5 = load i8, ptr %arr.5, align 1
+ %val.6 = load i8, ptr %arr.6, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ store i8 %val.5, ptr %dest.5, align 1
+ store i8 %val.6, ptr %dest.6, align 1
+ ret void
+}
+
+; The size of the overlap area equals the size of the non-overlap area
+define void @move_up_equal_overlap_nooverlap(i16 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up_equal_overlap_nooverlap(
+; CHECK-SAME: i16 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 2
+; CHECK-NEXT: store i16 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i16 0, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i16 1, ptr [[DEST_2]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [3 x i16], align 1
+ %arr.1 = getelementptr inbounds nuw i16, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i16, ptr %arr, i64 2
+ store i16 0, ptr %arr, align 1
+ store i16 1, ptr %arr.1, align 1
+ store i16 2, ptr %arr.2, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.1, align 1
+ store i16 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i16, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i16, ptr %dest, i64 2
+ %val.0 = load i16, ptr %arr, align 1
+ %val.1 = load i16, ptr %arr.1, align 1
+ %val.2 = load i16, ptr %arr.2, align 1
+ store i16 %val.0, ptr %dest, align 1
+ store i16 %val.1, ptr %dest.1, align 1
+ store i16 %val.2, ptr %dest.2, align 1
+ ret void
+}
+
+define void @move_down_equal_overlap_nooverlap(i16 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down_equal_overlap_nooverlap(
+; CHECK-SAME: i16 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 2
+; CHECK-NEXT: store i16 1, ptr [[DEST]], align 1
+; CHECK-NEXT: store i16 2, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i16 [[ARG]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [3 x i16], align 1
+ %arr.1 = getelementptr inbounds nuw i16, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i16, ptr %arr, i64 2
+ store i16 0, ptr %arr, align 1
+ store i16 1, ptr %arr.1, align 1
+ store i16 2, ptr %arr.2, align 1
+ %move = load i32, ptr %arr.1, align 1
+ store i32 %move, ptr %arr, align 1
+ store i16 %arg, ptr %arr.2, align 1
+ %dest.1 = getelementptr inbounds nuw i16, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i16, ptr %dest, i64 2
+ %val.0 = load i16, ptr %arr, align 1
+ %val.1 = load i16, ptr %arr.1, align 1
+ %val.2 = load i16, ptr %arr.2, align 1
+ store i16 %val.0, ptr %dest, align 1
+ store i16 %val.1, ptr %dest.1, align 1
+ store i16 %val.2, ptr %dest.2, align 1
+ ret void
+}
+
+define void @move_inside_loop(ptr %src, ptr %dest, i64 %n) {
+; CHECK-LABEL: define void @move_inside_loop(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_3_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_7_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_5_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_7_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_7_0]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_SROA_3_SROA_3_SROA_5_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_SROA_3_SROA_3_SROA_3_0]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[VAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_DEST:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 [[IV]]
+; CHECK-NEXT: [[VAL]] = load i8, ptr [[GEP_DEST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %arr = alloca [9 x i8], align 1
+ store i64 0, ptr %arr, align 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %move = load i64, ptr %arr, align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ store i64 %move, ptr %arr.1, align 1
+ %gep.dest = getelementptr inbounds nuw i8, ptr %dest, i64 %iv
+ %val = load i8, ptr %gep.dest, align 1
+ store i8 %val, ptr %arr, align 1
+ %iv.next = add i64 %iv, 1
+ %cmp = icmp eq i64 %iv.next, %n
+ br i1 %cmp, label %loop, label %end
+
+end:
+ ret void
+}
+
+; FIXME: SROA can't split memmove calls, so we can't optimize this
+define void @memmove_up(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @memmove_up(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT: store i8 0, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX1:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: store i8 1, ptr [[ARR_1_ARR_1_SROA_IDX1]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: store i8 2, ptr [[ARR_2_ARR_2_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: store i8 3, ptr [[ARR_3_ARR_3_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 4, ptr [[ARR_4_ARR_4_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i32(ptr align 1 [[ARR_1_ARR_1_SROA_IDX]], ptr align 1 [[ARR]], i32 4, i1 false)
+; CHECK-NEXT: store i8 [[ARG]], ptr [[ARR]], align 1
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[ARR_0_VAL_0:%.*]] = load i8, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX2:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: [[ARR_1_VAL_1:%.*]] = load i8, ptr [[ARR_1_ARR_1_SROA_IDX2]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX3:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: [[ARR_2_VAL_2:%.*]] = load i8, ptr [[ARR_2_ARR_2_SROA_IDX3]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX4:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: [[ARR_3_VAL_3:%.*]] = load i8, ptr [[ARR_3_ARR_3_SROA_IDX4]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX5:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: [[ARR_4_VAL_4:%.*]] = load i8, ptr [[ARR_4_ARR_4_SROA_IDX5]], align 1
+; CHECK-NEXT: store i8 [[ARR_0_VAL_0]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_1_VAL_1]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_2_VAL_2]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 [[ARR_3_VAL_3]], ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_4_VAL_4]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ call void @llvm.memmove.p0.p0.i32(ptr %arr.1, ptr %arr, i32 4, i1 false)
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @memmove_down(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @memmove_down(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT: store i8 0, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX1:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: store i8 1, ptr [[ARR_1_ARR_1_SROA_IDX1]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: store i8 2, ptr [[ARR_2_ARR_2_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: store i8 3, ptr [[ARR_3_ARR_3_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 4, ptr [[ARR_4_ARR_4_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i32(ptr align 1 [[ARR]], ptr align 1 [[ARR_1_ARR_1_SROA_IDX]], i32 4, i1 false)
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX5:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 [[ARG]], ptr [[ARR_4_ARR_4_SROA_IDX5]], align 1
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[ARR_0_VAL_0:%.*]] = load i8, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX2:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: [[ARR_1_VAL_1:%.*]] = load i8, ptr [[ARR_1_ARR_1_SROA_IDX2]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX3:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: [[ARR_2_VAL_2:%.*]] = load i8, ptr [[ARR_2_ARR_2_SROA_IDX3]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX4:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: [[ARR_3_VAL_3:%.*]] = load i8, ptr [[ARR_3_ARR_3_SROA_IDX4]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX6:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: [[ARR_4_VAL_4:%.*]] = load i8, ptr [[ARR_4_ARR_4_SROA_IDX6]], align 1
+; CHECK-NEXT: store i8 [[ARR_0_VAL_0]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_1_VAL_1]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_2_VAL_2]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 [[ARR_3_VAL_3]], ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_4_VAL_4]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ call void @llvm.memmove.p0.p0.i32(ptr %arr, ptr %arr.1, i32 4, i1 false)
+ store i8 %arg, ptr %arr.4, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
More information about the llvm-commits
mailing list