[llvm] 51b1c27 - [SROA] Pre-split overlapping move slices within a partition (#210061)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 02:59:01 PDT 2026
Author: John Brawn
Date: 2026-09-25T10:58:53+01:00
New Revision: 51b1c275718891743493f3ed5d9e54d91a027e2d
URL: https://github.com/llvm/llvm-project/commit/51b1c275718891743493f3ed5d9e54d91a027e2d
DIFF: https://github.com/llvm/llvm-project/commit/51b1c275718891743493f3ed5d9e54d91a027e2d.diff
LOG: [SROA] Pre-split overlapping move slices within a partition (#210061)
When we have an overlapping load and store within an alloca, e.g. due to
a memmove that has been transformed into a load+store by InstCombine,
then currently SROA can't do anything with it. By pre-splitting such
loads and stores we eliminate any overlap between slices and thus SROA
can optimize them.
Added:
llvm/test/Transforms/SROA/load-store-overlap.ll
Modified:
llvm/lib/Transforms/Scalar/SROA.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index f12ab1e590678..9178f8fb6ac9f 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -128,6 +128,7 @@ namespace {
class AllocaSliceRewriter;
class AllocaSlices;
class Partition;
+class Slice;
class SelectHandSpeculativity {
unsigned char Storage = 0; // None are speculatable by default.
@@ -152,6 +153,8 @@ using UnspeculatableStore = StoreInst *;
using RewriteableMemOp =
std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
using RewriteableMemOps = SmallVector<RewriteableMemOp, 2>;
+using InstructionSliceMap =
+ SmallDenseMap<Instruction *, SmallPtrSet<Slice *, 8>, 8>;
/// An optimization pass providing Scalar Replacement of Aggregates.
///
@@ -4937,6 +4940,43 @@ static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
return SubTy;
}
+/// Try to find a slice in the map that partially overlaps with S, i.e. some but
+/// not all of its range is contained within the range of S, and which forms a
+/// load/store pair with S (i.e. the two slices are used to perform a
+/// memmove-like copy). SliceMap is expected to contain the slices for a single
+/// Partition.
+static Slice *findOverlappingCopySlice(Slice &S,
+ InstructionSliceMap &SliceMap) {
+ // Single byte slices can't partially overlap anything
+ if (S.endOffset() - S.beginOffset() == 1)
+ return nullptr;
+ // The source/destination of this slice needs to be a memory instruction
+ // whose slice overlaps this one.
+ Instruction *I = cast<Instruction>(S.getUse()->getUser());
+ Instruction *J = nullptr;
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ if (!LI->hasOneUser())
+ return nullptr;
+ J = cast<Instruction>(*LI->user_begin());
+ } else if (auto *SI = dyn_cast<StoreInst>(I)) {
+ J = dyn_cast<Instruction>(SI->getValueOperand());
+ if (!J || !J->hasOneUser())
+ return nullptr;
+ } else {
+ return nullptr;
+ }
+ // Check if there's a slice that corresponds to J that overlaps this slice
+ for (Slice *JS : SliceMap[J]) {
+ if (S.beginOffset() > JS->beginOffset() &&
+ S.beginOffset() < JS->endOffset())
+ return JS;
+ if (JS->beginOffset() > S.beginOffset() &&
+ JS->beginOffset() < S.endOffset())
+ return JS;
+ }
+ return nullptr;
+}
+
/// Pre-split loads and stores to simplify rewriting.
///
/// We want to break up the splittable load+store pairs as much as
@@ -4997,9 +5037,15 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
for (auto &P : AS.partitions()) {
+ InstructionSliceMap SliceMap;
+ for (Slice &S : P) {
+ SliceMap[cast<Instruction>(S.getUse()->getUser())].insert(&S);
+ }
for (Slice &S : P) {
Instruction *I = cast<Instruction>(S.getUse()->getUser());
- if (!S.isSplittable() || S.endOffset() <= P.endOffset()) {
+ bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
+ Slice *CopyOverlap = findOverlappingCopySlice(S, SliceMap);
+ if (!S.isSplittable() || (!ExtendsPastPartitionEnd && !CopyOverlap)) {
// If this is a load we have to track that it can't participate in any
// pre-splitting. If this is a store of a load we have to track that
// that load also can't participate in any pre-splitting.
@@ -5055,7 +5101,51 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
assert(Offsets.Splits.empty() &&
"Should not have splits the first time we see an instruction!");
Offsets.S = &S;
- Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
+ if (CopyOverlap) {
+ // S is being moved to CopyOverlap, which overlaps with S, so we split S
+ // into three parts:
+ // * Initially outside the overlap and copied into the overlap.
+ // * Either initially inside the overlap and remains inside it after
+ // the copy, or initially outside and remains outside, depending on
+ // if the overlap or non-overlap area is larger, and which will be
+ // empty if they are equal.
+ // * Initially inside the overlap and copied outside the overlap.
+ // This should result in the first and last parts being promoted to
+ // scalars. If the middle part is an overlapping copy then we repeat
+ // this process until it isn't.
+ uint64_t OverlapStart =
+ std::max(S.beginOffset(), CopyOverlap->beginOffset());
+ uint64_t OverlapEnd = std::min(S.endOffset(), CopyOverlap->endOffset());
+ uint64_t OverlapSize = OverlapEnd - OverlapStart;
+ uint64_t SliceSize = S.endOffset() - S.beginOffset();
+ uint64_t NonOverlapSize = SliceSize - OverlapSize;
+ if (OverlapSize < NonOverlapSize) {
+ // When the overlap area is smaller the middle part is initially and
+ // remains outside the overlap, so splitting once is enough. The size
+ // of the first and last part both equal the overlap size, so the
+ // split points are that amount from the start and end of the slice.
+ Offsets.Splits.push_back(OverlapSize);
+ Offsets.Splits.push_back(SliceSize - OverlapSize);
+ } else if (OverlapSize > NonOverlapSize) {
+ // When the overlap area is larger the middle part is initially and
+ // remains inside the overlap, so we repeatedly split it.
+ for (uint64_t Split = NonOverlapSize; Split <= SliceSize / 2;
+ Split += NonOverlapSize) {
+ Offsets.Splits.push_back(Split);
+ if (Split != SliceSize - Split) {
+ Offsets.Splits.push_back(SliceSize - Split);
+ }
+ }
+ // Sort the splits as they need to be in ascending order.
+ llvm::sort(Offsets.Splits);
+ } else {
+ // Here the overlap and non-overlap size are the same, so the middle
+ // part is empty and the slice is split exactly in the center.
+ Offsets.Splits.push_back(OverlapSize);
+ }
+ } else {
+ Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
+ }
}
// Now scan the already split slices, and add a split for any of them which
diff --git a/llvm/test/Transforms/SROA/load-store-overlap.ll b/llvm/test/Transforms/SROA/load-store-overlap.ll
new file mode 100644
index 0000000000000..87340fd772e6a
--- /dev/null
+++ b/llvm/test/Transforms/SROA/load-store-overlap.ll
@@ -0,0 +1,474 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=sroa < %s | FileCheck %s
+
+; In these tests we have memmove-like loads and stores within an alloca, where
+; the load slice overlaps with the store slice.
+
+define void @move_up(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 0, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 1, ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 2, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.1, align 1
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @move_down(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: store i8 1, ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 2, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 4, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ %move = load i32, ptr %arr.1, align 1
+ store i32 %move, ptr %arr, align 1
+ store i8 %arg, ptr %arr.4, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @move_up_small_overlap(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up_small_overlap(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR_SROA_4_1_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_4_1_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_4_1_INSERT_MASK]], 1
+; CHECK-NEXT: [[ARR_SROA_4_2_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_4_1_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_4_2_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_4_2_INSERT_MASK]], 512
+; CHECK-NEXT: [[ARR_SROA_13_4_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_13_4_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_13_4_INSERT_MASK]], 4
+; CHECK-NEXT: [[ARR_SROA_13_5_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_13_4_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_13_5_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_13_5_INSERT_MASK]], 1280
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[DEST_5:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 5
+; CHECK-NEXT: [[DEST_6:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 6
+; CHECK-NEXT: [[ARR_SROA_4_1_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_4_2_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_4_2_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_4_2_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: [[ARR_SROA_13_4_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_4_2_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_13_5_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_4_2_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_13_5_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_13_5_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_4_1_EXTRACT_TRUNC]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_4_2_EXTRACT_TRUNC]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 0, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_13_4_EXTRACT_TRUNC]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_13_5_EXTRACT_TRUNC]], ptr [[DEST_5]], align 1
+; CHECK-NEXT: store i8 3, ptr [[DEST_6]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [8 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ %arr.5 = getelementptr inbounds nuw i8, ptr %arr, i64 5
+ %arr.6 = getelementptr inbounds nuw i8, ptr %arr, i64 6
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ store i8 5, ptr %arr.5, align 1
+ store i8 6, ptr %arr.6, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.3, align 1
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %dest.5 = getelementptr inbounds nuw i8, ptr %dest, i64 5
+ %dest.6 = getelementptr inbounds nuw i8, ptr %dest, i64 6
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ %val.5 = load i8, ptr %arr.5, align 1
+ %val.6 = load i8, ptr %arr.6, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ store i8 %val.5, ptr %dest.5, align 1
+ store i8 %val.6, ptr %dest.6, align 1
+ ret void
+}
+
+define void @move_down_small_overlap(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down_small_overlap(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR_SROA_3_1_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_3_1_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_3_1_INSERT_MASK]], 1
+; CHECK-NEXT: [[ARR_SROA_3_2_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_3_1_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_3_2_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_3_2_INSERT_MASK]], 512
+; CHECK-NEXT: [[ARR_SROA_12_4_INSERT_MASK:%.*]] = and i16 undef, -256
+; CHECK-NEXT: [[ARR_SROA_12_4_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_12_4_INSERT_MASK]], 4
+; CHECK-NEXT: [[ARR_SROA_12_5_INSERT_MASK:%.*]] = and i16 [[ARR_SROA_12_4_INSERT_INSERT]], 255
+; CHECK-NEXT: [[ARR_SROA_12_5_INSERT_INSERT:%.*]] = or i16 [[ARR_SROA_12_5_INSERT_MASK]], 1280
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[DEST_5:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 5
+; CHECK-NEXT: [[DEST_6:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 6
+; CHECK-NEXT: [[ARR_SROA_3_1_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_3_2_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_12_5_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_3_2_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_3_2_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: [[ARR_SROA_12_4_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_INSERT_INSERT]] to i8
+; CHECK-NEXT: [[ARR_SROA_12_5_EXTRACT_SHIFT:%.*]] = lshr i16 [[ARR_SROA_12_5_INSERT_INSERT]], 8
+; CHECK-NEXT: [[ARR_SROA_12_5_EXTRACT_TRUNC:%.*]] = trunc i16 [[ARR_SROA_12_5_EXTRACT_SHIFT]] to i8
+; CHECK-NEXT: store i8 3, ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_3_1_EXTRACT_TRUNC]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_3_2_EXTRACT_TRUNC]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 6, ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_12_4_EXTRACT_TRUNC]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: store i8 [[ARR_SROA_12_5_EXTRACT_TRUNC]], ptr [[DEST_5]], align 1
+; CHECK-NEXT: store i8 [[ARG]], ptr [[DEST_6]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [8 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ %arr.5 = getelementptr inbounds nuw i8, ptr %arr, i64 5
+ %arr.6 = getelementptr inbounds nuw i8, ptr %arr, i64 6
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ store i8 5, ptr %arr.5, align 1
+ store i8 6, ptr %arr.6, align 1
+ %move = load i32, ptr %arr.3, align 1
+ store i32 %move, ptr %arr, align 1
+ store i8 %arg, ptr %arr.6, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %dest.5 = getelementptr inbounds nuw i8, ptr %dest, i64 5
+ %dest.6 = getelementptr inbounds nuw i8, ptr %dest, i64 6
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ %val.5 = load i8, ptr %arr.5, align 1
+ %val.6 = load i8, ptr %arr.6, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ store i8 %val.5, ptr %dest.5, align 1
+ store i8 %val.6, ptr %dest.6, align 1
+ ret void
+}
+
+; The size of the overlap area equals the size of the non-overlap area
+define void @move_up_equal_overlap_nooverlap(i16 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_up_equal_overlap_nooverlap(
+; CHECK-SAME: i16 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 2
+; CHECK-NEXT: store i16 [[ARG]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i16 0, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i16 1, ptr [[DEST_2]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [3 x i16], align 1
+ %arr.1 = getelementptr inbounds nuw i16, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i16, ptr %arr, i64 2
+ store i16 0, ptr %arr, align 1
+ store i16 1, ptr %arr.1, align 1
+ store i16 2, ptr %arr.2, align 1
+ %move = load i32, ptr %arr, align 1
+ store i32 %move, ptr %arr.1, align 1
+ store i16 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i16, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i16, ptr %dest, i64 2
+ %val.0 = load i16, ptr %arr, align 1
+ %val.1 = load i16, ptr %arr.1, align 1
+ %val.2 = load i16, ptr %arr.2, align 1
+ store i16 %val.0, ptr %dest, align 1
+ store i16 %val.1, ptr %dest.1, align 1
+ store i16 %val.2, ptr %dest.2, align 1
+ ret void
+}
+
+define void @move_down_equal_overlap_nooverlap(i16 %arg, ptr %dest) {
+; CHECK-LABEL: define void @move_down_equal_overlap_nooverlap(
+; CHECK-SAME: i16 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i16, ptr [[DEST]], i64 2
+; CHECK-NEXT: store i16 1, ptr [[DEST]], align 1
+; CHECK-NEXT: store i16 2, ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i16 [[ARG]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [3 x i16], align 1
+ %arr.1 = getelementptr inbounds nuw i16, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i16, ptr %arr, i64 2
+ store i16 0, ptr %arr, align 1
+ store i16 1, ptr %arr.1, align 1
+ store i16 2, ptr %arr.2, align 1
+ %move = load i32, ptr %arr.1, align 1
+ store i32 %move, ptr %arr, align 1
+ store i16 %arg, ptr %arr.2, align 1
+ %dest.1 = getelementptr inbounds nuw i16, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i16, ptr %dest, i64 2
+ %val.0 = load i16, ptr %arr, align 1
+ %val.1 = load i16, ptr %arr.1, align 1
+ %val.2 = load i16, ptr %arr.2, align 1
+ store i16 %val.0, ptr %dest, align 1
+ store i16 %val.1, ptr %dest.1, align 1
+ store i16 %val.2, ptr %dest.2, align 1
+ ret void
+}
+
+define void @move_inside_loop(ptr %src, ptr %dest, i64 %n) {
+; CHECK-LABEL: define void @move_inside_loop(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[ARR_SROA_15_0:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_13_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_13_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_11_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_11_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_9_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_9_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_7_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_7_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_5_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_5_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_3_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_3_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[ARR_SROA_0_0:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ARR_SROA_0_0]] = phi i8 [ 0, %[[ENTRY]] ], [ [[VAL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP_DEST:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 [[IV]]
+; CHECK-NEXT: [[VAL]] = load i8, ptr [[GEP_DEST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[END:.*]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %arr = alloca [9 x i8], align 1
+ store i64 0, ptr %arr, align 1
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %move = load i64, ptr %arr, align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ store i64 %move, ptr %arr.1, align 1
+ %gep.dest = getelementptr inbounds nuw i8, ptr %dest, i64 %iv
+ %val = load i8, ptr %gep.dest, align 1
+ store i8 %val, ptr %arr, align 1
+ %iv.next = add i64 %iv, 1
+ %cmp = icmp eq i64 %iv.next, %n
+ br i1 %cmp, label %loop, label %end
+
+end:
+ ret void
+}
+
+; FIXME: SROA can't split memmove calls, so we can't optimize this
+define void @memmove_up(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @memmove_up(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT: store i8 0, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX1:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: store i8 1, ptr [[ARR_1_ARR_1_SROA_IDX1]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: store i8 2, ptr [[ARR_2_ARR_2_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: store i8 3, ptr [[ARR_3_ARR_3_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 4, ptr [[ARR_4_ARR_4_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i32(ptr align 1 [[ARR_1_ARR_1_SROA_IDX]], ptr align 1 [[ARR]], i32 4, i1 false)
+; CHECK-NEXT: store i8 [[ARG]], ptr [[ARR]], align 1
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[ARR_0_VAL_0:%.*]] = load i8, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX2:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: [[ARR_1_VAL_1:%.*]] = load i8, ptr [[ARR_1_ARR_1_SROA_IDX2]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX3:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: [[ARR_2_VAL_2:%.*]] = load i8, ptr [[ARR_2_ARR_2_SROA_IDX3]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX4:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: [[ARR_3_VAL_3:%.*]] = load i8, ptr [[ARR_3_ARR_3_SROA_IDX4]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX5:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: [[ARR_4_VAL_4:%.*]] = load i8, ptr [[ARR_4_ARR_4_SROA_IDX5]], align 1
+; CHECK-NEXT: store i8 [[ARR_0_VAL_0]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_1_VAL_1]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_2_VAL_2]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 [[ARR_3_VAL_3]], ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_4_VAL_4]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ call void @llvm.memmove.p0.p0.i32(ptr %arr.1, ptr %arr, i32 4, i1 false)
+ store i8 %arg, ptr %arr, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
+
+define void @memmove_down(i8 %arg, ptr %dest) {
+; CHECK-LABEL: define void @memmove_down(
+; CHECK-SAME: i8 [[ARG:%.*]], ptr [[DEST:%.*]]) {
+; CHECK-NEXT: [[ARR:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT: store i8 0, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX1:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: store i8 1, ptr [[ARR_1_ARR_1_SROA_IDX1]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: store i8 2, ptr [[ARR_2_ARR_2_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: store i8 3, ptr [[ARR_3_ARR_3_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 4, ptr [[ARR_4_ARR_4_SROA_IDX]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i32(ptr align 1 [[ARR]], ptr align 1 [[ARR_1_ARR_1_SROA_IDX]], i32 4, i1 false)
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX5:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: store i8 [[ARG]], ptr [[ARR_4_ARR_4_SROA_IDX5]], align 1
+; CHECK-NEXT: [[DEST_1:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 1
+; CHECK-NEXT: [[DEST_2:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 2
+; CHECK-NEXT: [[DEST_3:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 3
+; CHECK-NEXT: [[DEST_4:%.*]] = getelementptr inbounds nuw i8, ptr [[DEST]], i64 4
+; CHECK-NEXT: [[ARR_0_VAL_0:%.*]] = load i8, ptr [[ARR]], align 1
+; CHECK-NEXT: [[ARR_1_ARR_1_SROA_IDX2:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 1
+; CHECK-NEXT: [[ARR_1_VAL_1:%.*]] = load i8, ptr [[ARR_1_ARR_1_SROA_IDX2]], align 1
+; CHECK-NEXT: [[ARR_2_ARR_2_SROA_IDX3:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 2
+; CHECK-NEXT: [[ARR_2_VAL_2:%.*]] = load i8, ptr [[ARR_2_ARR_2_SROA_IDX3]], align 1
+; CHECK-NEXT: [[ARR_3_ARR_3_SROA_IDX4:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 3
+; CHECK-NEXT: [[ARR_3_VAL_3:%.*]] = load i8, ptr [[ARR_3_ARR_3_SROA_IDX4]], align 1
+; CHECK-NEXT: [[ARR_4_ARR_4_SROA_IDX6:%.*]] = getelementptr inbounds i8, ptr [[ARR]], i64 4
+; CHECK-NEXT: [[ARR_4_VAL_4:%.*]] = load i8, ptr [[ARR_4_ARR_4_SROA_IDX6]], align 1
+; CHECK-NEXT: store i8 [[ARR_0_VAL_0]], ptr [[DEST]], align 1
+; CHECK-NEXT: store i8 [[ARR_1_VAL_1]], ptr [[DEST_1]], align 1
+; CHECK-NEXT: store i8 [[ARR_2_VAL_2]], ptr [[DEST_2]], align 1
+; CHECK-NEXT: store i8 [[ARR_3_VAL_3]], ptr [[DEST_3]], align 1
+; CHECK-NEXT: store i8 [[ARR_4_VAL_4]], ptr [[DEST_4]], align 1
+; CHECK-NEXT: ret void
+;
+ %arr = alloca [5 x i8], align 1
+ %arr.1 = getelementptr inbounds nuw i8, ptr %arr, i64 1
+ %arr.2 = getelementptr inbounds nuw i8, ptr %arr, i64 2
+ %arr.3 = getelementptr inbounds nuw i8, ptr %arr, i64 3
+ %arr.4 = getelementptr inbounds nuw i8, ptr %arr, i64 4
+ store i8 0, ptr %arr, align 1
+ store i8 1, ptr %arr.1, align 1
+ store i8 2, ptr %arr.2, align 1
+ store i8 3, ptr %arr.3, align 1
+ store i8 4, ptr %arr.4, align 1
+ call void @llvm.memmove.p0.p0.i32(ptr %arr, ptr %arr.1, i32 4, i1 false)
+ store i8 %arg, ptr %arr.4, align 1
+ %dest.1 = getelementptr inbounds nuw i8, ptr %dest, i64 1
+ %dest.2 = getelementptr inbounds nuw i8, ptr %dest, i64 2
+ %dest.3 = getelementptr inbounds nuw i8, ptr %dest, i64 3
+ %dest.4 = getelementptr inbounds nuw i8, ptr %dest, i64 4
+ %val.0 = load i8, ptr %arr, align 1
+ %val.1 = load i8, ptr %arr.1, align 1
+ %val.2 = load i8, ptr %arr.2, align 1
+ %val.3 = load i8, ptr %arr.3, align 1
+ %val.4 = load i8, ptr %arr.4, align 1
+ store i8 %val.0, ptr %dest, align 1
+ store i8 %val.1, ptr %dest.1, align 1
+ store i8 %val.2, ptr %dest.2, align 1
+ store i8 %val.3, ptr %dest.3, align 1
+ store i8 %val.4, ptr %dest.4, align 1
+ ret void
+}
More information about the llvm-commits
mailing list