[llvm] [LoopPeel] Peel last iteration to enable load widening (PR #173420)
Guy David via llvm-commits
llvm-commits at lists.llvm.org
Sun May 10 04:04:20 PDT 2026
https://github.com/guy-david updated https://github.com/llvm/llvm-project/pull/173420
>From 719b58cc859b50f9c82f0ad53fbbb61cc78f1736 Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Wed, 10 Dec 2025 23:19:26 +0200
Subject: [PATCH 1/2] [LoopPeel] Peel last iteration to enable load widening
In loops that contain multiple consecutive small loads (e.g., 3 bytes
loading i8's), peeling the last iteration makes it safe to read beyond
the accessed region, enabling the use of a wider load (e.g., i32) for
all other N-1 iterations.
Patterns such as:
```
%a = load i8, ptr %p
%b = load i8, ptr %p+1
%c = load i8, ptr %p+2
...
%p.next = getelementptr i8, ptr %p, 3
```
Can be transformed to:
```
%wide = load i32, ptr %p ; Read 4 bytes
%a = trunc
%b = lshr + trunc
%c = lshr + trunc
...
```
This acts as a fallback strategy when vectorization fails and has
significant performance uplifts, most notably in image processing
where processing an RGB buffer is common.
---
llvm/include/llvm/Transforms/Utils/LoopPeel.h | 8 +-
.../llvm/Transforms/Utils/UnrollLoop.h | 3 +-
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 32 +-
llvm/lib/Transforms/Utils/LoopPeel.cpp | 231 ++++++-
.../peel-last-iteration-load-widening-be.ll | 104 +++
.../peel-last-iteration-load-widening.ll | 616 ++++++++++++++++++
6 files changed, 978 insertions(+), 16 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
diff --git a/llvm/include/llvm/Transforms/Utils/LoopPeel.h b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
index 5502da0ce0d09..205509debbe86 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopPeel.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
@@ -46,7 +46,13 @@ void computePeelCount(Loop *L, unsigned LoopSize,
unsigned TripCount, DominatorTree &DT,
ScalarEvolution &SE, const TargetTransformInfo &TTI,
AssumptionCache *AC = nullptr,
- unsigned Threshold = UINT_MAX);
+ unsigned Threshold = UINT_MAX,
+ bool AllowLoadWideningPeel = true);
+
+/// Combine load instructions in a loop into a wider one, given that we peeled
+/// the last iteration and can assume the bytes are dereferenceable.
+bool widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE, const DataLayout &DL,
+ const TargetTransformInfo &TTI, DominatorTree &DT);
} // end namespace llvm
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index 2ab4bac87901c..d7a7fdf0e8c89 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -187,7 +187,8 @@ computeUnrollCount(Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT,
unsigned MaxTripCount, bool MaxOrZero, unsigned TripMultiple,
const UnrollCostEstimator &UCE,
TargetTransformInfo::UnrollingPreferences &UP,
- TargetTransformInfo::PeelingPreferences &PP);
+ TargetTransformInfo::PeelingPreferences &PP,
+ bool AllowLoadWideningPeel = true);
LLVM_ABI std::optional<RecurrenceDescriptor>
canParallelizeReductionWhenUnrolling(PHINode &Phi, Loop *L,
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 0c5e3b5039309..77591b98c335a 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1033,17 +1033,15 @@ shouldPartialUnroll(const unsigned LoopSize, const unsigned TripCount,
// FIXME: This function is used by LoopUnroll and LoopUnrollAndJam, but consumes
// many LoopUnroll-specific options. The shared functionality should be
// refactored into it own function.
-void llvm::computeUnrollCount(Loop *L, const TargetTransformInfo &TTI,
- DominatorTree &DT, LoopInfo *LI,
- AssumptionCache *AC, ScalarEvolution &SE,
- const SmallPtrSetImpl<const Value *> &EphValues,
- OptimizationRemarkEmitter *ORE,
- const unsigned TripCount,
- const unsigned MaxTripCount, const bool MaxOrZero,
- const unsigned TripMultiple,
- const UnrollCostEstimator &UCE,
- TargetTransformInfo::UnrollingPreferences &UP,
- TargetTransformInfo::PeelingPreferences &PP) {
+void llvm::computeUnrollCount(
+ Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI,
+ AssumptionCache *AC, ScalarEvolution &SE,
+ const SmallPtrSetImpl<const Value *> &EphValues,
+ OptimizationRemarkEmitter *ORE, const unsigned TripCount,
+ const unsigned MaxTripCount, const bool MaxOrZero,
+ const unsigned TripMultiple, const UnrollCostEstimator &UCE,
+ TargetTransformInfo::UnrollingPreferences &UP,
+ TargetTransformInfo::PeelingPreferences &PP, bool AllowLoadWideningPeel) {
unsigned LoopSize = UCE.getRolledLoopSize();
@@ -1149,7 +1147,8 @@ void llvm::computeUnrollCount(Loop *L, const TargetTransformInfo &TTI,
// 5th priority is loop peeling.
LLVM_DEBUG(dbgs().indent(1) << "Trying loop peeling...\n");
- computePeelCount(L, LoopSize, PP, TripCount, DT, SE, TTI, AC, UP.Threshold);
+ computePeelCount(L, LoopSize, PP, TripCount, DT, SE, TTI, AC, UP.Threshold,
+ AllowLoadWideningPeel);
if (PP.PeelCount) {
LLVM_DEBUG(dbgs().indent(2)
<< "Peeling with count: " << PP.PeelCount << ".\n");
@@ -1417,8 +1416,10 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
// computeUnrollCount() decides whether it is beneficial to use upper bound to
// fully unroll the loop.
+ bool AllowLoadWideningPeel = !OnlyFullUnroll;
computeUnrollCount(L, TTI, DT, LI, &AC, SE, EphValues, &ORE, TripCount,
- MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP);
+ MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP,
+ AllowLoadWideningPeel);
if (!UP.Count) {
LLVM_DEBUG(dbgs().indent(1)
<< "Not unrolling: no viable strategy found.\n");
@@ -1450,6 +1451,11 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
VMap);
simplifyLoopAfterUnroll(L, true, LI, &SE, &DT, &AC, &TTI, L->getBlocks(),
nullptr);
+ // Widen consecutive loads after last-iteration peeling
+ if (PP.PeelLast) {
+ const DataLayout &DL = L->getHeader()->getDataLayout();
+ widenLoadsAfterPeel(*L, SE, DL, TTI, DT);
+ }
// If the loop was peeled, we already "used up" the profile information
// we had, so we don't want to unroll or peel again.
if (PP.PeelProfiledIterations)
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index 3ae2a78190082..8c60023cdf3c3 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -88,6 +88,11 @@ static cl::opt<bool> EnablePeelingForIV(
"enable-peeling-for-iv", cl::init(false), cl::Hidden,
cl::desc("Enable peeling to convert Phi nodes into IVs"));
+static cl::opt<bool> EnablePeelForLoadWidening(
+ "enable-peel-for-load-widening", cl::init(true), cl::Hidden,
+ cl::desc(
+ "Enable peeling last iteration to enable consecutive load widening"));
+
static const char *PeeledCountMetaData = "llvm.loop.peeled.count";
extern cl::opt<bool> ProfcheckDisableMetadataFixes;
@@ -748,13 +753,148 @@ static bool violatesLegacyMultiExitLoopCheck(Loop *L) {
});
}
+namespace {
+// Represents a group of loads in a loop that can be combined into a wider one.
+struct LoadGroup {
+ // Base object being read.
+ Value *BasePtr;
+ // First load instruction in the program order.
+ LoadInst *FirstLoad;
+ // Pairs of (load instruction, offset from base).
+ SmallVector<std::pair<LoadInst *, APInt>, 4> Loads;
+ // An applicable wider integer type to load as.
+ Type *WideType;
+};
+
+// Helper to compute load group span and validate for widening.
+static std::optional<LoadGroup> tryFormLoadGroupForWidening(
+ Value *Base, SmallVectorImpl<std::pair<LoadInst *, APInt>> &Loads, Loop &L,
+ ScalarEvolution &SE, const DataLayout &DL, const TargetTransformInfo &TTI) {
+ // Find the span of the loaded data.
+ int64_t Left = INT64_MAX;
+ int64_t Right = INT64_MIN;
+ for (const auto &[Load, Offset] : Loads) {
+ Left = std::min(Left, Offset.getSExtValue());
+ Right = std::max(
+ Right, Offset.getSExtValue() +
+ static_cast<int64_t>(DL.getTypeStoreSize(Load->getType())));
+ }
+ assert((Left < Right) && "Invalid load group span");
+ uint64_t TotalBytes = Right - Left;
+ uint64_t TotalBits = TotalBytes * 8;
+ // Powers of two are already natural for most targets.
+ if (isPowerOf2_64(TotalBits))
+ return std::nullopt;
+ Type *WideType =
+ DL.getSmallestLegalIntType(L.getHeader()->getContext(), TotalBits);
+ if (!WideType)
+ return std::nullopt;
+ unsigned WideBits = WideType->getIntegerBitWidth();
+ // Total size is already natural for the target.
+ if (WideBits == TotalBits)
+ return std::nullopt;
+ // Peeling doubles dereferenceable bytes, ensure wide type fits.
+ if (WideBits > TotalBits * 2)
+ return std::nullopt;
+ // Check alignment is unconstrained.
+ unsigned Fast = 0;
+ if (!TTI.allowsMisalignedMemoryAccesses(L.getHeader()->getContext(), WideBits,
+ DL.getDefaultGlobalsAddressSpace(),
+ Align(1), &Fast) &&
+ !Fast)
+ return std::nullopt;
+ // Validate pointer stride across iterations.
+ const SCEV *PtrSCEV = SE.getSCEV(Base);
+ const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrSCEV);
+ if (!AR || AR->getLoop() != &L)
+ return std::nullopt;
+ const SCEV *Step = AR->getStepRecurrence(SE);
+ auto *ConstStep = dyn_cast<SCEVConstant>(Step);
+ if (!ConstStep)
+ return std::nullopt;
+ int64_t StepVal = ConstStep->getValue()->getSExtValue();
+ if (StepVal != static_cast<int64_t>(TotalBytes))
+ return std::nullopt;
+
+ LoadInst *FirstLoad = Loads[0].first;
+ llvm::sort(Loads, [](const auto &A, const auto &B) {
+ return A.second.slt(B.second);
+ });
+ return LoadGroup{Base, FirstLoad, std::move(Loads), WideType};
+}
+
+// Find groups of consecutive loads in a basic block for peeling purposes
+static SmallVector<LoadGroup>
+findLoadGroupsForWidening(BasicBlock *BB, Loop &L, ScalarEvolution &SE,
+ const DataLayout &DL,
+ const TargetTransformInfo &TTI) {
+ SmallVector<LoadGroup> Groups;
+ // Mapping from base pointer to loads instructions and their offset from the
+ // base.
+ DenseMap<Value *, SmallVector<std::pair<LoadInst *, APInt>>> LoadsByBase;
+
+ auto ProcessCollectedLoads = [&]() {
+ for (auto &[Base, Loads] : LoadsByBase) {
+ if (Loads.size() <= 1)
+ continue;
+ if (auto Group = tryFormLoadGroupForWidening(Base, Loads, L, SE, DL, TTI))
+ Groups.emplace_back(*std::move(Group));
+ }
+ LoadsByBase.clear();
+ };
+
+ for (Instruction &I : *BB) {
+ if (auto *Load = dyn_cast<LoadInst>(&I)) {
+ if (Load->isVolatile() || Load->isAtomic() ||
+ !Load->getType()->isIntegerTy() ||
+ Load->getPointerAddressSpace() != DL.getDefaultGlobalsAddressSpace())
+ continue;
+ Value *Ptr = Load->getPointerOperand();
+ APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
+ Value *ActualBase = Ptr->stripAndAccumulateConstantOffsets(
+ DL, Offset, /*AllowNonInbounds=*/false);
+ LoadsByBase[ActualBase].emplace_back(Load, Offset);
+ } else if (I.mayHaveSideEffects())
+ ProcessCollectedLoads();
+ }
+ ProcessCollectedLoads();
+ return Groups;
+}
+
+// Returns 1 if peeling the last iteration would enable widening load groups to
+// natural sizes. Returns 0 otherwise.
+static unsigned peelLastForLoadWidening(Loop &L, ScalarEvolution &SE,
+ const DataLayout &DL,
+ const TargetTransformInfo &TTI,
+ DominatorTree &DT) {
+ if (!EnablePeelForLoadWidening)
+ return 0;
+ if (!L.isInnermost())
+ return 0;
+ if (!canPeelLastIteration(L, SE))
+ return 0;
+ BasicBlock *Latch = L.getLoopLatch();
+ if (!Latch)
+ return 0;
+ for (BasicBlock *BB : L.blocks()) {
+ // Look for consecutive loads in blocks that execute every iteration.
+ if (!DT.dominates(BB, Latch))
+ continue;
+ auto Groups = findLoadGroupsForWidening(BB, L, SE, DL, TTI);
+ if (!Groups.empty())
+ return 1;
+ }
+ return 0;
+}
+} // anonymous namespace
// Return the number of iterations we want to peel off.
void llvm::computePeelCount(Loop *L, unsigned LoopSize,
TargetTransformInfo::PeelingPreferences &PP,
unsigned TripCount, DominatorTree &DT,
ScalarEvolution &SE, const TargetTransformInfo &TTI,
- AssumptionCache *AC, unsigned Threshold) {
+ AssumptionCache *AC, unsigned Threshold,
+ bool AllowLoadWideningPeel) {
assert(LoopSize > 0 && "Zero loop size is not allowed!");
// Save the PP.PeelCount value set by the target in
// TTI.getPeelingPreferences or by the flag -unroll-peel-count.
@@ -854,6 +994,25 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
}
+ // Check for consecutive load widening opportunity.
+ // Skip this when running before vectorization (AllowLoadWideningPeel=false)
+ // to avoid peeling loops that could have been vectorized instead.
+ if (PP.PeelCount == 0 && AllowLoadWideningPeel) {
+ const DataLayout &DL = L->getHeader()->getDataLayout();
+ unsigned LoadWideningPeel = peelLastForLoadWidening(*L, SE, DL, TTI, DT);
+ if (LoadWideningPeel > 0) {
+ if (LoadWideningPeel + AlreadyPeeled <= UnrollPeelMaxCount) {
+ LLVM_DEBUG(
+ dbgs() << "Peel last " << LoadWideningPeel
+ << " iteration(s) to enable consecutive load widening.\n");
+ PP.PeelCount = LoadWideningPeel;
+ PP.PeelProfiledIterations = false;
+ PP.PeelLast = true;
+ return;
+ }
+ }
+ }
+
// Bail if we know the statically calculated trip count.
// In this case we rather prefer partial unrolling.
if (TripCount)
@@ -894,6 +1053,76 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
}
+bool llvm::widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE,
+ const DataLayout &DL,
+ const TargetTransformInfo &TTI,
+ DominatorTree &DT) {
+ BasicBlock *Latch = L.getLoopLatch();
+ if (!Latch)
+ return false;
+
+ bool Changed = false;
+
+ for (BasicBlock *BB : L.blocks()) {
+ if (!DT.dominates(BB, Latch))
+ continue;
+ SmallVector<LoadGroup> Groups =
+ findLoadGroupsForWidening(BB, L, SE, DL, TTI);
+ for (const LoadGroup &Group : Groups) {
+ LoadInst *InsertPoint = Group.FirstLoad;
+ IRBuilder<> Builder(InsertPoint);
+ Value *BasePtr = Group.BasePtr;
+ int64_t FirstOffset = Group.Loads[0].second.getSExtValue();
+ // If the first load doesn't start at offset 0, we need to adjust.
+ if (FirstOffset != 0) {
+ Value *OrigPtr = Group.BasePtr;
+ BasePtr = Builder.CreatePtrAdd(
+ OrigPtr,
+ ConstantInt::get(
+ Builder.getIndexTy(DL, DL.getDefaultGlobalsAddressSpace()),
+ FirstOffset));
+ }
+ // Merge AA metadata from all loads.
+ AAMDNodes AATags = InsertPoint->getAAMetadata();
+ for (const auto &[Load, Offset] : Group.Loads) {
+ if (Load != InsertPoint)
+ AATags = AATags.concat(Load->getAAMetadata());
+ }
+ // Create the wider load.
+ LoadInst *WideLoad = Builder.CreateLoad(Group.WideType, BasePtr);
+ unsigned SizeInBits = WideLoad->getType()->getScalarSizeInBits();
+ if (AATags)
+ WideLoad->setAAMetadata(AATags);
+ // For each original load, extract the corresponding bytes.
+ for (const auto &[Load, Offset] : Group.Loads) {
+ unsigned LoadBytes = DL.getTypeStoreSize(Load->getType());
+ Value *Extracted = WideLoad;
+ unsigned BitOffset =
+ DL.isBigEndian()
+ ? SizeInBits -
+ (Offset.getSExtValue() - FirstOffset + LoadBytes) * 8
+ : (Offset.getSExtValue() - FirstOffset) * 8;
+ if (BitOffset != 0)
+ Extracted = Builder.CreateLShr(
+ Extracted, ConstantInt::get(WideLoad->getType(), BitOffset));
+ unsigned TargetBits = Load->getType()->getScalarSizeInBits();
+ if (TargetBits < SizeInBits)
+ Extracted = Builder.CreateTrunc(Extracted, Load->getType());
+ Load->replaceAllUsesWith(Extracted);
+ }
+ // Delete the original loads.
+ for (auto &[Load, Offset] : Group.Loads)
+ Load->eraseFromParent();
+
+ LLVM_DEBUG(dbgs() << "Widened " << Group.Loads.size()
+ << " loads into a single " << *WideLoad << "\n");
+ Changed = true;
+ }
+ }
+
+ return Changed;
+}
+
/// Clones the body of the loop L, putting it between \p InsertTop and \p
/// InsertBot.
/// \param IterNumber The serial number of the iteration currently being
diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
new file mode 100644
index 0000000000000..b4599a19b47ce
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64_be -passes=loop-unroll -S %s | FileCheck %s
+
+; Test that loop peeling for load widening works correctly on big-endian targets.
+; The byte extraction shifts should be different from little-endian.
+
+target datalayout = "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; Test 3-byte consecutive loads on big-endian, should peel and use i32 load.
+; For big-endian with i32 load:
+; byte 0 (lowest address) is in bits [31:24]
+; byte 1 is in bits [23:16]
+; byte 2 is in bits [15:8]
+; byte 3 (unused) is in bits [7:0]
+define void @test_3_consecutive_loads_be(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_loads_be(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 24
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT: [[TMP5:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP4]], [[TMP6]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP8]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP9]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP10:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP11]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP10]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP10]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ ; Use the loaded values to prevent DCE
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+;.
diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
new file mode 100644
index 0000000000000..9b6368446757d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
@@ -0,0 +1,616 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -passes=loop-unroll -S %s | FileCheck %s
+
+; Test that loop peeling is applied for consecutive load widening opportunities
+; and that appropriate dereferenceable assumptions are added.
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; Test 3-byte consecutive loads, should peel last iteration and use an i32 load.
+define void @test_3_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP10]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP9]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ ; Use the loaded values to prevent DCE
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test 3-byte consecutive loads, should peel last iteration and use an i64 load.
+define void @test_5_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_5_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP14:%.*]] = lshr i64 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP14]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP2]], 24
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i64 [[TMP2]], 32
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP13]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[TMP9]]
+; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[TMP11]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM4]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP12]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1
+; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
+; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM4_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 5
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+
+ ; Use the loaded values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum4, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 5
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test 8-byte consecutive loads does not peel the loop, already natural size.
+define void @test_8_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_8_consecutive_loads_no_peel(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[D:%.*]] = load i8, ptr [[P3]], align 1
+; CHECK-NEXT: [[E:%.*]] = load i8, ptr [[P4]], align 1
+; CHECK-NEXT: [[F:%.*]] = load i8, ptr [[P5]], align 1
+; CHECK-NEXT: [[G:%.*]] = load i8, ptr [[P6]], align 1
+; CHECK-NEXT: [[H:%.*]] = load i8, ptr [[P7]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[D]]
+; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[E]]
+; CHECK-NEXT: [[SUM5:%.*]] = add i8 [[SUM4]], [[F]]
+; CHECK-NEXT: [[SUM6:%.*]] = add i8 [[SUM5]], [[G]]
+; CHECK-NEXT: [[SUM7:%.*]] = add i8 [[SUM6]], [[H]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM7]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 8
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+ %p5 = getelementptr inbounds i8, ptr %p, i64 5
+ %p6 = getelementptr inbounds i8, ptr %p, i64 6
+ %p7 = getelementptr inbounds i8, ptr %p, i64 7
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+ %f = load i8, ptr %p5, align 1
+ %g = load i8, ptr %p6, align 1
+ %h = load i8, ptr %p7, align 1
+
+ ; Use all values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %sum5 = add i8 %sum4, %f
+ %sum6 = add i8 %sum5, %g
+ %sum7 = add i8 %sum6, %h
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum7, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 8
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_non_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_non_consecutive_loads_no_peel(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 2 ; Skip offset 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 3
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 4
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test 9-byte consecutive loads, should not peel because i128 isn't a legal integer type.
+define void @test_9_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_9_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[P5_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 5
+; CHECK-NEXT: [[P6_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 6
+; CHECK-NEXT: [[P7_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 7
+; CHECK-NEXT: [[P8_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 8
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1
+; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1
+; CHECK-NEXT: [[F_PEEL:%.*]] = load i8, ptr [[P5_PEEL]], align 1
+; CHECK-NEXT: [[G_PEEL:%.*]] = load i8, ptr [[P6_PEEL]], align 1
+; CHECK-NEXT: [[H_PEEL:%.*]] = load i8, ptr [[P7_PEEL]], align 1
+; CHECK-NEXT: [[I_VAL_PEEL:%.*]] = load i8, ptr [[P8_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
+; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
+; CHECK-NEXT: [[SUM5_PEEL:%.*]] = add i8 [[SUM4_PEEL]], [[F_PEEL]]
+; CHECK-NEXT: [[SUM6_PEEL:%.*]] = add i8 [[SUM5_PEEL]], [[G_PEEL]]
+; CHECK-NEXT: [[SUM7_PEEL:%.*]] = add i8 [[SUM6_PEEL]], [[H_PEEL]]
+; CHECK-NEXT: [[SUM9_PEEL:%.*]] = add i8 [[SUM7_PEEL]], [[I_VAL_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM9_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP4]], i64 9
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+ %p5 = getelementptr inbounds i8, ptr %p, i64 5
+ %p6 = getelementptr inbounds i8, ptr %p, i64 6
+ %p7 = getelementptr inbounds i8, ptr %p, i64 7
+ %p8 = getelementptr inbounds i8, ptr %p, i64 8
+
+ %a = load i8, ptr %p, align 16
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+ %f = load i8, ptr %p5, align 1
+ %g = load i8, ptr %p6, align 1
+ %h = load i8, ptr %p7, align 1
+ %i.val = load i8, ptr %p8, align 1
+
+ ; Use all values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %sum5 = add i8 %sum4, %f
+ %sum6 = add i8 %sum5, %g
+ %sum7 = add i8 %sum6, %h
+ %sum8 = add i8 %sum7, %i.val
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum8, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 9
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_intervening_store(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_intervening_store(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: store i8 42, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ store i8 42, ptr %p1, align 1 ; Intervening store prevents optimization
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test 3 consecutive i16 loads (6 bytes), should peel and widen to i64.
+define void @test_3_consecutive_i16_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_i16_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i16
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP2]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i16
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 32
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i16
+; CHECK-NEXT: [[SUM1:%.*]] = add i16 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i16 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i16 [[SUM2]], ptr [[DST_I]], align 2
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 2
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i16, ptr [[P2_PEEL]], align 2
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i16 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i16 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT: store i16 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 2
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 6
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i16, ptr %p, i64 1
+ %p2 = getelementptr inbounds i16, ptr %p, i64 2
+
+ %a = load i16, ptr %p, align 2
+ %b = load i16, ptr %p1, align 2
+ %c = load i16, ptr %p2, align 2
+
+ %sum1 = add i16 %a, %b
+ %sum2 = add i16 %sum1, %c
+ %dst.i = getelementptr inbounds i16, ptr %dst, i32 %i
+ store i16 %sum2, ptr %dst.i, align 2
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 6
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Two i8 loads (or one i16) should not peel because 16-bits is already a natural size.
+define void @test_2_consecutive_loads_no_peel_legal_type(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_2_consecutive_loads_no_peel_legal_type(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 2
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM1]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+
+ %a = load i8, ptr %p, align 2
+ %b = load i8, ptr %p1, align 1
+
+ %sum1 = add i8 %a, %b
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum1, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 2
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+;.
>From 6b9b1c2054fe29d24abe122c49f7fe7f31f975dc Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Thu, 15 Jan 2026 11:56:58 +0200
Subject: [PATCH 2/2] Address comments 1
---
llvm/include/llvm/IR/DataLayout.h | 8 +
llvm/include/llvm/Transforms/Utils/LoopPeel.h | 2 +-
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 8 +-
llvm/lib/Transforms/Utils/LoopPeel.cpp | 194 +-
.../Transforms/Vectorize/VectorCombine.cpp | 6 +-
...l-last-iteration-load-widening-disabled.ll | 56 +
.../peel-last-iteration-load-widening.ll | 1694 +++++++++++++++++
.../peel-last-iteration-load-widening-be.ll | 67 +
.../peel-last-iteration-load-widening-be.ll | 104 -
.../peel-last-iteration-load-widening.ll | 616 ------
10 files changed, 1947 insertions(+), 808 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-disabled.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/PowerPC/peel-last-iteration-load-widening-be.ll
delete mode 100644 llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
delete mode 100644 llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 934c838782417..35e8f10f10984 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -220,6 +220,14 @@ class DataLayout {
/// Whether vectors are element aligned, rather than naturally aligned.
bool vectorsAreElementAligned() const { return VectorsAreElementAligned; }
+ /// Compute the bit offset to extract an element from a packed integer,
+ /// accounting for endianness.
+ uint64_t getElementBitOffset(uint64_t BitPosition, uint64_t ElementBits,
+ uint64_t ContainerBits) const {
+ return isBigEndian() ? ContainerBits - ElementBits - BitPosition
+ : BitPosition;
+ }
+
/// Returns the string representation of the DataLayout.
///
/// This representation is in the same format accepted by the string
diff --git a/llvm/include/llvm/Transforms/Utils/LoopPeel.h b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
index 205509debbe86..2152a6def3523 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopPeel.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
@@ -51,7 +51,7 @@ void computePeelCount(Loop *L, unsigned LoopSize,
/// Combine load instructions in a loop into a wider one, given that we peeled
/// the last iteration and can assume the bytes are dereferenceable.
-bool widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE, const DataLayout &DL,
+bool widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE,
const TargetTransformInfo &TTI, DominatorTree &DT);
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 77591b98c335a..184282247b55b 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1449,13 +1449,11 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
ValueToValueMapTy VMap;
peelLoop(L, PP.PeelCount, PP.PeelLast, LI, &SE, DT, &AC, PreserveLCSSA,
VMap);
+ // Widen consecutive loads after last-iteration peeling
+ if (PP.PeelLast)
+ widenLoadsAfterPeel(*L, SE, TTI, DT);
simplifyLoopAfterUnroll(L, true, LI, &SE, &DT, &AC, &TTI, L->getBlocks(),
nullptr);
- // Widen consecutive loads after last-iteration peeling
- if (PP.PeelLast) {
- const DataLayout &DL = L->getHeader()->getDataLayout();
- widenLoadsAfterPeel(*L, SE, DL, TTI, DT);
- }
// If the loop was peeled, we already "used up" the profile information
// we had, so we don't want to unroll or peel again.
if (PP.PeelProfiledIterations)
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index 8c60023cdf3c3..4ecaa9a464860 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -45,6 +45,7 @@
#include <algorithm>
#include <cassert>
#include <cstdint>
+#include <limits>
#include <optional>
using namespace llvm;
@@ -511,16 +512,12 @@ bool llvm::canPeelLastIteration(const Loop &L, ScalarEvolution &SE) {
m_scev_AffineAddRec(m_SCEV(), m_scev_One(), m_SpecificLoop(&L)));
}
-/// Returns true if the last iteration can be peeled off and the condition (Pred
-/// LeftAR, RightSCEV) is known at the last iteration and the inverse condition
-/// is known at the second-to-last.
+/// Returns true if the condition (Pred LeftAR, RightSCEV) is known at the last
+/// iteration and the inverse condition is known at the second-to-last.
static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
const SCEVAddRecExpr *LeftAR,
const SCEV *RightSCEV, ScalarEvolution &SE,
const TargetTransformInfo &TTI) {
- if (!canPeelLastIteration(L, SE))
- return false;
-
const SCEV *BTC = SE.getBackedgeTakenCount(&L);
SCEVExpander Expander(SE, "loop-peel");
if (!SE.isKnownNonZero(BTC) &&
@@ -553,7 +550,7 @@ static bool shouldPeelLastIteration(Loop &L, CmpPredicate Pred,
// }
static std::pair<unsigned, unsigned>
countToEliminateCompares(Loop &L, unsigned MaxPeelCount, ScalarEvolution &SE,
- const TargetTransformInfo &TTI) {
+ const TargetTransformInfo &TTI, bool CanPeelLast) {
assert(L.isLoopSimplifyForm() && "Loop needs to be in loop simplify form");
unsigned DesiredPeelCount = 0;
unsigned DesiredPeelCountLast = 0;
@@ -641,7 +638,8 @@ countToEliminateCompares(Loop &L, unsigned MaxPeelCount, ScalarEvolution &SE,
const SCEV *Step = LeftAR->getStepRecurrence(SE);
if (!PeelWhilePredicateIsKnown(NewPeelCount, IterVal, RightSCEV, Step,
Pred)) {
- if (shouldPeelLastIteration(L, Pred, LeftAR, RightSCEV, SE, TTI))
+ if (CanPeelLast &&
+ shouldPeelLastIteration(L, Pred, LeftAR, RightSCEV, SE, TTI))
DesiredPeelCountLast = 1;
return;
}
@@ -700,7 +698,8 @@ countToEliminateCompares(Loop &L, unsigned MaxPeelCount, ScalarEvolution &SE,
SE.getConstant(AddRec->getType(), NewPeelCount), SE);
if (!PeelWhilePredicateIsKnown(NewPeelCount, IterVal, BoundSCEV, Step,
Pred)) {
- if (shouldPeelLastIteration(L, Pred, AddRec, BoundSCEV, SE, TTI))
+ if (CanPeelLast &&
+ shouldPeelLastIteration(L, Pred, AddRec, BoundSCEV, SE, TTI))
DesiredPeelCountLast = 1;
return;
}
@@ -760,7 +759,7 @@ struct LoadGroup {
Value *BasePtr;
// First load instruction in the program order.
LoadInst *FirstLoad;
- // Pairs of (load instruction, offset from base).
+ // Pairs of (load instruction, offset from base) sorted by offset.
SmallVector<std::pair<LoadInst *, APInt>, 4> Loads;
// An applicable wider integer type to load as.
Type *WideType;
@@ -770,56 +769,96 @@ struct LoadGroup {
static std::optional<LoadGroup> tryFormLoadGroupForWidening(
Value *Base, SmallVectorImpl<std::pair<LoadInst *, APInt>> &Loads, Loop &L,
ScalarEvolution &SE, const DataLayout &DL, const TargetTransformInfo &TTI) {
+ // Verify all loads use the same address space.
+ unsigned AddrSpace = Loads[0].first->getPointerAddressSpace();
+ for (const auto &[Load, Offset] : Loads) {
+ if (Load->getPointerAddressSpace() != AddrSpace)
+ return std::nullopt;
+ }
+
// Find the span of the loaded data.
- int64_t Left = INT64_MAX;
- int64_t Right = INT64_MIN;
+ int64_t Left = std::numeric_limits<int64_t>::max();
+ int64_t Right = std::numeric_limits<int64_t>::min();
for (const auto &[Load, Offset] : Loads) {
- Left = std::min(Left, Offset.getSExtValue());
- Right = std::max(
- Right, Offset.getSExtValue() +
- static_cast<int64_t>(DL.getTypeStoreSize(Load->getType())));
+ int64_t OffsetVal = Offset.getSExtValue();
+ int64_t StoreSize =
+ static_cast<int64_t>(DL.getTypeStoreSize(Load->getType()));
+ if (OffsetVal > std::numeric_limits<int64_t>::max() - StoreSize)
+ return std::nullopt;
+ Left = std::min(Left, OffsetVal);
+ Right = std::max(Right, OffsetVal + StoreSize);
}
assert((Left < Right) && "Invalid load group span");
uint64_t TotalBytes = Right - Left;
uint64_t TotalBits = TotalBytes * 8;
- // Powers of two are already natural for most targets.
- if (isPowerOf2_64(TotalBits))
- return std::nullopt;
Type *WideType =
DL.getSmallestLegalIntType(L.getHeader()->getContext(), TotalBits);
if (!WideType)
return std::nullopt;
unsigned WideBits = WideType->getIntegerBitWidth();
- // Total size is already natural for the target.
+ // Total size is already natural for the target, no benefit from widening.
if (WideBits == TotalBits)
return std::nullopt;
// Peeling doubles dereferenceable bytes, ensure wide type fits.
if (WideBits > TotalBits * 2)
return std::nullopt;
- // Check alignment is unconstrained.
+ // Check alignment is unconstrained and without penalty.
unsigned Fast = 0;
if (!TTI.allowsMisalignedMemoryAccesses(L.getHeader()->getContext(), WideBits,
- DL.getDefaultGlobalsAddressSpace(),
- Align(1), &Fast) &&
+ AddrSpace, Align(1), &Fast) ||
!Fast)
return std::nullopt;
// Validate pointer stride across iterations.
- const SCEV *PtrSCEV = SE.getSCEV(Base);
- const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrSCEV);
- if (!AR || AR->getLoop() != &L)
- return std::nullopt;
- const SCEV *Step = AR->getStepRecurrence(SE);
- auto *ConstStep = dyn_cast<SCEVConstant>(Step);
- if (!ConstStep)
+ const SCEVConstant *ConstStep = nullptr;
+ if (!match(SE.getSCEV(Base),
+ m_scev_AffineAddRec(m_SCEV(), m_SCEVConstant(ConstStep),
+ m_SpecificLoop(&L))))
return std::nullopt;
int64_t StepVal = ConstStep->getValue()->getSExtValue();
if (StepVal != static_cast<int64_t>(TotalBytes))
return std::nullopt;
LoadInst *FirstLoad = Loads[0].first;
+ // Cost model: compare cost of individual loads vs wide load + extraction ops.
+ // Sort loads by offset first since we need this for cost calculation.
llvm::sort(Loads, [](const auto &A, const auto &B) {
return A.second.slt(B.second);
});
+ int64_t FirstOffset = Loads[0].second.getSExtValue();
+
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+
+ // Cost of original individual loads.
+ InstructionCost OriginalCost = 0;
+ for (const auto &[Load, Offset] : Loads) {
+ OriginalCost += TTI.getMemoryOpCost(Instruction::Load, Load->getType(),
+ Load->getAlign(), AddrSpace, CostKind);
+ }
+
+ // Cost of wide load + extraction operations (shift + trunc for each load).
+ InstructionCost WidenedCost = TTI.getMemoryOpCost(
+ Instruction::Load, WideType, Align(1), AddrSpace, CostKind);
+ for (const auto &[Load, Offset] : Loads) {
+ unsigned LoadBits = Load->getType()->getScalarSizeInBits();
+ uint64_t BitPosition = (Offset.getSExtValue() - FirstOffset) * 8;
+ uint64_t BitOffset =
+ DL.getElementBitOffset(BitPosition, LoadBits, WideBits);
+ if (BitOffset != 0)
+ WidenedCost +=
+ TTI.getArithmeticInstrCost(Instruction::LShr, WideType, CostKind);
+ if (LoadBits < WideBits)
+ WidenedCost +=
+ TTI.getCastInstrCost(Instruction::Trunc, Load->getType(), WideType,
+ TTI::CastContextHint::None, CostKind);
+ }
+
+ LLVM_DEBUG(dbgs() << "Load widening cost: original=" << OriginalCost
+ << " widened=" << WidenedCost << "\n");
+
+ // Bail if the wider load and extractions have a greater cost.
+ if (WidenedCost > OriginalCost)
+ return std::nullopt;
+
return LoadGroup{Base, FirstLoad, std::move(Loads), WideType};
}
@@ -845,46 +884,52 @@ findLoadGroupsForWidening(BasicBlock *BB, Loop &L, ScalarEvolution &SE,
for (Instruction &I : *BB) {
if (auto *Load = dyn_cast<LoadInst>(&I)) {
+ // Skip volatile/atomic loads. Also skip non-integer types since the
+ // widening uses integer shift/truncate operations to extract bytes.
+ // Supporting floats would require additional bitcasts and consecutive
+ // float loads are less probable to create odd amounts of bytes.
if (Load->isVolatile() || Load->isAtomic() ||
- !Load->getType()->isIntegerTy() ||
- Load->getPointerAddressSpace() != DL.getDefaultGlobalsAddressSpace())
+ !Load->getType()->isIntegerTy())
continue;
Value *Ptr = Load->getPointerOperand();
APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
Value *ActualBase = Ptr->stripAndAccumulateConstantOffsets(
DL, Offset, /*AllowNonInbounds=*/false);
LoadsByBase[ActualBase].emplace_back(Load, Offset);
- } else if (I.mayHaveSideEffects())
+ } else if (I.mayHaveSideEffects()) {
+ // Side effects may clobber memory, so we can only group loads that
+ // appear between side effects. Process what we've collected so far.
ProcessCollectedLoads();
+ }
}
ProcessCollectedLoads();
return Groups;
}
-// Returns 1 if peeling the last iteration would enable widening load groups to
-// natural sizes. Returns 0 otherwise.
-static unsigned peelLastForLoadWidening(Loop &L, ScalarEvolution &SE,
- const DataLayout &DL,
- const TargetTransformInfo &TTI,
- DominatorTree &DT) {
+// Returns true if peeling the last iteration would enable widening load groups
+// to natural sizes. Returns false otherwise.
+static bool peelLastForLoadWidening(Loop &L, ScalarEvolution &SE,
+ const TargetTransformInfo &TTI,
+ DominatorTree &DT) {
if (!EnablePeelForLoadWidening)
- return 0;
+ return false;
if (!L.isInnermost())
- return 0;
- if (!canPeelLastIteration(L, SE))
- return 0;
+ return false;
BasicBlock *Latch = L.getLoopLatch();
if (!Latch)
- return 0;
+ return false;
+ const DataLayout &DL = L.getHeader()->getDataLayout();
for (BasicBlock *BB : L.blocks()) {
- // Look for consecutive loads in blocks that execute every iteration.
+ // Only consider blocks that dominate the latch, ensuring they execute on
+ // every iteration that continues the loop. It is guaranteed that the latch
+ // is the only exiting block (no early exits).
if (!DT.dominates(BB, Latch))
continue;
auto Groups = findLoadGroupsForWidening(BB, L, SE, DL, TTI);
if (!Groups.empty())
- return 1;
+ return true;
}
- return 0;
+ return false;
}
} // anonymous namespace
@@ -943,6 +988,9 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
// in TTI.getPeelingPreferences or by the flag -unroll-peel-count.
unsigned DesiredPeelCount = TargetPeelCount;
+ // Check once if we can peel the last iteration to avoid repeated checks.
+ bool CanPeelLast = canPeelLastIteration(*L, SE);
+
// Here we try to get rid of Phis which become invariants or inductions after
// 1, 2, ..., N iterations of the loop. For this we compute the number for
// iterations after which every Phi is guaranteed to become an invariant or an
@@ -957,7 +1005,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
const auto &[CountToEliminateCmps, CountToEliminateCmpsLast] =
- countToEliminateCompares(*L, MaxPeelCount, SE, TTI);
+ countToEliminateCompares(*L, MaxPeelCount, SE, TTI, CanPeelLast);
DesiredPeelCount = std::max(DesiredPeelCount, CountToEliminateCmps);
if (DesiredPeelCount == 0)
@@ -997,19 +1045,15 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
// Check for consecutive load widening opportunity.
// Skip this when running before vectorization (AllowLoadWideningPeel=false)
// to avoid peeling loops that could have been vectorized instead.
- if (PP.PeelCount == 0 && AllowLoadWideningPeel) {
- const DataLayout &DL = L->getHeader()->getDataLayout();
- unsigned LoadWideningPeel = peelLastForLoadWidening(*L, SE, DL, TTI, DT);
- if (LoadWideningPeel > 0) {
- if (LoadWideningPeel + AlreadyPeeled <= UnrollPeelMaxCount) {
- LLVM_DEBUG(
- dbgs() << "Peel last " << LoadWideningPeel
- << " iteration(s) to enable consecutive load widening.\n");
- PP.PeelCount = LoadWideningPeel;
- PP.PeelProfiledIterations = false;
- PP.PeelLast = true;
- return;
- }
+ if (PP.PeelCount == 0 && AllowLoadWideningPeel && CanPeelLast &&
+ peelLastForLoadWidening(*L, SE, TTI, DT)) {
+ if (1 + AlreadyPeeled <= UnrollPeelMaxCount) {
+ LLVM_DEBUG(dbgs() << "Peel last iteration to enable consecutive load "
+ "widening.\n");
+ PP.PeelCount = 1;
+ PP.PeelProfiledIterations = false;
+ PP.PeelLast = true;
+ return;
}
}
@@ -1054,13 +1098,11 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
bool llvm::widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE,
- const DataLayout &DL,
const TargetTransformInfo &TTI,
DominatorTree &DT) {
BasicBlock *Latch = L.getLoopLatch();
- if (!Latch)
- return false;
-
+ assert(Latch && "Loop should have a latch after peeling");
+ const DataLayout &DL = L.getHeader()->getDataLayout();
bool Changed = false;
for (BasicBlock *BB : L.blocks()) {
@@ -1073,20 +1115,19 @@ bool llvm::widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE,
IRBuilder<> Builder(InsertPoint);
Value *BasePtr = Group.BasePtr;
int64_t FirstOffset = Group.Loads[0].second.getSExtValue();
+ unsigned AddrSpace = Group.FirstLoad->getPointerAddressSpace();
// If the first load doesn't start at offset 0, we need to adjust.
if (FirstOffset != 0) {
Value *OrigPtr = Group.BasePtr;
BasePtr = Builder.CreatePtrAdd(
OrigPtr,
- ConstantInt::get(
- Builder.getIndexTy(DL, DL.getDefaultGlobalsAddressSpace()),
- FirstOffset));
+ ConstantInt::get(Builder.getIndexTy(DL, AddrSpace), FirstOffset));
}
- // Merge AA metadata from all loads.
+ // Merge AA metadata from all loads using intersection for correctness.
AAMDNodes AATags = InsertPoint->getAAMetadata();
for (const auto &[Load, Offset] : Group.Loads) {
if (Load != InsertPoint)
- AATags = AATags.concat(Load->getAAMetadata());
+ AATags = AATags.merge(Load->getAAMetadata());
}
// Create the wider load.
LoadInst *WideLoad = Builder.CreateLoad(Group.WideType, BasePtr);
@@ -1095,18 +1136,15 @@ bool llvm::widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE,
WideLoad->setAAMetadata(AATags);
// For each original load, extract the corresponding bytes.
for (const auto &[Load, Offset] : Group.Loads) {
- unsigned LoadBytes = DL.getTypeStoreSize(Load->getType());
+ unsigned LoadBits = Load->getType()->getScalarSizeInBits();
Value *Extracted = WideLoad;
- unsigned BitOffset =
- DL.isBigEndian()
- ? SizeInBits -
- (Offset.getSExtValue() - FirstOffset + LoadBytes) * 8
- : (Offset.getSExtValue() - FirstOffset) * 8;
+ uint64_t BitPosition = (Offset.getSExtValue() - FirstOffset) * 8;
+ uint64_t BitOffset =
+ DL.getElementBitOffset(BitPosition, LoadBits, SizeInBits);
if (BitOffset != 0)
Extracted = Builder.CreateLShr(
Extracted, ConstantInt::get(WideLoad->getType(), BitOffset));
- unsigned TargetBits = Load->getType()->getScalarSizeInBits();
- if (TargetBits < SizeInBits)
+ if (LoadBits < SizeInBits)
Extracted = Builder.CreateTrunc(Extracted, Load->getType());
Load->replaceAllUsesWith(Extracted);
}
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 10ad3a71c73de..5efb4498523f3 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -2300,10 +2300,8 @@ bool VectorCombine::scalarizeExtExtract(Instruction &I) {
auto *Extract = cast<ExtractElementInst>(U);
uint64_t Idx =
cast<ConstantInt>(Extract->getIndexOperand())->getZExtValue();
- uint64_t ShiftAmt =
- DL->isBigEndian()
- ? (TotalBits - SrcEltSizeInBits - Idx * SrcEltSizeInBits)
- : (Idx * SrcEltSizeInBits);
+ uint64_t ShiftAmt = DL->getElementBitOffset(Idx * SrcEltSizeInBits,
+ SrcEltSizeInBits, TotalBits);
Value *LShr = Builder.CreateLShr(ScalarV, ShiftAmt);
Value *And = Builder.CreateAnd(LShr, Mask);
U->replaceAllUsesWith(And);
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-disabled.ll b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-disabled.ll
new file mode 100644
index 0000000000000..3d1ef68a81032
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening-disabled.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -passes=loop-unroll -enable-peel-for-load-widening=false -S %s | FileCheck %s
+
+define void @test_3_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+;
+; CHECK-LABEL: define void @test_3_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ ; Use the loaded values to prevent DCE
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
new file mode 100644
index 0000000000000..f96ce6b31eae7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/peel-last-iteration-load-widening.ll
@@ -0,0 +1,1694 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -passes=loop-unroll -S %s | FileCheck %s
+
+; Test that loop peeling is applied for consecutive load widening opportunities
+; and that the loads are widened to natural sizes.
+
+define void @test_3_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP10]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP9]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ ; Use the loaded values to prevent DCE
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_5_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_5_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP14:%.*]] = lshr i64 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP14]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP2]], 24
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i64 [[TMP2]], 32
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP13]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[TMP9]]
+; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[TMP11]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM4]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP12]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1
+; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
+; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM4_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 5
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+
+ ; Use the loaded values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum4, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 5
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_8_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_8_consecutive_loads_no_peel(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[D:%.*]] = load i8, ptr [[P3]], align 1
+; CHECK-NEXT: [[E:%.*]] = load i8, ptr [[P4]], align 1
+; CHECK-NEXT: [[F:%.*]] = load i8, ptr [[P5]], align 1
+; CHECK-NEXT: [[G:%.*]] = load i8, ptr [[P6]], align 1
+; CHECK-NEXT: [[H:%.*]] = load i8, ptr [[P7]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[D]]
+; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[E]]
+; CHECK-NEXT: [[SUM5:%.*]] = add i8 [[SUM4]], [[F]]
+; CHECK-NEXT: [[SUM6:%.*]] = add i8 [[SUM5]], [[G]]
+; CHECK-NEXT: [[SUM7:%.*]] = add i8 [[SUM6]], [[H]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM7]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 8
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+ %p5 = getelementptr inbounds i8, ptr %p, i64 5
+ %p6 = getelementptr inbounds i8, ptr %p, i64 6
+ %p7 = getelementptr inbounds i8, ptr %p, i64 7
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+ %f = load i8, ptr %p5, align 1
+ %g = load i8, ptr %p6, align 1
+ %h = load i8, ptr %p7, align 1
+
+ ; Use all values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %sum5 = add i8 %sum4, %f
+ %sum6 = add i8 %sum5, %g
+ %sum7 = add i8 %sum6, %h
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum7, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 8
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_non_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_non_consecutive_loads_no_peel(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 2 ; Skip offset 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 3
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 4
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_9_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_9_consecutive_loads_no_peel(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[P5_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 5
+; CHECK-NEXT: [[P6_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 6
+; CHECK-NEXT: [[P7_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 7
+; CHECK-NEXT: [[P8_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 8
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1
+; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1
+; CHECK-NEXT: [[F_PEEL:%.*]] = load i8, ptr [[P5_PEEL]], align 1
+; CHECK-NEXT: [[G_PEEL:%.*]] = load i8, ptr [[P6_PEEL]], align 1
+; CHECK-NEXT: [[H_PEEL:%.*]] = load i8, ptr [[P7_PEEL]], align 1
+; CHECK-NEXT: [[I_VAL_PEEL:%.*]] = load i8, ptr [[P8_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
+; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
+; CHECK-NEXT: [[SUM5_PEEL:%.*]] = add i8 [[SUM4_PEEL]], [[F_PEEL]]
+; CHECK-NEXT: [[SUM6_PEEL:%.*]] = add i8 [[SUM5_PEEL]], [[G_PEEL]]
+; CHECK-NEXT: [[SUM7_PEEL:%.*]] = add i8 [[SUM6_PEEL]], [[H_PEEL]]
+; CHECK-NEXT: [[SUM9_PEEL:%.*]] = add i8 [[SUM7_PEEL]], [[I_VAL_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM9_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP4]], i64 9
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+ %p5 = getelementptr inbounds i8, ptr %p, i64 5
+ %p6 = getelementptr inbounds i8, ptr %p, i64 6
+ %p7 = getelementptr inbounds i8, ptr %p, i64 7
+ %p8 = getelementptr inbounds i8, ptr %p, i64 8
+
+ %a = load i8, ptr %p, align 16
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+ %f = load i8, ptr %p5, align 1
+ %g = load i8, ptr %p6, align 1
+ %h = load i8, ptr %p7, align 1
+ %i.val = load i8, ptr %p8, align 1
+
+ ; Use all values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %sum5 = add i8 %sum4, %f
+ %sum6 = add i8 %sum5, %g
+ %sum7 = add i8 %sum6, %h
+ %sum8 = add i8 %sum7, %i.val
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum8, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 9
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_intervening_store(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_intervening_store(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: store i8 42, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ store i8 42, ptr %p1, align 1 ; Intervening store prevents optimization
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_3_consecutive_i16_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_i16_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i16
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP2]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i16
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 32
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i16
+; CHECK-NEXT: [[SUM1:%.*]] = add i16 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i16 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i16 [[SUM2]], ptr [[DST_I]], align 2
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 2
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i16, ptr [[P2_PEEL]], align 2
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i16 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i16 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT: store i16 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 2
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 6
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i16, ptr %p, i64 1
+ %p2 = getelementptr inbounds i16, ptr %p, i64 2
+
+ %a = load i16, ptr %p, align 2
+ %b = load i16, ptr %p1, align 2
+ %c = load i16, ptr %p2, align 2
+
+ %sum1 = add i16 %a, %b
+ %sum2 = add i16 %sum1, %c
+ %dst.i = getelementptr inbounds i16, ptr %dst, i32 %i
+ store i16 %sum2, ptr %dst.i, align 2
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 6
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Two consecutive i8 loads (2 bytes) are widened to i32 on AArch64 because
+; the smallest legal integer type is i32 (i16 isn't in the legal int widths).
+; This test verifies that 2-byte spans are still profitable to widen.
+define void @test_2_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_2_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I1:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P2:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[DST_I1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I1]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I1]], align 1
+; CHECK-NEXT: [[P_NEXT1]] = getelementptr inbounds i8, ptr [[P2]], i64 2
+; CHECK-NEXT: [[I_NEXT1]] = add nuw i32 [[I1]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND1:%.*]] = icmp ne i32 [[I_NEXT1]], [[TMP6]]
+; CHECK-NEXT: br i1 [[COND1]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT1]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT1]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 2
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM1]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[I_NEXT:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+
+ %a = load i8, ptr %p, align 2
+ %b = load i8, ptr %p1, align 1
+
+ %sum1 = add i8 %a, %b
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum1, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 2
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_aa_metadata_preserved(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_aa_metadata_preserved(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4, !tbaa [[TBAA5:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP10]], align 1, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !tbaa !0
+ %b = load i8, ptr %p1, align 1, !tbaa !0
+ %c = load i8, ptr %p2, align 1, !tbaa !0
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_aa_metadata_different(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_aa_metadata_different(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP10]], align 1, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1, !tbaa [[TBAA10:![0-9]+]]
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1, !tbaa [[TBAA5]]
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1, !tbaa !0
+ %b = load i8, ptr %p1, align 1, !tbaa !3
+ %c = load i8, ptr %p2, align 1, !tbaa !0
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_negative_offsets(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_negative_offsets(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SRC_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 2
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC_OFFSET]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[P]], i64 -2
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT: [[TMP5:%.*]] = lshr i32 [[TMP3]], 8
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP3]], 16
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP4]], [[TMP6]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP8]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP9]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP10:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi ptr [ [[SRC_OFFSET]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P_NEG2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 -2
+; CHECK-NEXT: [[P_NEG1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 -1
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[P_NEG2_PEEL]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P_NEG1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[TMP11]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP10]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP10]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ ; Start at offset 2 so negative offsets are valid
+ %src.offset = getelementptr inbounds i8, ptr %src, i64 2
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src.offset, %entry ], [ %p.next, %loop ]
+
+ ; Negative offsets from current pointer
+ %p_neg2 = getelementptr inbounds i8, ptr %p, i64 -2
+ %p_neg1 = getelementptr inbounds i8, ptr %p, i64 -1
+
+ %a = load i8, ptr %p_neg2, align 1
+ %b = load i8, ptr %p_neg1, align 1
+ %c = load i8, ptr %p, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_unordered_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_unordered_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ ; Loads in non-sequential order: offset 2, then 0, then 1
+ %c = load i8, ptr %p2, align 1
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_multiple_exits(ptr %src, ptr %dst, i32 %n, i1 %cond_early) {
+; CHECK-LABEL: define void @test_multiple_exits(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]], i1 [[COND_EARLY:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: br i1 [[COND_EARLY]], label %[[EARLY_EXIT:.*]], label %[[LOOP_BODY:.*]]
+; CHECK: [[LOOP_BODY]]:
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: br label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop.latch ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop.latch ]
+
+ ; Early exit condition
+ br i1 %cond_early, label %early_exit, label %loop.body
+
+loop.body:
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+ br label %loop.latch
+
+loop.latch:
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+early_exit:
+ ret void
+
+exit:
+ ret void
+}
+
+define void @test_side_effects_between_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_side_effects_between_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: store i8 42, ptr [[P1]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ ; Store between loads - prevents grouping
+ store i8 42, ptr %p1, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_non_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_non_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP2]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 32
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 4
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 5
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ ; Non-consecutive: 0, 2, 4 (gaps at 1, 3)
+ %p1 = getelementptr inbounds i8, ptr %p, i64 2
+ %p2 = getelementptr inbounds i8, ptr %p, i64 4
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ ; Step doesn't match the span
+ %p.next = getelementptr inbounds i8, ptr %p, i64 5
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_float_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_float_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load float, ptr [[P]], align 4
+; CHECK-NEXT: [[B:%.*]] = load float, ptr [[P1]], align 4
+; CHECK-NEXT: [[C:%.*]] = load float, ptr [[P2]], align 4
+; CHECK-NEXT: [[SUM1:%.*]] = fadd float [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = fadd float [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds float, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store float [[SUM2]], ptr [[DST_I]], align 4
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds float, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds float, ptr %p, i64 1
+ %p2 = getelementptr inbounds float, ptr %p, i64 2
+
+ %a = load float, ptr %p, align 4
+ %b = load float, ptr %p1, align 4
+ %c = load float, ptr %p2, align 4
+
+ %sum1 = fadd float %a, %b
+ %sum2 = fadd float %sum1, %c
+ %dst.i = getelementptr inbounds float, ptr %dst, i32 %i
+ store float %sum2, ptr %dst.i, align 4
+
+ %p.next = getelementptr inbounds float, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_i24_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_i24_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[A:%.*]] = load i24, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i24, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i24, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i24 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i24 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i24, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i24 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 9
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 3
+ %p2 = getelementptr inbounds i8, ptr %p, i64 6
+
+ %a = load i24, ptr %p, align 1
+ %b = load i24, ptr %p1, align 1
+ %c = load i24, ptr %p2, align 1
+
+ %sum1 = add i24 %a, %b
+ %sum2 = add i24 %sum1, %c
+ %dst.i = getelementptr inbounds i24, ptr %dst, i32 %i
+ store i24 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 9
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_non_inbounds_geps(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_non_inbounds_geps(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ ; Non-inbounds GEPs
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %p2 = getelementptr i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ ; Non-inbounds for pointer advancement
+ %p.next = getelementptr i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_base_plus_iv_pattern(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_base_plus_iv_pattern(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[I_EXT:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT: [[STRIDE:%.*]] = mul i64 [[I_EXT]], 3
+; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[STRIDE]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[STRIDE]]
+; CHECK-NEXT: [[P1_OFF:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[STRIDE]]
+; CHECK-NEXT: [[P2_OFF:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1_OFF]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2_OFF]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+
+ ; Compute pointer as base + i * 3 + offset
+ %i.ext = zext i32 %i to i64
+ %stride = mul i64 %i.ext, 3
+ %p = getelementptr inbounds i8, ptr %src, i64 %stride
+ %p1 = getelementptr inbounds i8, ptr %src, i64 %stride
+ %p1.off = getelementptr inbounds i8, ptr %p1, i64 1
+ %p2 = getelementptr inbounds i8, ptr %src, i64 %stride
+ %p2.off = getelementptr inbounds i8, ptr %p2, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1.off, align 1
+ %c = load i8, ptr %p2.off, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_mixed_size_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_mixed_size_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i16, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[A_EXT:%.*]] = zext i8 [[A]] to i16
+; CHECK-NEXT: [[SUM1:%.*]] = add i16 [[A_EXT]], [[B]]
+; CHECK-NEXT: [[C_EXT:%.*]] = zext i8 [[C]] to i16
+; CHECK-NEXT: [[SUM2:%.*]] = add i16 [[SUM1]], [[C_EXT]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i16 [[SUM2]], ptr [[DST_I]], align 2
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ ; Mixed sizes: i8, i16, i8
+ %a = load i8, ptr %p, align 1
+ %b = load i16, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %a.ext = zext i8 %a to i16
+ %sum1 = add i16 %a.ext, %b
+ %c.ext = zext i8 %c to i16
+ %sum2 = add i16 %sum1, %c.ext
+ %dst.i = getelementptr inbounds i16, ptr %dst, i32 %i
+ store i16 %sum2, ptr %dst.i, align 2
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 4
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_volatile_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_volatile_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load volatile i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load volatile i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load volatile i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load volatile i8, ptr %p, align 1
+ %b = load volatile i8, ptr %p1, align 1
+ %c = load volatile i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_atomic_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_atomic_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load atomic i8, ptr [[P]] monotonic, align 1
+; CHECK-NEXT: [[B:%.*]] = load atomic i8, ptr [[P1]] monotonic, align 1
+; CHECK-NEXT: [[C:%.*]] = load atomic i8, ptr [[P2]] monotonic, align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load atomic i8, ptr %p monotonic, align 1
+ %b = load atomic i8, ptr %p1 monotonic, align 1
+ %c = load atomic i8, ptr %p2 monotonic, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_different_address_spaces(ptr %src, ptr addrspace(1) %src2, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_different_address_spaces(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P2:%.*]] = phi ptr addrspace(1) [ [[SRC2]], %[[ENTRY]] ], [ [[P2_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2_OFF:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P2]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr addrspace(1) [[P2_OFF]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[P2_NEXT]] = getelementptr inbounds i8, ptr addrspace(1) [[P2]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+ %p2 = phi ptr addrspace(1) [ %src2, %entry ], [ %p2.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2.off = getelementptr inbounds i8, ptr addrspace(1) %p2, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr addrspace(1) %p2.off, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %p2.next = getelementptr inbounds i8, ptr addrspace(1) %p2, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_step_mismatch(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_step_mismatch(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ ; Step is 4 but span is only 3 - mismatch prevents widening
+ %p.next = getelementptr inbounds i8, ptr %p, i64 4
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"char", !2, i64 0}
+!2 = !{!"root"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !2, i64 0}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
+; CHECK: [[TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
+; CHECK: [[META6]] = !{!"char", [[META7:![0-9]+]], i64 0}
+; CHECK: [[META7]] = !{!"root"}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
+; CHECK: [[TBAA10]] = !{[[META11:![0-9]+]], [[META11]], i64 0}
+; CHECK: [[META11]] = !{!"int", [[META7]], i64 0}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/peel-last-iteration-load-widening-be.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/peel-last-iteration-load-widening-be.ll
new file mode 100644
index 0000000000000..40ba011557c0a
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/PowerPC/peel-last-iteration-load-widening-be.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=powerpc64-unknown-linux-gnu -passes=loop-unroll -S %s | FileCheck %s
+
+; Test that loop peeling for load widening correctly handles big-endian targets.
+; On PowerPC64 (big-endian), the cost model determines that widening 3 consecutive
+; i8 loads to an i32 load + shifts + truncs is NOT profitable (widened cost > original),
+; so the optimization correctly bails out. This test verifies the cost model check
+; works on big-endian targets.
+;
+; Note: The big-endian extraction logic (different shift directions from LE) is
+; tested implicitly - if cost model ever changes to favor widening on BE targets,
+; this test's CHECK lines would need updating to verify correct shift amounts.
+
+; Test that 3-byte consecutive loads are NOT widened on PowerPC due to cost model.
+define void @test_3_consecutive_loads_be(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_loads_be(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ ; Use the loaded values to prevent DCE
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
deleted file mode 100644
index b4599a19b47ce..0000000000000
--- a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
+++ /dev/null
@@ -1,104 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -mtriple=aarch64_be -passes=loop-unroll -S %s | FileCheck %s
-
-; Test that loop peeling for load widening works correctly on big-endian targets.
-; The byte extraction shifts should be different from little-endian.
-
-target datalayout = "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-
-; Test 3-byte consecutive loads on big-endian, should peel and use i32 load.
-; For big-endian with i32 load:
-; byte 0 (lowest address) is in bits [31:24]
-; byte 1 is in bits [23:16]
-; byte 2 is in bits [15:8]
-; byte 3 (unused) is in bits [7:0]
-define void @test_3_consecutive_loads_be(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_3_consecutive_loads_be(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
-; CHECK: [[ENTRY_SPLIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 24
-; CHECK-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
-; CHECK-NEXT: [[TMP5:%.*]] = lshr i32 [[TMP2]], 16
-; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
-; CHECK-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP2]], 8
-; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i8
-; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP4]], [[TMP6]]
-; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP8]]
-; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
-; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
-; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
-; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[N]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP9]]
-; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
-; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
-; CHECK: [[EXIT_PEEL_BEGIN]]:
-; CHECK-NEXT: [[TMP10:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
-; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
-; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 1
-; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP11]], align 1
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
-; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
-; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
-; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP10]]
-; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP10]], 1
-; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
-; CHECK: [[EXIT_PEEL_NEXT]]:
-; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
-; CHECK: [[LOOP_PEEL_NEXT]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i8, ptr %p, i64 1
- %p2 = getelementptr inbounds i8, ptr %p, i64 2
-
- %a = load i8, ptr %p, align 1
- %b = load i8, ptr %p1, align 1
- %c = load i8, ptr %p2, align 1
-
- ; Use the loaded values to prevent DCE
- %sum1 = add i8 %a, %b
- %sum2 = add i8 %sum1, %c
- %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
- store i8 %sum2, ptr %dst.i, align 1
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 3
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
-;.
diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
deleted file mode 100644
index 9b6368446757d..0000000000000
--- a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
+++ /dev/null
@@ -1,616 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -mtriple=aarch64 -passes=loop-unroll -S %s | FileCheck %s
-
-; Test that loop peeling is applied for consecutive load widening opportunities
-; and that appropriate dereferenceable assumptions are added.
-
-target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-
-; Test 3-byte consecutive loads, should peel last iteration and use an i32 load.
-define void @test_3_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_3_consecutive_loads(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
-; CHECK: [[ENTRY_SPLIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP2]] to i8
-; CHECK-NEXT: [[TMP10:%.*]] = lshr i32 [[TMP2]], 8
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP10]] to i8
-; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP2]], 16
-; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
-; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP9]], [[TMP5]]
-; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
-; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
-; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
-; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
-; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
-; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
-; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
-; CHECK: [[EXIT_PEEL_BEGIN]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
-; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
-; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
-; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
-; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
-; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
-; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
-; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
-; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP3]], 1
-; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
-; CHECK: [[EXIT_PEEL_NEXT]]:
-; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
-; CHECK: [[LOOP_PEEL_NEXT]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i8, ptr %p, i64 1
- %p2 = getelementptr inbounds i8, ptr %p, i64 2
-
- %a = load i8, ptr %p, align 1
- %b = load i8, ptr %p1, align 1
- %c = load i8, ptr %p2, align 1
-
- ; Use the loaded values to prevent DCE
- %sum1 = add i8 %a, %b
- %sum2 = add i8 %sum1, %c
- %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
- store i8 %sum2, ptr %dst.i, align 1
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 3
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-; Test 3-byte consecutive loads, should peel last iteration and use an i64 load.
-define void @test_5_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_5_consecutive_loads(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
-; CHECK: [[ENTRY_SPLIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP2]] to i8
-; CHECK-NEXT: [[TMP14:%.*]] = lshr i64 [[TMP2]], 8
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP14]] to i8
-; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 16
-; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i8
-; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP2]], 24
-; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8
-; CHECK-NEXT: [[TMP10:%.*]] = lshr i64 [[TMP2]], 32
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i8
-; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP13]], [[TMP5]]
-; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
-; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[TMP9]]
-; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[TMP11]]
-; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
-; CHECK-NEXT: store i8 [[SUM4]], ptr [[DST_I]], align 1
-; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 5
-; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = sub i32 [[N]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP12]]
-; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP2:![0-9]+]]
-; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
-; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
-; CHECK: [[EXIT_PEEL_BEGIN]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
-; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
-; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
-; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
-; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
-; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 4
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
-; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1
-; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1
-; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
-; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
-; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
-; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
-; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
-; CHECK-NEXT: store i8 [[SUM4_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 5
-; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP3]], 1
-; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
-; CHECK: [[EXIT_PEEL_NEXT]]:
-; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
-; CHECK: [[LOOP_PEEL_NEXT]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i8, ptr %p, i64 1
- %p2 = getelementptr inbounds i8, ptr %p, i64 2
- %p3 = getelementptr inbounds i8, ptr %p, i64 3
- %p4 = getelementptr inbounds i8, ptr %p, i64 4
-
- %a = load i8, ptr %p, align 1
- %b = load i8, ptr %p1, align 1
- %c = load i8, ptr %p2, align 1
- %d = load i8, ptr %p3, align 1
- %e = load i8, ptr %p4, align 1
-
- ; Use the loaded values
- %sum1 = add i8 %a, %b
- %sum2 = add i8 %sum1, %c
- %sum3 = add i8 %sum2, %d
- %sum4 = add i8 %sum3, %e
- %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
- store i8 %sum4, ptr %dst.i, align 1
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 5
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-; Test 8-byte consecutive loads does not peel the loop, already natural size.
-define void @test_8_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_8_consecutive_loads_no_peel(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
-; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
-; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
-; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
-; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
-; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
-; CHECK-NEXT: [[D:%.*]] = load i8, ptr [[P3]], align 1
-; CHECK-NEXT: [[E:%.*]] = load i8, ptr [[P4]], align 1
-; CHECK-NEXT: [[F:%.*]] = load i8, ptr [[P5]], align 1
-; CHECK-NEXT: [[G:%.*]] = load i8, ptr [[P6]], align 1
-; CHECK-NEXT: [[H:%.*]] = load i8, ptr [[P7]], align 1
-; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
-; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
-; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[D]]
-; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[E]]
-; CHECK-NEXT: [[SUM5:%.*]] = add i8 [[SUM4]], [[F]]
-; CHECK-NEXT: [[SUM6:%.*]] = add i8 [[SUM5]], [[G]]
-; CHECK-NEXT: [[SUM7:%.*]] = add i8 [[SUM6]], [[H]]
-; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
-; CHECK-NEXT: store i8 [[SUM7]], ptr [[DST_I]], align 1
-; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 8
-; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i8, ptr %p, i64 1
- %p2 = getelementptr inbounds i8, ptr %p, i64 2
- %p3 = getelementptr inbounds i8, ptr %p, i64 3
- %p4 = getelementptr inbounds i8, ptr %p, i64 4
- %p5 = getelementptr inbounds i8, ptr %p, i64 5
- %p6 = getelementptr inbounds i8, ptr %p, i64 6
- %p7 = getelementptr inbounds i8, ptr %p, i64 7
-
- %a = load i8, ptr %p, align 1
- %b = load i8, ptr %p1, align 1
- %c = load i8, ptr %p2, align 1
- %d = load i8, ptr %p3, align 1
- %e = load i8, ptr %p4, align 1
- %f = load i8, ptr %p5, align 1
- %g = load i8, ptr %p6, align 1
- %h = load i8, ptr %p7, align 1
-
- ; Use all values
- %sum1 = add i8 %a, %b
- %sum2 = add i8 %sum1, %c
- %sum3 = add i8 %sum2, %d
- %sum4 = add i8 %sum3, %e
- %sum5 = add i8 %sum4, %f
- %sum6 = add i8 %sum5, %g
- %sum7 = add i8 %sum6, %h
- %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
- store i8 %sum7, ptr %dst.i, align 1
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 8
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-define void @test_non_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_non_consecutive_loads_no_peel(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
-; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
-; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
-; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
-; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
-; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
-; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 4
-; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i8, ptr %p, i64 2 ; Skip offset 1
- %p2 = getelementptr inbounds i8, ptr %p, i64 3
-
- %a = load i8, ptr %p, align 1
- %b = load i8, ptr %p1, align 1
- %c = load i8, ptr %p2, align 1
-
- %sum1 = add i8 %a, %b
- %sum2 = add i8 %sum1, %c
- %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
- store i8 %sum2, ptr %dst.i, align 1
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 4
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-; Test 9-byte consecutive loads, should not peel because i128 isn't a legal integer type.
-define void @test_9_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_9_consecutive_loads(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
-; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
-; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
-; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 4
-; CHECK-NEXT: [[P5_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 5
-; CHECK-NEXT: [[P6_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 6
-; CHECK-NEXT: [[P7_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 7
-; CHECK-NEXT: [[P8_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 8
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 16
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
-; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1
-; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1
-; CHECK-NEXT: [[F_PEEL:%.*]] = load i8, ptr [[P5_PEEL]], align 1
-; CHECK-NEXT: [[G_PEEL:%.*]] = load i8, ptr [[P6_PEEL]], align 1
-; CHECK-NEXT: [[H_PEEL:%.*]] = load i8, ptr [[P7_PEEL]], align 1
-; CHECK-NEXT: [[I_VAL_PEEL:%.*]] = load i8, ptr [[P8_PEEL]], align 1
-; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
-; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
-; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
-; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
-; CHECK-NEXT: [[SUM5_PEEL:%.*]] = add i8 [[SUM4_PEEL]], [[F_PEEL]]
-; CHECK-NEXT: [[SUM6_PEEL:%.*]] = add i8 [[SUM5_PEEL]], [[G_PEEL]]
-; CHECK-NEXT: [[SUM7_PEEL:%.*]] = add i8 [[SUM6_PEEL]], [[H_PEEL]]
-; CHECK-NEXT: [[SUM9_PEEL:%.*]] = add i8 [[SUM7_PEEL]], [[I_VAL_PEEL]]
-; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
-; CHECK-NEXT: store i8 [[SUM9_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP4]], i64 9
-; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP3]], 1
-; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i8, ptr %p, i64 1
- %p2 = getelementptr inbounds i8, ptr %p, i64 2
- %p3 = getelementptr inbounds i8, ptr %p, i64 3
- %p4 = getelementptr inbounds i8, ptr %p, i64 4
- %p5 = getelementptr inbounds i8, ptr %p, i64 5
- %p6 = getelementptr inbounds i8, ptr %p, i64 6
- %p7 = getelementptr inbounds i8, ptr %p, i64 7
- %p8 = getelementptr inbounds i8, ptr %p, i64 8
-
- %a = load i8, ptr %p, align 16
- %b = load i8, ptr %p1, align 1
- %c = load i8, ptr %p2, align 1
- %d = load i8, ptr %p3, align 1
- %e = load i8, ptr %p4, align 1
- %f = load i8, ptr %p5, align 1
- %g = load i8, ptr %p6, align 1
- %h = load i8, ptr %p7, align 1
- %i.val = load i8, ptr %p8, align 1
-
- ; Use all values
- %sum1 = add i8 %a, %b
- %sum2 = add i8 %sum1, %c
- %sum3 = add i8 %sum2, %d
- %sum4 = add i8 %sum3, %e
- %sum5 = add i8 %sum4, %f
- %sum6 = add i8 %sum5, %g
- %sum7 = add i8 %sum6, %h
- %sum8 = add i8 %sum7, %i.val
- %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
- store i8 %sum8, ptr %dst.i, align 1
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 9
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-define void @test_intervening_store(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_intervening_store(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
-; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 1
-; CHECK-NEXT: store i8 42, ptr [[P1_PEEL]], align 1
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
-; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
-; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
-; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
-; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
-; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
-; CHECK-NEXT: [[I_NEXT]] = add i32 [[TMP3]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i8, ptr %p, i64 1
- %p2 = getelementptr inbounds i8, ptr %p, i64 2
-
- %a = load i8, ptr %p, align 1
- store i8 42, ptr %p1, align 1 ; Intervening store prevents optimization
- %b = load i8, ptr %p1, align 1
- %c = load i8, ptr %p2, align 1
-
- %sum1 = add i8 %a, %b
- %sum2 = add i8 %sum1, %c
- %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
- store i8 %sum2, ptr %dst.i, align 1
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 3
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-; Test 3 consecutive i16 loads (6 bytes), should peel and widen to i64.
-define void @test_3_consecutive_i16_loads(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_3_consecutive_i16_loads(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
-; CHECK: [[ENTRY_SPLIT]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i16
-; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP2]], 16
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i16
-; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 32
-; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i16
-; CHECK-NEXT: [[SUM1:%.*]] = add i16 [[TMP3]], [[TMP5]]
-; CHECK-NEXT: [[SUM2:%.*]] = add i16 [[SUM1]], [[TMP7]]
-; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[I]]
-; CHECK-NEXT: store i16 [[SUM2]], ptr [[DST_I]], align 2
-; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 6
-; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
-; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
-; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
-; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
-; CHECK: [[EXIT_PEEL_BEGIN]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
-; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
-; CHECK: [[LOOP_PEEL]]:
-; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 1
-; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 2
-; CHECK-NEXT: [[A_PEEL:%.*]] = load i16, ptr [[TMP10]], align 2
-; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 2
-; CHECK-NEXT: [[C_PEEL:%.*]] = load i16, ptr [[P2_PEEL]], align 2
-; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i16 [[A_PEEL]], [[B_PEEL]]
-; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i16 [[SUM1_PEEL]], [[C_PEEL]]
-; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP9]]
-; CHECK-NEXT: store i16 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 2
-; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 6
-; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
-; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
-; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
-; CHECK: [[EXIT_PEEL_NEXT]]:
-; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
-; CHECK: [[LOOP_PEEL_NEXT]]:
-; CHECK-NEXT: br label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i16, ptr %p, i64 1
- %p2 = getelementptr inbounds i16, ptr %p, i64 2
-
- %a = load i16, ptr %p, align 2
- %b = load i16, ptr %p1, align 2
- %c = load i16, ptr %p2, align 2
-
- %sum1 = add i16 %a, %b
- %sum2 = add i16 %sum1, %c
- %dst.i = getelementptr inbounds i16, ptr %dst, i32 %i
- store i16 %sum2, ptr %dst.i, align 2
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 6
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-; Two i8 loads (or one i16) should not peel because 16-bits is already a natural size.
-define void @test_2_consecutive_loads_no_peel_legal_type(ptr %src, ptr %dst, i32 %n) {
-; CHECK-LABEL: define void @test_2_consecutive_loads_no_peel_legal_type(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
-; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 2
-; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
-; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
-; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
-; CHECK-NEXT: store i8 [[SUM1]], ptr [[DST_I]], align 1
-; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 2
-; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
- %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
-
- %p1 = getelementptr inbounds i8, ptr %p, i64 1
-
- %a = load i8, ptr %p, align 2
- %b = load i8, ptr %p1, align 1
-
- %sum1 = add i8 %a, %b
- %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
- store i8 %sum1, ptr %dst.i, align 1
-
- %p.next = getelementptr inbounds i8, ptr %p, i64 2
- %i.next = add i32 %i, 1
- %cond = icmp ne i32 %i.next, %n
- br i1 %cond, label %loop, label %exit
-
-exit:
- ret void
-}
-
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
-; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
-;.
More information about the llvm-commits
mailing list