[llvm] [LoopPeel] Peel last iteration to enable load widening (PR #173420)
Guy David via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 23 13:36:03 PST 2025
https://github.com/guy-david created https://github.com/llvm/llvm-project/pull/173420
In loops that contain multiple consecutive small loads (e.g., 3 bytes loading i8's), peeling the last iteration makes it safe to read beyond the accessed region, enabling the use of a wider load (e.g., i32) for all other N-1 iterations.
Patterns such as:
```
%a = load i8, ptr %p
%b = load i8, ptr %p+1
%c = load i8, ptr %p+2
...
%p.next = getelementptr i8, ptr %p, 3
```
Can be transformed to:
```
%wide = load i32, ptr %p ; Read 4 bytes
%a = trunc
%b = lshr + trunc
%c = lshr + trunc
...
```
This acts as a fallback strategy when vectorization fails and has significant performance uplifts, most notably in image processing where processing an RGB buffer is common.
>From c3be13eef3b89f6ffa06fdb93de06d94f762cb13 Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Wed, 10 Dec 2025 23:19:26 +0200
Subject: [PATCH] [LoopPeel] Peel last iteration to enable load widening
In loops that contain multiple consecutive small loads (e.g., 3 bytes
loading i8's), peeling the last iteration makes it safe to read beyond
the accessed region, enabling the use of a wider load (e.g., i32) for
all other N-1 iterations.
Patterns such as:
```
%a = load i8, ptr %p
%b = load i8, ptr %p+1
%c = load i8, ptr %p+2
...
%p.next = getelementptr i8, ptr %p, 3
```
Can be transformed to:
```
%wide = load i32, ptr %p ; Read 4 bytes
%a = trunc
%b = lshr + trunc
%c = lshr + trunc
...
```
This acts as a fallback strategy when vectorization fails and has
significant performance uplifts, most notably in image processing
where processing an RGB buffer is common.
---
llvm/include/llvm/Transforms/Utils/LoopPeel.h | 8 +-
.../llvm/Transforms/Utils/UnrollLoop.h | 18 +-
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 34 +-
llvm/lib/Transforms/Utils/LoopPeel.cpp | 231 ++++++-
.../peel-last-iteration-load-widening-be.ll | 104 +++
.../peel-last-iteration-load-widening.ll | 616 ++++++++++++++++++
6 files changed, 991 insertions(+), 20 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
diff --git a/llvm/include/llvm/Transforms/Utils/LoopPeel.h b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
index 49dbc9aa1f2a9..9daf4789082cc 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopPeel.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
@@ -46,7 +46,13 @@ void computePeelCount(Loop *L, unsigned LoopSize,
unsigned TripCount, DominatorTree &DT,
ScalarEvolution &SE, const TargetTransformInfo &TTI,
AssumptionCache *AC = nullptr,
- unsigned Threshold = UINT_MAX);
+ unsigned Threshold = UINT_MAX,
+ bool AllowLoadWideningPeel = true);
+
+/// Combine load instructions in a loop into a wider one, given that we peeled
+/// the last iteration and can assume the bytes are dereferenceable.
+bool widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE, const DataLayout &DL,
+ const TargetTransformInfo &TTI, DominatorTree &DT);
} // end namespace llvm
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index a3efc43c62dc3..073f8c26c664e 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -157,14 +157,16 @@ class UnrollCostEstimator {
unsigned CountOverwrite = 0) const;
};
-LLVM_ABI bool computeUnrollCount(
- Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI,
- AssumptionCache *AC, ScalarEvolution &SE,
- const SmallPtrSetImpl<const Value *> &EphValues,
- OptimizationRemarkEmitter *ORE, unsigned TripCount, unsigned MaxTripCount,
- bool MaxOrZero, unsigned TripMultiple, const UnrollCostEstimator &UCE,
- TargetTransformInfo::UnrollingPreferences &UP,
- TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound);
+LLVM_ABI bool
+computeUnrollCount(Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT,
+ LoopInfo *LI, AssumptionCache *AC, ScalarEvolution &SE,
+ const SmallPtrSetImpl<const Value *> &EphValues,
+ OptimizationRemarkEmitter *ORE, unsigned TripCount,
+ unsigned MaxTripCount, bool MaxOrZero, unsigned TripMultiple,
+ const UnrollCostEstimator &UCE,
+ TargetTransformInfo::UnrollingPreferences &UP,
+ TargetTransformInfo::PeelingPreferences &PP,
+ bool &UseUpperBound, bool AllowLoadWideningPeel = true);
LLVM_ABI std::optional<RecurrenceDescriptor>
canParallelizeReductionWhenUnrolling(PHINode &Phi, Loop *L,
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 802ae4e9c28e3..6ecc50d741ce2 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -921,14 +921,17 @@ shouldPartialUnroll(const unsigned LoopSize, const unsigned TripCount,
// FIXME: This function is used by LoopUnroll and LoopUnrollAndJam, but consumes
// many LoopUnroll-specific options. The shared functionality should be
// refactored into it own function.
-bool llvm::computeUnrollCount(
- Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI,
- AssumptionCache *AC, ScalarEvolution &SE,
- const SmallPtrSetImpl<const Value *> &EphValues,
- OptimizationRemarkEmitter *ORE, unsigned TripCount, unsigned MaxTripCount,
- bool MaxOrZero, unsigned TripMultiple, const UnrollCostEstimator &UCE,
- TargetTransformInfo::UnrollingPreferences &UP,
- TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound) {
+bool llvm::computeUnrollCount(Loop *L, const TargetTransformInfo &TTI,
+ DominatorTree &DT, LoopInfo *LI,
+ AssumptionCache *AC, ScalarEvolution &SE,
+ const SmallPtrSetImpl<const Value *> &EphValues,
+ OptimizationRemarkEmitter *ORE,
+ unsigned TripCount, unsigned MaxTripCount,
+ bool MaxOrZero, unsigned TripMultiple,
+ const UnrollCostEstimator &UCE,
+ TargetTransformInfo::UnrollingPreferences &UP,
+ TargetTransformInfo::PeelingPreferences &PP,
+ bool &UseUpperBound, bool AllowLoadWideningPeel) {
unsigned LoopSize = UCE.getRolledLoopSize();
@@ -1014,7 +1017,8 @@ bool llvm::computeUnrollCount(
}
// 5th priority is loop peeling.
- computePeelCount(L, LoopSize, PP, TripCount, DT, SE, TTI, AC, UP.Threshold);
+ computePeelCount(L, LoopSize, PP, TripCount, DT, SE, TTI, AC, UP.Threshold,
+ AllowLoadWideningPeel);
if (PP.PeelCount) {
UP.Runtime = false;
UP.Count = 1;
@@ -1293,10 +1297,14 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
// computeUnrollCount() decides whether it is beneficial to use upper bound to
// fully unroll the loop.
+ // When OnlyFullUnroll is true, we're running before vectorization
+ // (LoopFullUnrollPass), so disable load widening peeling to avoid peeling
+ // loops that could have been vectorized instead.
bool UseUpperBound = false;
bool IsCountSetExplicitly = computeUnrollCount(
L, TTI, DT, LI, &AC, SE, EphValues, &ORE, TripCount, MaxTripCount,
- MaxOrZero, TripMultiple, UCE, UP, PP, UseUpperBound);
+ MaxOrZero, TripMultiple, UCE, UP, PP, UseUpperBound,
+ /*AllowLoadWideningPeel=*/!OnlyFullUnroll);
if (!UP.Count)
return LoopUnrollResult::Unmodified;
@@ -1316,6 +1324,12 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
ValueToValueMapTy VMap;
if (peelLoop(L, PP.PeelCount, PP.PeelLast, LI, &SE, DT, &AC, PreserveLCSSA,
VMap)) {
+ // Widen consecutive loads after last-iteration peeling
+ if (PP.PeelLast) {
+ const DataLayout &DL = L->getHeader()->getDataLayout();
+ widenLoadsAfterPeel(*L, SE, DL, TTI, DT);
+ }
+
simplifyLoopAfterUnroll(L, true, LI, &SE, &DT, &AC, &TTI, nullptr);
// If the loop was peeled, we already "used up" the profile information
// we had, so we don't want to unroll or peel again.
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index 960ec9d4c7d6e..b86f34a41ab6b 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -86,6 +86,11 @@ static cl::opt<bool> EnablePeelingForIV(
"enable-peeling-for-iv", cl::init(false), cl::Hidden,
cl::desc("Enable peeling to convert Phi nodes into IVs"));
+static cl::opt<bool> EnablePeelForLoadWidening(
+ "enable-peel-for-load-widening", cl::init(true), cl::Hidden,
+ cl::desc(
+ "Enable peeling last iteration to enable consecutive load widening"));
+
static const char *PeeledCountMetaData = "llvm.loop.peeled.count";
extern cl::opt<bool> ProfcheckDisableMetadataFixes;
@@ -746,13 +751,148 @@ static bool violatesLegacyMultiExitLoopCheck(Loop *L) {
});
}
+namespace {
+// Represents a group of loads in a loop that can be combined into a wider one.
+struct LoadGroup {
+ // Base object being read.
+ Value *BasePtr;
+ // First load instruction in the program order.
+ LoadInst *FirstLoad;
+ // Pairs of (load instruction, offset from base).
+ SmallVector<std::pair<LoadInst *, APInt>, 4> Loads;
+ // An applicable wider integer type to load as.
+ Type *WideType;
+};
+
+// Helper to compute load group span and validate for widening.
+static std::optional<LoadGroup> tryFormLoadGroupForWidening(
+ Value *Base, SmallVectorImpl<std::pair<LoadInst *, APInt>> &Loads, Loop &L,
+ ScalarEvolution &SE, const DataLayout &DL, const TargetTransformInfo &TTI) {
+ // Find the span of the loaded data.
+ int64_t Left = INT64_MAX;
+ int64_t Right = INT64_MIN;
+ for (const auto &[Load, Offset] : Loads) {
+ Left = std::min(Left, Offset.getSExtValue());
+ Right = std::max(
+ Right, Offset.getSExtValue() +
+ static_cast<int64_t>(DL.getTypeStoreSize(Load->getType())));
+ }
+ assert((Left < Right) && "Invalid load group span");
+ uint64_t TotalBytes = Right - Left;
+ uint64_t TotalBits = TotalBytes * 8;
+ // Powers of two are already natural for most targets.
+ if (isPowerOf2_64(TotalBits))
+ return std::nullopt;
+ Type *WideType =
+ DL.getSmallestLegalIntType(L.getHeader()->getContext(), TotalBits);
+ if (!WideType)
+ return std::nullopt;
+ unsigned WideBits = WideType->getIntegerBitWidth();
+ // Total size is already natural for the target.
+ if (WideBits == TotalBits)
+ return std::nullopt;
+ // Peeling doubles dereferenceable bytes, ensure wide type fits.
+ if (WideBits > TotalBits * 2)
+ return std::nullopt;
+ // Check alignment is unconstrained.
+ unsigned Fast = 0;
+ if (!TTI.allowsMisalignedMemoryAccesses(L.getHeader()->getContext(), WideBits,
+ DL.getDefaultGlobalsAddressSpace(),
+ Align(1), &Fast) &&
+ !Fast)
+ return std::nullopt;
+ // Validate pointer stride across iterations.
+ const SCEV *PtrSCEV = SE.getSCEV(Base);
+ const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrSCEV);
+ if (!AR || AR->getLoop() != &L)
+ return std::nullopt;
+ const SCEV *Step = AR->getStepRecurrence(SE);
+ auto *ConstStep = dyn_cast<SCEVConstant>(Step);
+ if (!ConstStep)
+ return std::nullopt;
+ int64_t StepVal = ConstStep->getValue()->getSExtValue();
+ if (StepVal != static_cast<int64_t>(TotalBytes))
+ return std::nullopt;
+
+ LoadInst *FirstLoad = Loads[0].first;
+ llvm::sort(Loads, [](const auto &A, const auto &B) {
+ return A.second.slt(B.second);
+ });
+ return LoadGroup{Base, FirstLoad, std::move(Loads), WideType};
+}
+
+// Find groups of consecutive loads in a basic block for peeling purposes
+static SmallVector<LoadGroup>
+findLoadGroupsForWidening(BasicBlock *BB, Loop &L, ScalarEvolution &SE,
+ const DataLayout &DL,
+ const TargetTransformInfo &TTI) {
+ SmallVector<LoadGroup> Groups;
+ // Mapping from base pointer to loads instructions and their offset from the
+ // base.
+ DenseMap<Value *, SmallVector<std::pair<LoadInst *, APInt>>> LoadsByBase;
+
+ auto ProcessCollectedLoads = [&]() {
+ for (auto &[Base, Loads] : LoadsByBase) {
+ if (Loads.size() <= 1)
+ continue;
+ if (auto Group = tryFormLoadGroupForWidening(Base, Loads, L, SE, DL, TTI))
+ Groups.emplace_back(*std::move(Group));
+ }
+ LoadsByBase.clear();
+ };
+
+ for (Instruction &I : *BB) {
+ if (auto *Load = dyn_cast<LoadInst>(&I)) {
+ if (Load->isVolatile() || Load->isAtomic() ||
+ !Load->getType()->isIntegerTy() ||
+ Load->getPointerAddressSpace() != DL.getDefaultGlobalsAddressSpace())
+ continue;
+ Value *Ptr = Load->getPointerOperand();
+ APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
+ Value *ActualBase = Ptr->stripAndAccumulateConstantOffsets(
+ DL, Offset, /*AllowNonInbounds=*/false);
+ LoadsByBase[ActualBase].emplace_back(Load, Offset);
+ } else if (I.mayHaveSideEffects())
+ ProcessCollectedLoads();
+ }
+ ProcessCollectedLoads();
+ return Groups;
+}
+
+// Returns 1 if peeling the last iteration would enable widening load groups to
+// natural sizes. Returns 0 otherwise.
+static unsigned peelLastForLoadWidening(Loop &L, ScalarEvolution &SE,
+ const DataLayout &DL,
+ const TargetTransformInfo &TTI,
+ DominatorTree &DT) {
+ if (!EnablePeelForLoadWidening)
+ return 0;
+ if (!L.isInnermost())
+ return 0;
+ if (!canPeelLastIteration(L, SE))
+ return 0;
+ BasicBlock *Latch = L.getLoopLatch();
+ if (!Latch)
+ return 0;
+ for (BasicBlock *BB : L.blocks()) {
+ // Look for consecutive loads in blocks that execute every iteration.
+ if (!DT.dominates(BB, Latch))
+ continue;
+ auto Groups = findLoadGroupsForWidening(BB, L, SE, DL, TTI);
+ if (!Groups.empty())
+ return 1;
+ }
+ return 0;
+}
+} // anonymous namespace
// Return the number of iterations we want to peel off.
void llvm::computePeelCount(Loop *L, unsigned LoopSize,
TargetTransformInfo::PeelingPreferences &PP,
unsigned TripCount, DominatorTree &DT,
ScalarEvolution &SE, const TargetTransformInfo &TTI,
- AssumptionCache *AC, unsigned Threshold) {
+ AssumptionCache *AC, unsigned Threshold,
+ bool AllowLoadWideningPeel) {
assert(LoopSize > 0 && "Zero loop size is not allowed!");
// Save the PP.PeelCount value set by the target in
// TTI.getPeelingPreferences or by the flag -unroll-peel-count.
@@ -852,6 +992,25 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
}
+ // Check for consecutive load widening opportunity.
+ // Skip this when running before vectorization (AllowLoadWideningPeel=false)
+ // to avoid peeling loops that could have been vectorized instead.
+ if (PP.PeelCount == 0 && AllowLoadWideningPeel) {
+ const DataLayout &DL = L->getHeader()->getDataLayout();
+ unsigned LoadWideningPeel = peelLastForLoadWidening(*L, SE, DL, TTI, DT);
+ if (LoadWideningPeel > 0) {
+ if (LoadWideningPeel + AlreadyPeeled <= UnrollPeelMaxCount) {
+ LLVM_DEBUG(
+ dbgs() << "Peel last " << LoadWideningPeel
+ << " iteration(s) to enable consecutive load widening.\n");
+ PP.PeelCount = LoadWideningPeel;
+ PP.PeelProfiledIterations = false;
+ PP.PeelLast = true;
+ return;
+ }
+ }
+ }
+
// Bail if we know the statically calculated trip count.
// In this case we rather prefer partial unrolling.
if (TripCount)
@@ -890,6 +1049,76 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
}
}
+bool llvm::widenLoadsAfterPeel(Loop &L, ScalarEvolution &SE,
+ const DataLayout &DL,
+ const TargetTransformInfo &TTI,
+ DominatorTree &DT) {
+ BasicBlock *Latch = L.getLoopLatch();
+ if (!Latch)
+ return false;
+
+ bool Changed = false;
+
+ for (BasicBlock *BB : L.blocks()) {
+ if (!DT.dominates(BB, Latch))
+ continue;
+ SmallVector<LoadGroup> Groups =
+ findLoadGroupsForWidening(BB, L, SE, DL, TTI);
+ for (const LoadGroup &Group : Groups) {
+ LoadInst *InsertPoint = Group.FirstLoad;
+ IRBuilder<> Builder(InsertPoint);
+ Value *BasePtr = Group.BasePtr;
+ int64_t FirstOffset = Group.Loads[0].second.getSExtValue();
+ // If the first load doesn't start at offset 0, we need to adjust.
+ if (FirstOffset != 0) {
+ Value *OrigPtr = Group.BasePtr;
+ BasePtr = Builder.CreatePtrAdd(
+ OrigPtr,
+ ConstantInt::get(
+ Builder.getIndexTy(DL, DL.getDefaultGlobalsAddressSpace()),
+ FirstOffset));
+ }
+ // Merge AA metadata from all loads.
+ AAMDNodes AATags = InsertPoint->getAAMetadata();
+ for (const auto &[Load, Offset] : Group.Loads) {
+ if (Load != InsertPoint)
+ AATags = AATags.concat(Load->getAAMetadata());
+ }
+ // Create the wider load.
+ LoadInst *WideLoad = Builder.CreateLoad(Group.WideType, BasePtr);
+ unsigned SizeInBits = WideLoad->getType()->getScalarSizeInBits();
+ if (AATags)
+ WideLoad->setAAMetadata(AATags);
+ // For each original load, extract the corresponding bytes.
+ for (const auto &[Load, Offset] : Group.Loads) {
+ unsigned LoadBytes = DL.getTypeStoreSize(Load->getType());
+ Value *Extracted = WideLoad;
+ unsigned BitOffset =
+ DL.isBigEndian()
+ ? SizeInBits -
+ (Offset.getSExtValue() - FirstOffset + LoadBytes) * 8
+ : (Offset.getSExtValue() - FirstOffset) * 8;
+ if (BitOffset != 0)
+ Extracted = Builder.CreateLShr(
+ Extracted, ConstantInt::get(WideLoad->getType(), BitOffset));
+ unsigned TargetBits = Load->getType()->getScalarSizeInBits();
+ if (TargetBits < SizeInBits)
+ Extracted = Builder.CreateTrunc(Extracted, Load->getType());
+ Load->replaceAllUsesWith(Extracted);
+ }
+ // Delete the original loads.
+ for (auto &[Load, Offset] : Group.Loads)
+ Load->eraseFromParent();
+
+ LLVM_DEBUG(dbgs() << "Widened " << Group.Loads.size()
+ << " loads into a single " << *WideLoad << "\n");
+ Changed = true;
+ }
+ }
+
+ return Changed;
+}
+
/// Clones the body of the loop L, putting it between \p InsertTop and \p
/// InsertBot.
/// \param IterNumber The serial number of the iteration currently being
diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
new file mode 100644
index 0000000000000..173130deae048
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening-be.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64_be -passes=loop-unroll -S %s | FileCheck %s
+
+; Test that loop peeling for load widening works correctly on big-endian targets.
+; The byte extraction shifts should be different from little-endian.
+
+target datalayout = "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; Test 3-byte consecutive loads on big-endian, should peel and use i32 load.
+; For big-endian with i32 load:
+; byte 0 (lowest address) is in bits [31:24]
+; byte 1 is in bits [23:16]
+; byte 2 is in bits [15:8]
+; byte 3 (unused) is in bits [7:0]
+define void @test_3_consecutive_loads_be(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_loads_be(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 24
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i32 [[TMP3]] to i8
+; CHECK-NEXT: [[TMP5:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[TMP5]] to i8
+; CHECK-NEXT: [[TMP7:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP4]], [[TMP6]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP8]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP9]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP10:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP11]], align 4
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP10]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP10]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 4
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ ; Use the loaded values to prevent DCE
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+;.
diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
new file mode 100644
index 0000000000000..3166539e02e19
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-load-widening.ll
@@ -0,0 +1,616 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -passes=loop-unroll -S %s | FileCheck %s
+
+; Test that loop peeling is applied for consecutive load widening opportunities
+; and that appropriate dereferenceable assumptions are added.
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+; Test 3-byte consecutive loads, should peel last iteration and use an i32 load.
+define void @test_3_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i32 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i32 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP10]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP9]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 4
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ ; Use the loaded values to prevent DCE
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test 3-byte consecutive loads, should peel last iteration and use an i64 load.
+define void @test_5_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_5_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP13:%.*]] = trunc i64 [[TMP2]] to i8
+; CHECK-NEXT: [[TMP14:%.*]] = lshr i64 [[TMP2]], 8
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP14]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i8
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP2]], 24
+; CHECK-NEXT: [[TMP9:%.*]] = trunc i64 [[TMP8]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = lshr i64 [[TMP2]], 32
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i8
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[TMP13]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[TMP9]]
+; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[TMP11]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM4]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP12:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP12]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 8
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1
+; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
+; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM4_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 5
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+
+ %a = load i8, ptr %p, align 8
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+
+ ; Use the loaded values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum4, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 5
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test 8-byte consecutive loads does not peel the loop, already natural size.
+define void @test_8_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_8_consecutive_loads_no_peel(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 5
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 7
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 8
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[D:%.*]] = load i8, ptr [[P3]], align 1
+; CHECK-NEXT: [[E:%.*]] = load i8, ptr [[P4]], align 1
+; CHECK-NEXT: [[F:%.*]] = load i8, ptr [[P5]], align 1
+; CHECK-NEXT: [[G:%.*]] = load i8, ptr [[P6]], align 1
+; CHECK-NEXT: [[H:%.*]] = load i8, ptr [[P7]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[SUM3:%.*]] = add i8 [[SUM2]], [[D]]
+; CHECK-NEXT: [[SUM4:%.*]] = add i8 [[SUM3]], [[E]]
+; CHECK-NEXT: [[SUM5:%.*]] = add i8 [[SUM4]], [[F]]
+; CHECK-NEXT: [[SUM6:%.*]] = add i8 [[SUM5]], [[G]]
+; CHECK-NEXT: [[SUM7:%.*]] = add i8 [[SUM6]], [[H]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM7]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 8
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+ %p5 = getelementptr inbounds i8, ptr %p, i64 5
+ %p6 = getelementptr inbounds i8, ptr %p, i64 6
+ %p7 = getelementptr inbounds i8, ptr %p, i64 7
+
+ %a = load i8, ptr %p, align 8
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+ %f = load i8, ptr %p5, align 1
+ %g = load i8, ptr %p6, align 1
+ %h = load i8, ptr %p7, align 1
+
+ ; Use all values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %sum5 = add i8 %sum4, %f
+ %sum6 = add i8 %sum5, %g
+ %sum7 = add i8 %sum6, %h
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum7, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 8
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_non_consecutive_loads_no_peel(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_non_consecutive_loads_no_peel(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 3
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 1
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[C:%.*]] = load i8, ptr [[P2]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i8 [[SUM1]], [[C]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM2]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 2 ; Skip offset 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 3
+
+ %a = load i8, ptr %p, align 1
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 4
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test 9-byte consecutive loads, should not peel because i128 isn't a legal integer type.
+define void @test_9_consecutive_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_9_consecutive_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[P3_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[P4_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 4
+; CHECK-NEXT: [[P5_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 5
+; CHECK-NEXT: [[P6_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 6
+; CHECK-NEXT: [[P7_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 7
+; CHECK-NEXT: [[P8_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 8
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 16
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[D_PEEL:%.*]] = load i8, ptr [[P3_PEEL]], align 1
+; CHECK-NEXT: [[E_PEEL:%.*]] = load i8, ptr [[P4_PEEL]], align 1
+; CHECK-NEXT: [[F_PEEL:%.*]] = load i8, ptr [[P5_PEEL]], align 1
+; CHECK-NEXT: [[G_PEEL:%.*]] = load i8, ptr [[P6_PEEL]], align 1
+; CHECK-NEXT: [[H_PEEL:%.*]] = load i8, ptr [[P7_PEEL]], align 1
+; CHECK-NEXT: [[I_VAL_PEEL:%.*]] = load i8, ptr [[P8_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[SUM3_PEEL:%.*]] = add i8 [[SUM2_PEEL]], [[D_PEEL]]
+; CHECK-NEXT: [[SUM4_PEEL:%.*]] = add i8 [[SUM3_PEEL]], [[E_PEEL]]
+; CHECK-NEXT: [[SUM5_PEEL:%.*]] = add i8 [[SUM4_PEEL]], [[F_PEEL]]
+; CHECK-NEXT: [[SUM6_PEEL:%.*]] = add i8 [[SUM5_PEEL]], [[G_PEEL]]
+; CHECK-NEXT: [[SUM7_PEEL:%.*]] = add i8 [[SUM6_PEEL]], [[H_PEEL]]
+; CHECK-NEXT: [[SUM9_PEEL:%.*]] = add i8 [[SUM7_PEEL]], [[I_VAL_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM9_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP4]], i64 9
+; CHECK-NEXT: [[I_NEXT_PEEL]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+ %p3 = getelementptr inbounds i8, ptr %p, i64 3
+ %p4 = getelementptr inbounds i8, ptr %p, i64 4
+ %p5 = getelementptr inbounds i8, ptr %p, i64 5
+ %p6 = getelementptr inbounds i8, ptr %p, i64 6
+ %p7 = getelementptr inbounds i8, ptr %p, i64 7
+ %p8 = getelementptr inbounds i8, ptr %p, i64 8
+
+ %a = load i8, ptr %p, align 16
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+ %d = load i8, ptr %p3, align 1
+ %e = load i8, ptr %p4, align 1
+ %f = load i8, ptr %p5, align 1
+ %g = load i8, ptr %p6, align 1
+ %h = load i8, ptr %p7, align 1
+ %i.val = load i8, ptr %p8, align 1
+
+ ; Use all values
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %sum3 = add i8 %sum2, %d
+ %sum4 = add i8 %sum3, %e
+ %sum5 = add i8 %sum4, %f
+ %sum6 = add i8 %sum5, %g
+ %sum7 = add i8 %sum6, %h
+ %sum8 = add i8 %sum7, %i.val
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum8, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 9
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @test_intervening_store(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_intervening_store(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT_PEEL:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i8, ptr [[TMP4]], align 4
+; CHECK-NEXT: store i8 42, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i8, ptr [[P1_PEEL]], align 1
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i8, ptr [[P2_PEEL]], align 1
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i8 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i8 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT: store i8 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 1
+; CHECK-NEXT: [[P_NEXT_PEEL]] = getelementptr inbounds i8, ptr [[TMP4]], i64 3
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[TMP3]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p, i64 2
+
+ %a = load i8, ptr %p, align 4
+ store i8 42, ptr %p1, align 1 ; Intervening store prevents optimization
+ %b = load i8, ptr %p1, align 1
+ %c = load i8, ptr %p2, align 1
+
+ %sum1 = add i8 %a, %b
+ %sum2 = add i8 %sum1, %c
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum2, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 3
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Test 3 consecutive i16 loads (6 bytes), should peel and widen to i64.
+define void @test_3_consecutive_i16_loads(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_3_consecutive_i16_loads(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[ENTRY_SPLIT:.*]], label %[[EXIT_PEEL_BEGIN:.*]]
+; CHECK: [[ENTRY_SPLIT]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY_SPLIT]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY_SPLIT]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i16
+; CHECK-NEXT: [[TMP4:%.*]] = lshr i64 [[TMP2]], 16
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i16
+; CHECK-NEXT: [[TMP6:%.*]] = lshr i64 [[TMP2]], 32
+; CHECK-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i16
+; CHECK-NEXT: [[SUM1:%.*]] = add i16 [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[SUM2:%.*]] = add i16 [[SUM1]], [[TMP7]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i16 [[SUM2]], ptr [[DST_I]], align 2
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 6
+; CHECK-NEXT: [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = sub i32 [[N]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[TMP8]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT_PEEL_BEGIN_LOOPEXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT_PEEL_BEGIN_LOOPEXIT]]:
+; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[I_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: [[DOTPH1:%.*]] = phi ptr [ [[P_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT: br label %[[EXIT_PEEL_BEGIN]]
+; CHECK: [[EXIT_PEEL_BEGIN]]:
+; CHECK-NEXT: [[TMP9:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DOTPH]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[DOTPH1]], %[[EXIT_PEEL_BEGIN_LOOPEXIT]] ]
+; CHECK-NEXT: br label %[[LOOP_PEEL:.*]]
+; CHECK: [[LOOP_PEEL]]:
+; CHECK-NEXT: [[P1_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 1
+; CHECK-NEXT: [[P2_PEEL:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 2
+; CHECK-NEXT: [[A_PEEL:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT: [[B_PEEL:%.*]] = load i16, ptr [[P1_PEEL]], align 2
+; CHECK-NEXT: [[C_PEEL:%.*]] = load i16, ptr [[P2_PEEL]], align 2
+; CHECK-NEXT: [[SUM1_PEEL:%.*]] = add i16 [[A_PEEL]], [[B_PEEL]]
+; CHECK-NEXT: [[SUM2_PEEL:%.*]] = add i16 [[SUM1_PEEL]], [[C_PEEL]]
+; CHECK-NEXT: [[DST_I_PEEL:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT: store i16 [[SUM2_PEEL]], ptr [[DST_I_PEEL]], align 2
+; CHECK-NEXT: [[P_NEXT_PEEL:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 6
+; CHECK-NEXT: [[I_NEXT_PEEL:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT: [[COND_PEEL:%.*]] = icmp ne i32 [[I_NEXT_PEEL]], [[N]]
+; CHECK-NEXT: br i1 [[COND_PEEL]], label %[[EXIT_PEEL_NEXT:.*]], label %[[EXIT_PEEL_NEXT]]
+; CHECK: [[EXIT_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[LOOP_PEEL_NEXT:.*]]
+; CHECK: [[LOOP_PEEL_NEXT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i16, ptr %p, i64 1
+ %p2 = getelementptr inbounds i16, ptr %p, i64 2
+
+ %a = load i16, ptr %p, align 2
+ %b = load i16, ptr %p1, align 2
+ %c = load i16, ptr %p2, align 2
+
+ %sum1 = add i16 %a, %b
+ %sum2 = add i16 %sum1, %c
+ %dst.i = getelementptr inbounds i16, ptr %dst, i32 %i
+ store i16 %sum2, ptr %dst.i, align 2
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 6
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Two i8 loads (or one i16) should not peel because 16-bits is already a natural size.
+define void @test_2_consecutive_loads_no_peel_legal_type(ptr %src, ptr %dst, i32 %n) {
+; CHECK-LABEL: define void @test_2_consecutive_loads_no_peel_legal_type(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[P_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 1
+; CHECK-NEXT: [[A:%.*]] = load i8, ptr [[P]], align 2
+; CHECK-NEXT: [[B:%.*]] = load i8, ptr [[P1]], align 1
+; CHECK-NEXT: [[SUM1:%.*]] = add i8 [[A]], [[B]]
+; CHECK-NEXT: [[DST_I:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[I]]
+; CHECK-NEXT: store i8 [[SUM1]], ptr [[DST_I]], align 1
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds i8, ptr [[P]], i64 2
+; CHECK-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT: [[COND:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %p = phi ptr [ %src, %entry ], [ %p.next, %loop ]
+
+ %p1 = getelementptr inbounds i8, ptr %p, i64 1
+
+ %a = load i8, ptr %p, align 2
+ %b = load i8, ptr %p1, align 1
+
+ %sum1 = add i8 %a, %b
+ %dst.i = getelementptr inbounds i8, ptr %dst, i32 %i
+ store i8 %sum1, ptr %dst.i, align 1
+
+ %p.next = getelementptr inbounds i8, ptr %p, i64 2
+ %i.next = add i32 %i, 1
+ %cond = icmp ne i32 %i.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+;.
More information about the llvm-commits
mailing list