[llvm] [LoopIdiom] Use SCEV for deciding memmove validity (PR #211274)
John Brawn via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 06:19:47 PDT 2026
https://github.com/john-brawn-arm updated https://github.com/llvm/llvm-project/pull/211274
>From 54b05d9d1b6d6a6232178793979096558b5a10e5 Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn at arm.com>
Date: Mon, 20 Jul 2026 13:48:36 +0100
Subject: [PATCH 1/3] [LoopIdiom] Use SCEV for deciding memmove validity
When the loop index variable is smaller than the GEP offset size, and
is thus sign or zero extended before being used, then the IR that is
expanded from the SCEV expressions for the load and store locations
will be in a form that means GetPointerBaseWithConstantOffset can't
deduce the base and offset, meaning we can't generate memmove.
Solve this be deciding memmove validity based on the SCEV expressions
instead of the IR that is expanded from them. This means we also need
to insert a check to handle a null base pointer, as that was
previously handled implicitly due to how SCEVExpander expands
expressions involving null pointers.
---
.../Transforms/Scalar/LoopIdiomRecognize.cpp | 51 +--
llvm/test/Transforms/LoopIdiom/memmove-ext.ll | 341 ++++++++++++++++++
2 files changed, 367 insertions(+), 25 deletions(-)
create mode 100644 llvm/test/Transforms/LoopIdiom/memmove-ext.ll
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index e674143099a53..a17d909567602 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1290,43 +1290,44 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
namespace {
class MemmoveVerifier {
public:
- explicit MemmoveVerifier(const Value &LoadBasePtr, const Value &StoreBasePtr,
- const DataLayout &DL)
- : DL(DL), BP1(llvm::GetPointerBaseWithConstantOffset(
- LoadBasePtr.stripPointerCasts(), LoadOff, DL)),
- BP2(llvm::GetPointerBaseWithConstantOffset(
- StoreBasePtr.stripPointerCasts(), StoreOff, DL)),
- IsSameObject(BP1 == BP2) {}
+ explicit MemmoveVerifier(const SCEV &LoadStart, const SCEV &StoreStart,
+ const DataLayout &DL, ScalarEvolution &SE)
+ : DL(DL),
+ Off(dyn_cast<SCEVConstant>(SE.getMinusSCEV(&StoreStart, &LoadStart))),
+ BasePtr(dyn_cast<SCEVUnknown>(SE.getPointerBase(&StoreStart))),
+ IsSameObject(Off != nullptr) {}
bool loadAndStoreMayFormMemmove(unsigned StoreSize, bool IsNegStride,
const Instruction &TheLoad,
bool IsMemCpy) const {
+ // The store must be at a constant offset from the load, and there must be
+ // an underlying pointer.
+ if (!Off || !BasePtr)
+ return false;
+ int64_t OffVal = Off->getValue()->getSExtValue();
+ // If null is defined then the base pointer can't be null
+ if (TheLoad.getParent()->getParent()->nullPointerIsDefined() &&
+ isa<ConstantPointerNull>(BasePtr->getValue()))
+ return false;
+ int64_t LoadSize;
if (IsMemCpy) {
- // Ensure that LoadBasePtr is after StoreBasePtr or before StoreBasePtr
- // for negative stride.
- if ((!IsNegStride && LoadOff <= StoreOff) ||
- (IsNegStride && LoadOff >= StoreOff))
- return false;
+ LoadSize = 1;
} else {
- // Ensure that LoadBasePtr is after StoreBasePtr or before StoreBasePtr
- // for negative stride. LoadBasePtr shouldn't overlap with StoreBasePtr.
- int64_t LoadSize =
- DL.getTypeSizeInBits(TheLoad.getType()).getFixedValue() / 8;
- if (BP1 != BP2 || LoadSize != int64_t(StoreSize))
- return false;
- if ((!IsNegStride && LoadOff < StoreOff + int64_t(StoreSize)) ||
- (IsNegStride && LoadOff + LoadSize > StoreOff))
+ LoadSize = DL.getTypeSizeInBits(TheLoad.getType()).getFixedValue() / 8;
+ if (LoadSize != StoreSize)
return false;
}
+ // Ensure that LoadBasePtr is after StoreBasePtr or before StoreBasePtr
+ // for negative stride. LoadBasePtr shouldn't overlap with StoreBasePtr.
+ if (IsNegStride ? OffVal < LoadSize : OffVal > -LoadSize)
+ return false;
return true;
}
private:
const DataLayout &DL;
- int64_t LoadOff = 0;
- int64_t StoreOff = 0;
- const Value *BP1;
- const Value *BP2;
+ const SCEVConstant *Off;
+ const SCEVUnknown *BasePtr;
public:
const bool IsSameObject;
@@ -1437,7 +1438,7 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
// If the store is a memcpy instruction, we must check if it will write to
// the load memory locations. So remove it from the ignored stores.
- MemmoveVerifier Verifier(*LoadBasePtr, *StoreBasePtr, *DL);
+ MemmoveVerifier Verifier(*LdStart, *StrStart, *DL, *SE);
if (IsMemCpy && !Verifier.IsSameObject)
IgnoredInsts.erase(TheStore);
if (mayLoopAccessLocation(LoadBasePtr, ModRefInfo::Mod, CurLoop, BECount,
diff --git a/llvm/test/Transforms/LoopIdiom/memmove-ext.ll b/llvm/test/Transforms/LoopIdiom/memmove-ext.ll
new file mode 100644
index 0000000000000..ac576ebc93f81
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/memmove-ext.ll
@@ -0,0 +1,341 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes="loop-idiom" < %s -S | FileCheck %s
+
+; Check that we can form memmove when the loop index variable is sign or zero
+; extended.
+
+define void @move_up_dec_loop_zext(i16 %n, ptr %p) {
+; CHECK-LABEL: define void @move_up_dec_loop_zext(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i16 [[N]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = add i16 [[N]], -1
+; CHECK-NEXT: [[SMIN:%.*]] = call i16 @llvm.smin.i16(i16 [[TMP1]], i16 0)
+; CHECK-NEXT: [[TMP2:%.*]] = sub i16 [[TMP1]], [[SMIN]]
+; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP0]], [[TMP3]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP0]], -1
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], [[TMP3]]
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[SCEVGEP]], ptr align 1 [[SCEVGEP1]], i64 [[TMP7]], i1 false)
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[N]], %[[ENTRY]] ]
+; CHECK-NEXT: [[IV_EXT:%.*]] = zext nneg i16 [[IV]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub nuw nsw i64 [[IV_EXT]], 1
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[SUB]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IV_EXT]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], -1
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i16 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ %iv.next, %loop ], [ %n, %entry ]
+ %iv.ext = zext nneg i16 %iv to i64
+ %sub = sub nuw nsw i64 %iv.ext, 1
+ %src = getelementptr inbounds nuw i8, ptr %p, i64 %sub
+ %dst = getelementptr inbounds nuw i8, ptr %p, i64 %iv.ext
+ %val = load i8, ptr %src, align 1
+ store i8 %val, ptr %dst, align 1
+ %iv.next = add i16 %iv, -1
+ %cmp = icmp sgt i16 %iv.next, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @move_down_inc_loop_zext(i16 %n, ptr %p) {
+; CHECK-LABEL: define void @move_down_inc_loop_zext(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P]], i64 -1
+; CHECK-NEXT: [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 1)
+; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i16 [[SMAX]] to i64
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[SCEVGEP]], ptr align 1 [[P]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[IV_EXT:%.*]] = zext nneg i16 [[IV]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub nuw nsw i64 [[IV_EXT]], 1
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IV_EXT]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[SUB]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ %iv.next, %loop ], [ 0, %entry ]
+ %iv.ext = zext nneg i16 %iv to i64
+ %sub = sub nuw nsw i64 %iv.ext, 1
+ %src = getelementptr inbounds nuw i8, ptr %p, i64 %iv.ext
+ %dst = getelementptr inbounds nuw i8, ptr %p, i64 %sub
+ %val = load i8, ptr %src, align 1
+ store i8 %val, ptr %dst, align 1
+ %iv.next = add i16 %iv, 1
+ %cmp = icmp slt i16 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @move_up_dec_loop_sext(i16 %n, ptr %p) {
+; CHECK-LABEL: define void @move_up_dec_loop_sext(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT: [[TMP1:%.*]] = add i16 [[N]], -1
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw i16 [[N]], -1
+; CHECK-NEXT: [[SMIN:%.*]] = call i16 @llvm.smin.i16(i16 [[TMP8]], i16 0)
+; CHECK-NEXT: [[TMP2:%.*]] = sub i16 [[TMP1]], [[SMIN]]
+; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[TMP2]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP0]], [[TMP3]]
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP0]], -1
+; CHECK-NEXT: [[TMP6:%.*]] = sub i64 [[TMP5]], [[TMP3]]
+; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[SCEVGEP]], ptr align 1 [[SCEVGEP1]], i64 [[TMP7]], i1 false)
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[N]], %[[ENTRY]] ]
+; CHECK-NEXT: [[IV_EXT:%.*]] = sext i16 [[IV]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub nuw nsw i64 [[IV_EXT]], 1
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[SUB]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IV_EXT]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i16 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ %iv.next, %loop ], [ %n, %entry ]
+ %iv.ext = sext i16 %iv to i64
+ %sub = sub nuw nsw i64 %iv.ext, 1
+ %src = getelementptr inbounds nuw i8, ptr %p, i64 %sub
+ %dst = getelementptr inbounds nuw i8, ptr %p, i64 %iv.ext
+ %val = load i8, ptr %src, align 1
+ store i8 %val, ptr %dst, align 1
+ %iv.next = add nsw i16 %iv, -1
+ %cmp = icmp sgt i16 %iv.next, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @move_down_inc_loop_sext(i16 %n, ptr %p) {
+; CHECK-LABEL: define void @move_down_inc_loop_sext(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[P]], i64 -1
+; CHECK-NEXT: [[SMAX:%.*]] = call i16 @llvm.smax.i16(i16 [[N]], i16 1)
+; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i16 [[SMAX]] to i64
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[SCEVGEP]], ptr align 1 [[P]], i64 [[TMP0]], i1 false)
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[IV_EXT:%.*]] = sext i16 [[IV]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub nuw nsw i64 [[IV_EXT]], 1
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IV_EXT]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[SUB]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ %iv.next, %loop ], [ 0, %entry ]
+ %iv.ext = sext i16 %iv to i64
+ %sub = sub nuw nsw i64 %iv.ext, 1
+ %src = getelementptr inbounds nuw i8, ptr %p, i64 %iv.ext
+ %dst = getelementptr inbounds nuw i8, ptr %p, i64 %sub
+ %val = load i8, ptr %src, align 1
+ store i8 %val, ptr %dst, align 1
+ %iv.next = add nsw i16 %iv, 1
+ %cmp = icmp slt i16 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; In the following tests the load reads from the location that was stored to in
+; the previous loop iteration, so using memmove is not valid.
+
+define void @move_down_dec_loop_zext(i16 %n, ptr %p) {
+; CHECK-LABEL: define void @move_down_dec_loop_zext(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[N]], %[[ENTRY]] ]
+; CHECK-NEXT: [[IV_EXT:%.*]] = zext nneg i16 [[IV]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub nuw nsw i64 [[IV_EXT]], 1
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IV_EXT]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[SUB]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: store i8 [[VAL]], ptr [[DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], -1
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i16 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ %iv.next, %loop ], [ %n, %entry ]
+ %iv.ext = zext nneg i16 %iv to i64
+ %sub = sub nuw nsw i64 %iv.ext, 1
+ %src = getelementptr inbounds nuw i8, ptr %p, i64 %iv.ext
+ %dst = getelementptr inbounds nuw i8, ptr %p, i64 %sub
+ %val = load i8, ptr %src, align 1
+ store i8 %val, ptr %dst, align 1
+ %iv.next = add i16 %iv, -1
+ %cmp = icmp sgt i16 %iv.next, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @move_up_inc_loop_zext(i16 %n, ptr %p) {
+; CHECK-LABEL: define void @move_up_inc_loop_zext(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[IV_EXT:%.*]] = zext nneg i16 [[IV]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub nuw nsw i64 [[IV_EXT]], 1
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[SUB]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IV_EXT]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: store i8 [[VAL]], ptr [[DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add i16 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ %iv.next, %loop ], [ 0, %entry ]
+ %iv.ext = zext nneg i16 %iv to i64
+ %sub = sub nuw nsw i64 %iv.ext, 1
+ %src = getelementptr inbounds nuw i8, ptr %p, i64 %sub
+ %dst = getelementptr inbounds nuw i8, ptr %p, i64 %iv.ext
+ %val = load i8, ptr %src, align 1
+ store i8 %val, ptr %dst, align 1
+ %iv.next = add i16 %iv, 1
+ %cmp = icmp slt i16 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @move_down_dec_loop_sext(i16 %n, ptr %p) {
+; CHECK-LABEL: define void @move_down_dec_loop_sext(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[N]], %[[ENTRY]] ]
+; CHECK-NEXT: [[IV_EXT:%.*]] = sext i16 [[IV]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub nuw nsw i64 [[IV_EXT]], 1
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IV_EXT]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[SUB]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: store i8 [[VAL]], ptr [[DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i16 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ %iv.next, %loop ], [ %n, %entry ]
+ %iv.ext = sext i16 %iv to i64
+ %sub = sub nuw nsw i64 %iv.ext, 1
+ %src = getelementptr inbounds nuw i8, ptr %p, i64 %iv.ext
+ %dst = getelementptr inbounds nuw i8, ptr %p, i64 %sub
+ %val = load i8, ptr %src, align 1
+ store i8 %val, ptr %dst, align 1
+ %iv.next = add nsw i16 %iv, -1
+ %cmp = icmp sgt i16 %iv.next, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @move_up_inc_loop_sext(i16 %n, ptr %p) {
+; CHECK-LABEL: define void @move_up_inc_loop_sext(
+; CHECK-SAME: i16 [[N:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i16 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[IV_EXT:%.*]] = sext i16 [[IV]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub nuw nsw i64 [[IV_EXT]], 1
+; CHECK-NEXT: [[SRC:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[SUB]]
+; CHECK-NEXT: [[DST:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[IV_EXT]]
+; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[SRC]], align 1
+; CHECK-NEXT: store i8 [[VAL]], ptr [[DST]], align 1
+; CHECK-NEXT: [[IV_NEXT]] = add nsw i16 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i16 [ %iv.next, %loop ], [ 0, %entry ]
+ %iv.ext = sext i16 %iv to i64
+ %sub = sub nuw nsw i64 %iv.ext, 1
+ %src = getelementptr inbounds nuw i8, ptr %p, i64 %sub
+ %dst = getelementptr inbounds nuw i8, ptr %p, i64 %iv.ext
+ %val = load i8, ptr %src, align 1
+ store i8 %val, ptr %dst, align 1
+ %iv.next = add nsw i16 %iv, 1
+ %cmp = icmp slt i16 %iv.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret void
+}
>From d179ba8462eb3b071cb87879d4918b60d4aaf64f Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn at arm.com>
Date: Tue, 28 Jul 2026 17:37:37 +0100
Subject: [PATCH 2/3] Use OffVal as APInt, instead of extracting the SExt value
This avoids the potential problem of the bitwidth of Off being > 64
---
llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index a17d909567602..7b0a6ece4fb60 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1304,7 +1304,7 @@ class MemmoveVerifier {
// an underlying pointer.
if (!Off || !BasePtr)
return false;
- int64_t OffVal = Off->getValue()->getSExtValue();
+ const APInt &OffVal = Off->getAPInt();
// If null is defined then the base pointer can't be null
if (TheLoad.getParent()->getParent()->nullPointerIsDefined() &&
isa<ConstantPointerNull>(BasePtr->getValue()))
@@ -1319,7 +1319,7 @@ class MemmoveVerifier {
}
// Ensure that LoadBasePtr is after StoreBasePtr or before StoreBasePtr
// for negative stride. LoadBasePtr shouldn't overlap with StoreBasePtr.
- if (IsNegStride ? OffVal < LoadSize : OffVal > -LoadSize)
+ if (IsNegStride ? OffVal.slt(LoadSize) : OffVal.sgt(-LoadSize))
return false;
return true;
}
>From b5595eec13746f88a6a1ded23dd3dcb781776943 Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn at arm.com>
Date: Wed, 29 Jul 2026 12:05:22 +0100
Subject: [PATCH 3/3] Adjust based on review comments
Remove DataLayout argument
Add comment
---
llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 7b0a6ece4fb60..dc59673073ba8 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1291,8 +1291,8 @@ namespace {
class MemmoveVerifier {
public:
explicit MemmoveVerifier(const SCEV &LoadStart, const SCEV &StoreStart,
- const DataLayout &DL, ScalarEvolution &SE)
- : DL(DL),
+ ScalarEvolution &SE)
+ : DL(SE.getDataLayout()),
Off(dyn_cast<SCEVConstant>(SE.getMinusSCEV(&StoreStart, &LoadStart))),
BasePtr(dyn_cast<SCEVUnknown>(SE.getPointerBase(&StoreStart))),
IsSameObject(Off != nullptr) {}
@@ -1311,6 +1311,7 @@ class MemmoveVerifier {
return false;
int64_t LoadSize;
if (IsMemCpy) {
+ // memcpy is equivalent to a sequence of byte loads and stores
LoadSize = 1;
} else {
LoadSize = DL.getTypeSizeInBits(TheLoad.getType()).getFixedValue() / 8;
@@ -1438,7 +1439,7 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
// If the store is a memcpy instruction, we must check if it will write to
// the load memory locations. So remove it from the ignored stores.
- MemmoveVerifier Verifier(*LdStart, *StrStart, *DL, *SE);
+ MemmoveVerifier Verifier(*LdStart, *StrStart, *SE);
if (IsMemCpy && !Verifier.IsSameObject)
IgnoredInsts.erase(TheStore);
if (mayLoopAccessLocation(LoadBasePtr, ModRefInfo::Mod, CurLoop, BECount,
More information about the llvm-commits
mailing list