[llvm] [MemCpyOpt] Drop dead `memmove` calls on `memset`'d source data (PR #101930)
Antonio Frighetto via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 3 11:04:03 PDT 2024
https://github.com/antoniofrighetto updated https://github.com/llvm/llvm-project/pull/101930
>From d25e401ed14c8ec42b1d74a3c70932a1460da3a1 Mon Sep 17 00:00:00 2001
From: Antonio Frighetto <me at antoniofrighetto.com>
Date: Mon, 5 Aug 2024 09:35:37 +0200
Subject: [PATCH 1/2] [MemCpyOpt] Introduce test for PR101930 (NFC)
---
.../memset-memmove-redundant-memmove.ll | 89 +++++++++++++++++++
1 file changed, 89 insertions(+)
create mode 100644 llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll
diff --git a/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll b/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll
new file mode 100644
index 00000000000000..d91b4023c15e12
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=memcpyopt -S %s -verify-memoryssa | FileCheck %s
+
+; Redundant memmove.
+define i32 @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT: [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 [[ARRAY]], i8 0, i64 104, i1 false)
+; CHECK-NEXT: [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 [[ARRAY]], ptr noundef nonnull align 4 [[ARRAY_IDX]], i64 100, i1 false)
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[ARRAY]], align 16
+; CHECK-NEXT: ret i32 [[VAL]]
+;
+ %array = alloca [26 x i32], align 16
+ call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 %array, i8 0, i64 104, i1 false)
+ %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+ call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 %array, ptr noundef nonnull align 4 %array.idx, i64 100, i1 false)
+ %val = load i32, ptr %array, align 16
+ ret i32 %val
+}
+
+; Used memmmove, buffer is reset to zero.
+define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 [[ARRAY]], i8 0, i64 104, i1 false)
+; CHECK-NEXT: [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT: store i32 1, ptr [[ARRAY_IDX]], align 4
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 [[ARRAY]], ptr noundef nonnull align 4 [[ARRAY_IDX]], i64 100, i1 false)
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[ARRAY_IDX]], align 4
+; CHECK-NEXT: ret i32 [[VAL]]
+;
+ %array = alloca [26 x i32], align 16
+ call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 %array, i8 0, i64 104, i1 false)
+ %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+ store i32 1, ptr %array.idx
+ call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 %array, ptr noundef nonnull align 4 %array.idx, i64 100, i1 false)
+ %val = load i32, ptr %array.idx, align 4
+ ret i32 %val
+}
+
+; Used memmove, buffer clobbered by opaque.
+define i32 @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: [[ARRAY:%.*]] = alloca [25 x i32], align 16
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 dereferenceable(100) [[ARRAY]], i8 0, i64 100, i1 false)
+; CHECK-NEXT: call void @opaque(ptr noundef nonnull [[ARRAY]])
+; CHECK-NEXT: [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) [[ARRAY]], ptr noundef nonnull align 4 dereferenceable(96) [[ARRAY_IDX]], i64 96, i1 false)
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[ARRAY]], align 16
+; CHECK-NEXT: ret i32 [[VAL]]
+;
+ %array = alloca [25 x i32], align 16
+ call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 dereferenceable(100) %array, i8 0, i64 100, i1 false)
+ call void @opaque(ptr noundef nonnull %array)
+ %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+ call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %array, ptr noundef nonnull align 4 dereferenceable(96) %array.idx, i64 96, i1 false)
+ %val = load i32, ptr %array, align 16
+ ret i32 %val
+}
+
+; Redundant memmove, but not within the same basic block.
+define i32 @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ARRAY:%.*]] = alloca [26 x i32], align 16
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 [[ARRAY]], i8 0, i64 104, i1 false)
+; CHECK-NEXT: [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
+; CHECK-NEXT: br label [[USE:%.*]]
+; CHECK: use:
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 [[ARRAY]], ptr noundef nonnull align 4 [[ARRAY_IDX]], i64 100, i1 false)
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[ARRAY]], align 16
+; CHECK-NEXT: ret i32 [[VAL]]
+;
+entry:
+ %array = alloca [26 x i32], align 16
+ call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 %array, i8 0, i64 104, i1 false)
+ %array.idx = getelementptr inbounds i8, ptr %array, i64 4
+ br label %use
+
+use: ; preds = %entry
+ call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 %array, ptr noundef nonnull align 4 %array.idx, i64 100, i1 false)
+ %val = load i32, ptr %array, align 16
+ ret i32 %val
+}
+
+declare void @opaque(ptr)
+declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1)
+declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
>From ff821fb3e2a9d99e5679b690f05c77a9f04e4f57 Mon Sep 17 00:00:00 2001
From: Antonio Frighetto <me at antoniofrighetto.com>
Date: Mon, 5 Aug 2024 09:36:22 +0200
Subject: [PATCH 2/2] [MemCpyOpt] Drop dead `memmove` calls on `memset`'d
source data
When a memmove happens to clobber source data, and such data have
been previously memset'd, the memmove may be redundant.
---
.../llvm/Transforms/Scalar/MemCpyOptimizer.h | 3 +-
.../lib/Transforms/Scalar/MemCpyOptimizer.cpp | 49 +++++++++++++++++--
.../memset-memmove-redundant-memmove.ll | 1 -
3 files changed, 48 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h b/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
index 8ed03d7f3ddbff..63e6fa8e805eca 100644
--- a/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
+++ b/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
@@ -66,7 +66,7 @@ class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
BasicBlock::iterator &BBI);
bool processMemSet(MemSetInst *SI, BasicBlock::iterator &BBI);
bool processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI);
- bool processMemMove(MemMoveInst *M);
+ bool processMemMove(MemMoveInst *M, BasicBlock::iterator &BBI);
bool performCallSlotOptzn(Instruction *cpyLoad, Instruction *cpyStore,
Value *cpyDst, Value *cpySrc, TypeSize cpyLen,
Align cpyAlign, BatchAAResults &BAA,
@@ -85,6 +85,7 @@ class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
bool performStackMoveOptzn(Instruction *Load, Instruction *Store,
AllocaInst *DestAlloca, AllocaInst *SrcAlloca,
TypeSize Size, BatchAAResults &BAA);
+ bool isMemMoveMemSetDependency(MemMoveInst *M);
void eraseInstruction(Instruction *I);
bool iterateOnFunction(Function &F);
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 3f15fa2163d270..3b1e507f244df4 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -69,6 +69,7 @@ static cl::opt<bool> EnableMemCpyOptWithoutLibcalls(
cl::desc("Enable memcpyopt even when libcalls are disabled"));
STATISTIC(NumMemCpyInstr, "Number of memcpy instructions deleted");
+STATISTIC(NumMemMoveInstr, "Number of memmove instructions deleted");
STATISTIC(NumMemSetInfer, "Number of memsets inferred");
STATISTIC(NumMoveToCpy, "Number of memmoves converted to memcpy");
STATISTIC(NumCpyToSet, "Number of memcpys converted to memset");
@@ -1853,12 +1854,54 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
return false;
}
+/// Memmove calls with overlapping src/dest buffers that come after a memset may
+/// be removed.
+bool MemCpyOptPass::isMemMoveMemSetDependency(MemMoveInst *M) {
+ MemoryUseOrDef *MemMoveAccess = MSSA->getMemoryAccess(M);
+ if (!MemMoveAccess)
+ return false;
+
+ BatchAAResults BAA(*AA);
+ MemoryAccess *FirstDef = MemMoveAccess->getDefiningAccess();
+ MemoryLocation SourceLoc = MemoryLocation::getForSource(M);
+ MemoryAccess *SourceClobber =
+ MSSA->getWalker()->getClobberingMemoryAccess(FirstDef, SourceLoc, BAA);
+
+ MemSetInst *MS = nullptr;
+ if (auto *Def = dyn_cast<MemoryDef>(SourceClobber))
+ MS = dyn_cast_or_null<MemSetInst>(Def->getMemoryInst());
+
+ // Limit the memset to be within the same basic block.
+ if (!MS || MS->getParent() != M->getParent())
+ return false;
+
+ // The destination buffer must have been memset'd.
+ if (!BAA.isMustAlias(MS->getDest(), M->getDest()))
+ return false;
+
+ // No clobbering writes in between.
+ if (writtenBetween(MSSA, BAA, SourceLoc, MSSA->getMemoryAccess(MS),
+ MemMoveAccess))
+ return false;
+ return true;
+}
+
/// Transforms memmove calls to memcpy calls when the src/dst are guaranteed
/// not to alias.
-bool MemCpyOptPass::processMemMove(MemMoveInst *M) {
+bool MemCpyOptPass::processMemMove(MemMoveInst *M, BasicBlock::iterator &BBI) {
// See if the source could be modified by this memmove potentially.
- if (isModSet(AA->getModRefInfo(M, MemoryLocation::getForSource(M))))
+ if (isModSet(AA->getModRefInfo(M, MemoryLocation::getForSource(M)))) {
+ // On the off-chance the memmove clobbers src with previously memset'd
+ // bytes, the memmove may be redundant.
+ if (!M->isVolatile() && isMemMoveMemSetDependency(M)) {
+ LLVM_DEBUG(dbgs() << "Removed redundant memmove.\n");
+ ++BBI;
+ eraseInstruction(M);
+ ++NumMemMoveInstr;
+ return true;
+ }
return false;
+ }
LLVM_DEBUG(dbgs() << "MemCpyOptPass: Optimizing memmove -> memcpy: " << *M
<< "\n");
@@ -2067,7 +2110,7 @@ bool MemCpyOptPass::iterateOnFunction(Function &F) {
else if (auto *M = dyn_cast<MemCpyInst>(I))
RepeatInstruction = processMemCpy(M, BI);
else if (auto *M = dyn_cast<MemMoveInst>(I))
- RepeatInstruction = processMemMove(M);
+ RepeatInstruction = processMemMove(M, BI);
else if (auto *CB = dyn_cast<CallBase>(I)) {
for (unsigned i = 0, e = CB->arg_size(); i != e; ++i) {
if (CB->isByValArgument(i))
diff --git a/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll b/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll
index d91b4023c15e12..c6fdaa3489a565 100644
--- a/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memset-memmove-redundant-memmove.ll
@@ -7,7 +7,6 @@ define i32 @test() {
; CHECK-NEXT: [[ARRAY:%.*]] = alloca [26 x i32], align 16
; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 [[ARRAY]], i8 0, i64 104, i1 false)
; CHECK-NEXT: [[ARRAY_IDX:%.*]] = getelementptr inbounds i8, ptr [[ARRAY]], i64 4
-; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 16 [[ARRAY]], ptr noundef nonnull align 4 [[ARRAY_IDX]], i64 100, i1 false)
; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[ARRAY]], align 16
; CHECK-NEXT: ret i32 [[VAL]]
;
More information about the llvm-commits
mailing list