[llvm] [InstCombine][CVP] Fold length-one memset with variable fill (PR #213240)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 05:22:29 PDT 2026
https://github.com/hsnaveen2u updated https://github.com/llvm/llvm-project/pull/213240
>From a6749c6e84db521f0060c896763f384fbebbe289 Mon Sep 17 00:00:00 2001
From: Naveen <naveen.siddegowda at oss.qualcomm.com>
Date: Fri, 31 Jul 2026 03:01:29 -0700
Subject: [PATCH 1/2] Fold length-one memset with variable fill
A one-byte memset does not require replicating the fill byte into a
wider integer value. Allow a nonconstant i8 fill value to be stored
directly when the memset length is one.
Keep the existing constant-fill handling for lengths 1, 2, 4 and 8.
Preserve volatility and unordered atomic ordering on the generated
store.
Allow volatile AnyMemSetInst operations to reach SimplifyAnyMemSet
while continuing to block other volatile memory-intrinsic
transformations.
This is the InstCombine prerequisite for #213027.
Signed-off-by: Naveen <naveen.siddegowda at oss.qualcomm.com>
---
.../InstCombine/InstCombineCalls.cpp | 49 +++++++++++----
.../InstCombine/memset-variable-fill.ll | 59 +++++++++++++++++++
2 files changed, 97 insertions(+), 11 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/memset-variable-fill.ll
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 5ee5009bd0262..5c5522cadd0b9 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -221,6 +221,24 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
}
Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
+ ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength());
+ Value *Fill = MI->getValue();
+
+ // Keep volatile memset scalarization limited to the single-byte case
+ // where the replacement is exactly one volatile byte store.
+ if (MI->isVolatile()) {
+ if (!LenC || !LenC->isOne() || !Fill->getType()->isIntegerTy(8))
+ return nullptr;
+
+ StoreInst *S = Builder.CreateStore(Fill, MI->getDest(), true);
+ S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
+ S->setAlignment(MI->getDestAlign().valueOrOne());
+
+ // Set the size of the copy to 0 and will be deleted on the next iteration.
+ MI->setLength((uint64_t)0);
+ return MI;
+ }
+
const Align KnownAlignment =
getKnownAlignment(MI->getDest(), DL, MI, &AC, &DT);
MaybeAlign MemSetAlign = MI->getDestAlign();
@@ -247,10 +265,8 @@ Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
return MI;
}
- // Extract the length and alignment and fill if they are constant.
- ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength());
- ConstantInt *FillC = dyn_cast<ConstantInt>(MI->getValue());
- if (!LenC || !FillC || !FillC->getType()->isIntegerTy(8))
+ // Extract the length and validate the fill type.
+ if (!LenC || !Fill->getType()->isIntegerTy(8))
return nullptr;
const uint64_t Len = LenC->getLimitedValue();
assert(Len && "0-sized memory setting should be removed already.");
@@ -267,14 +283,22 @@ Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
Value *Dest = MI->getDest();
- // Extract the fill value and store.
- Constant *FillVal = ConstantInt::get(
- MI->getContext(), APInt::getSplat(Len * 8, FillC->getValue()));
+ // Extract the fill value and store. A one-byte memset does not need
+ // replication so a nonconstant i8 fill can be stored directly.
+ Value *FillVal;
+ if (auto *FillC = dyn_cast<ConstantInt>(Fill))
+ FillVal = ConstantInt::get(MI->getContext(),
+ APInt::getSplat(Len * 8, FillC->getValue()));
+ else if (Len == 1)
+ FillVal = Fill;
+ else
+ return nullptr;
+
StoreInst *S = Builder.CreateStore(FillVal, Dest, MI->isVolatile());
S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(S)) {
- if (llvm::is_contained(DbgAssign->location_ops(), FillC))
- DbgAssign->replaceVariableLocationOp(FillC, FillVal);
+ if (llvm::is_contained(DbgAssign->location_ops(), Fill))
+ DbgAssign->replaceVariableLocationOp(Fill, FillVal);
}
S->setAlignment(Alignment);
@@ -2024,8 +2048,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
}
}
- // No other transformations apply to volatile transfers.
- if (MI->isVolatile())
+ // Apart from memset-to-store scalarization below no other transformations
+ // apply to volatile transfers.
+ if (MI->isVolatile() && !isa<AnyMemSetInst>(MI))
return nullptr;
if (AnyMemTransferInst *MTI = dyn_cast<AnyMemTransferInst>(MI)) {
@@ -2050,6 +2075,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
} else if (auto *MSI = dyn_cast<AnyMemSetInst>(MI)) {
if (Instruction *I = SimplifyAnyMemSet(MSI))
return I;
+ if (MI->isVolatile())
+ return nullptr;
}
// If src/dest is null, this memory intrinsic must be a noop.
diff --git a/llvm/test/Transforms/InstCombine/memset-variable-fill.ll b/llvm/test/Transforms/InstCombine/memset-variable-fill.ll
new file mode 100644
index 0000000000000..b5547071a05d5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memset-variable-fill.ll
@@ -0,0 +1,59 @@
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
+declare void @llvm.memset.p1.i64(ptr addrspace(1) nocapture writeonly, i8, i64, i1 immarg)
+declare void @llvm.memset.element.unordered.atomic.p0.i64(ptr nocapture writeonly, i8, i64, i32 immarg)
+
+define void @variable_fill_len1(ptr %dst, i8 %value) {
+; CHECK-LABEL: define void @variable_fill_len1(
+; CHECK-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]]) {
+; CHECK-NEXT: store i8 [[VALUE]], ptr [[DST]], align 1
+; CHECK-NEXT: ret void
+ call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value, i64 1, i1 false)
+ ret void
+}
+
+define void @variable_fill_len1_volatile(ptr %dst, i8 %value) {
+; CHECK-LABEL: define void @variable_fill_len1_volatile(
+; CHECK-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]]) {
+; CHECK-NEXT: store volatile i8 [[VALUE]], ptr [[DST]], align 1
+; CHECK-NEXT: ret void
+ call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value, i64 1, i1 true)
+ ret void
+}
+
+define void @variable_fill_len1_align8(ptr %dst, i8 %value) {
+; CHECK-LABEL: define void @variable_fill_len1_align8(
+; CHECK-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]]) {
+; CHECK-NEXT: store i8 [[VALUE]], ptr [[DST]], align 8
+; CHECK-NEXT: ret void
+ call void @llvm.memset.p0.i64(ptr align 8 %dst, i8 %value, i64 1, i1 false)
+ ret void
+}
+
+define void @variable_fill_len1_addrspace(ptr addrspace(1) %dst, i8 %value) {
+; CHECK-LABEL: define void @variable_fill_len1_addrspace(
+; CHECK-SAME: ptr addrspace(1) [[DST:%.*]], i8 [[VALUE:%.*]]) {
+; CHECK-NEXT: store i8 [[VALUE]], ptr addrspace(1) [[DST]], align 1
+; CHECK-NEXT: ret void
+ call void @llvm.memset.p1.i64(ptr addrspace(1) align 1 %dst, i8 %value, i64 1, i1 false)
+ ret void
+}
+
+define void @variable_fill_len1_atomic(ptr %dst, i8 %value) {
+; CHECK-LABEL: define void @variable_fill_len1_atomic(
+; CHECK-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]]) {
+; CHECK-NEXT: store atomic i8 [[VALUE]], ptr [[DST]] unordered, align 1
+; CHECK-NEXT: ret void
+ call void @llvm.memset.element.unordered.atomic.p0.i64(ptr align 1 %dst, i8 %value, i64 1, i32 1)
+ ret void
+}
+
+define void @variable_fill_len2(ptr %dst, i8 %value) {
+; CHECK-LABEL: define void @variable_fill_len2(
+; CHECK-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]]) {
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr {{.*}}[[DST]], i8 [[VALUE]], i64 2, i1 false)
+; CHECK-NEXT: ret void
+ call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value, i64 2, i1 false)
+ ret void
+}
>From 21fd1ed6252db9a37e91f9a0962805b0ac508fc8 Mon Sep 17 00:00:00 2001
From: Naveen <naveen.siddegowda at oss.qualcomm.com>
Date: Mon, 3 Aug 2026 13:12:41 +0900
Subject: [PATCH 2/2] Guard memset with length in [0, 1]
Use computeKnownBits to identify nonconstant memset lengths whose
possible values are limited to zero and one. The check is integrated
into the existing instruction loop in foldUnusualPatterns.
Insert a conditional branch around the memset and specialize the
executed path to a constant length of one. A following InstCombine pass
can then replace it with a byte store, including for a nonconstant fill
value.
Do not transform wider ranges such as [0, 2].
Fixes #213027.
Assisted by GPT-5
Signed-off-by: Naveen <naveen.siddegowda at oss.qualcomm.com>
---
.../AggressiveInstCombine.cpp | 34 ++++++
.../AggressiveInstCombine/memset.ll | 104 ++++++++++++++++++
2 files changed, 138 insertions(+)
create mode 100644 llvm/test/Transforms/AggressiveInstCombine/memset.ll
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index fc8a079ed41d4..a926a065d3b1f 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -29,6 +29,7 @@
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ProfDataUtils.h"
@@ -57,6 +58,7 @@ STATISTIC(NumSelectCTTZFolded,
"Number of select-based split cttz patterns folded");
STATISTIC(NumSelectCTLZFolded,
"Number of select-based split ctlz patterns folded");
+STATISTIC(NumMemSetsGuarded, "Number of memsets guarded for a zero length");
static cl::opt<unsigned> MaxInstrsToScan(
"aggressive-instcombine-max-scan-instrs", cl::init(64), cl::Hidden,
@@ -2467,6 +2469,7 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
TargetLibraryInfo &TLI, AliasAnalysis &AA,
AssumptionCache &AC, bool &MadeCFGChange) {
bool MadeChange = false;
+ SmallVector<MemSetInst *, 4> MemSetsToGuard;
for (BasicBlock &BB : F) {
// Ignore unreachable basic blocks.
if (!DT.isReachableFromEntry(&BB))
@@ -2492,6 +2495,18 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
MadeChange |= foldPatternedLoads(I, DL);
MadeChange |= foldICmpOrChain(I, DL, TTI, AA, DT);
MadeChange |= foldMulHigh(I);
+ // Collect nonconstant-length memsets whose length is known to be in
+ // [0, 1]; the guard is inserted after the main loop to avoid
+ // invalidating the instruction iterator.
+ if (auto *MI = dyn_cast<MemSetInst>(&I))
+ if (!isa<ConstantInt>(MI->getLength())) {
+ SimplifyQuery SQ(DL, &TLI, &DT, &AC, nullptr,
+ /*UseInstrInfo=*/true, /*CanUseUndef=*/false);
+ KnownBits KnownLen =
+ computeKnownBits(MI->getLength(), SQ.getWithInstruction(MI));
+ if (KnownLen.getMaxValue().isOne())
+ MemSetsToGuard.push_back(MI);
+ }
// NOTE: This function introduces erasing of the instruction `I`, so it
// needs to be called at the end of this sequence, otherwise we may make
// bugs.
@@ -2502,6 +2517,25 @@ static bool foldUnusualPatterns(Function &F, DominatorTree &DT,
MadeChange |= foldConsecutiveStores(BB, DL, TTI, AA);
}
+ // Guard memsets whose nonconstant length is known to be in [0, 1]. All
+ // candidates are collected above before any CFG changes are made.
+ if (!MemSetsToGuard.empty()) {
+ DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+ for (MemSetInst *MI : MemSetsToGuard) {
+ IRBuilder<> B(MI);
+ B.SetCurrentDebugLocation(MI->getDebugLoc());
+ Value *IsNonZero = B.CreateIsNotNull(MI->getLength(), "memset.notzero");
+ Instruction *ThenTerm = SplitBlockAndInsertIfThen(
+ IsNonZero, MI->getIterator(), /*Unreachable=*/false,
+ /*BranchWeights=*/nullptr, &DTU);
+ MI->moveBefore(ThenTerm->getIterator());
+ MI->setLength((uint64_t)1);
+ ++NumMemSetsGuarded;
+ }
+ MadeChange = true;
+ MadeCFGChange = true;
+ }
+
// We're done with transforms, so remove dead instructions.
if (MadeChange)
for (BasicBlock &BB : F)
diff --git a/llvm/test/Transforms/AggressiveInstCombine/memset.ll b/llvm/test/Transforms/AggressiveInstCombine/memset.ll
new file mode 100644
index 0000000000000..6cf8a27d14460
--- /dev/null
+++ b/llvm/test/Transforms/AggressiveInstCombine/memset.ll
@@ -0,0 +1,104 @@
+; RUN: opt -passes=aggressive-instcombine -S < %s | FileCheck %s --check-prefix=AIC
+; RUN: opt -passes='aggressive-instcombine,instcombine' -S < %s | FileCheck %s --check-prefix=COMBINED
+
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
+
+define void @range_0_1(ptr %dst, i8 %value, i64 %n) {
+; AIC-LABEL: define void @range_0_1(
+; AIC-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]], i64 [[N:%.*]]) {
+; AIC: entry:
+; AIC-NEXT: [[LEN:%.*]] = and i64 [[N]], 1
+; AIC-NEXT: [[MEMSET_NOTZERO:%.*]] = icmp ne i64 [[LEN]], 0
+; AIC-NEXT: br i1 [[MEMSET_NOTZERO]], label %[[DO_MEMSET:.*]], label %[[END:.*]]
+; AIC: [[DO_MEMSET]]:
+; AIC-NEXT: call void @llvm.memset.p0.i64(ptr align 1 [[DST]], i8 [[VALUE]], i64 1, i1 false)
+; AIC-NEXT: br label %[[END]]
+; AIC: [[END]]:
+; AIC-NEXT: ret void
+;
+; COMBINED-LABEL: define void @range_0_1(
+; COMBINED-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]], i64 [[N:%.*]]) {
+; COMBINED: [[LEN:%.*]] = and i64 [[N]], 1
+; COMBINED: icmp {{eq|ne}} i64 [[LEN]], 0
+; COMBINED: br i1
+; COMBINED: store i8 [[VALUE]], ptr [[DST]], align 1
+; COMBINED-NOT: call void @llvm.memset
+; COMBINED: ret void
+entry:
+ %len = and i64 %n, 1
+ call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value, i64 %len, i1 false)
+ ret void
+}
+
+define void @range_0_1_zext(ptr %dst, i8 %value, i32 %n) {
+; AIC-LABEL: define void @range_0_1_zext(
+; AIC-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]], i32 [[N:%.*]]) {
+; AIC: entry:
+; AIC-NEXT: [[MASKED:%.*]] = and i32 [[N]], 1
+; AIC-NEXT: [[LEN:%.*]] = zext i32 [[MASKED]] to i64
+; AIC-NEXT: [[MEMSET_NOTZERO:%.*]] = icmp ne i64 [[LEN]], 0
+; AIC-NEXT: br i1 [[MEMSET_NOTZERO]], label %[[DO_MEMSET:.*]], label %[[END:.*]]
+; AIC: [[DO_MEMSET]]:
+; AIC-NEXT: call void @llvm.memset.p0.i64(ptr align 1 [[DST]], i8 [[VALUE]], i64 1, i1 false)
+; AIC-NEXT: br label %[[END]]
+; AIC: [[END]]:
+; AIC-NEXT: ret void
+;
+; COMBINED-LABEL: define void @range_0_1_zext(
+; COMBINED-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]], i32 [[N:%.*]]) {
+; COMBINED: [[MASKED:%.*]] = and i32 [[N]], 1
+; COMBINED: icmp {{eq|ne}} i32 [[MASKED]], 0
+; COMBINED: br i1
+; COMBINED: store i8 [[VALUE]], ptr [[DST]], align 1
+; COMBINED-NOT: call void @llvm.memset
+; COMBINED: ret void
+entry:
+ %masked = and i32 %n, 1
+ %len = zext i32 %masked to i64
+ call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value, i64 %len, i1 false)
+ ret void
+}
+
+define void @range_0_1_volatile(ptr %dst, i8 %value, i64 %n) {
+; AIC-LABEL: define void @range_0_1_volatile(
+; AIC-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]], i64 [[N:%.*]]) {
+; AIC: entry:
+; AIC-NEXT: [[LEN:%.*]] = and i64 [[N]], 1
+; AIC-NEXT: [[MEMSET_NOTZERO:%.*]] = icmp ne i64 [[LEN]], 0
+; AIC-NEXT: br i1 [[MEMSET_NOTZERO]], label %[[DO_MEMSET:.*]], label %[[END:.*]]
+; AIC: [[DO_MEMSET]]:
+; AIC-NEXT: call void @llvm.memset.p0.i64(ptr align 1 [[DST]], i8 [[VALUE]], i64 1, i1 true)
+; AIC-NEXT: br label %[[END]]
+; AIC: [[END]]:
+; AIC-NEXT: ret void
+;
+; COMBINED-LABEL: define void @range_0_1_volatile(
+; COMBINED-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]], i64 [[N:%.*]]) {
+; COMBINED: br i1
+; COMBINED: store volatile i8 [[VALUE]], ptr [[DST]], align 1
+; COMBINED-NOT: call void @llvm.memset
+; COMBINED: ret void
+entry:
+ %len = and i64 %n, 1
+ call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value, i64 %len, i1 true)
+ ret void
+}
+
+define void @range_0_2(ptr %dst, i8 %value, i64 %n) {
+; AIC-LABEL: define void @range_0_2(
+; AIC-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]], i64 [[N:%.*]]) {
+; AIC: entry:
+; AIC-NEXT: [[LEN:%.*]] = urem i64 [[N]], 3
+; AIC-NEXT: call void @llvm.memset.p0.i64(ptr align 1 [[DST]], i8 [[VALUE]], i64 [[LEN]], i1 false)
+; AIC-NEXT: ret void
+;
+; COMBINED-LABEL: define void @range_0_2(
+; COMBINED-SAME: ptr [[DST:%.*]], i8 [[VALUE:%.*]], i64 [[N:%.*]]) {
+; COMBINED: [[LEN:%.*]] = urem i64 [[N]], 3
+; COMBINED-NEXT: call void @llvm.memset.p0.i64(ptr align 1 [[DST]], i8 [[VALUE]], i64 [[LEN]], i1 false)
+; COMBINED-NEXT: ret void
+entry:
+ %len = urem i64 %n, 3
+ call void @llvm.memset.p0.i64(ptr align 1 %dst, i8 %value, i64 %len, i1 false)
+ ret void
+}
More information about the llvm-commits
mailing list