[llvm] [PreISelIntrinsicLowering] Lower bounded memcpy/memmove to masked loa… (PR #212710)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 07:28:43 PDT 2026
https://github.com/Harishankar14 updated https://github.com/llvm/llvm-project/pull/212710
>From f4e467008aa3d3561be00a8f51e735b20049389a Mon Sep 17 00:00:00 2001
From: Harishankar <harishankarpp7 at gmail.com>
Date: Wed, 29 Jul 2026 14:04:03 +0530
Subject: [PATCH] [PreISelIntrinsicLowering] Lower bounded memcpy/memmove to
masked load/store
When a memcpy or memmove has a non-constant length that a preceding
llvm.assume proves is small ( assume(size <= 64)), emit a single
masked load + masked store instead of falling through to a libcall.
The vector width is derived from the length's known range and gated on
the target actually supporting byte-masked load/store, so a tighter bound
uses a narrower vector (<=16 -> XMM, <=32 -> YMM, <=64 -> ZMM). Bounds
wider than one register, or when no masked support is available, fall
through to the existing expansion unchanged.
The AssumptionCache is used only if available, so this doesn't perturb
pipelines ( -O0) that don't already provide it.
Fixes #202883
---
.../Transforms/Utils/LowerMemIntrinsics.h | 2 +
llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp | 61 ++++-
.../Transforms/Utils/LowerMemIntrinsics.cpp | 24 ++
.../X86/masked-memcpy-bounded.ll | 223 ++++++++++++++++++
4 files changed, 306 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/Transforms/PreISelIntrinsicLowering/X86/masked-memcpy-bounded.ll
diff --git a/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
index 55c7119cd8fdb..767e1da28a1d3 100644
--- a/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
+++ b/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
@@ -14,6 +14,7 @@
#ifndef LLVM_TRANSFORMS_UTILS_LOWERMEMINTRINSICS_H
#define LLVM_TRANSFORMS_UTILS_LOWERMEMINTRINSICS_H
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/Support/Compiler.h"
#include <cstdint>
#include <optional>
@@ -55,6 +56,7 @@ LLVM_ABI void expandMemCpyAsLoop(MemCpyInst *MemCpy,
const TargetTransformInfo &TTI,
ScalarEvolution *SE = nullptr);
+LLVM_ABI void emitBoundedMaskedMemcpy(MemTransferInst *MemCpy, unsigned VF);
/// Expand \p MemMove as a loop. \p MemMove is not deleted. Returns true if the
/// memmove was lowered.
LLVM_ABI bool expandMemMoveAsLoop(MemMoveInst *MemMove,
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 4b5c97a23b091..08f7bb0df176a 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -13,10 +13,12 @@
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/ObjCARCInstKind.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ExpandVectorPredication.h"
#include "llvm/CodeGen/LibcallLoweringInfo.h"
#include "llvm/CodeGen/Passes.h"
@@ -61,6 +63,7 @@ struct PreISelIntrinsicLowering {
const TargetMachine *TM;
const ModuleLibcallLoweringInfo &ModuleLibcalls;
const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
+ const function_ref<AssumptionCache *(Function &)> LookupAC;
const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
/// If this is true, assume it's preferably to leave memory intrinsic calls
@@ -72,11 +75,12 @@ struct PreISelIntrinsicLowering {
const TargetMachine *TM_,
const ModuleLibcallLoweringInfo &ModuleLibcalls_,
function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
+ function_ref<AssumptionCache *(Function &)> LookupAC_,
function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
bool UseMemIntrinsicLibFunc_ = true)
: TM(TM_), ModuleLibcalls(ModuleLibcalls_), LookupTTI(LookupTTI_),
- LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {
- }
+ LookupAC(LookupAC_), LookupTLI(LookupTLI_),
+ UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
static bool shouldExpandMemIntrinsicWithSize(Value *Size,
const TargetTransformInfo &TTI);
@@ -240,7 +244,33 @@ bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
// intrinsics, including size 0.
return SizeVal > Threshold || Threshold == 0;
}
+static bool tryEmitBoundedMaskedMemcpy(MemTransferInst *Memcpy,
+ const TargetTransformInfo &TTI,
+ AssumptionCache *AC) {
+ Value *Len = Memcpy->getLength();
+ if (isa<ConstantInt>(Len)) {
+ return false;
+ }
+
+ const DataLayout &DL = Memcpy->getModule()->getDataLayout();
+ SimplifyQuery SQ(DL, Memcpy);
+ SQ.AC = AC;
+ ConstantRange CR = computeConstantRange(Len, false, SQ);
+
+ uint64_t MaxLen = CR.getUnsignedMax().getZExtValue();
+ if (MaxLen == 0 || MaxLen > 64)
+ return false;
+ unsigned VF = PowerOf2Ceil(MaxLen);
+
+ Type *DataVecTy =
+ FixedVectorType::get(Type::getInt8Ty(Memcpy->getContext()), VF);
+ if (!TTI.isLegalMaskedLoad(DataVecTy, Align(1), 0) ||
+ !TTI.isLegalMaskedStore(DataVecTy, Align(1), 0))
+ return false;
+ emitBoundedMaskedMemcpy(Memcpy, VF);
+ return true;
+}
static bool canEmitLibcall(const ModuleLibcallLoweringInfo &ModuleLowering,
const TargetMachine *TM, Function *F,
RTLIB::Libcall LC) {
@@ -331,6 +361,12 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
auto *Memcpy = cast<MemCpyInst>(Inst);
Function *ParentFunc = Memcpy->getFunction();
const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
+ AssumptionCache *AC = LookupAC(*ParentFunc);
+ if (tryEmitBoundedMaskedMemcpy(Memcpy, TTI, AC)) {
+ Changed = true;
+ Memcpy->eraseFromParent();
+ break;
+ }
if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
if (UseMemIntrinsicLibFunc &&
canEmitMemcpy(ModuleLibcalls, TM, ParentFunc))
@@ -363,6 +399,12 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
auto *Memmove = cast<MemMoveInst>(Inst);
Function *ParentFunc = Memmove->getFunction();
const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
+ AssumptionCache *AC = LookupAC(*ParentFunc);
+ if (tryEmitBoundedMaskedMemcpy(Memmove, TTI, AC)) {
+ Changed = true;
+ Memmove->eraseFromParent();
+ break;
+ }
if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
if (UseMemIntrinsicLibFunc &&
canEmitLibcall(ModuleLibcalls, TM, ParentFunc, RTLIB::MEMMOVE))
@@ -843,6 +885,7 @@ class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
AU.addRequired<TargetLibraryInfoWrapperPass>();
AU.addRequired<LibcallLoweringInfoWrapper>();
AU.addRequired<TargetPassConfig>();
+ AU.addUsedIfAvailable<AssumptionCacheTracker>();
}
bool runOnModule(Module &M) override {
@@ -852,12 +895,17 @@ class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
};
+ auto LookupAC = [this](Function &F) -> AssumptionCache * {
+ auto *ACT = this->getAnalysisIfAvailable<AssumptionCacheTracker>();
+ return ACT ? &ACT->getAssumptionCache(F) : nullptr;
+ };
auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
};
const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
- PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI);
+ PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupAC,
+ LookupTLI);
return Lowering.lowerIntrinsics(M);
}
};
@@ -874,6 +922,7 @@ INITIALIZE_PASS_DEPENDENCY(RuntimeLibraryInfoWrapper)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
"pre-isel-intrinsic-lowering",
"Pre-ISel Intrinsic Lowering", false, false)
@@ -892,11 +941,15 @@ PreISelIntrinsicLoweringPass::run(Module &M, ModuleAnalysisManager &MAM) {
auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
return FAM.getResult<TargetIRAnalysis>(F);
};
+ auto LookupAC = [&FAM](Function &F) -> AssumptionCache * {
+ return &FAM.getResult<AssumptionAnalysis>(F);
+ };
auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
return FAM.getResult<TargetLibraryAnalysis>(F);
};
- PreISelIntrinsicLowering Lowering(TM, LibcallLowering, LookupTTI, LookupTLI);
+ PreISelIntrinsicLowering Lowering(TM, LibcallLowering, LookupTTI, LookupAC,
+ LookupTLI);
if (!Lowering.lowerIntrinsics(M))
return PreservedAnalyses::all();
else
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index 198a950c7c961..df23d8ca046cf 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -1591,3 +1591,27 @@ void llvm::expandAtomicMemCpyAsLoop(AnyMemCpyInst *AtomicMemcpy,
/*AtomicElementSize=*/AtomicMemcpy->getElementSizeInBytes());
}
}
+void llvm::emitBoundedMaskedMemcpy(MemTransferInst *MemCpy, unsigned VF) {
+ IRBuilder<> Builder(MemCpy);
+ Value *Dst = MemCpy->getRawDest();
+ Value *Src = MemCpy->getRawSource();
+ Value *Size = MemCpy->getLength();
+
+ Type *MaskIntTy = Builder.getIntNTy(VF);
+
+ Type *WideTy = Builder.getInt128Ty();
+ Value *SizeWide = Builder.CreateZExtOrTrunc(Size, WideTy);
+ Value *One = ConstantInt::get(WideTy, 1);
+ Value *Shifted = Builder.CreateShl(One, SizeWide);
+ Value *MinusOne = Builder.CreateSub(Shifted, ConstantInt::get(WideTy, 1));
+ Value *MaskBits = Builder.CreateTrunc(MinusOne, MaskIntTy);
+
+ Type *MaskVecTy = FixedVectorType::get(Builder.getInt1Ty(), VF);
+ Value *Mask = Builder.CreateBitCast(MaskBits, MaskVecTy);
+
+ Type *DataVecTy = FixedVectorType::get(Builder.getInt8Ty(), VF);
+ Align A(1);
+ Value *Loaded =
+ Builder.CreateMaskedLoad(DataVecTy, Src, A, Mask, nullptr, "masked.copy");
+ Builder.CreateMaskedStore(Loaded, Dst, A, Mask);
+}
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/X86/masked-memcpy-bounded.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/X86/masked-memcpy-bounded.ll
new file mode 100644
index 0000000000000..4212d70c2a127
--- /dev/null
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/X86/masked-memcpy-bounded.ll
@@ -0,0 +1,223 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 6
+; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; A memcpy with a non-constant length bounded by assume(size <= 64), on a target
+; with byte-masked load/store (AVX512BW), lowers to a single masked load + masked
+; store rather than a memcpy libcall.
+define void @memcpy_bounded_assume(ptr %dst, ptr %src, i64 %size) #0 {
+; CHECK-LABEL: define void @memcpy_bounded_assume(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 65
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[Z:%.*]] = icmp eq i64 [[SIZE]], 0
+; CHECK-NEXT: br i1 [[Z]], label %[[RET:.*]], label %[[COPY:.*]]
+; CHECK: [[COPY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i64 [[SIZE]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = shl i128 1, [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = sub i128 [[TMP4]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[TMP1]] to <64 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[SRC]], <64 x i1> [[TMP2]], <64 x i8> poison)
+; CHECK-NEXT: call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP3]], ptr align 1 [[DST]], <64 x i1> [[TMP2]])
+; CHECK-NEXT: br label %[[RET]]
+; CHECK: [[RET]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp = icmp ult i64 %size, 65
+ tail call void @llvm.assume(i1 %cmp)
+ %z = icmp eq i64 %size, 0
+ br i1 %z, label %ret, label %copy
+copy:
+ tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ br label %ret
+ret:
+ ret void
+}
+
+; A memcpy WITHOUT the assume bound must NOT be rewritten - the length range is
+; unknown, so the masked path is unsafe and the intrinsic is left for the backend.
+define void @memcpy_unbounded(ptr %dst, ptr %src, i64 %size) #0 {
+; CHECK-LABEL: define void @memcpy_unbounded(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: tail call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+declare void @llvm.assume(i1)
+declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
+
+attributes #0 = { "target-features"="+avx512bw,+avx512vl" }
+
+; memmove with the same bound lowers identically - the single masked load into a
+; register before any store makes this overlap-safe, so memmove uses the same
+; sequence as memcpy.
+define void @memmove_bounded_assume(ptr %dst, ptr %src, i64 %size) #0 {
+; CHECK-LABEL: define void @memmove_bounded_assume(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 65
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[Z:%.*]] = icmp eq i64 [[SIZE]], 0
+; CHECK-NEXT: br i1 [[Z]], label %[[RET:.*]], label %[[MOVE:.*]]
+; CHECK: [[MOVE]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i64 [[SIZE]] to i128
+; CHECK-NEXT: [[TMP4:%.*]] = shl i128 1, [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = sub i128 [[TMP4]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i128 [[TMP5]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[TMP1]] to <64 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[SRC]], <64 x i1> [[TMP2]], <64 x i8> poison)
+; CHECK-NEXT: call void @llvm.masked.store.v64i8.p0(<64 x i8> [[TMP3]], ptr align 1 [[DST]], <64 x i1> [[TMP2]])
+; CHECK-NEXT: br label %[[RET]]
+; CHECK: [[RET]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp = icmp ult i64 %size, 65
+ tail call void @llvm.assume(i1 %cmp)
+ %z = icmp eq i64 %size, 0
+ br i1 %z, label %ret, label %move
+move:
+ tail call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ br label %ret
+ret:
+ ret void
+}
+
+; memmove without a bound is left alone.
+define void @memmove_unbounded(ptr %dst, ptr %src, i64 %size) #0 {
+; CHECK-LABEL: define void @memmove_unbounded(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: tail call void @llvm.memmove.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; A tighter bound (<= 32) uses a narrower 32-byte (YMM) masked op, not 64.
+define void @memcpy_bound32(ptr %dst, ptr %src, i64 %size) #0 {
+; CHECK-LABEL: define void @memcpy_bound32(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[Z:%.*]] = icmp eq i64 [[SIZE]], 0
+; CHECK-NEXT: br i1 [[Z]], label %[[RET:.*]], label %[[COPY:.*]]
+; CHECK: [[COPY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i64 [[SIZE]] to i128
+; CHECK-NEXT: [[TMP1:%.*]] = shl i128 1, [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i128 [[TMP1]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i128 [[TMP2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
+; CHECK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[SRC]], <32 x i1> [[TMP4]], <32 x i8> poison)
+; CHECK-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <32 x i1> [[TMP4]])
+; CHECK-NEXT: br label %[[RET]]
+; CHECK: [[RET]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp = icmp ult i64 %size, 33
+ tail call void @llvm.assume(i1 %cmp)
+ %z = icmp eq i64 %size, 0
+ br i1 %z, label %ret, label %copy
+copy:
+ tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ br label %ret
+ret:
+ ret void
+}
+
+; An even tighter bound (<= 16) uses a 16-byte (XMM) masked op.
+define void @memcpy_bound16(ptr %dst, ptr %src, i64 %size) #0 {
+; CHECK-LABEL: define void @memcpy_bound16(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 17
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[Z:%.*]] = icmp eq i64 [[SIZE]], 0
+; CHECK-NEXT: br i1 [[Z]], label %[[RET:.*]], label %[[COPY:.*]]
+; CHECK: [[COPY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = zext i64 [[SIZE]] to i128
+; CHECK-NEXT: [[TMP1:%.*]] = shl i128 1, [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i128 [[TMP1]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i128 [[TMP2]] to i16
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
+; CHECK-NEXT: [[MASKED_COPY:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[SRC]], <16 x i1> [[TMP4]], <16 x i8> poison)
+; CHECK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <16 x i1> [[TMP4]])
+; CHECK-NEXT: br label %[[RET]]
+; CHECK: [[RET]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp = icmp ult i64 %size, 17
+ tail call void @llvm.assume(i1 %cmp)
+ %z = icmp eq i64 %size, 0
+ br i1 %z, label %ret, label %copy
+copy:
+ tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ br label %ret
+ret:
+ ret void
+}
+
+; A bound wider than one register (<= 100 > 64) is NOT rewritten - it falls
+; through to the existing loop/libcall expansion.
+define void @memcpy_bound_too_wide(ptr %dst, ptr %src, i64 %size) #0 {
+; CHECK-LABEL: define void @memcpy_bound_too_wide(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 101
+; CHECK-NEXT: tail call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[Z:%.*]] = icmp eq i64 [[SIZE]], 0
+; CHECK-NEXT: br i1 [[Z]], label %[[RET:.*]], label %[[COPY:.*]]
+; CHECK: [[COPY]]:
+; CHECK-NEXT: tail call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: br label %[[RET]]
+; CHECK: [[RET]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp = icmp ult i64 %size, 101
+ tail call void @llvm.assume(i1 %cmp)
+ %z = icmp eq i64 %size, 0
+ br i1 %z, label %ret, label %copy
+copy:
+ tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ br label %ret
+ret:
+ ret void
+}
+
+; The length can be bounded by ValueTracking without any assume - here an `and`
+; mask proves the length is <= 63, so no AssumptionCache is needed to rewrite.
+define void @memcpy_and_mask(ptr %dst, ptr %src, i64 %size) #0 {
+; CHECK-LABEL: define void @memcpy_and_mask(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SIZE_MOD:%.*]] = and i64 [[SIZE]], 63
+; CHECK-NEXT: [[TMP1:%.*]] = zext i64 [[SIZE_MOD]] to i128
+; CHECK-NEXT: [[TMP2:%.*]] = shl i128 1, [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = sub i128 [[TMP2]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i128 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i64 [[TMP4]] to <64 x i1>
+; CHECK-NEXT: [[MASKED_COPY:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[SRC]], <64 x i1> [[TMP5]], <64 x i8> poison)
+; CHECK-NEXT: call void @llvm.masked.store.v64i8.p0(<64 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <64 x i1> [[TMP5]])
+; CHECK-NEXT: ret void
+;
+ %size.mod = and i64 %size, 63
+ tail call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size.mod, i1 false)
+ ret void
+}
+;.
+; CHECK: attributes #[[ATTR0]] = { "target-features"="+avx512bw,+avx512vl" }
+; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; CHECK: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+; CHECK: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
+; CHECK: attributes #[[ATTR4:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
+;.
More information about the llvm-commits
mailing list