[llvm] [PreISelIntrinsicLowering] Lower bounded memcpy/memmove to masked load/store (PR #212710)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 26 13:58:42 PDT 2026
https://github.com/Harishankar14 updated https://github.com/llvm/llvm-project/pull/212710
>From 73f7a3b6f50a25185c8897e8a20a545132396a43 Mon Sep 17 00:00:00 2001
From: Harishankar <harishankarpp7 at gmail.com>
Date: Wed, 29 Jul 2026 14:04:03 +0530
Subject: [PATCH] [PreISelIntrinsicLowering] Lower bounded memcpy/memmove to
masked load/store
When a memcpy or memmove has a non-constant length that a preceding
llvm.assume proves is small ( assume(size <= 64)), emit a single
masked load + masked store instead of falling through to a libcall.
The vector width is derived from the length's known range and gated on
the target actually supporting byte-masked load/store, so a tighter bound
uses a narrower vector (<=16 -> XMM, <=32 -> YMM, <=64 -> ZMM). Bounds
wider than one register, or when no masked support is available, fall
through to the existing expansion unchanged.
The AssumptionCache is used only if available, so this doesn't perturb
pipelines ( -O0) that don't already provide it.
Fixes #202883
---
.../Transforms/Utils/LowerMemIntrinsics.h | 9 +-
llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp | 88 +++-
.../Transforms/Utils/LowerMemIntrinsics.cpp | 20 +
.../AArch64/masked-memcpy-bounded.ll | 102 +++++
.../X86/masked-memcpy-bounded.ll | 418 ++++++++++++++++++
5 files changed, 632 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/masked-memcpy-bounded.ll
create mode 100644 llvm/test/Transforms/PreISelIntrinsicLowering/X86/masked-memcpy-bounded.ll
diff --git a/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
index 55c7119cd8fdb..5b14f36b7847f 100644
--- a/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
+++ b/llvm/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
@@ -27,6 +27,7 @@ class MemCpyInst;
class MemMoveInst;
class MemSetInst;
class MemSetPatternInst;
+class MemTransferInst;
class ScalarEvolution;
class TargetTransformInfo;
class Value;
@@ -54,7 +55,13 @@ LLVM_ABI void createMemCpyLoopKnownSize(
LLVM_ABI void expandMemCpyAsLoop(MemCpyInst *MemCpy,
const TargetTransformInfo &TTI,
ScalarEvolution *SE = nullptr);
-
+/// \p VF is the vectorization factor: the number of i8 lanes in the masked
+/// operations. Lane i is active iff i is less than the transfer length, so
+/// exactly the requested bytes are read and written and nothing beyond them.
+/// \p VF must be a power of two and at least the maximum possible length of
+/// the transfer; the caller is responsible for establishing that bound and
+/// for checking that masked load/store of <VF x i8> is legal on the target.
+LLVM_ABI void emitBoundedMaskedMemcpy(MemTransferInst *MemCpy, unsigned VF);
/// Expand \p MemMove as a loop. \p MemMove is not deleted. Returns true if the
/// memmove was lowered.
LLVM_ABI bool expandMemMoveAsLoop(MemMoveInst *MemMove,
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 4b5c97a23b091..6c081b2b1e998 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -13,10 +13,12 @@
#include "llvm/CodeGen/PreISelIntrinsicLowering.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/ObjCARCInstKind.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ExpandVectorPredication.h"
#include "llvm/CodeGen/LibcallLoweringInfo.h"
#include "llvm/CodeGen/Passes.h"
@@ -61,6 +63,7 @@ struct PreISelIntrinsicLowering {
const TargetMachine *TM;
const ModuleLibcallLoweringInfo &ModuleLibcalls;
const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
+ const function_ref<AssumptionCache *(Function &)> LookupAC;
const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
/// If this is true, assume it's preferably to leave memory intrinsic calls
@@ -72,11 +75,12 @@ struct PreISelIntrinsicLowering {
const TargetMachine *TM_,
const ModuleLibcallLoweringInfo &ModuleLibcalls_,
function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
+ function_ref<AssumptionCache *(Function &)> LookupAC_,
function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
bool UseMemIntrinsicLibFunc_ = true)
: TM(TM_), ModuleLibcalls(ModuleLibcalls_), LookupTTI(LookupTTI_),
- LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {
- }
+ LookupAC(LookupAC_), LookupTLI(LookupTLI_),
+ UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
static bool shouldExpandMemIntrinsicWithSize(Value *Size,
const TargetTransformInfo &TTI);
@@ -241,6 +245,59 @@ bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
return SizeVal > Threshold || Threshold == 0;
}
+// Try to replace a variable-length memcpy or memmove with a single masked
+// load and masked store.
+
+// A libcall is a poor trade when the length is small, and value tracking can
+// often prove that it is: `[[assume(n <= 64)]]`, an `and` with a small
+// constant, or anything else that narrows the range. If the whole transfer
+// fits in one vector register, one masked load and one masked store cover
+// every length in that range without a branch, a loop, or a call.
+//
+// This is safe for both intrinsics. Masked lanes are never accessed, so the
+// load cannot fault past the end of the source and the store cannot clobber
+// bytes past the length. Overlap is fine too: the load completes into a
+// register before the store issues, which is what memmove requires.
+
+/// Returns true if \p Memcpy was replaced.
+static bool tryEmitBoundedMaskedMemcpy(MemTransferInst *Memcpy,
+ const TargetTransformInfo &TTI,
+ AssumptionCache *AC) {
+ Value *Len = Memcpy->getLength();
+ if (isa<ConstantInt>(Len))
+ return false;
+ if (Memcpy->isVolatile())
+ return false;
+
+ const DataLayout &DL = Memcpy->getDataLayout();
+ SimplifyQuery SQ(DL, Memcpy);
+ SQ.AC = AC;
+ ConstantRange CR = computeConstantRange(Len, false, SQ);
+
+ Type *I8Ty = Type::getInt8Ty(Memcpy->getContext());
+ TypeSize VecRegBits =
+ TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector);
+ if (VecRegBits.isScalable() || VecRegBits.getKnownMinValue() == 0)
+ return false;
+ uint64_t MaxBytes = VecRegBits.getFixedValue() / 8;
+ const APInt &MaxLenAP = CR.getUnsignedMax();
+ if (MaxLenAP.isZero() || MaxLenAP.ugt(MaxBytes))
+ return false;
+ uint64_t MaxLen = MaxLenAP.getZExtValue();
+ unsigned VF = PowerOf2Ceil(MaxLen);
+
+ Type *DataVecTy = FixedVectorType::get(I8Ty, VF);
+ unsigned SourceAddress =
+ Memcpy->getRawSource()->getType()->getPointerAddressSpace();
+ unsigned DestAddress =
+ Memcpy->getRawDest()->getType()->getPointerAddressSpace();
+ if (!TTI.isLegalMaskedLoad(DataVecTy, Align(1), SourceAddress) ||
+ !TTI.isLegalMaskedStore(DataVecTy, Align(1), DestAddress))
+ return false;
+
+ emitBoundedMaskedMemcpy(Memcpy, VF);
+ return true;
+}
static bool canEmitLibcall(const ModuleLibcallLoweringInfo &ModuleLowering,
const TargetMachine *TM, Function *F,
RTLIB::Libcall LC) {
@@ -331,6 +388,12 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
auto *Memcpy = cast<MemCpyInst>(Inst);
Function *ParentFunc = Memcpy->getFunction();
const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
+ AssumptionCache *AC = LookupAC(*ParentFunc);
+ if (tryEmitBoundedMaskedMemcpy(Memcpy, TTI, AC)) {
+ Changed = true;
+ Memcpy->eraseFromParent();
+ break;
+ }
if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
if (UseMemIntrinsicLibFunc &&
canEmitMemcpy(ModuleLibcalls, TM, ParentFunc))
@@ -363,6 +426,12 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(
auto *Memmove = cast<MemMoveInst>(Inst);
Function *ParentFunc = Memmove->getFunction();
const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
+ AssumptionCache *AC = LookupAC(*ParentFunc);
+ if (tryEmitBoundedMaskedMemcpy(Memmove, TTI, AC)) {
+ Changed = true;
+ Memmove->eraseFromParent();
+ break;
+ }
if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
if (UseMemIntrinsicLibFunc &&
canEmitLibcall(ModuleLibcalls, TM, ParentFunc, RTLIB::MEMMOVE))
@@ -843,6 +912,7 @@ class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
AU.addRequired<TargetLibraryInfoWrapperPass>();
AU.addRequired<LibcallLoweringInfoWrapper>();
AU.addRequired<TargetPassConfig>();
+ AU.addUsedIfAvailable<AssumptionCacheTracker>();
}
bool runOnModule(Module &M) override {
@@ -852,12 +922,17 @@ class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
};
+ auto LookupAC = [this](Function &F) -> AssumptionCache * {
+ auto *ACT = this->getAnalysisIfAvailable<AssumptionCacheTracker>();
+ return ACT ? &ACT->getAssumptionCache(F) : nullptr;
+ };
auto LookupTLI = [this](Function &F) -> TargetLibraryInfo & {
return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
};
const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
- PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI);
+ PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupAC,
+ LookupTLI);
return Lowering.lowerIntrinsics(M);
}
};
@@ -874,6 +949,7 @@ INITIALIZE_PASS_DEPENDENCY(RuntimeLibraryInfoWrapper)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
"pre-isel-intrinsic-lowering",
"Pre-ISel Intrinsic Lowering", false, false)
@@ -892,11 +968,15 @@ PreISelIntrinsicLoweringPass::run(Module &M, ModuleAnalysisManager &MAM) {
auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
return FAM.getResult<TargetIRAnalysis>(F);
};
+ auto LookupAC = [&FAM](Function &F) -> AssumptionCache * {
+ return &FAM.getResult<AssumptionAnalysis>(F);
+ };
auto LookupTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
return FAM.getResult<TargetLibraryAnalysis>(F);
};
- PreISelIntrinsicLowering Lowering(TM, LibcallLowering, LookupTTI, LookupTLI);
+ PreISelIntrinsicLowering Lowering(TM, LibcallLowering, LookupTTI, LookupAC,
+ LookupTLI);
if (!Lowering.lowerIntrinsics(M))
return PreservedAnalyses::all();
else
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index 198a950c7c961..38229e1c37e09 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -1591,3 +1591,23 @@ void llvm::expandAtomicMemCpyAsLoop(AnyMemCpyInst *AtomicMemcpy,
/*AtomicElementSize=*/AtomicMemcpy->getElementSizeInBytes());
}
}
+void llvm::emitBoundedMaskedMemcpy(MemTransferInst *MemCpy, unsigned VF) {
+ IRBuilder<> Builder(MemCpy);
+ Value *Dst = MemCpy->getRawDest();
+ Value *Src = MemCpy->getRawSource();
+ Value *Size = MemCpy->getLength();
+
+ // Lane i is active iff i < Size. Expressed as a lane-indexed mask rather than
+ // a bitcast from an integer bit pattern, which would be endian-dependent.
+ Type *MaskVecTy = FixedVectorType::get(Builder.getInt1Ty(), VF);
+ Value *Mask = Builder.CreateIntrinsic(
+ Intrinsic::get_active_lane_mask, {MaskVecTy, Size->getType()},
+ {ConstantInt::get(Size->getType(), 0), Size});
+
+ Type *DataVecTy = FixedVectorType::get(Builder.getInt8Ty(), VF);
+ Align DstAlign = MemCpy->getDestAlign().valueOrOne();
+ Align SrcAlign = MemCpy->getSourceAlign().valueOrOne();
+ Value *Loaded = Builder.CreateMaskedLoad(DataVecTy, Src, SrcAlign, Mask,
+ nullptr, "masked.copy");
+ Builder.CreateMaskedStore(Loaded, Dst, DstAlign, Mask);
+}
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/masked-memcpy-bounded.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/masked-memcpy-bounded.ll
new file mode 100644
index 0000000000000..8d18265249338
--- /dev/null
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/masked-memcpy-bounded.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -mtriple=aarch64-- -mattr=+sve -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+; RUN: opt -mtriple=aarch64_be-- -mattr=+sve -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+; +sve is required: AArch64 gates masked load/store on
+; isSVEorStreamingSVEAvailable(). For fixed-width vector types it then accepts
+; only exactly 64- or 128-bit data, so <8 x i8> and <16 x i8> transform while
+; <4 x i8> and <32 x i8> are declined.
+;
+; Everything lands under CHECK because the mask comes from
+; llvm.get.active.lane.mask, which is lane-indexed, so the emitted IR is
+; identical on big- and little-endian targets. CHECK-LE and CHECK-BE exist to
+; catch any future divergence — an endian-sensitive construct such as a bitcast
+; from an integer bit pattern would show up there.
+
+; A 4-byte bound gives <4 x i8> at 32 bits, which is below the legal fixed-width
+; sizes and is declined.
+define void @memcpy_bound4(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memcpy_bound4(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 5
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 5
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; An 8-byte bound gives <8 x i8> at 64 bits, which is legal.
+define void @memcpy_bound8(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memcpy_bound8(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 9
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 0, i64 [[SIZE]])
+; CHECK-NEXT: [[MASKED_COPY:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[SRC]], <8 x i1> [[TMP1]], <8 x i8> poison)
+; CHECK-NEXT: call void @llvm.masked.store.v8i8.p0(<8 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <8 x i1> [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 9
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+define void @memcpy_bound16(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memcpy_bound16(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 17
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[SIZE]])
+; CHECK-NEXT: [[MASKED_COPY:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[SRC]], <16 x i1> [[TMP1]], <16 x i8> poison)
+; CHECK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <16 x i1> [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 17
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+define void @memmove_bound16(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memmove_bound16(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 17
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[SIZE]])
+; CHECK-NEXT: [[MASKED_COPY:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[SRC]], <16 x i1> [[TMP1]], <16 x i8> poison)
+; CHECK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <16 x i1> [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 17
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; A 32-byte bound gives <32 x i8> at 256 bits, which is above the legal
+; fixed-width sizes and is declined.
+define void @memcpy_bound32(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memcpy_bound32(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 33
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+declare void @llvm.assume(i1)
+declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
+declare void @llvm.memmove.p0.p0.i64(ptr, ptr, i64, i1)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-BE: {{.*}}
+; CHECK-LE: {{.*}}
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/X86/masked-memcpy-bounded.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/X86/masked-memcpy-bounded.ll
new file mode 100644
index 0000000000000..7643c41441c2d
--- /dev/null
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/X86/masked-memcpy-bounded.ll
@@ -0,0 +1,418 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -mcpu=x86-64 -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefixes=CHECK,NOMASK
+; RUN: opt -mcpu=x86-64-v2 -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefixes=CHECK,NOMASK
+; RUN: opt -mcpu=x86-64-v3 -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefixes=CHECK,NOMASK
+; RUN: opt -mcpu=x86-64-v4 -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefixes=CHECK,MASK,MASK256
+; RUN: opt -mcpu=znver4 -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s --check-prefixes=CHECK,MASK,MASK512
+
+; The bounded memcpy/memmove -> masked load/store transform is gated on two
+; things: the widest fixed-width vector register the subtarget prefers, which
+; caps the bound we accept, and byte-granular masked load/store legality.
+;
+; x86-64 SSE2 128-bit regs, no masked load/store -> never fires
+; x86-64-v2 SSE4.2 128-bit regs, no masked load/store -> never fires
+; x86-64-v3 AVX2 256-bit regs, but vpmaskmov handles 32/64-bit elements
+; only, so <N x i8> is illegal -> never fires
+; x86-64-v4 AVX512BW+VL, up to 32 bytes
+; znver4 AVX512BW+VL, up to 64 bytes
+;
+; The split between the last two is prefer-256-bit, which is a tuning decision
+; rather than an ISA one: x86-64-v4, sapphirerapids and skylake-avx512 all set
+; it and so cap getRegisterBitWidth at 256, while znver4 does not. znver4 is
+; therefore the configuration that covers the 64-byte case from the bug report.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; A 16-byte bound needs only XMM.
+define void @memcpy_bound16(ptr %dst, ptr %src, i64 %size) {
+; NOMASK-LABEL: define void @memcpy_bound16(
+; NOMASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
+; NOMASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 17
+; NOMASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; NOMASK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memcpy_bound16(
+; MASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
+; MASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 17
+; MASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK-NEXT: [[TMP1:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 0, i64 [[SIZE]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[SRC]], <16 x i1> [[TMP1]], <16 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <16 x i1> [[TMP1]])
+; MASK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 17
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; A 32-byte bound uses YMM. Still within the prefer-256-bit cap.
+define void @memcpy_bound32(ptr %dst, ptr %src, i64 %size) {
+; NOMASK-LABEL: define void @memcpy_bound32(
+; NOMASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; NOMASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; NOMASK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memcpy_bound32(
+; MASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; MASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK-NEXT: [[TMP1:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[SIZE]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[SRC]], <32 x i1> [[TMP1]], <32 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <32 x i1> [[TMP1]])
+; MASK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 33
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; Alignment on the intrinsic is carried through to the masked ops.
+define void @memcpy_aligned_bound32(ptr align 32 %dst, ptr align 32 %src, i64 %size) {
+; NOMASK-LABEL: define void @memcpy_aligned_bound32(
+; NOMASK-SAME: ptr align 32 [[DST:%.*]], ptr align 32 [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; NOMASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; NOMASK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 32 [[DST]], ptr align 32 [[SRC]], i64 [[SIZE]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memcpy_aligned_bound32(
+; MASK-SAME: ptr align 32 [[DST:%.*]], ptr align 32 [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; MASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK-NEXT: [[TMP1:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[SIZE]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 32 [[SRC]], <32 x i1> [[TMP1]], <32 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[MASKED_COPY]], ptr align 32 [[DST]], <32 x i1> [[TMP1]])
+; MASK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 33
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr align 32 %dst, ptr align 32 %src, i64 %size, i1 false)
+ ret void
+}
+
+; The case from the bug report: assume(size <= 64) on AVX512. Needs ZMM, so it
+; only fires on a subtarget that does not prefer 256-bit vectors.
+define void @memcpy_bound64(ptr %dst, ptr %src, i64 %size) {
+; NOMASK-LABEL: define void @memcpy_bound64(
+; NOMASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 65
+; NOMASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; NOMASK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK256-LABEL: define void @memcpy_bound64(
+; MASK256-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK256-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 65
+; MASK256-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK256-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; MASK256-NEXT: ret void
+;
+; MASK512-LABEL: define void @memcpy_bound64(
+; MASK512-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK512-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 65
+; MASK512-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK512-NEXT: [[TMP5:%.*]] = call <64 x i1> @llvm.get.active.lane.mask.v64i1.i64(i64 0, i64 [[SIZE]])
+; MASK512-NEXT: [[MASKED_COPY:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[SRC]], <64 x i1> [[TMP5]], <64 x i8> poison)
+; MASK512-NEXT: call void @llvm.masked.store.v64i8.p0(<64 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <64 x i1> [[TMP5]])
+; MASK512-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 65
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; A non-power-of-two bound rounds the vector width up: <= 20 bytes uses a
+; 32-lane masked op, not a 20-lane one.
+define void @memcpy_bound_not_pow2(ptr %dst, ptr %src, i64 %size) {
+; NOMASK-LABEL: define void @memcpy_bound_not_pow2(
+; NOMASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 21
+; NOMASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; NOMASK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memcpy_bound_not_pow2(
+; MASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 21
+; MASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK-NEXT: [[TMP1:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[SIZE]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[SRC]], <32 x i1> [[TMP1]], <32 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <32 x i1> [[TMP1]])
+; MASK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 21
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; A bound wider than one vector register is declined on every configuration and
+; falls through to the existing expansion.
+define void @memcpy_bound_too_wide(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memcpy_bound_too_wide(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 101
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 101
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; A provably-zero length gives an unsigned max of zero, so it is declined on the
+; isZero() arm rather than lowered to an all-zero mask.
+define void @memcpy_bound_zero(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memcpy_bound_zero(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 1
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 1
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; No bound at all: the length range is unknown, so the masked path is unsafe.
+define void @memcpy_unbounded(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memcpy_unbounded(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; Constant lengths are handled by the existing lowering and rejected up front.
+define void @memcpy_constant_length(ptr %dst, ptr %src) {
+; CHECK-LABEL: define void @memcpy_constant_length(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 32, i1 false)
+; CHECK-NEXT: ret void
+;
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 32, i1 false)
+ ret void
+}
+
+; The assume does not have to sit in the same block as the memcpy. No dominator
+; tree is threaded into the SimplifyQuery, so this relies on the single-
+; predecessor fallback in isValidAssumeForContext.
+define void @memcpy_assume_in_predecessor(ptr %dst, ptr %src, i64 %size) {
+; NOMASK-LABEL: define void @memcpy_assume_in_predecessor(
+; NOMASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[ENTRY:.*:]]
+; NOMASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; NOMASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; NOMASK-NEXT: [[Z:%.*]] = icmp eq i64 [[SIZE]], 0
+; NOMASK-NEXT: br i1 [[Z]], label %[[RET:.*]], label %[[COPY:.*]]
+; NOMASK: [[COPY]]:
+; NOMASK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; NOMASK-NEXT: br label %[[RET]]
+; NOMASK: [[RET]]:
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memcpy_assume_in_predecessor(
+; MASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK-NEXT: [[ENTRY:.*:]]
+; MASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; MASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK-NEXT: [[Z:%.*]] = icmp eq i64 [[SIZE]], 0
+; MASK-NEXT: br i1 [[Z]], label %[[RET:.*]], label %[[COPY:.*]]
+; MASK: [[COPY]]:
+; MASK-NEXT: [[TMP0:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[SIZE]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[SRC]], <32 x i1> [[TMP0]], <32 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <32 x i1> [[TMP0]])
+; MASK-NEXT: br label %[[RET]]
+; MASK: [[RET]]:
+; MASK-NEXT: ret void
+;
+entry:
+ %cmp = icmp ult i64 %size, 33
+ call void @llvm.assume(i1 %cmp)
+ %z = icmp eq i64 %size, 0
+ br i1 %z, label %ret, label %copy
+
+copy:
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ br label %ret
+
+ret:
+ ret void
+}
+
+; An assume that does not reach the memcpy must not be used. %copy has two
+; predecessors, so the fallback above does not apply and the bound is unknown.
+define void @memcpy_assume_does_not_dominate(ptr %dst, ptr %src, i64 %size, i1 %c) {
+; CHECK-LABEL: define void @memcpy_assume_does_not_dominate(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]], i1 [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br i1 [[C]], label %[[BOUND:.*]], label %[[COPY:.*]]
+; CHECK: [[BOUND]]:
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: br label %[[COPY]]
+; CHECK: [[COPY]]:
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+entry:
+ br i1 %c, label %bound, label %copy
+
+bound:
+ %cmp = icmp ult i64 %size, 33
+ call void @llvm.assume(i1 %cmp)
+ br label %copy
+
+copy:
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; memmove uses the same sequence: the masked load completes into a register
+; before the masked store issues, so overlapping src/dst is handled.
+define void @memmove_bound32(ptr %dst, ptr %src, i64 %size) {
+; NOMASK-LABEL: define void @memmove_bound32(
+; NOMASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; NOMASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; NOMASK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memmove_bound32(
+; MASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; MASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK-NEXT: [[TMP1:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[SIZE]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[SRC]], <32 x i1> [[TMP1]], <32 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <32 x i1> [[TMP1]])
+; MASK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 33
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+define void @memmove_unbounded(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memmove_unbounded(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 false)
+ ret void
+}
+
+; The bound can come from plain ValueTracking rather than an assume, so this
+; fires even when no AssumptionCache is available.
+define void @memcpy_and_mask(ptr %dst, ptr %src, i64 %size) {
+; NOMASK-LABEL: define void @memcpy_and_mask(
+; NOMASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[SIZE_MOD:%.*]] = and i64 [[SIZE]], 31
+; NOMASK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE_MOD]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memcpy_and_mask(
+; MASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK-NEXT: [[SIZE_MOD:%.*]] = and i64 [[SIZE]], 31
+; MASK-NEXT: [[TMP1:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[SIZE_MOD]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[SRC]], <32 x i1> [[TMP1]], <32 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <32 x i1> [[TMP1]])
+; MASK-NEXT: ret void
+;
+ %size.mod = and i64 %size, 31
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size.mod, i1 false)
+ ret void
+}
+
+; A length type wider than i64 is fine as long as the range itself is narrow.
+define void @memcpy_i128_bounded(ptr %dst, ptr %src, i128 %size) {
+; NOMASK-LABEL: define void @memcpy_i128_bounded(
+; NOMASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i128 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[SIZE_MOD:%.*]] = and i128 [[SIZE]], 31
+; NOMASK-NEXT: call void @llvm.memcpy.p0.p0.i128(ptr [[DST]], ptr [[SRC]], i128 [[SIZE_MOD]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memcpy_i128_bounded(
+; MASK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i128 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK-NEXT: [[SIZE_MOD:%.*]] = and i128 [[SIZE]], 31
+; MASK-NEXT: [[TMP1:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i128(i128 0, i128 [[SIZE_MOD]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 [[SRC]], <32 x i1> [[TMP1]], <32 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v32i8.p0(<32 x i8> [[MASKED_COPY]], ptr align 1 [[DST]], <32 x i1> [[TMP1]])
+; MASK-NEXT: ret void
+;
+ %size.mod = and i128 %size, 31
+ call void @llvm.memcpy.p0.p0.i128(ptr %dst, ptr %src, i128 %size.mod, i1 false)
+ ret void
+}
+
+; Regression test: an unbounded i128 length must be declined on the APInt
+; comparison, before anything calls getZExtValue on an out-of-range maximum.
+define void @memcpy_i128_unbounded(ptr %dst, ptr %src, i128 %size) {
+; CHECK-LABEL: define void @memcpy_i128_unbounded(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i128 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i128(ptr [[DST]], ptr [[SRC]], i128 [[SIZE]], i1 false)
+; CHECK-NEXT: ret void
+;
+ call void @llvm.memcpy.p0.p0.i128(ptr %dst, ptr %src, i128 %size, i1 false)
+ ret void
+}
+
+; A volatile transfer cannot be expressed as masked load/store, which carry no
+; volatility marker, so it is declined on every configuration.
+define void @memcpy_volatile_bounded(ptr %dst, ptr %src, i64 %size) {
+; CHECK-LABEL: define void @memcpy_volatile_bounded(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[DST]], ptr [[SRC]], i64 [[SIZE]], i1 true)
+; CHECK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 33
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 %size, i1 true)
+ ret void
+}
+
+; Masked load/store legality is queried against the pointers' own address
+; spaces, not address space 0.
+define void @memcpy_addrspace_bounded(ptr addrspace(256) %dst, ptr addrspace(256) %src, i64 %size) {
+; NOMASK-LABEL: define void @memcpy_addrspace_bounded(
+; NOMASK-SAME: ptr addrspace(256) [[DST:%.*]], ptr addrspace(256) [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; NOMASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; NOMASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; NOMASK-NEXT: call void @llvm.memcpy.p256.p256.i64(ptr addrspace(256) [[DST]], ptr addrspace(256) [[SRC]], i64 [[SIZE]], i1 false)
+; NOMASK-NEXT: ret void
+;
+; MASK-LABEL: define void @memcpy_addrspace_bounded(
+; MASK-SAME: ptr addrspace(256) [[DST:%.*]], ptr addrspace(256) [[SRC:%.*]], i64 [[SIZE:%.*]]) #[[ATTR0]] {
+; MASK-NEXT: [[CMP:%.*]] = icmp ult i64 [[SIZE]], 33
+; MASK-NEXT: call void @llvm.assume(i1 [[CMP]])
+; MASK-NEXT: [[TMP1:%.*]] = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 0, i64 [[SIZE]])
+; MASK-NEXT: [[MASKED_COPY:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p256(ptr addrspace(256) align 1 [[SRC]], <32 x i1> [[TMP1]], <32 x i8> poison)
+; MASK-NEXT: call void @llvm.masked.store.v32i8.p256(<32 x i8> [[MASKED_COPY]], ptr addrspace(256) align 1 [[DST]], <32 x i1> [[TMP1]])
+; MASK-NEXT: ret void
+;
+ %cmp = icmp ult i64 %size, 33
+ call void @llvm.assume(i1 %cmp)
+ call void @llvm.memcpy.p256.p256.i64(ptr addrspace(256) %dst, ptr addrspace(256) %src, i64 %size, i1 false)
+ ret void
+}
+
+declare void @llvm.assume(i1)
+declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
+declare void @llvm.memcpy.p0.p0.i128(ptr, ptr, i128, i1)
+declare void @llvm.memmove.p0.p0.i64(ptr, ptr, i64, i1)
+declare void @llvm.memcpy.p256.p256.i64(ptr addrspace(256), ptr addrspace(256), i64, i1)
More information about the llvm-commits
mailing list