[llvm] 12fb0d4 - [DirectX] Legalize memcpy (#139173)
via llvm-commits
llvm-commits at lists.llvm.org
Mon May 19 09:35:02 PDT 2025
Author: Deric C.
Date: 2025-05-19T09:34:58-07:00
New Revision: 12fb0d451bbdd1d1e8035197e4f5e47e8afa6141
URL: https://github.com/llvm/llvm-project/commit/12fb0d451bbdd1d1e8035197e4f5e47e8afa6141
DIFF: https://github.com/llvm/llvm-project/commit/12fb0d451bbdd1d1e8035197e4f5e47e8afa6141.diff
LOG: [DirectX] Legalize memcpy (#139173)
Fixes #137188
This PR legalizes memcpy for DXIL in cases where:
- the src and dst arguments are from Alloca or a GlobalVariable,
- the src and dst are pointers to an ArrayType,
- the array element types of src and dst must be equivalent, and
- the len param is a ConstantInt
These assumptions simplify the legalization and, with the addition of
#138991, covers the currently-known cases of memcpy that appear when
compiling DML shaders.
This PR may be unnecessary if #138788 determines that memset and memcpy
can be eliminated entirely.
---------
Co-authored-by: Finn Plummer <canadienfinn at gmail.com>
Co-authored-by: Greg Roth <grroth at microsoft.com>
Added:
llvm/test/CodeGen/DirectX/legalize-memcpy.ll
Modified:
llvm/lib/Target/DirectX/DXILLegalizePass.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
index be77a70fa46ba..7f8903a9d5925 100644
--- a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
+++ b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
@@ -246,6 +246,67 @@ downcastI64toI32InsertExtractElements(Instruction &I,
}
}
+static void emitMemcpyExpansion(IRBuilder<> &Builder, Value *Dst, Value *Src,
+ ConstantInt *Length) {
+
+ uint64_t ByteLength = Length->getZExtValue();
+ // If length to copy is zero, no memcpy is needed.
+ if (ByteLength == 0)
+ return;
+
+ LLVMContext &Ctx = Builder.getContext();
+ const DataLayout &DL = Builder.GetInsertBlock()->getModule()->getDataLayout();
+
+ auto GetArrTyFromVal = [](Value *Val) -> ArrayType * {
+ assert(isa<AllocaInst>(Val) ||
+ isa<GlobalVariable>(Val) &&
+ "Expected Val to be an Alloca or Global Variable");
+ if (auto *Alloca = dyn_cast<AllocaInst>(Val))
+ return dyn_cast<ArrayType>(Alloca->getAllocatedType());
+ if (auto *GlobalVar = dyn_cast<GlobalVariable>(Val))
+ return dyn_cast<ArrayType>(GlobalVar->getValueType());
+ return nullptr;
+ };
+
+ ArrayType *DstArrTy = GetArrTyFromVal(Dst);
+ assert(DstArrTy && "Expected Dst of memcpy to be a Pointer to an Array Type");
+ if (auto *DstGlobalVar = dyn_cast<GlobalVariable>(Dst))
+ assert(!DstGlobalVar->isConstant() &&
+ "The Dst of memcpy must not be a constant Global Variable");
+ [[maybe_unused]] ArrayType *SrcArrTy = GetArrTyFromVal(Src);
+ assert(SrcArrTy && "Expected Src of memcpy to be a Pointer to an Array Type");
+
+ Type *DstElemTy = DstArrTy->getElementType();
+ uint64_t DstElemByteSize = DL.getTypeStoreSize(DstElemTy);
+ assert(DstElemByteSize > 0 && "Dst element type store size must be set");
+ Type *SrcElemTy = SrcArrTy->getElementType();
+ [[maybe_unused]] uint64_t SrcElemByteSize = DL.getTypeStoreSize(SrcElemTy);
+ assert(SrcElemByteSize > 0 && "Src element type store size must be set");
+
+ // This assumption simplifies implementation and covers currently-known
+ // use-cases for DXIL. It may be relaxed in the future if required.
+ assert(DstElemTy == SrcElemTy &&
+ "The element types of Src and Dst arrays must match");
+
+ [[maybe_unused]] uint64_t DstArrNumElems = DstArrTy->getArrayNumElements();
+ assert(DstElemByteSize * DstArrNumElems >= ByteLength &&
+ "Dst array size must be at least as large as the memcpy length");
+ [[maybe_unused]] uint64_t SrcArrNumElems = SrcArrTy->getArrayNumElements();
+ assert(SrcElemByteSize * SrcArrNumElems >= ByteLength &&
+ "Src array size must be at least as large as the memcpy length");
+
+ uint64_t NumElemsToCopy = ByteLength / DstElemByteSize;
+ assert(ByteLength % DstElemByteSize == 0 &&
+ "memcpy length must be divisible by array element type");
+ for (uint64_t I = 0; I < NumElemsToCopy; ++I) {
+ Value *Offset = ConstantInt::get(Type::getInt32Ty(Ctx), I);
+ Value *SrcPtr = Builder.CreateInBoundsGEP(SrcElemTy, Src, Offset, "gep");
+ Value *SrcVal = Builder.CreateLoad(SrcElemTy, SrcPtr);
+ Value *DstPtr = Builder.CreateInBoundsGEP(DstElemTy, Dst, Offset, "gep");
+ Builder.CreateStore(SrcVal, DstPtr);
+ }
+}
+
static void emitMemsetExpansion(IRBuilder<> &Builder, Value *Dst, Value *Val,
ConstantInt *SizeCI,
DenseMap<Value *, Value *> &ReplacedValues) {
@@ -296,6 +357,33 @@ static void emitMemsetExpansion(IRBuilder<> &Builder, Value *Dst, Value *Val,
}
}
+// Expands the instruction `I` into corresponding loads and stores if it is a
+// memcpy call. In that case, the call instruction is added to the `ToRemove`
+// vector. `ReplacedValues` is unused.
+static void legalizeMemCpy(Instruction &I,
+ SmallVectorImpl<Instruction *> &ToRemove,
+ DenseMap<Value *, Value *> &ReplacedValues) {
+
+ CallInst *CI = dyn_cast<CallInst>(&I);
+ if (!CI)
+ return;
+
+ Intrinsic::ID ID = CI->getIntrinsicID();
+ if (ID != Intrinsic::memcpy)
+ return;
+
+ IRBuilder<> Builder(&I);
+ Value *Dst = CI->getArgOperand(0);
+ Value *Src = CI->getArgOperand(1);
+ ConstantInt *Length = dyn_cast<ConstantInt>(CI->getArgOperand(2));
+ assert(Length && "Expected Length to be a ConstantInt");
+ ConstantInt *IsVolatile = dyn_cast<ConstantInt>(CI->getArgOperand(3));
+ assert(IsVolatile && "Expected IsVolatile to be a ConstantInt");
+ assert(IsVolatile->getZExtValue() == 0 && "Expected IsVolatile to be false");
+ emitMemcpyExpansion(Builder, Dst, Src, Length);
+ ToRemove.push_back(CI);
+}
+
static void removeMemSet(Instruction &I,
SmallVectorImpl<Instruction *> &ToRemove,
DenseMap<Value *, Value *> &ReplacedValues) {
@@ -348,6 +436,7 @@ class DXILLegalizationPipeline {
LegalizationPipeline.push_back(fixI8UseChain);
LegalizationPipeline.push_back(downcastI64toI32InsertExtractElements);
LegalizationPipeline.push_back(legalizeFreeze);
+ LegalizationPipeline.push_back(legalizeMemCpy);
LegalizationPipeline.push_back(removeMemSet);
}
};
diff --git a/llvm/test/CodeGen/DirectX/legalize-memcpy.ll b/llvm/test/CodeGen/DirectX/legalize-memcpy.ll
new file mode 100644
index 0000000000000..0cf9d83b3af25
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/legalize-memcpy.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -dxil-legalize -dxil-finalize-linkage -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+
+define void @replace_int_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_int_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [1 x i32], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [1 x i32], align 4
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store i32 [[TMP3]], ptr [[GEP1]], align 4
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [1 x i32], align 4
+ %2 = alloca [1 x i32], align 4
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(4) %2, ptr align 4 dereferenceable(4) %1, i32 4, i1 false)
+ ret void
+}
+
+define void @replace_3int_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_3int_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store i32 [[TMP3]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store i32 [[TMP4]], ptr [[GEP3]], align 4
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[GEP4]], align 4
+; CHECK-NEXT: [[GEP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 2
+; CHECK-NEXT: store i32 [[TMP5]], ptr [[GEP5]], align 4
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [3 x i32], align 4
+ %2 = alloca [3 x i32], align 4
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(12) %2, ptr align 4 dereferenceable(12) %1, i32 12, i1 false)
+ ret void
+}
+
+define void @replace_mismatched_size_int_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_mismatched_size_int_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x i32], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store i32 [[TMP3]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store i32 [[TMP4]], ptr [[GEP3]], align 4
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x i32], align 4
+ %2 = alloca [3 x i32], align 4
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(12) %2, ptr align 4 dereferenceable(8) %1, i32 8, i1 false)
+ ret void
+}
+
+define void @replace_int16_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_int16_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x i16], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x i16], align 2
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i16, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store i16 [[TMP3]], ptr [[GEP1]], align 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[GEP2]], align 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i16, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store i16 [[TMP4]], ptr [[GEP3]], align 2
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x i16], align 2
+ %2 = alloca [2 x i16], align 2
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 2 dereferenceable(4) %2, ptr align 2 dereferenceable(4) %1, i32 4, i1 false)
+ ret void
+}
+
+define void @replace_float_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_float_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x float], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x float], align 4
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store float [[TMP3]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store float [[TMP4]], ptr [[GEP3]], align 4
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x float], align 4
+ %2 = alloca [2 x float], align 4
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(8) %2, ptr align 4 dereferenceable(8) %1, i32 8, i1 false)
+ ret void
+}
+
+define void @replace_double_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_double_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x double], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x double], align 4
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load double, ptr [[GEP]], align 8
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds double, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store double [[TMP3]], ptr [[GEP1]], align 8
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load double, ptr [[GEP2]], align 8
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds double, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store double [[TMP4]], ptr [[GEP3]], align 8
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x double], align 4
+ %2 = alloca [2 x double], align 4
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(8) %2, ptr align 4 dereferenceable(8) %1, i32 16, i1 false)
+ ret void
+}
+
+define void @replace_half_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_half_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x half], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x half], align 2
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds half, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load half, ptr [[GEP]], align 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds half, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store half [[TMP3]], ptr [[GEP1]], align 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds half, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load half, ptr [[GEP2]], align 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds half, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store half [[TMP4]], ptr [[GEP3]], align 2
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x half], align 2
+ %2 = alloca [2 x half], align 2
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 2 dereferenceable(4) %2, ptr align 2 dereferenceable(4) %1, i32 4, i1 false)
+ ret void
+}
+
+attributes #0 = {"hlsl.export"}
+
+declare void @llvm.memcpy.p0.p2.i32(ptr noalias, ptr addrspace(2) noalias readonly, i32, i1)
+declare void @llvm.memcpy.p0.p0.i32(ptr noalias, ptr noalias readonly, i32, i1)
More information about the llvm-commits
mailing list