[llvm] [LowerMemIntrinsics][AMDGPU] Propagate Debug Value (PR #178131)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 26 23:57:39 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Abhishek Kaushik (abhishek-kaushik22)
<details>
<summary>Changes</summary>
Propagate debug value to expanded loops for `memcpy`, `memmove` and `memset` intrinsics.
---
Patch is 38.04 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/178131.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp (+34-6)
- (added) llvm/test/Transforms/PreISelIntrinsicLowering/AMDGPU/expand-mem-intrinsics.ll (+330)
``````````diff
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index e294a3e29c28d..e77cdbe7a3876 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -140,7 +140,9 @@ insertLoopExpansion(Instruction *InsertBefore, Value *Len,
InsertBefore, BBNamePrefix + "-post-expansion");
Function *ParentFunc = PreLoopBB->getParent();
LLVMContext &Ctx = PreLoopBB->getContext();
+ const DebugLoc &DbgLoc = InsertBefore->getStableDebugLoc();
IRBuilder<> PreLoopBuilder(PreLoopBB->getTerminator());
+ PreLoopBuilder.SetCurrentDebugLocation(DbgLoc);
// Calculate the main loop trip count and remaining units to cover after the
// loop.
@@ -179,6 +181,7 @@ insertLoopExpansion(Instruction *InsertBefore, Value *Len,
BasicBlock *MainLoopBB = BasicBlock::Create(
Ctx, BBNamePrefix + "-expansion-main-body", ParentFunc, PostLoopBB);
IRBuilder<> LoopBuilder(MainLoopBB);
+ LoopBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *LoopIndex = LoopBuilder.CreatePHI(LenType, 2, "loop-index");
LEI.MainLoopIndex = LoopIndex;
@@ -228,10 +231,12 @@ insertLoopExpansion(Instruction *InsertBefore, Value *Len,
// Determine if we need to branch to the residual loop or bypass it.
IRBuilder<> RCBuilder(ResidualCondBB);
+ RCBuilder.SetCurrentDebugLocation(DbgLoc);
RCBuilder.CreateCondBr(RCBuilder.CreateICmpNE(ResidualUnits, Zero),
ResLoopBB, PostLoopBB);
IRBuilder<> ResBuilder(ResLoopBB);
+ ResBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *ResidualIndex =
ResBuilder.CreatePHI(LenType, 2, "residual-loop-index");
ResidualIndex->addIncoming(Zero, ResidualCondBB);
@@ -589,7 +594,9 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
ConstantInt::get(ILengthType, ResidualLoopOpSize);
ConstantInt *Zero = ConstantInt::get(ILengthType, 0);
+ const DebugLoc &DbgLoc = InsertBefore->getStableDebugLoc();
IRBuilder<> PLBuilder(InsertBefore);
+ PLBuilder.SetCurrentDebugLocation(DbgLoc);
Value *RuntimeLoopBytes = CopyLen;
Value *RuntimeLoopRemainder = nullptr;
@@ -675,6 +682,7 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
BasicBlock *ResidualLoopBB = BasicBlock::Create(
F->getContext(), "memmove_bwd_residual_loop", F, MainLoopBB);
IRBuilder<> ResidualLoopBuilder(ResidualLoopBB);
+ ResidualLoopBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *ResidualLoopPhi = ResidualLoopBuilder.CreatePHI(ILengthType, 0);
Value *ResidualIndex = ResidualLoopBuilder.CreateSub(
ResidualLoopPhi, CIResidualLoopOpSize, "bwd_residual_index");
@@ -697,6 +705,7 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
F->getContext(), "memmove_bwd_middle", F, MainLoopBB);
// Later code expects a terminator in the PredBB.
IRBuilder<> IntermediateBuilder(IntermediateBB);
+ IntermediateBuilder.SetCurrentDebugLocation(DbgLoc);
IntermediateBuilder.CreateUnreachable();
ResidualLoopBuilder.CreateCondBr(
ResidualLoopBuilder.CreateICmpEQ(ResidualIndex, RuntimeLoopBytes),
@@ -706,8 +715,10 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
ResidualLoopPhi->addIncoming(CopyLen, CopyBackwardsBB);
// How to get to the residual:
- BranchInst::Create(IntermediateBB, ResidualLoopBB, SkipResidualCondition,
- ThenTerm->getIterator());
+ BranchInst *BrInst =
+ BranchInst::Create(IntermediateBB, ResidualLoopBB,
+ SkipResidualCondition, ThenTerm->getIterator());
+ BrInst->setDebugLoc(DbgLoc);
ThenTerm->eraseFromParent();
PredBB = IntermediateBB;
@@ -715,6 +726,7 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
// main loop
IRBuilder<> MainLoopBuilder(MainLoopBB);
+ MainLoopBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *MainLoopPhi = MainLoopBuilder.CreatePHI(ILengthType, 0);
Value *MainIndex =
MainLoopBuilder.CreateSub(MainLoopPhi, CILoopOpSize, "bwd_main_index");
@@ -733,8 +745,9 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
// How to get to the main loop:
Instruction *PredBBTerm = PredBB->getTerminator();
- BranchInst::Create(ExitBB, MainLoopBB, SkipMainCondition,
- PredBBTerm->getIterator());
+ BranchInst *BrInst = BranchInst::Create(
+ ExitBB, MainLoopBB, SkipMainCondition, PredBBTerm->getIterator());
+ BrInst->setDebugLoc(DbgLoc);
PredBBTerm->eraseFromParent();
}
@@ -744,6 +757,7 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
BasicBlock *MainLoopBB =
BasicBlock::Create(F->getContext(), "memmove_fwd_main_loop", F, ExitBB);
IRBuilder<> MainLoopBuilder(MainLoopBB);
+ MainLoopBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *MainLoopPhi =
MainLoopBuilder.CreatePHI(ILengthType, 0, "fwd_main_index");
Value *LoadGEP =
@@ -770,13 +784,16 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
MainLoopBB);
// getting in or skipping the main loop
- BranchInst::Create(SuccessorBB, MainLoopBB, SkipMainCondition,
- CopyFwdBBTerm->getIterator());
+ BranchInst *BrInst =
+ BranchInst::Create(SuccessorBB, MainLoopBB, SkipMainCondition,
+ CopyFwdBBTerm->getIterator());
+ BrInst->setDebugLoc(DbgLoc);
CopyFwdBBTerm->eraseFromParent();
if (RequiresResidual) {
BasicBlock *IntermediateBB = SuccessorBB;
IRBuilder<> IntermediateBuilder(IntermediateBB);
+ IntermediateBuilder.SetCurrentDebugLocation(DbgLoc);
BasicBlock *ResidualLoopBB = BasicBlock::Create(
F->getContext(), "memmove_fwd_residual_loop", F, ExitBB);
IntermediateBuilder.CreateCondBr(SkipResidualCondition, ExitBB,
@@ -784,6 +801,7 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
// Residual loop
IRBuilder<> ResidualLoopBuilder(ResidualLoopBB);
+ ResidualLoopBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *ResidualLoopPhi =
ResidualLoopBuilder.CreatePHI(ILengthType, 0, "fwd_residual_index");
Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
@@ -841,7 +859,9 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
ConstantInt *LoopBound = ConstantInt::get(ILengthType, BytesCopiedInLoop);
ConstantInt *CILoopOpSize = ConstantInt::get(ILengthType, LoopOpSize);
+ const DebugLoc &DbgLoc = InsertBefore->getStableDebugLoc();
IRBuilder<> PLBuilder(InsertBefore);
+ PLBuilder.SetCurrentDebugLocation(DbgLoc);
auto [CmpSrcAddr, CmpDstAddr] =
tryInsertCastToCommonAddrSpace(PLBuilder, SrcAddr, DstAddr, TTI);
@@ -894,6 +914,7 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
// instead of after it.
IRBuilder<> BwdResBuilder(CopyBackwardsBB,
CopyBackwardsBB->getFirstNonPHIIt());
+ BwdResBuilder.SetCurrentDebugLocation(DbgLoc);
SmallVector<Type *, 5> RemainingOps;
TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
SrcAS, DstAS, PartSrcAlign,
@@ -917,6 +938,7 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
CopyBackwardsBB->setName("memmove_bwd_loop");
}
IRBuilder<> LoopBuilder(LoopBB->getTerminator());
+ LoopBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *LoopPhi = LoopBuilder.CreatePHI(ILengthType, 0);
Value *Index = LoopBuilder.CreateSub(LoopPhi, CILoopOpSize, "bwd_index");
Value *LoadGEP = LoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr, Index);
@@ -950,6 +972,7 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
FwdResidualBB = SuccBB;
}
IRBuilder<> LoopBuilder(LoopBB->getTerminator());
+ LoopBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *LoopPhi = LoopBuilder.CreatePHI(ILengthType, 0, "fwd_index");
Value *LoadGEP = LoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr, LoopPhi);
Value *Element = LoopBuilder.CreateAlignedLoad(
@@ -974,6 +997,7 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
// Residual code is required to move the remaining bytes. In the forward
// case, we emit it in the normal order.
IRBuilder<> FwdResBuilder(FwdResidualBB->getTerminator());
+ FwdResBuilder.SetCurrentDebugLocation(DbgLoc);
SmallVector<Type *, 5> RemainingOps;
TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
SrcAS, DstAS, PartSrcAlign,
@@ -996,7 +1020,9 @@ static void createMemSetLoop(Instruction *InsertBefore, Value *DstAddr,
BasicBlock *LoopBB
= BasicBlock::Create(F->getContext(), "loadstoreloop", F, NewBB);
+ const DebugLoc &DbgLoc = InsertBefore->getStableDebugLoc();
IRBuilder<> Builder(OrigBB->getTerminator());
+ Builder.SetCurrentDebugLocation(DbgLoc);
auto *ToLoopBR = Builder.CreateCondBr(
Builder.CreateICmpEQ(ConstantInt::get(TypeOfCopyLen, 0), CopyLen), NewBB,
@@ -1014,6 +1040,7 @@ static void createMemSetLoop(Instruction *InsertBefore, Value *DstAddr,
Align PartAlign(commonAlignment(DstAlign, PartSize));
IRBuilder<> LoopBuilder(LoopBB);
+ LoopBuilder.SetCurrentDebugLocation(DbgLoc);
PHINode *LoopIndex = LoopBuilder.CreatePHI(TypeOfCopyLen, 0);
LoopIndex->addIncoming(ConstantInt::get(TypeOfCopyLen, 0), OrigBB);
@@ -1092,6 +1119,7 @@ bool llvm::expandMemMoveAsLoop(MemMoveInst *Memmove,
bool SrcIsVolatile = Memmove->isVolatile();
bool DstIsVolatile = SrcIsVolatile;
IRBuilder<> CastBuilder(Memmove);
+ CastBuilder.SetCurrentDebugLocation(Memmove->getStableDebugLoc());
unsigned SrcAS = SrcAddr->getType()->getPointerAddressSpace();
unsigned DstAS = DstAddr->getType()->getPointerAddressSpace();
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/AMDGPU/expand-mem-intrinsics.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/AMDGPU/expand-mem-intrinsics.ll
new file mode 100644
index 0000000000000..0da7b1494ef9f
--- /dev/null
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/AMDGPU/expand-mem-intrinsics.ll
@@ -0,0 +1,330 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -p=debugify,pre-isel-intrinsic-lowering,check-debugify -S < %s | FileCheck %s
+
+define protected amdgpu_kernel void @memcpy(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 noundef %n) {
+; CHECK-LABEL: define protected amdgpu_kernel void @memcpy(
+; CHECK-SAME: ptr addrspace(1) [[DST:%.*]], ptr addrspace(1) [[SRC:%.*]], i64 noundef [[N:%.*]]) !dbg [[DBG5:![0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[N]], 15, !dbg [[DBG11:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[TMP1]], !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne i64 [[TMP2]], 0, !dbg [[DBG11]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[DYNAMIC_MEMCPY_EXPANSION_MAIN_BODY:.*]], label %[[DYNAMIC_MEMCPY_EXPANSION_RESIDUAL_COND:.*]], !dbg [[DBG11]]
+; CHECK: [[DYNAMIC_MEMCPY_EXPANSION_MAIN_BODY]]:
+; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP7:%.*]], %[[DYNAMIC_MEMCPY_EXPANSION_MAIN_BODY]] ], !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[LOOP_INDEX]], !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP4]], align 1, !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[LOOP_INDEX]], !dbg [[DBG11]]
+; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr addrspace(1) [[TMP6]], align 1, !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP7]] = add i64 [[LOOP_INDEX]], 16, !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i64 [[TMP7]], [[TMP2]], !dbg [[DBG11]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[DYNAMIC_MEMCPY_EXPANSION_MAIN_BODY]], label %[[DYNAMIC_MEMCPY_EXPANSION_RESIDUAL_COND]], !dbg [[DBG11]]
+; CHECK: [[DYNAMIC_MEMCPY_EXPANSION_RESIDUAL_COND]]:
+; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i64 [[TMP1]], 0, !dbg [[DBG11]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[DYNAMIC_MEMCPY_EXPANSION_RESIDUAL_BODY:.*]], label %[[DYNAMIC_MEMCPY_POST_EXPANSION:.*]], !dbg [[DBG11]]
+; CHECK: [[DYNAMIC_MEMCPY_EXPANSION_RESIDUAL_BODY]]:
+; CHECK-NEXT: [[RESIDUAL_LOOP_INDEX:%.*]] = phi i64 [ 0, %[[DYNAMIC_MEMCPY_EXPANSION_RESIDUAL_COND]] ], [ [[TMP14:%.*]], %[[DYNAMIC_MEMCPY_EXPANSION_RESIDUAL_BODY]] ], !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP2]], [[RESIDUAL_LOOP_INDEX]], !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[TMP10]], !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr addrspace(1) [[TMP11]], align 1, !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[TMP10]], !dbg [[DBG11]]
+; CHECK-NEXT: store i8 [[TMP12]], ptr addrspace(1) [[TMP13]], align 1, !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP14]] = add i64 [[RESIDUAL_LOOP_INDEX]], 1, !dbg [[DBG11]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp ult i64 [[TMP14]], [[TMP1]], !dbg [[DBG11]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[DYNAMIC_MEMCPY_EXPANSION_RESIDUAL_BODY]], label %[[DYNAMIC_MEMCPY_POST_EXPANSION]], !dbg [[DBG11]]
+; CHECK: [[DYNAMIC_MEMCPY_POST_EXPANSION]]:
+; CHECK-NEXT: #dbg_value(i32 0, [[META9:![0-9]+]], !DIExpression(), [[META12:![0-9]+]])
+; CHECK-NEXT: ret void, !dbg [[META12]]
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) align 1 %dst, ptr addrspace(1) align 1 %src, i64 %n, i1 false)
+ ret void
+}
+
+define protected amdgpu_kernel void @memcpy_1025_bytes(ptr addrspace(1) %dst, ptr addrspace(1) %src) {
+; CHECK-LABEL: define protected amdgpu_kernel void @memcpy_1025_bytes(
+; CHECK-SAME: ptr addrspace(1) [[DST:%.*]], ptr addrspace(1) [[SRC:%.*]]) !dbg [[DBG13:![0-9]+]] {
+; CHECK-NEXT: br label %[[STATIC_MEMCPY_EXPANSION_MAIN_BODY:.*]], !dbg [[DBG16:![0-9]+]]
+; CHECK: [[STATIC_MEMCPY_EXPANSION_MAIN_BODY]]:
+; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_EXPANSION_MAIN_BODY]] ], !dbg [[DBG16]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[LOOP_INDEX]], !dbg [[DBG16]]
+; CHECK-NEXT: [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 1, !dbg [[DBG16]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[LOOP_INDEX]], !dbg [[DBG16]]
+; CHECK-NEXT: store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1, !dbg [[DBG16]]
+; CHECK-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 256, !dbg [[DBG16]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 1024, !dbg [[DBG16]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_EXPANSION:.*]], !dbg [[DBG16]]
+; CHECK: [[STATIC_MEMCPY_POST_EXPANSION]]:
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1024, !dbg [[DBG16]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr addrspace(1) [[TMP6]], align 1, !dbg [[DBG16]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1024, !dbg [[DBG16]]
+; CHECK-NEXT: store i8 [[TMP7]], ptr addrspace(1) [[TMP8]], align 1, !dbg [[DBG16]]
+; CHECK-NEXT: #dbg_value(i32 0, [[META15:![0-9]+]], !DIExpression(), [[META17:![0-9]+]])
+; CHECK-NEXT: ret void, !dbg [[META17]]
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 1 dereferenceable(1025) %dst, ptr addrspace(1) noundef align 1 dereferenceable(1025) %src, i64 1025, i1 false)
+ ret void
+}
+
+define protected amdgpu_kernel void @memcpy_2048_bytes(ptr addrspace(1) %dst, ptr addrspace(1) %src) {
+; CHECK-LABEL: define protected amdgpu_kernel void @memcpy_2048_bytes(
+; CHECK-SAME: ptr addrspace(1) [[DST:%.*]], ptr addrspace(1) [[SRC:%.*]]) !dbg [[DBG18:![0-9]+]] {
+; CHECK-NEXT: br label %[[STATIC_MEMCPY_EXPANSION_MAIN_BODY:.*]], !dbg [[DBG21:![0-9]+]]
+; CHECK: [[STATIC_MEMCPY_EXPANSION_MAIN_BODY]]:
+; CHECK-NEXT: [[LOOP_INDEX:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP4:%.*]], %[[STATIC_MEMCPY_EXPANSION_MAIN_BODY]] ], !dbg [[DBG21]]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[LOOP_INDEX]], !dbg [[DBG21]]
+; CHECK-NEXT: [[TMP2:%.*]] = load <64 x i32>, ptr addrspace(1) [[TMP1]], align 1, !dbg [[DBG21]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[LOOP_INDEX]], !dbg [[DBG21]]
+; CHECK-NEXT: store <64 x i32> [[TMP2]], ptr addrspace(1) [[TMP3]], align 1, !dbg [[DBG21]]
+; CHECK-NEXT: [[TMP4]] = add i64 [[LOOP_INDEX]], 256, !dbg [[DBG21]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP4]], 2048, !dbg [[DBG21]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[STATIC_MEMCPY_EXPANSION_MAIN_BODY]], label %[[STATIC_MEMCPY_POST_EXPANSION:.*]], !dbg [[DBG21]]
+; CHECK: [[STATIC_MEMCPY_POST_EXPANSION]]:
+; CHECK-NEXT: #dbg_value(i32 0, [[META20:![0-9]+]], !DIExpression(), [[META22:![0-9]+]])
+; CHECK-NEXT: ret void, !dbg [[META22]]
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 1 dereferenceable(2048) %dst, ptr addrspace(1) noundef align 1 dereferenceable(2048) %src, i64 2048, i1 false)
+ ret void
+}
+
+define protected amdgpu_kernel void @memmove(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 noundef %n) {
+; CHECK-LABEL: define protected amdgpu_kernel void @memmove(
+; CHECK-SAME: ptr addrspace(1) [[DST:%.*]], ptr addrspace(1) [[SRC:%.*]], i64 noundef [[N:%.*]]) !dbg [[DBG23:![0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = and i64 [[N]], 15, !dbg [[DBG26:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[TMP1]], !dbg [[DBG26]]
+; CHECK-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i64 [[TMP1]], 0, !dbg [[DBG26]]
+; CHECK-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i64 [[TMP2]], 0, !dbg [[DBG26]]
+; CHECK-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC]], [[DST]], !dbg [[DBG26]]
+; CHECK-NEXT: br i1 [[COMPARE_SRC_DST]], label %[[MEMMOVE_COPY_BACKWARDS:.*]], label %[[MEMMOVE_COPY_FORWARD:.*]], !dbg [[DBG26]]
+; CHECK: [[MEMMOVE_COPY_BACKWARDS]]:
+; CHECK-NEXT: br i1 [[SKIP_RESIDUAL]], label %[[MEMMOVE_BWD_MIDDLE:.*]], label %[[MEMMOVE_BWD_RESIDUAL_LOOP:.*]], !dbg [[DBG26]]
+; CHECK: [[MEMMOVE_BWD_RESIDUAL_LOOP]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], %[[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[N]], %[[MEMMOVE_COPY_BACKWARDS]] ], !dbg [[DBG26]]
+; CHECK-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP3]], 1, !dbg [[DBG26]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]], !dbg [[DBG26]]
+; CHECK-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(1) [[TMP4]], align 1, !dbg [[DBG26]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]], !dbg [[DBG26]]
+; CHECK-NEXT: store i8 [[ELEMENT]], ptr addrspace(1) [[TMP5]], align 1, !dbg [[DBG26]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], [[TMP2]], !dbg [[DBG26]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MEMMOVE_BWD_MIDDLE]], label %[[MEMMOVE_BWD_RESIDUAL_LOOP]], !dbg [[DBG26]]
+; CHECK: [[MEMMOVE_BWD_MIDDLE]]:
+; CHECK-NEXT: br i1 [[SKIP_MAIN]], label %[[MEMMOVE_DONE:.*]], label %[[MEMMOVE_BWD_MAIN_LOOP:.*]], !dbg [[DBG26]]
+; CHECK: [[MEMMOVE_BWD_MAIN_LOOP]]:
+; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], %[[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP2]], %[[MEMMOVE_BWD_MIDDLE]] ], !dbg [[DBG26]]
+; CHECK-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP7]], 16, !dbg [[DBG26]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]], !dbg [[DBG26]]
+; CHECK-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP8]], align 1, !dbg [[DBG26]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]], !dbg [[...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/178131
More information about the llvm-commits
mailing list