[llvm] [LowerMemIntrinsics] Lower llvm.memmove to wide memory accesses (PR #100122)
Fabian Ritter via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 24 00:47:33 PDT 2024
https://github.com/ritter-x2a updated https://github.com/llvm/llvm-project/pull/100122
>From af1c0f8ccf73d1587d1ab590b4a6bdd2cf63ba44 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Tue, 23 Jul 2024 09:09:44 -0400
Subject: [PATCH 1/4] [LowerMemIntrinsics] Lower llvm.memmove to wide memory
accesses
So far, the IR-level lowering of llvm.memmove intrinsics generates loops
that copy each byte individually. This can be wasteful for targets that
provide wider memory access operations.
This patch makes the memmove lowering more similar to the lowering of
memcpy with unknown length.
TargetTransformInfo::getMemcpyLoopLoweringType() is queried for an
adequate type for the memory accesses, and if it is wider than a single
byte, the greatest multiple of the type's size that is less than or
equal to the length is copied with corresponding wide memory accesses.
A residual loop with byte-wise accesses is introduced for the remaining
bytes.
For memmove, this construct is required in two variants: one for copying
forward and one for copying backwards, to handle overlapping memory
ranges. For the backwards case, the residual loop still covers the bytes
at the end of the copied region and is therefore executed before the
wide main loop. This implementation choice is based on the assumption
that we are more likely to encounter memory ranges whose start aligns
with the access width than ones whose end does.
In microbenchmarks on gfx1030 (AMDGPU), this change yields speedups up
to 16x for memmoves with variable or large constant lengths.
---
.../Transforms/Utils/LowerMemIntrinsics.cpp | 270 ++++++++---
.../CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll | 15 +-
.../CodeGen/AMDGPU/lower-mem-intrinsics.ll | 455 ++++++++++++------
llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll | 2 +-
4 files changed, 519 insertions(+), 223 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index b38db412f786a..55cd61746d19c 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -369,6 +369,10 @@ void llvm::createMemCpyLoopUnknownSize(
// }
// return dst;
// }
+//
+// If the TargetTransformInfo specifies a wider MemcpyLoopLoweringType, it is
+// used for the memory accesses in the loops. Then, additional loops with
+// byte-wise accesses are added for the remaining bytes.
static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
Value *DstAddr, Value *CopyLen, Align SrcAlign,
Align DstAlign, bool SrcIsVolatile,
@@ -378,8 +382,46 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
BasicBlock *OrigBB = InsertBefore->getParent();
Function *F = OrigBB->getParent();
const DataLayout &DL = F->getDataLayout();
- // TODO: Use different element type if possible?
- Type *EltTy = Type::getInt8Ty(F->getContext());
+ LLVMContext &Ctx = OrigBB->getContext();
+ unsigned SrcAS = cast<PointerType>(SrcAddr->getType())->getAddressSpace();
+ unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
+
+ Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
+ Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value());
+ unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
+ Type *Int8Type = Type::getInt8Ty(Ctx);
+ bool LoopOpIsInt8 = LoopOpType == Int8Type;
+
+ // If the memory accesses are wider than one byte, residual loops with
+ // i8-accesses are required to move remaining bytes.
+ bool RequiresResidual = !LoopOpIsInt8;
+
+ // Calculate the loop trip count and remaining bytes to copy after the loop.
+ IntegerType *ILengthType = dyn_cast<IntegerType>(TypeOfCopyLen);
+ assert(ILengthType &&
+ "expected size argument to memcpy to be an integer type!");
+ ConstantInt *CILoopOpSize = ConstantInt::get(ILengthType, LoopOpSize);
+ ConstantInt *Zero = ConstantInt::get(ILengthType, 0);
+ ConstantInt *One = ConstantInt::get(ILengthType, 1);
+
+ IRBuilder<> PLBuilder(InsertBefore);
+
+ Value *RuntimeLoopCount = CopyLen;
+ Value *RuntimeLoopRemainder = nullptr;
+ Value *RuntimeBytesCopiedMainLoop = CopyLen;
+ Value *SkipResidualCondition = nullptr;
+ if (RequiresResidual) {
+ RuntimeLoopCount =
+ getRuntimeLoopCount(DL, PLBuilder, CopyLen, CILoopOpSize, LoopOpSize);
+ RuntimeLoopRemainder = getRuntimeLoopRemainder(DL, PLBuilder, CopyLen,
+ CILoopOpSize, LoopOpSize);
+ RuntimeBytesCopiedMainLoop =
+ PLBuilder.CreateSub(CopyLen, RuntimeLoopRemainder);
+ SkipResidualCondition =
+ PLBuilder.CreateICmpEQ(RuntimeLoopRemainder, Zero, "skip_residual");
+ }
+ Value *SkipMainCondition =
+ PLBuilder.CreateICmpEQ(RuntimeLoopCount, Zero, "skip_main");
// Create the a comparison of src and dst, based on which we jump to either
// the forward-copy part of the function (if src >= dst) or the backwards-copy
@@ -387,76 +429,182 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
// SplitBlockAndInsertIfThenElse conveniently creates the basic if-then-else
// structure. Its block terminators (unconditional branches) are replaced by
// the appropriate conditional branches when the loop is built.
- ICmpInst *PtrCompare = new ICmpInst(InsertBefore->getIterator(), ICmpInst::ICMP_ULT,
- SrcAddr, DstAddr, "compare_src_dst");
+ Value *PtrCompare =
+ PLBuilder.CreateICmpULT(SrcAddr, DstAddr, "compare_src_dst");
Instruction *ThenTerm, *ElseTerm;
- SplitBlockAndInsertIfThenElse(PtrCompare, InsertBefore->getIterator(), &ThenTerm,
- &ElseTerm);
-
- // Each part of the function consists of two blocks:
- // copy_backwards: used to skip the loop when n == 0
- // copy_backwards_loop: the actual backwards loop BB
- // copy_forward: used to skip the loop when n == 0
- // copy_forward_loop: the actual forward loop BB
+ SplitBlockAndInsertIfThenElse(PtrCompare, InsertBefore->getIterator(),
+ &ThenTerm, &ElseTerm);
+
+ // If the LoopOpSize is greater than 1, each part of the function consist of
+ // four blocks:
+ // memmove_copy_backwards:
+ // skip the residual loop when 0 iterations are required
+ // memmove_bwd_residual_loop:
+ // copy the last few bytes individually so that the remaining length is
+ // a multiple of the LoopOpSize
+ // memmove_bwd_middle: skip the main loop when 0 iterations are required
+ // memmove_bwd_main_loop: the actual backwards loop BB with wide accesses
+ // memmove_copy_forward: skip the main loop when 0 iterations are required
+ // memmove_fwd_main_loop: the actual forward loop BB with wide accesses
+ // memmove_fwd_middle: skip the residual loop when 0 iterations are required
+ // memmove_fwd_residual_loop: copy the last few bytes individually
+ //
+ // The main and residual loop are switched between copying forward and
+ // backward so that the residual loop always operates on the end of the moved
+ // range. This is based on the assumption that buffers whose start is aligned
+ // with the LoopOpSize are more common than buffers whose end is.
+ //
+ // If the LoopOpSize is 1, each part of the function consists of two blocks:
+ // memmove_copy_backwards: skip the loop when 0 iterations are required
+ // memmove_bwd_main_loop: the actual backwards loop BB
+ // memmove_copy_forward: skip the loop when 0 iterations are required
+ // memmove_fwd_main_loop: the actual forward loop BB
BasicBlock *CopyBackwardsBB = ThenTerm->getParent();
- CopyBackwardsBB->setName("copy_backwards");
+ CopyBackwardsBB->setName("memmove_copy_backwards");
BasicBlock *CopyForwardBB = ElseTerm->getParent();
- CopyForwardBB->setName("copy_forward");
+ CopyForwardBB->setName("memmove_copy_forward");
BasicBlock *ExitBB = InsertBefore->getParent();
ExitBB->setName("memmove_done");
- unsigned PartSize = DL.getTypeStoreSize(EltTy);
- Align PartSrcAlign(commonAlignment(SrcAlign, PartSize));
- Align PartDstAlign(commonAlignment(DstAlign, PartSize));
-
- // Initial comparison of n == 0 that lets us skip the loops altogether. Shared
- // between both backwards and forward copy clauses.
- ICmpInst *CompareN =
- new ICmpInst(OrigBB->getTerminator()->getIterator(), ICmpInst::ICMP_EQ, CopyLen,
- ConstantInt::get(TypeOfCopyLen, 0), "compare_n_to_0");
+ Align PartSrcAlign(commonAlignment(SrcAlign, LoopOpSize));
+ Align PartDstAlign(commonAlignment(DstAlign, LoopOpSize));
// Copying backwards.
- BasicBlock *LoopBB =
- BasicBlock::Create(F->getContext(), "copy_backwards_loop", F, CopyForwardBB);
- IRBuilder<> LoopBuilder(LoopBB);
+ {
+ BasicBlock *MainLoopBB = BasicBlock::Create(
+ F->getContext(), "memmove_bwd_main_loop", F, CopyForwardBB);
+
+ // The predecessor of the memmove_bwd_main_loop. Updated in the
+ // following if a residual loop is emitted first.
+ BasicBlock *PredBB = CopyBackwardsBB;
+
+ if (RequiresResidual) {
+ // backwards residual loop
+ BasicBlock *ResidualLoopBB = BasicBlock::Create(
+ F->getContext(), "memmove_bwd_residual_loop", F, MainLoopBB);
+ IRBuilder<> ResidualLoopBuilder(ResidualLoopBB);
+ PHINode *ResidualLoopPhi = ResidualLoopBuilder.CreatePHI(ILengthType, 0);
+ Value *ResidualIndex = ResidualLoopBuilder.CreateSub(
+ ResidualLoopPhi, One, "bwd_residual_index");
+ Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
+ ResidualIndex);
+ Value *Element = ResidualLoopBuilder.CreateLoad(Int8Type, LoadGEP,
+ SrcIsVolatile, "element");
+ Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, DstAddr,
+ ResidualIndex);
+ ResidualLoopBuilder.CreateStore(Element, StoreGEP, DstIsVolatile);
+
+ // After the residual loop, go to an intermediate block.
+ BasicBlock *IntermediateBB = BasicBlock::Create(
+ F->getContext(), "memmove_bwd_middle", F, MainLoopBB);
+ // Later code expects a terminator in the PredBB.
+ IRBuilder<> IntermediateBuilder(IntermediateBB);
+ IntermediateBuilder.CreateUnreachable();
+ ResidualLoopBuilder.CreateCondBr(
+ ResidualLoopBuilder.CreateICmpEQ(ResidualIndex,
+ RuntimeBytesCopiedMainLoop),
+ IntermediateBB, ResidualLoopBB);
+
+ ResidualLoopPhi->addIncoming(ResidualIndex, ResidualLoopBB);
+ ResidualLoopPhi->addIncoming(CopyLen, CopyBackwardsBB);
+
+ // How to get to the residual:
+ BranchInst::Create(IntermediateBB, ResidualLoopBB, SkipResidualCondition,
+ ThenTerm->getIterator());
+ ThenTerm->eraseFromParent();
+
+ PredBB = IntermediateBB;
+ }
- PHINode *LoopPhi = LoopBuilder.CreatePHI(TypeOfCopyLen, 0);
- Value *IndexPtr = LoopBuilder.CreateSub(
- LoopPhi, ConstantInt::get(TypeOfCopyLen, 1), "index_ptr");
- Value *Element = LoopBuilder.CreateAlignedLoad(
- EltTy, LoopBuilder.CreateInBoundsGEP(EltTy, SrcAddr, IndexPtr),
- PartSrcAlign, SrcIsVolatile, "element");
- LoopBuilder.CreateAlignedStore(
- Element, LoopBuilder.CreateInBoundsGEP(EltTy, DstAddr, IndexPtr),
- PartDstAlign, DstIsVolatile);
- LoopBuilder.CreateCondBr(
- LoopBuilder.CreateICmpEQ(IndexPtr, ConstantInt::get(TypeOfCopyLen, 0)),
- ExitBB, LoopBB);
- LoopPhi->addIncoming(IndexPtr, LoopBB);
- LoopPhi->addIncoming(CopyLen, CopyBackwardsBB);
- BranchInst::Create(ExitBB, LoopBB, CompareN, ThenTerm->getIterator());
- ThenTerm->eraseFromParent();
+ // main loop
+ IRBuilder<> MainLoopBuilder(MainLoopBB);
+ PHINode *MainLoopPhi = MainLoopBuilder.CreatePHI(ILengthType, 0);
+ Value *MainIndex =
+ MainLoopBuilder.CreateSub(MainLoopPhi, One, "bwd_main_index");
+ Value *LoadGEP =
+ MainLoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, MainIndex);
+ Value *Element = MainLoopBuilder.CreateAlignedLoad(
+ LoopOpType, LoadGEP, PartSrcAlign, SrcIsVolatile, "element");
+ Value *StoreGEP =
+ MainLoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, MainIndex);
+ MainLoopBuilder.CreateAlignedStore(Element, StoreGEP, PartDstAlign,
+ DstIsVolatile);
+ MainLoopBuilder.CreateCondBr(MainLoopBuilder.CreateICmpEQ(MainIndex, Zero),
+ ExitBB, MainLoopBB);
+ MainLoopPhi->addIncoming(MainIndex, MainLoopBB);
+ MainLoopPhi->addIncoming(RuntimeLoopCount, PredBB);
+
+ // How to get to the main loop:
+ Instruction *PredBBTerm = PredBB->getTerminator();
+ BranchInst::Create(ExitBB, MainLoopBB, SkipMainCondition,
+ PredBBTerm->getIterator());
+ PredBBTerm->eraseFromParent();
+ }
// Copying forward.
- BasicBlock *FwdLoopBB =
- BasicBlock::Create(F->getContext(), "copy_forward_loop", F, ExitBB);
- IRBuilder<> FwdLoopBuilder(FwdLoopBB);
- PHINode *FwdCopyPhi = FwdLoopBuilder.CreatePHI(TypeOfCopyLen, 0, "index_ptr");
- Value *SrcGEP = FwdLoopBuilder.CreateInBoundsGEP(EltTy, SrcAddr, FwdCopyPhi);
- Value *FwdElement = FwdLoopBuilder.CreateAlignedLoad(
- EltTy, SrcGEP, PartSrcAlign, SrcIsVolatile, "element");
- Value *DstGEP = FwdLoopBuilder.CreateInBoundsGEP(EltTy, DstAddr, FwdCopyPhi);
- FwdLoopBuilder.CreateAlignedStore(FwdElement, DstGEP, PartDstAlign,
- DstIsVolatile);
- Value *FwdIndexPtr = FwdLoopBuilder.CreateAdd(
- FwdCopyPhi, ConstantInt::get(TypeOfCopyLen, 1), "index_increment");
- FwdLoopBuilder.CreateCondBr(FwdLoopBuilder.CreateICmpEQ(FwdIndexPtr, CopyLen),
- ExitBB, FwdLoopBB);
- FwdCopyPhi->addIncoming(FwdIndexPtr, FwdLoopBB);
- FwdCopyPhi->addIncoming(ConstantInt::get(TypeOfCopyLen, 0), CopyForwardBB);
-
- BranchInst::Create(ExitBB, FwdLoopBB, CompareN, ElseTerm->getIterator());
- ElseTerm->eraseFromParent();
+ // main loop
+ {
+ BasicBlock *MainLoopBB =
+ BasicBlock::Create(F->getContext(), "memmove_fwd_main_loop", F, ExitBB);
+ IRBuilder<> MainLoopBuilder(MainLoopBB);
+ PHINode *MainLoopPhi =
+ MainLoopBuilder.CreatePHI(ILengthType, 0, "fwd_main_index");
+ Value *LoadGEP =
+ MainLoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, MainLoopPhi);
+ Value *Element = MainLoopBuilder.CreateAlignedLoad(
+ LoopOpType, LoadGEP, PartSrcAlign, SrcIsVolatile, "element");
+ Value *StoreGEP =
+ MainLoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, MainLoopPhi);
+ MainLoopBuilder.CreateAlignedStore(Element, StoreGEP, PartDstAlign,
+ DstIsVolatile);
+ Value *MainIndex = MainLoopBuilder.CreateAdd(MainLoopPhi, One);
+ MainLoopPhi->addIncoming(MainIndex, MainLoopBB);
+ MainLoopPhi->addIncoming(Zero, CopyForwardBB);
+
+ Instruction *CopyFwdBBTerm = CopyForwardBB->getTerminator();
+ BasicBlock *SuccessorBB = ExitBB;
+ if (RequiresResidual)
+ SuccessorBB =
+ BasicBlock::Create(F->getContext(), "memmove_fwd_middle", F, ExitBB);
+
+ // leaving or staying in the main loop
+ MainLoopBuilder.CreateCondBr(
+ MainLoopBuilder.CreateICmpEQ(MainIndex, RuntimeLoopCount), SuccessorBB,
+ MainLoopBB);
+
+ // getting in or skipping the main loop
+ BranchInst::Create(SuccessorBB, MainLoopBB, SkipMainCondition,
+ CopyFwdBBTerm->getIterator());
+ CopyFwdBBTerm->eraseFromParent();
+
+ if (RequiresResidual) {
+ BasicBlock *IntermediateBB = SuccessorBB;
+ IRBuilder<> IntermediateBuilder(IntermediateBB);
+ BasicBlock *ResidualLoopBB = BasicBlock::Create(
+ F->getContext(), "memmove_fwd_residual_loop", F, ExitBB);
+ IntermediateBuilder.CreateCondBr(SkipResidualCondition, ExitBB,
+ ResidualLoopBB);
+
+ // Residual loop
+ IRBuilder<> ResidualLoopBuilder(ResidualLoopBB);
+ PHINode *ResidualLoopPhi =
+ ResidualLoopBuilder.CreatePHI(ILengthType, 0, "fwd_residual_index");
+ Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
+ ResidualLoopPhi);
+ Value *Element = ResidualLoopBuilder.CreateLoad(Int8Type, LoadGEP,
+ SrcIsVolatile, "element");
+ Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, DstAddr,
+ ResidualLoopPhi);
+ ResidualLoopBuilder.CreateStore(Element, StoreGEP, DstIsVolatile);
+ Value *ResidualIndex =
+ ResidualLoopBuilder.CreateAdd(ResidualLoopPhi, One);
+ ResidualLoopBuilder.CreateCondBr(
+ ResidualLoopBuilder.CreateICmpEQ(ResidualIndex, CopyLen), ExitBB,
+ ResidualLoopBB);
+ ResidualLoopPhi->addIncoming(ResidualIndex, ResidualLoopBB);
+ ResidualLoopPhi->addIncoming(RuntimeBytesCopiedMainLoop, IntermediateBB);
+ }
+ }
}
static void createMemSetLoop(Instruction *InsertBefore, Value *DstAddr,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
index 4d4da869d7507..f59b549c0f88a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
@@ -11,14 +11,14 @@ define amdgpu_cs void @memmove_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src
; LOOP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; LOOP-NEXT: s_xor_b64 s[4:5], exec, s[0:1]
; LOOP-NEXT: s_cbranch_execz .LBB0_3
-; LOOP-NEXT: ; %bb.1: ; %copy_forward
+; LOOP-NEXT: ; %bb.1: ; %memmove_fwd_middle
; LOOP-NEXT: s_mov_b64 s[6:7], 0
; LOOP-NEXT: s_mov_b32 s2, 0
; LOOP-NEXT: s_mov_b32 s3, 0xf000
; LOOP-NEXT: s_mov_b64 s[0:1], 0
; LOOP-NEXT: v_mov_b32_e32 v4, s6
; LOOP-NEXT: v_mov_b32_e32 v5, s7
-; LOOP-NEXT: .LBB0_2: ; %copy_forward_loop
+; LOOP-NEXT: .LBB0_2: ; %memmove_fwd_residual_loop
; LOOP-NEXT: ; =>This Inner Loop Header: Depth=1
; LOOP-NEXT: v_add_i32_e32 v6, vcc, v2, v4
; LOOP-NEXT: v_addc_u32_e32 v7, vcc, v3, v5, vcc
@@ -32,10 +32,10 @@ define amdgpu_cs void @memmove_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src
; LOOP-NEXT: s_waitcnt vmcnt(0)
; LOOP-NEXT: buffer_store_byte v8, v[6:7], s[0:3], 0 addr64
; LOOP-NEXT: s_cbranch_vccnz .LBB0_2
-; LOOP-NEXT: .LBB0_3: ; %Flow17
+; LOOP-NEXT: .LBB0_3: ; %Flow25
; LOOP-NEXT: s_andn2_saveexec_b64 s[0:1], s[4:5]
; LOOP-NEXT: s_cbranch_execz .LBB0_6
-; LOOP-NEXT: ; %bb.4: ; %copy_backwards
+; LOOP-NEXT: ; %bb.4: ; %memmove_copy_backwards
; LOOP-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; LOOP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; LOOP-NEXT: v_add_i32_e32 v2, vcc, 3, v2
@@ -45,19 +45,20 @@ define amdgpu_cs void @memmove_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src
; LOOP-NEXT: s_mov_b32 s7, 0xf000
; LOOP-NEXT: s_mov_b64 s[4:5], 0
; LOOP-NEXT: v_mov_b32_e32 v4, s0
-; LOOP-NEXT: .LBB0_5: ; %copy_backwards_loop
+; LOOP-NEXT: .LBB0_5: ; %memmove_bwd_residual_loop
; LOOP-NEXT: ; =>This Inner Loop Header: Depth=1
; LOOP-NEXT: s_waitcnt expcnt(0)
; LOOP-NEXT: buffer_load_ubyte v5, v[2:3], s[4:7], 0 addr64
; LOOP-NEXT: v_add_i32_e32 v4, vcc, 1, v4
-; LOOP-NEXT: s_and_b64 vcc, vcc, exec
+; LOOP-NEXT: s_xor_b64 s[0:1], vcc, -1
+; LOOP-NEXT: s_and_b64 vcc, s[0:1], exec
; LOOP-NEXT: s_waitcnt vmcnt(0)
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[4:7], 0 addr64
; LOOP-NEXT: v_add_i32_e64 v0, s[0:1], -1, v0
; LOOP-NEXT: v_addc_u32_e64 v1, s[0:1], -1, v1, s[0:1]
; LOOP-NEXT: v_add_i32_e64 v2, s[0:1], -1, v2
; LOOP-NEXT: v_addc_u32_e64 v3, s[0:1], -1, v3, s[0:1]
-; LOOP-NEXT: s_cbranch_vccz .LBB0_5
+; LOOP-NEXT: s_cbranch_vccnz .LBB0_5
; LOOP-NEXT: .LBB0_6: ; %memmove_done
; LOOP-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index 5cb57ee112b3a..1e60ba415e414 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -87,30 +87,51 @@ define amdgpu_kernel void @max_size_small_static_memmove_caller0(ptr addrspace(1
;
; ALL-LABEL: @max_size_small_static_memmove_caller0(
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 1024, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 1024, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP1]], 1
-; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(1) [[TMP2]], align 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr addrspace(1) [[TMP5]], align 1
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr addrspace(1) [[TMP6]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 1024
-; ALL-NEXT: br i1 [[TMP7]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 1024
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 64, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
+; ALL-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 64
+; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(1) [[TMP13]], align 1
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
+; ALL-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 1024
+; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -121,30 +142,51 @@ define amdgpu_kernel void @max_size_small_static_memmove_caller0(ptr addrspace(1
define amdgpu_kernel void @min_size_large_static_memmove_caller0(ptr addrspace(1) %dst, ptr addrspace(1) %src) #0 {
; OPT-LABEL: @min_size_large_static_memmove_caller0(
; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; OPT-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 1025, 0
-; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; OPT: copy_backwards:
-; OPT-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; OPT: copy_backwards_loop:
-; OPT-NEXT: [[TMP1:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 1025, [[COPY_BACKWARDS]] ]
-; OPT-NEXT: [[INDEX_PTR]] = sub i64 [[TMP1]], 1
-; OPT-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 false, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 1025, [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
+; OPT-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(1) [[TMP2]], align 1
-; OPT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR]]
+; OPT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; OPT-NEXT: store i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; OPT-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; OPT-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; OPT: copy_forward:
-; OPT-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; OPT: copy_forward_loop:
-; OPT-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; OPT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR1]]
-; OPT-NEXT: [[ELEMENT2:%.*]] = load i8, ptr addrspace(1) [[TMP5]], align 1
-; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR1]]
-; OPT-NEXT: store i8 [[ELEMENT2]], ptr addrspace(1) [[TMP6]], align 1
-; OPT-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; OPT-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 1025
-; OPT-NEXT: br i1 [[TMP7]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; OPT-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 1024
+; OPT-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 64, [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
+; OPT-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 64
+; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 false, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(1) [[TMP13]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
+; OPT-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 1025
+; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; OPT: memmove_done:
; OPT-NEXT: ret void
;
@@ -1333,30 +1375,51 @@ define amdgpu_kernel void @memmove_flat_align1_global_align1(ptr %dst, ptr addrs
; ALL-LABEL: @memmove_flat_align1_global_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[SRC:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 256, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 256, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr [[TMP7]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 256
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1372,30 +1435,51 @@ define amdgpu_kernel void @memmove_global_align1_flat_align1(ptr addrspace(1) %d
; ALL-LABEL: @memmove_global_align1_flat_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[DST:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 256, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 256, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr [[TMP7]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 256
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1411,30 +1495,51 @@ define amdgpu_kernel void @memmove_flat_align1_private_align1(ptr %dst, ptr addr
; ALL-LABEL: @memmove_flat_align1_private_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(5) [[SRC:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 256, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 256, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr [[TMP7]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 256
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1450,30 +1555,51 @@ define amdgpu_kernel void @memmove_private_align1_flat_align1(ptr addrspace(5) %
; ALL-LABEL: @memmove_private_align1_flat_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(5) [[DST:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 256, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 256, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr [[TMP7]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 256
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1783,30 +1909,51 @@ define amdgpu_kernel void @memmove_volatile(ptr addrspace(1) %dst, ptr addrspace
;
; ALL-LABEL: @memmove_volatile(
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 64, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 64, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP1]], 1
-; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 64, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load volatile i8, ptr addrspace(1) [[TMP2]], align 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store volatile i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load volatile i8, ptr addrspace(1) [[TMP5]], align 1
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store volatile i8 [[ELEMENT2]], ptr addrspace(1) [[TMP6]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 64
-; ALL-NEXT: br i1 [[TMP7]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 64
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 4, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store volatile <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store volatile <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
+; ALL-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 4
+; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 64, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load volatile i8, ptr addrspace(1) [[TMP13]], align 1
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store volatile i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
+; ALL-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 64
+; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
diff --git a/llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll b/llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll
index afa7fde6c842b..59e755aee215c 100644
--- a/llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll
@@ -154,7 +154,7 @@ entry:
; IR-LABEL: @memmove_caller
; IR: icmp ult ptr %src, %dst
; IR: [[PHIVAL:%[0-9a-zA-Z_]+]] = phi i64
-; IR-NEXT: %index_ptr = sub i64 [[PHIVAL]], 1
+; IR-NEXT: %bwd_main_index = sub i64 [[PHIVAL]], 1
; IR: [[FWDPHIVAL:%[0-9a-zA-Z_]+]] = phi i64
; IR: {{%[0-9a-zA-Z_]+}} = add i64 [[FWDPHIVAL]], 1
>From dbe8d4c178fcbfe4b3783203968e8af3ae14b5a2 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Tue, 23 Jul 2024 10:32:16 -0400
Subject: [PATCH 2/4] fixup! [LowerMemIntrinsics] Lower llvm.memmove to wide
memory accesses
Use cast instead of dyn_cast + assert.
---
llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index 55cd61746d19c..2f8abd264135e 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -397,9 +397,7 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
bool RequiresResidual = !LoopOpIsInt8;
// Calculate the loop trip count and remaining bytes to copy after the loop.
- IntegerType *ILengthType = dyn_cast<IntegerType>(TypeOfCopyLen);
- assert(ILengthType &&
- "expected size argument to memcpy to be an integer type!");
+ IntegerType *ILengthType = cast<IntegerType>(TypeOfCopyLen);
ConstantInt *CILoopOpSize = ConstantInt::get(ILengthType, LoopOpSize);
ConstantInt *Zero = ConstantInt::get(ILengthType, 0);
ConstantInt *One = ConstantInt::get(ILengthType, 1);
>From aa87bceb18f5e81390507ecfb82b0e08f1fba8ed Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 24 Jul 2024 03:13:47 -0400
Subject: [PATCH 3/4] fixup! fixup! [LowerMemIntrinsics] Lower llvm.memmove to
wide memory accesses
Add more memmove lowering tests.
---
.../CodeGen/AMDGPU/lower-mem-intrinsics.ll | 484 ++++++++++++++++++
1 file changed, 484 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index 1e60ba415e414..5c6716cf5be16 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -17,6 +17,9 @@ declare void @llvm.memcpy.p3.p3.i32(ptr addrspace(3) nocapture, ptr addrspace(3)
declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) nocapture, ptr addrspace(1) nocapture readonly, i64, i1) #1
declare void @llvm.memmove.p1.p3.i32(ptr addrspace(1) nocapture, ptr addrspace(3) nocapture readonly, i32, i1) #1
+declare void @llvm.memmove.p0.p3.i32(ptr nocapture writeonly, ptr addrspace(3) nocapture readonly, i32, i1 immarg) #1
+declare void @llvm.memmove.p3.p0.i32(ptr addrspace(3) nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) #1
+declare void @llvm.memmove.p3.p3.i32(ptr addrspace(3) nocapture writeonly, ptr addrspace(3) nocapture readonly, i32, i1 immarg) #1
declare void @llvm.memmove.p5.p5.i32(ptr addrspace(5) nocapture, ptr addrspace(5) nocapture readonly, i32, i1) #1
declare void @llvm.memmove.p3.p5.i32(ptr addrspace(3) nocapture, ptr addrspace(5) nocapture readonly, i32, i1) #1
declare void @llvm.memmove.p5.p3.i32(ptr addrspace(5) nocapture, ptr addrspace(3) nocapture readonly, i32, i1) #1
@@ -1860,6 +1863,487 @@ define amdgpu_kernel void @memmove_private_align1_local_align1_unknown_size(ptr
ret void
}
+
+define amdgpu_kernel void @memmove_flat_align1_local_align1(ptr addrspace(0) %dst, ptr addrspace(3) %src) {
+; MAX1024-LABEL: @memmove_flat_align1_local_align1(
+; MAX1024-NEXT: call void @llvm.memmove.p0.p3.i32(ptr [[DST:%.*]], ptr addrspace(3) [[SRC:%.*]], i32 256, i1 false)
+; MAX1024-NEXT: ret void
+;
+; ALL-LABEL: @memmove_flat_align1_local_align1(
+; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[SRC:%.*]] to ptr
+; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i32 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL: memmove_done:
+; ALL-NEXT: ret void
+;
+ call void @llvm.memmove.p0.p3.i32(ptr addrspace(0) %dst, ptr addrspace(3) %src, i32 256, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_flat_align1_local_align1_unknown_size(ptr addrspace(0) %dst, ptr addrspace(3) %src, i32 %size) {
+; OPT-LABEL: @memmove_flat_align1_local_align1_unknown_size(
+; OPT-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[SRC:%.*]] to ptr
+; OPT-NEXT: [[TMP2:%.*]] = lshr i32 [[SIZE:%.*]], 4
+; OPT-NEXT: [[TMP3:%.*]] = and i32 [[SIZE]], 15
+; OPT-NEXT: [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP3]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP5]], 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr [[TMP7]], align 1
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP4]]
+; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP2]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
+; OPT-NEXT: [[TMP12:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
+; OPT-NEXT: [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP2]]
+; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP19:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP4]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP17]], align 1
+; OPT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr [[TMP18]], align 1
+; OPT-NEXT: [[TMP19]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP20:%.*]] = icmp eq i32 [[TMP19]], [[SIZE]]
+; OPT-NEXT: br i1 [[TMP20]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p0.p3.i32(ptr addrspace(0) %dst, ptr addrspace(3) %src, i32 %size, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_local_align1_flat_align1(ptr addrspace(3) %dst, ptr addrspace(0) %src) {
+; MAX1024-LABEL: @memmove_local_align1_flat_align1(
+; MAX1024-NEXT: call void @llvm.memmove.p3.p0.i32(ptr addrspace(3) [[DST:%.*]], ptr [[SRC:%.*]], i32 256, i1 false)
+; MAX1024-NEXT: ret void
+;
+; ALL-LABEL: @memmove_local_align1_flat_align1(
+; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[DST:%.*]] to ptr
+; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i32 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL: memmove_done:
+; ALL-NEXT: ret void
+;
+ call void @llvm.memmove.p3.p0.i32(ptr addrspace(3) %dst, ptr addrspace(0) %src, i32 256, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_local_align1_flat_align1_unknown_size(ptr addrspace(3) %dst, ptr addrspace(0) %src, i32 %size) {
+; OPT-LABEL: @memmove_local_align1_flat_align1_unknown_size(
+; OPT-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[DST:%.*]] to ptr
+; OPT-NEXT: [[TMP2:%.*]] = lshr i32 [[SIZE:%.*]], 4
+; OPT-NEXT: [[TMP3:%.*]] = and i32 [[SIZE]], 15
+; OPT-NEXT: [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP3]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP5]], 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr [[TMP7]], align 1
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP4]]
+; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP2]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
+; OPT-NEXT: [[TMP12:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
+; OPT-NEXT: [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP2]]
+; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP19:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP4]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP17]], align 1
+; OPT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr [[TMP18]], align 1
+; OPT-NEXT: [[TMP19]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP20:%.*]] = icmp eq i32 [[TMP19]], [[SIZE]]
+; OPT-NEXT: br i1 [[TMP20]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p3.p0.i32(ptr addrspace(3) %dst, ptr addrspace(0) %src, i32 %size, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_local_align1_local_align1(ptr addrspace(3) %dst, ptr addrspace(3) %src) {
+; MAX1024-LABEL: @memmove_local_align1_local_align1(
+; MAX1024-NEXT: call void @llvm.memmove.p3.p3.i32(ptr addrspace(3) [[DST:%.*]], ptr addrspace(3) [[SRC:%.*]], i32 256, i1 false)
+; MAX1024-NEXT: ret void
+;
+; ALL-LABEL: @memmove_local_align1_local_align1(
+; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(3) [[SRC:%.*]], [[DST:%.*]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(3) [[TMP2]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(3) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 32, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP5]], 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP9]], align 1
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP10]], align 1
+; ALL-NEXT: [[TMP11]] = add i32 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 32
+; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(3) [[TMP13]], align 1
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(3) [[TMP14]], align 1
+; ALL-NEXT: [[TMP15]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 256
+; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL: memmove_done:
+; ALL-NEXT: ret void
+;
+ call void @llvm.memmove.p3.p3.i32(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 256, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_local_align1_local_align1_unknown_size(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 %size) {
+; OPT-LABEL: @memmove_local_align1_local_align1_unknown_size(
+; OPT-NEXT: [[TMP1:%.*]] = lshr i32 [[SIZE:%.*]], 3
+; OPT-NEXT: [[TMP2:%.*]] = and i32 [[SIZE]], 7
+; OPT-NEXT: [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP1]], 0
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(3) [[SRC:%.*]], [[DST:%.*]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP4:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP4]], 1
+; OPT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(3) [[TMP5]], align 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr addrspace(3) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP3]]
+; OPT-NEXT: br i1 [[TMP7]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP8:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP1]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP8]], 1
+; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP9]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP11]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP14:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP12:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP12]], align 1
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <2 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP13]], align 1
+; OPT-NEXT: [[TMP14]] = add i32 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP15:%.*]] = icmp eq i32 [[TMP14]], [[TMP1]]
+; OPT-NEXT: br i1 [[TMP15]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP18:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP3]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(3) [[TMP16]], align 1
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr addrspace(3) [[TMP17]], align 1
+; OPT-NEXT: [[TMP18]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP18]], [[SIZE]]
+; OPT-NEXT: br i1 [[TMP19]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p3.p3.i32(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 %size, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_private_align1_private_align1(ptr addrspace(5) %dst, ptr addrspace(5) %src) {
+; MAX1024-LABEL: @memmove_private_align1_private_align1(
+; MAX1024-NEXT: call void @llvm.memmove.p5.p5.i32(ptr addrspace(5) [[DST:%.*]], ptr addrspace(5) [[SRC:%.*]], i32 256, i1 false)
+; MAX1024-NEXT: ret void
+;
+; ALL-LABEL: @memmove_private_align1_private_align1(
+; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(5) [[SRC:%.*]], [[DST:%.*]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(5) [[TMP2]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(5) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP5]], 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP9]], align 1
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(5) [[TMP10]], align 1
+; ALL-NEXT: [[TMP11]] = add i32 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 16
+; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(5) [[TMP13]], align 1
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(5) [[TMP14]], align 1
+; ALL-NEXT: [[TMP15]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 256
+; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL: memmove_done:
+; ALL-NEXT: ret void
+;
+ call void @llvm.memmove.p5.p5.i32(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 256, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_private_align1_private_align1_unknown_size(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 %size) {
+; OPT-LABEL: @memmove_private_align1_private_align1_unknown_size(
+; OPT-NEXT: [[TMP1:%.*]] = lshr i32 [[SIZE:%.*]], 4
+; OPT-NEXT: [[TMP2:%.*]] = and i32 [[SIZE]], 15
+; OPT-NEXT: [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP1]], 0
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(5) [[SRC:%.*]], [[DST:%.*]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP4:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP4]], 1
+; OPT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(5) [[TMP5]], align 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr addrspace(5) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP3]]
+; OPT-NEXT: br i1 [[TMP7]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP8:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP1]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP8]], 1
+; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP9]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP11]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP14:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP12:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP12]], align 1
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(5) [[TMP13]], align 1
+; OPT-NEXT: [[TMP14]] = add i32 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP15:%.*]] = icmp eq i32 [[TMP14]], [[TMP1]]
+; OPT-NEXT: br i1 [[TMP15]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP18:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP3]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(5) [[TMP16]], align 1
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr addrspace(5) [[TMP17]], align 1
+; OPT-NEXT: [[TMP18]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP18]], [[SIZE]]
+; OPT-NEXT: br i1 [[TMP19]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p5.p5.i32(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 %size, i1 false)
+ ret void
+}
+
define void @test_umin(i64 %0, i64 %idxprom, ptr %x, ptr %y) {
; OPT-LABEL: @test_umin(
; OPT-NEXT: entry:
>From b491cb637825798ac358189b490bfa7f3980fff4 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 24 Jul 2024 03:43:28 -0400
Subject: [PATCH 4/4] fixup! fixup! fixup! [LowerMemIntrinsics] Lower
llvm.memmove to wide memory accesses
Specify alignment for residual accesses.
---
.../Transforms/Utils/LowerMemIntrinsics.cpp | 40 ++++++++++++-------
1 file changed, 26 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index 2f8abd264135e..a6bf72789ed21 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -396,6 +396,9 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
// i8-accesses are required to move remaining bytes.
bool RequiresResidual = !LoopOpIsInt8;
+ Type *ResidualLoopOpType = Int8Type;
+ unsigned ResidualLoopOpSize = DL.getTypeStoreSize(ResidualLoopOpType);
+
// Calculate the loop trip count and remaining bytes to copy after the loop.
IntegerType *ILengthType = cast<IntegerType>(TypeOfCopyLen);
ConstantInt *CILoopOpSize = ConstantInt::get(ILengthType, LoopOpSize);
@@ -467,6 +470,11 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
Align PartSrcAlign(commonAlignment(SrcAlign, LoopOpSize));
Align PartDstAlign(commonAlignment(DstAlign, LoopOpSize));
+ // Accesses in the residual loops do not share the same alignment as those in
+ // the main loops.
+ Align ResidualSrcAlign(commonAlignment(PartSrcAlign, ResidualLoopOpSize));
+ Align ResidualDstAlign(commonAlignment(PartDstAlign, ResidualLoopOpSize));
+
// Copying backwards.
{
BasicBlock *MainLoopBB = BasicBlock::Create(
@@ -484,13 +492,15 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
PHINode *ResidualLoopPhi = ResidualLoopBuilder.CreatePHI(ILengthType, 0);
Value *ResidualIndex = ResidualLoopBuilder.CreateSub(
ResidualLoopPhi, One, "bwd_residual_index");
- Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
- ResidualIndex);
- Value *Element = ResidualLoopBuilder.CreateLoad(Int8Type, LoadGEP,
- SrcIsVolatile, "element");
- Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, DstAddr,
- ResidualIndex);
- ResidualLoopBuilder.CreateStore(Element, StoreGEP, DstIsVolatile);
+ Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(
+ ResidualLoopOpType, SrcAddr, ResidualIndex);
+ Value *Element = ResidualLoopBuilder.CreateAlignedLoad(
+ ResidualLoopOpType, LoadGEP, ResidualSrcAlign, SrcIsVolatile,
+ "element");
+ Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(
+ ResidualLoopOpType, DstAddr, ResidualIndex);
+ ResidualLoopBuilder.CreateAlignedStore(Element, StoreGEP,
+ ResidualDstAlign, DstIsVolatile);
// After the residual loop, go to an intermediate block.
BasicBlock *IntermediateBB = BasicBlock::Create(
@@ -587,13 +597,15 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
IRBuilder<> ResidualLoopBuilder(ResidualLoopBB);
PHINode *ResidualLoopPhi =
ResidualLoopBuilder.CreatePHI(ILengthType, 0, "fwd_residual_index");
- Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
- ResidualLoopPhi);
- Value *Element = ResidualLoopBuilder.CreateLoad(Int8Type, LoadGEP,
- SrcIsVolatile, "element");
- Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, DstAddr,
- ResidualLoopPhi);
- ResidualLoopBuilder.CreateStore(Element, StoreGEP, DstIsVolatile);
+ Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(
+ ResidualLoopOpType, SrcAddr, ResidualLoopPhi);
+ Value *Element = ResidualLoopBuilder.CreateAlignedLoad(
+ ResidualLoopOpType, LoadGEP, ResidualSrcAlign, SrcIsVolatile,
+ "element");
+ Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(
+ ResidualLoopOpType, DstAddr, ResidualLoopPhi);
+ ResidualLoopBuilder.CreateAlignedStore(Element, StoreGEP,
+ ResidualDstAlign, DstIsVolatile);
Value *ResidualIndex =
ResidualLoopBuilder.CreateAdd(ResidualLoopPhi, One);
ResidualLoopBuilder.CreateCondBr(
More information about the llvm-commits
mailing list