[llvm] [LowerMemIntrinsics] Lower llvm.memmove to wide memory accesses (PR #100122)
Fabian Ritter via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 25 02:27:22 PDT 2024
https://github.com/ritter-x2a updated https://github.com/llvm/llvm-project/pull/100122
>From af1c0f8ccf73d1587d1ab590b4a6bdd2cf63ba44 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Tue, 23 Jul 2024 09:09:44 -0400
Subject: [PATCH 1/6] [LowerMemIntrinsics] Lower llvm.memmove to wide memory
accesses
So far, the IR-level lowering of llvm.memmove intrinsics generates loops
that copy each byte individually. This can be wasteful for targets that
provide wider memory access operations.
This patch makes the memmove lowering more similar to the lowering of
memcpy with unknown length.
TargetTransformInfo::getMemcpyLoopLoweringType() is queried for an
adequate type for the memory accesses, and if it is wider than a single
byte, the greatest multiple of the type's size that is less than or
equal to the length is copied with corresponding wide memory accesses.
A residual loop with byte-wise accesses is introduced for the remaining
bytes.
For memmove, this construct is required in two variants: one for copying
forward and one for copying backwards, to handle overlapping memory
ranges. For the backwards case, the residual loop still covers the bytes
at the end of the copied region and is therefore executed before the
wide main loop. This implementation choice is based on the assumption
that we are more likely to encounter memory ranges whose start aligns
with the access width than ones whose end does.
In microbenchmarks on gfx1030 (AMDGPU), this change yields speedups up
to 16x for memmoves with variable or large constant lengths.
---
.../Transforms/Utils/LowerMemIntrinsics.cpp | 270 ++++++++---
.../CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll | 15 +-
.../CodeGen/AMDGPU/lower-mem-intrinsics.ll | 455 ++++++++++++------
llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll | 2 +-
4 files changed, 519 insertions(+), 223 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index b38db412f786a..55cd61746d19c 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -369,6 +369,10 @@ void llvm::createMemCpyLoopUnknownSize(
// }
// return dst;
// }
+//
+// If the TargetTransformInfo specifies a wider MemcpyLoopLoweringType, it is
+// used for the memory accesses in the loops. Then, additional loops with
+// byte-wise accesses are added for the remaining bytes.
static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
Value *DstAddr, Value *CopyLen, Align SrcAlign,
Align DstAlign, bool SrcIsVolatile,
@@ -378,8 +382,46 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
BasicBlock *OrigBB = InsertBefore->getParent();
Function *F = OrigBB->getParent();
const DataLayout &DL = F->getDataLayout();
- // TODO: Use different element type if possible?
- Type *EltTy = Type::getInt8Ty(F->getContext());
+ LLVMContext &Ctx = OrigBB->getContext();
+ unsigned SrcAS = cast<PointerType>(SrcAddr->getType())->getAddressSpace();
+ unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
+
+ Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
+ Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value());
+ unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
+ Type *Int8Type = Type::getInt8Ty(Ctx);
+ bool LoopOpIsInt8 = LoopOpType == Int8Type;
+
+ // If the memory accesses are wider than one byte, residual loops with
+ // i8-accesses are required to move remaining bytes.
+ bool RequiresResidual = !LoopOpIsInt8;
+
+ // Calculate the loop trip count and remaining bytes to copy after the loop.
+ IntegerType *ILengthType = dyn_cast<IntegerType>(TypeOfCopyLen);
+ assert(ILengthType &&
+ "expected size argument to memcpy to be an integer type!");
+ ConstantInt *CILoopOpSize = ConstantInt::get(ILengthType, LoopOpSize);
+ ConstantInt *Zero = ConstantInt::get(ILengthType, 0);
+ ConstantInt *One = ConstantInt::get(ILengthType, 1);
+
+ IRBuilder<> PLBuilder(InsertBefore);
+
+ Value *RuntimeLoopCount = CopyLen;
+ Value *RuntimeLoopRemainder = nullptr;
+ Value *RuntimeBytesCopiedMainLoop = CopyLen;
+ Value *SkipResidualCondition = nullptr;
+ if (RequiresResidual) {
+ RuntimeLoopCount =
+ getRuntimeLoopCount(DL, PLBuilder, CopyLen, CILoopOpSize, LoopOpSize);
+ RuntimeLoopRemainder = getRuntimeLoopRemainder(DL, PLBuilder, CopyLen,
+ CILoopOpSize, LoopOpSize);
+ RuntimeBytesCopiedMainLoop =
+ PLBuilder.CreateSub(CopyLen, RuntimeLoopRemainder);
+ SkipResidualCondition =
+ PLBuilder.CreateICmpEQ(RuntimeLoopRemainder, Zero, "skip_residual");
+ }
+ Value *SkipMainCondition =
+ PLBuilder.CreateICmpEQ(RuntimeLoopCount, Zero, "skip_main");
// Create the a comparison of src and dst, based on which we jump to either
// the forward-copy part of the function (if src >= dst) or the backwards-copy
@@ -387,76 +429,182 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
// SplitBlockAndInsertIfThenElse conveniently creates the basic if-then-else
// structure. Its block terminators (unconditional branches) are replaced by
// the appropriate conditional branches when the loop is built.
- ICmpInst *PtrCompare = new ICmpInst(InsertBefore->getIterator(), ICmpInst::ICMP_ULT,
- SrcAddr, DstAddr, "compare_src_dst");
+ Value *PtrCompare =
+ PLBuilder.CreateICmpULT(SrcAddr, DstAddr, "compare_src_dst");
Instruction *ThenTerm, *ElseTerm;
- SplitBlockAndInsertIfThenElse(PtrCompare, InsertBefore->getIterator(), &ThenTerm,
- &ElseTerm);
-
- // Each part of the function consists of two blocks:
- // copy_backwards: used to skip the loop when n == 0
- // copy_backwards_loop: the actual backwards loop BB
- // copy_forward: used to skip the loop when n == 0
- // copy_forward_loop: the actual forward loop BB
+ SplitBlockAndInsertIfThenElse(PtrCompare, InsertBefore->getIterator(),
+ &ThenTerm, &ElseTerm);
+
+ // If the LoopOpSize is greater than 1, each part of the function consist of
+ // four blocks:
+ // memmove_copy_backwards:
+ // skip the residual loop when 0 iterations are required
+ // memmove_bwd_residual_loop:
+ // copy the last few bytes individually so that the remaining length is
+ // a multiple of the LoopOpSize
+ // memmove_bwd_middle: skip the main loop when 0 iterations are required
+ // memmove_bwd_main_loop: the actual backwards loop BB with wide accesses
+ // memmove_copy_forward: skip the main loop when 0 iterations are required
+ // memmove_fwd_main_loop: the actual forward loop BB with wide accesses
+ // memmove_fwd_middle: skip the residual loop when 0 iterations are required
+ // memmove_fwd_residual_loop: copy the last few bytes individually
+ //
+ // The main and residual loop are switched between copying forward and
+ // backward so that the residual loop always operates on the end of the moved
+ // range. This is based on the assumption that buffers whose start is aligned
+ // with the LoopOpSize are more common than buffers whose end is.
+ //
+ // If the LoopOpSize is 1, each part of the function consists of two blocks:
+ // memmove_copy_backwards: skip the loop when 0 iterations are required
+ // memmove_bwd_main_loop: the actual backwards loop BB
+ // memmove_copy_forward: skip the loop when 0 iterations are required
+ // memmove_fwd_main_loop: the actual forward loop BB
BasicBlock *CopyBackwardsBB = ThenTerm->getParent();
- CopyBackwardsBB->setName("copy_backwards");
+ CopyBackwardsBB->setName("memmove_copy_backwards");
BasicBlock *CopyForwardBB = ElseTerm->getParent();
- CopyForwardBB->setName("copy_forward");
+ CopyForwardBB->setName("memmove_copy_forward");
BasicBlock *ExitBB = InsertBefore->getParent();
ExitBB->setName("memmove_done");
- unsigned PartSize = DL.getTypeStoreSize(EltTy);
- Align PartSrcAlign(commonAlignment(SrcAlign, PartSize));
- Align PartDstAlign(commonAlignment(DstAlign, PartSize));
-
- // Initial comparison of n == 0 that lets us skip the loops altogether. Shared
- // between both backwards and forward copy clauses.
- ICmpInst *CompareN =
- new ICmpInst(OrigBB->getTerminator()->getIterator(), ICmpInst::ICMP_EQ, CopyLen,
- ConstantInt::get(TypeOfCopyLen, 0), "compare_n_to_0");
+ Align PartSrcAlign(commonAlignment(SrcAlign, LoopOpSize));
+ Align PartDstAlign(commonAlignment(DstAlign, LoopOpSize));
// Copying backwards.
- BasicBlock *LoopBB =
- BasicBlock::Create(F->getContext(), "copy_backwards_loop", F, CopyForwardBB);
- IRBuilder<> LoopBuilder(LoopBB);
+ {
+ BasicBlock *MainLoopBB = BasicBlock::Create(
+ F->getContext(), "memmove_bwd_main_loop", F, CopyForwardBB);
+
+ // The predecessor of the memmove_bwd_main_loop. Updated in the
+ // following if a residual loop is emitted first.
+ BasicBlock *PredBB = CopyBackwardsBB;
+
+ if (RequiresResidual) {
+ // backwards residual loop
+ BasicBlock *ResidualLoopBB = BasicBlock::Create(
+ F->getContext(), "memmove_bwd_residual_loop", F, MainLoopBB);
+ IRBuilder<> ResidualLoopBuilder(ResidualLoopBB);
+ PHINode *ResidualLoopPhi = ResidualLoopBuilder.CreatePHI(ILengthType, 0);
+ Value *ResidualIndex = ResidualLoopBuilder.CreateSub(
+ ResidualLoopPhi, One, "bwd_residual_index");
+ Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
+ ResidualIndex);
+ Value *Element = ResidualLoopBuilder.CreateLoad(Int8Type, LoadGEP,
+ SrcIsVolatile, "element");
+ Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, DstAddr,
+ ResidualIndex);
+ ResidualLoopBuilder.CreateStore(Element, StoreGEP, DstIsVolatile);
+
+ // After the residual loop, go to an intermediate block.
+ BasicBlock *IntermediateBB = BasicBlock::Create(
+ F->getContext(), "memmove_bwd_middle", F, MainLoopBB);
+ // Later code expects a terminator in the PredBB.
+ IRBuilder<> IntermediateBuilder(IntermediateBB);
+ IntermediateBuilder.CreateUnreachable();
+ ResidualLoopBuilder.CreateCondBr(
+ ResidualLoopBuilder.CreateICmpEQ(ResidualIndex,
+ RuntimeBytesCopiedMainLoop),
+ IntermediateBB, ResidualLoopBB);
+
+ ResidualLoopPhi->addIncoming(ResidualIndex, ResidualLoopBB);
+ ResidualLoopPhi->addIncoming(CopyLen, CopyBackwardsBB);
+
+ // How to get to the residual:
+ BranchInst::Create(IntermediateBB, ResidualLoopBB, SkipResidualCondition,
+ ThenTerm->getIterator());
+ ThenTerm->eraseFromParent();
+
+ PredBB = IntermediateBB;
+ }
- PHINode *LoopPhi = LoopBuilder.CreatePHI(TypeOfCopyLen, 0);
- Value *IndexPtr = LoopBuilder.CreateSub(
- LoopPhi, ConstantInt::get(TypeOfCopyLen, 1), "index_ptr");
- Value *Element = LoopBuilder.CreateAlignedLoad(
- EltTy, LoopBuilder.CreateInBoundsGEP(EltTy, SrcAddr, IndexPtr),
- PartSrcAlign, SrcIsVolatile, "element");
- LoopBuilder.CreateAlignedStore(
- Element, LoopBuilder.CreateInBoundsGEP(EltTy, DstAddr, IndexPtr),
- PartDstAlign, DstIsVolatile);
- LoopBuilder.CreateCondBr(
- LoopBuilder.CreateICmpEQ(IndexPtr, ConstantInt::get(TypeOfCopyLen, 0)),
- ExitBB, LoopBB);
- LoopPhi->addIncoming(IndexPtr, LoopBB);
- LoopPhi->addIncoming(CopyLen, CopyBackwardsBB);
- BranchInst::Create(ExitBB, LoopBB, CompareN, ThenTerm->getIterator());
- ThenTerm->eraseFromParent();
+ // main loop
+ IRBuilder<> MainLoopBuilder(MainLoopBB);
+ PHINode *MainLoopPhi = MainLoopBuilder.CreatePHI(ILengthType, 0);
+ Value *MainIndex =
+ MainLoopBuilder.CreateSub(MainLoopPhi, One, "bwd_main_index");
+ Value *LoadGEP =
+ MainLoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, MainIndex);
+ Value *Element = MainLoopBuilder.CreateAlignedLoad(
+ LoopOpType, LoadGEP, PartSrcAlign, SrcIsVolatile, "element");
+ Value *StoreGEP =
+ MainLoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, MainIndex);
+ MainLoopBuilder.CreateAlignedStore(Element, StoreGEP, PartDstAlign,
+ DstIsVolatile);
+ MainLoopBuilder.CreateCondBr(MainLoopBuilder.CreateICmpEQ(MainIndex, Zero),
+ ExitBB, MainLoopBB);
+ MainLoopPhi->addIncoming(MainIndex, MainLoopBB);
+ MainLoopPhi->addIncoming(RuntimeLoopCount, PredBB);
+
+ // How to get to the main loop:
+ Instruction *PredBBTerm = PredBB->getTerminator();
+ BranchInst::Create(ExitBB, MainLoopBB, SkipMainCondition,
+ PredBBTerm->getIterator());
+ PredBBTerm->eraseFromParent();
+ }
// Copying forward.
- BasicBlock *FwdLoopBB =
- BasicBlock::Create(F->getContext(), "copy_forward_loop", F, ExitBB);
- IRBuilder<> FwdLoopBuilder(FwdLoopBB);
- PHINode *FwdCopyPhi = FwdLoopBuilder.CreatePHI(TypeOfCopyLen, 0, "index_ptr");
- Value *SrcGEP = FwdLoopBuilder.CreateInBoundsGEP(EltTy, SrcAddr, FwdCopyPhi);
- Value *FwdElement = FwdLoopBuilder.CreateAlignedLoad(
- EltTy, SrcGEP, PartSrcAlign, SrcIsVolatile, "element");
- Value *DstGEP = FwdLoopBuilder.CreateInBoundsGEP(EltTy, DstAddr, FwdCopyPhi);
- FwdLoopBuilder.CreateAlignedStore(FwdElement, DstGEP, PartDstAlign,
- DstIsVolatile);
- Value *FwdIndexPtr = FwdLoopBuilder.CreateAdd(
- FwdCopyPhi, ConstantInt::get(TypeOfCopyLen, 1), "index_increment");
- FwdLoopBuilder.CreateCondBr(FwdLoopBuilder.CreateICmpEQ(FwdIndexPtr, CopyLen),
- ExitBB, FwdLoopBB);
- FwdCopyPhi->addIncoming(FwdIndexPtr, FwdLoopBB);
- FwdCopyPhi->addIncoming(ConstantInt::get(TypeOfCopyLen, 0), CopyForwardBB);
-
- BranchInst::Create(ExitBB, FwdLoopBB, CompareN, ElseTerm->getIterator());
- ElseTerm->eraseFromParent();
+ // main loop
+ {
+ BasicBlock *MainLoopBB =
+ BasicBlock::Create(F->getContext(), "memmove_fwd_main_loop", F, ExitBB);
+ IRBuilder<> MainLoopBuilder(MainLoopBB);
+ PHINode *MainLoopPhi =
+ MainLoopBuilder.CreatePHI(ILengthType, 0, "fwd_main_index");
+ Value *LoadGEP =
+ MainLoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, MainLoopPhi);
+ Value *Element = MainLoopBuilder.CreateAlignedLoad(
+ LoopOpType, LoadGEP, PartSrcAlign, SrcIsVolatile, "element");
+ Value *StoreGEP =
+ MainLoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, MainLoopPhi);
+ MainLoopBuilder.CreateAlignedStore(Element, StoreGEP, PartDstAlign,
+ DstIsVolatile);
+ Value *MainIndex = MainLoopBuilder.CreateAdd(MainLoopPhi, One);
+ MainLoopPhi->addIncoming(MainIndex, MainLoopBB);
+ MainLoopPhi->addIncoming(Zero, CopyForwardBB);
+
+ Instruction *CopyFwdBBTerm = CopyForwardBB->getTerminator();
+ BasicBlock *SuccessorBB = ExitBB;
+ if (RequiresResidual)
+ SuccessorBB =
+ BasicBlock::Create(F->getContext(), "memmove_fwd_middle", F, ExitBB);
+
+ // leaving or staying in the main loop
+ MainLoopBuilder.CreateCondBr(
+ MainLoopBuilder.CreateICmpEQ(MainIndex, RuntimeLoopCount), SuccessorBB,
+ MainLoopBB);
+
+ // getting in or skipping the main loop
+ BranchInst::Create(SuccessorBB, MainLoopBB, SkipMainCondition,
+ CopyFwdBBTerm->getIterator());
+ CopyFwdBBTerm->eraseFromParent();
+
+ if (RequiresResidual) {
+ BasicBlock *IntermediateBB = SuccessorBB;
+ IRBuilder<> IntermediateBuilder(IntermediateBB);
+ BasicBlock *ResidualLoopBB = BasicBlock::Create(
+ F->getContext(), "memmove_fwd_residual_loop", F, ExitBB);
+ IntermediateBuilder.CreateCondBr(SkipResidualCondition, ExitBB,
+ ResidualLoopBB);
+
+ // Residual loop
+ IRBuilder<> ResidualLoopBuilder(ResidualLoopBB);
+ PHINode *ResidualLoopPhi =
+ ResidualLoopBuilder.CreatePHI(ILengthType, 0, "fwd_residual_index");
+ Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
+ ResidualLoopPhi);
+ Value *Element = ResidualLoopBuilder.CreateLoad(Int8Type, LoadGEP,
+ SrcIsVolatile, "element");
+ Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, DstAddr,
+ ResidualLoopPhi);
+ ResidualLoopBuilder.CreateStore(Element, StoreGEP, DstIsVolatile);
+ Value *ResidualIndex =
+ ResidualLoopBuilder.CreateAdd(ResidualLoopPhi, One);
+ ResidualLoopBuilder.CreateCondBr(
+ ResidualLoopBuilder.CreateICmpEQ(ResidualIndex, CopyLen), ExitBB,
+ ResidualLoopBB);
+ ResidualLoopPhi->addIncoming(ResidualIndex, ResidualLoopBB);
+ ResidualLoopPhi->addIncoming(RuntimeBytesCopiedMainLoop, IntermediateBB);
+ }
+ }
}
static void createMemSetLoop(Instruction *InsertBefore, Value *DstAddr,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
index 4d4da869d7507..f59b549c0f88a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
@@ -11,14 +11,14 @@ define amdgpu_cs void @memmove_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src
; LOOP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; LOOP-NEXT: s_xor_b64 s[4:5], exec, s[0:1]
; LOOP-NEXT: s_cbranch_execz .LBB0_3
-; LOOP-NEXT: ; %bb.1: ; %copy_forward
+; LOOP-NEXT: ; %bb.1: ; %memmove_fwd_middle
; LOOP-NEXT: s_mov_b64 s[6:7], 0
; LOOP-NEXT: s_mov_b32 s2, 0
; LOOP-NEXT: s_mov_b32 s3, 0xf000
; LOOP-NEXT: s_mov_b64 s[0:1], 0
; LOOP-NEXT: v_mov_b32_e32 v4, s6
; LOOP-NEXT: v_mov_b32_e32 v5, s7
-; LOOP-NEXT: .LBB0_2: ; %copy_forward_loop
+; LOOP-NEXT: .LBB0_2: ; %memmove_fwd_residual_loop
; LOOP-NEXT: ; =>This Inner Loop Header: Depth=1
; LOOP-NEXT: v_add_i32_e32 v6, vcc, v2, v4
; LOOP-NEXT: v_addc_u32_e32 v7, vcc, v3, v5, vcc
@@ -32,10 +32,10 @@ define amdgpu_cs void @memmove_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src
; LOOP-NEXT: s_waitcnt vmcnt(0)
; LOOP-NEXT: buffer_store_byte v8, v[6:7], s[0:3], 0 addr64
; LOOP-NEXT: s_cbranch_vccnz .LBB0_2
-; LOOP-NEXT: .LBB0_3: ; %Flow17
+; LOOP-NEXT: .LBB0_3: ; %Flow25
; LOOP-NEXT: s_andn2_saveexec_b64 s[0:1], s[4:5]
; LOOP-NEXT: s_cbranch_execz .LBB0_6
-; LOOP-NEXT: ; %bb.4: ; %copy_backwards
+; LOOP-NEXT: ; %bb.4: ; %memmove_copy_backwards
; LOOP-NEXT: v_add_i32_e32 v0, vcc, 3, v0
; LOOP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; LOOP-NEXT: v_add_i32_e32 v2, vcc, 3, v2
@@ -45,19 +45,20 @@ define amdgpu_cs void @memmove_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src
; LOOP-NEXT: s_mov_b32 s7, 0xf000
; LOOP-NEXT: s_mov_b64 s[4:5], 0
; LOOP-NEXT: v_mov_b32_e32 v4, s0
-; LOOP-NEXT: .LBB0_5: ; %copy_backwards_loop
+; LOOP-NEXT: .LBB0_5: ; %memmove_bwd_residual_loop
; LOOP-NEXT: ; =>This Inner Loop Header: Depth=1
; LOOP-NEXT: s_waitcnt expcnt(0)
; LOOP-NEXT: buffer_load_ubyte v5, v[2:3], s[4:7], 0 addr64
; LOOP-NEXT: v_add_i32_e32 v4, vcc, 1, v4
-; LOOP-NEXT: s_and_b64 vcc, vcc, exec
+; LOOP-NEXT: s_xor_b64 s[0:1], vcc, -1
+; LOOP-NEXT: s_and_b64 vcc, s[0:1], exec
; LOOP-NEXT: s_waitcnt vmcnt(0)
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[4:7], 0 addr64
; LOOP-NEXT: v_add_i32_e64 v0, s[0:1], -1, v0
; LOOP-NEXT: v_addc_u32_e64 v1, s[0:1], -1, v1, s[0:1]
; LOOP-NEXT: v_add_i32_e64 v2, s[0:1], -1, v2
; LOOP-NEXT: v_addc_u32_e64 v3, s[0:1], -1, v3, s[0:1]
-; LOOP-NEXT: s_cbranch_vccz .LBB0_5
+; LOOP-NEXT: s_cbranch_vccnz .LBB0_5
; LOOP-NEXT: .LBB0_6: ; %memmove_done
; LOOP-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index 5cb57ee112b3a..1e60ba415e414 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -87,30 +87,51 @@ define amdgpu_kernel void @max_size_small_static_memmove_caller0(ptr addrspace(1
;
; ALL-LABEL: @max_size_small_static_memmove_caller0(
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 1024, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 1024, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP1]], 1
-; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(1) [[TMP2]], align 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr addrspace(1) [[TMP5]], align 1
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr addrspace(1) [[TMP6]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 1024
-; ALL-NEXT: br i1 [[TMP7]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 1024
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 64, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
+; ALL-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 64
+; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(1) [[TMP13]], align 1
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
+; ALL-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 1024
+; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -121,30 +142,51 @@ define amdgpu_kernel void @max_size_small_static_memmove_caller0(ptr addrspace(1
define amdgpu_kernel void @min_size_large_static_memmove_caller0(ptr addrspace(1) %dst, ptr addrspace(1) %src) #0 {
; OPT-LABEL: @min_size_large_static_memmove_caller0(
; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; OPT-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 1025, 0
-; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; OPT: copy_backwards:
-; OPT-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; OPT: copy_backwards_loop:
-; OPT-NEXT: [[TMP1:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 1025, [[COPY_BACKWARDS]] ]
-; OPT-NEXT: [[INDEX_PTR]] = sub i64 [[TMP1]], 1
-; OPT-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 false, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 1025, [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
+; OPT-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(1) [[TMP2]], align 1
-; OPT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR]]
+; OPT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; OPT-NEXT: store i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; OPT-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; OPT-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; OPT: copy_forward:
-; OPT-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; OPT: copy_forward_loop:
-; OPT-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; OPT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR1]]
-; OPT-NEXT: [[ELEMENT2:%.*]] = load i8, ptr addrspace(1) [[TMP5]], align 1
-; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR1]]
-; OPT-NEXT: store i8 [[ELEMENT2]], ptr addrspace(1) [[TMP6]], align 1
-; OPT-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; OPT-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 1025
-; OPT-NEXT: br i1 [[TMP7]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; OPT-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 1024
+; OPT-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 64, [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
+; OPT-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 64
+; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 false, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(1) [[TMP13]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
+; OPT-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 1025
+; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; OPT: memmove_done:
; OPT-NEXT: ret void
;
@@ -1333,30 +1375,51 @@ define amdgpu_kernel void @memmove_flat_align1_global_align1(ptr %dst, ptr addrs
; ALL-LABEL: @memmove_flat_align1_global_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[SRC:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 256, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 256, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr [[TMP7]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 256
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1372,30 +1435,51 @@ define amdgpu_kernel void @memmove_global_align1_flat_align1(ptr addrspace(1) %d
; ALL-LABEL: @memmove_global_align1_flat_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[DST:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 256, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 256, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr [[TMP7]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 256
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1411,30 +1495,51 @@ define amdgpu_kernel void @memmove_flat_align1_private_align1(ptr %dst, ptr addr
; ALL-LABEL: @memmove_flat_align1_private_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(5) [[SRC:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 256, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 256, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr [[TMP7]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 256
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1450,30 +1555,51 @@ define amdgpu_kernel void @memmove_private_align1_flat_align1(ptr addrspace(5) %
; ALL-LABEL: @memmove_private_align1_flat_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(5) [[DST:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 256, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 256, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load i8, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store i8 [[ELEMENT2]], ptr [[TMP7]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 256
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1783,30 +1909,51 @@ define amdgpu_kernel void @memmove_volatile(ptr addrspace(1) %dst, ptr addrspace
;
; ALL-LABEL: @memmove_volatile(
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; ALL-NEXT: [[COMPARE_N_TO_0:%.*]] = icmp eq i64 64, 0
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[COPY_BACKWARDS:%.*]], label [[COPY_FORWARD:%.*]]
-; ALL: copy_backwards:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE:%.*]], label [[COPY_BACKWARDS_LOOP:%.*]]
-; ALL: copy_backwards_loop:
-; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[INDEX_PTR:%.*]], [[COPY_BACKWARDS_LOOP]] ], [ 64, [[COPY_BACKWARDS]] ]
-; ALL-NEXT: [[INDEX_PTR]] = sub i64 [[TMP1]], 1
-; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 64, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load volatile i8, ptr addrspace(1) [[TMP2]], align 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR]]
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
; ALL-NEXT: store volatile i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_PTR]], 0
-; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE]], label [[COPY_BACKWARDS_LOOP]]
-; ALL: copy_forward:
-; ALL-NEXT: br i1 [[COMPARE_N_TO_0]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP:%.*]]
-; ALL: copy_forward_loop:
-; ALL-NEXT: [[INDEX_PTR1:%.*]] = phi i64 [ [[INDEX_INCREMENT:%.*]], [[COPY_FORWARD_LOOP]] ], [ 0, [[COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load volatile i8, ptr addrspace(1) [[TMP5]], align 1
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[INDEX_PTR1]]
-; ALL-NEXT: store volatile i8 [[ELEMENT2]], ptr addrspace(1) [[TMP6]], align 1
-; ALL-NEXT: [[INDEX_INCREMENT]] = add i64 [[INDEX_PTR1]], 1
-; ALL-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_INCREMENT]], 64
-; ALL-NEXT: br i1 [[TMP7]], label [[MEMMOVE_DONE]], label [[COPY_FORWARD_LOOP]]
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 64
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 4, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store volatile <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store volatile <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
+; ALL-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 4
+; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 64, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load volatile i8, ptr addrspace(1) [[TMP13]], align 1
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store volatile i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
+; ALL-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 64
+; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
diff --git a/llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll b/llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll
index afa7fde6c842b..59e755aee215c 100644
--- a/llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-aggr-copies.ll
@@ -154,7 +154,7 @@ entry:
; IR-LABEL: @memmove_caller
; IR: icmp ult ptr %src, %dst
; IR: [[PHIVAL:%[0-9a-zA-Z_]+]] = phi i64
-; IR-NEXT: %index_ptr = sub i64 [[PHIVAL]], 1
+; IR-NEXT: %bwd_main_index = sub i64 [[PHIVAL]], 1
; IR: [[FWDPHIVAL:%[0-9a-zA-Z_]+]] = phi i64
; IR: {{%[0-9a-zA-Z_]+}} = add i64 [[FWDPHIVAL]], 1
>From dbe8d4c178fcbfe4b3783203968e8af3ae14b5a2 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Tue, 23 Jul 2024 10:32:16 -0400
Subject: [PATCH 2/6] fixup! [LowerMemIntrinsics] Lower llvm.memmove to wide
memory accesses
Use cast instead of dyn_cast + assert.
---
llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index 55cd61746d19c..2f8abd264135e 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -397,9 +397,7 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
bool RequiresResidual = !LoopOpIsInt8;
// Calculate the loop trip count and remaining bytes to copy after the loop.
- IntegerType *ILengthType = dyn_cast<IntegerType>(TypeOfCopyLen);
- assert(ILengthType &&
- "expected size argument to memcpy to be an integer type!");
+ IntegerType *ILengthType = cast<IntegerType>(TypeOfCopyLen);
ConstantInt *CILoopOpSize = ConstantInt::get(ILengthType, LoopOpSize);
ConstantInt *Zero = ConstantInt::get(ILengthType, 0);
ConstantInt *One = ConstantInt::get(ILengthType, 1);
>From aa87bceb18f5e81390507ecfb82b0e08f1fba8ed Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 24 Jul 2024 03:13:47 -0400
Subject: [PATCH 3/6] fixup! fixup! [LowerMemIntrinsics] Lower llvm.memmove to
wide memory accesses
Add more memmove lowering tests.
---
.../CodeGen/AMDGPU/lower-mem-intrinsics.ll | 484 ++++++++++++++++++
1 file changed, 484 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index 1e60ba415e414..5c6716cf5be16 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -17,6 +17,9 @@ declare void @llvm.memcpy.p3.p3.i32(ptr addrspace(3) nocapture, ptr addrspace(3)
declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) nocapture, ptr addrspace(1) nocapture readonly, i64, i1) #1
declare void @llvm.memmove.p1.p3.i32(ptr addrspace(1) nocapture, ptr addrspace(3) nocapture readonly, i32, i1) #1
+declare void @llvm.memmove.p0.p3.i32(ptr nocapture writeonly, ptr addrspace(3) nocapture readonly, i32, i1 immarg) #1
+declare void @llvm.memmove.p3.p0.i32(ptr addrspace(3) nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) #1
+declare void @llvm.memmove.p3.p3.i32(ptr addrspace(3) nocapture writeonly, ptr addrspace(3) nocapture readonly, i32, i1 immarg) #1
declare void @llvm.memmove.p5.p5.i32(ptr addrspace(5) nocapture, ptr addrspace(5) nocapture readonly, i32, i1) #1
declare void @llvm.memmove.p3.p5.i32(ptr addrspace(3) nocapture, ptr addrspace(5) nocapture readonly, i32, i1) #1
declare void @llvm.memmove.p5.p3.i32(ptr addrspace(5) nocapture, ptr addrspace(3) nocapture readonly, i32, i1) #1
@@ -1860,6 +1863,487 @@ define amdgpu_kernel void @memmove_private_align1_local_align1_unknown_size(ptr
ret void
}
+
+define amdgpu_kernel void @memmove_flat_align1_local_align1(ptr addrspace(0) %dst, ptr addrspace(3) %src) {
+; MAX1024-LABEL: @memmove_flat_align1_local_align1(
+; MAX1024-NEXT: call void @llvm.memmove.p0.p3.i32(ptr [[DST:%.*]], ptr addrspace(3) [[SRC:%.*]], i32 256, i1 false)
+; MAX1024-NEXT: ret void
+;
+; ALL-LABEL: @memmove_flat_align1_local_align1(
+; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[SRC:%.*]] to ptr
+; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i32 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL: memmove_done:
+; ALL-NEXT: ret void
+;
+ call void @llvm.memmove.p0.p3.i32(ptr addrspace(0) %dst, ptr addrspace(3) %src, i32 256, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_flat_align1_local_align1_unknown_size(ptr addrspace(0) %dst, ptr addrspace(3) %src, i32 %size) {
+; OPT-LABEL: @memmove_flat_align1_local_align1_unknown_size(
+; OPT-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[SRC:%.*]] to ptr
+; OPT-NEXT: [[TMP2:%.*]] = lshr i32 [[SIZE:%.*]], 4
+; OPT-NEXT: [[TMP3:%.*]] = and i32 [[SIZE]], 15
+; OPT-NEXT: [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP3]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP5]], 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr [[TMP7]], align 1
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP4]]
+; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP2]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
+; OPT-NEXT: [[TMP12:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
+; OPT-NEXT: [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP2]]
+; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP19:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP4]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP17]], align 1
+; OPT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr [[TMP18]], align 1
+; OPT-NEXT: [[TMP19]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP20:%.*]] = icmp eq i32 [[TMP19]], [[SIZE]]
+; OPT-NEXT: br i1 [[TMP20]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p0.p3.i32(ptr addrspace(0) %dst, ptr addrspace(3) %src, i32 %size, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_local_align1_flat_align1(ptr addrspace(3) %dst, ptr addrspace(0) %src) {
+; MAX1024-LABEL: @memmove_local_align1_flat_align1(
+; MAX1024-NEXT: call void @llvm.memmove.p3.p0.i32(ptr addrspace(3) [[DST:%.*]], ptr [[SRC:%.*]], i32 256, i1 false)
+; MAX1024-NEXT: ret void
+;
+; ALL-LABEL: @memmove_local_align1_flat_align1(
+; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[DST:%.*]] to ptr
+; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP6:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP6]], 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
+; ALL-NEXT: [[TMP12]] = add i32 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 16
+; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
+; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
+; ALL-NEXT: [[TMP16]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 256
+; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL: memmove_done:
+; ALL-NEXT: ret void
+;
+ call void @llvm.memmove.p3.p0.i32(ptr addrspace(3) %dst, ptr addrspace(0) %src, i32 256, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_local_align1_flat_align1_unknown_size(ptr addrspace(3) %dst, ptr addrspace(0) %src, i32 %size) {
+; OPT-LABEL: @memmove_local_align1_flat_align1_unknown_size(
+; OPT-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[DST:%.*]] to ptr
+; OPT-NEXT: [[TMP2:%.*]] = lshr i32 [[SIZE:%.*]], 4
+; OPT-NEXT: [[TMP3:%.*]] = and i32 [[SIZE]], 15
+; OPT-NEXT: [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP3]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP5]], 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr [[TMP7]], align 1
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP4]]
+; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP2]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
+; OPT-NEXT: [[TMP12:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
+; OPT-NEXT: [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP2]]
+; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP19:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP4]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP17]], align 1
+; OPT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr [[TMP18]], align 1
+; OPT-NEXT: [[TMP19]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP20:%.*]] = icmp eq i32 [[TMP19]], [[SIZE]]
+; OPT-NEXT: br i1 [[TMP20]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p3.p0.i32(ptr addrspace(3) %dst, ptr addrspace(0) %src, i32 %size, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_local_align1_local_align1(ptr addrspace(3) %dst, ptr addrspace(3) %src) {
+; MAX1024-LABEL: @memmove_local_align1_local_align1(
+; MAX1024-NEXT: call void @llvm.memmove.p3.p3.i32(ptr addrspace(3) [[DST:%.*]], ptr addrspace(3) [[SRC:%.*]], i32 256, i1 false)
+; MAX1024-NEXT: ret void
+;
+; ALL-LABEL: @memmove_local_align1_local_align1(
+; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(3) [[SRC:%.*]], [[DST:%.*]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(3) [[TMP2]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(3) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 32, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP5]], 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP9]], align 1
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP10]], align 1
+; ALL-NEXT: [[TMP11]] = add i32 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 32
+; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(3) [[TMP13]], align 1
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(3) [[TMP14]], align 1
+; ALL-NEXT: [[TMP15]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 256
+; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL: memmove_done:
+; ALL-NEXT: ret void
+;
+ call void @llvm.memmove.p3.p3.i32(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 256, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_local_align1_local_align1_unknown_size(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 %size) {
+; OPT-LABEL: @memmove_local_align1_local_align1_unknown_size(
+; OPT-NEXT: [[TMP1:%.*]] = lshr i32 [[SIZE:%.*]], 3
+; OPT-NEXT: [[TMP2:%.*]] = and i32 [[SIZE]], 7
+; OPT-NEXT: [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP1]], 0
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(3) [[SRC:%.*]], [[DST:%.*]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP4:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP4]], 1
+; OPT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(3) [[TMP5]], align 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr addrspace(3) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP3]]
+; OPT-NEXT: br i1 [[TMP7]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP8:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP1]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP8]], 1
+; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP9]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP11]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP14:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP12:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP12]], align 1
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <2 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP13]], align 1
+; OPT-NEXT: [[TMP14]] = add i32 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP15:%.*]] = icmp eq i32 [[TMP14]], [[TMP1]]
+; OPT-NEXT: br i1 [[TMP15]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP18:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP3]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(3) [[TMP16]], align 1
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr addrspace(3) [[TMP17]], align 1
+; OPT-NEXT: [[TMP18]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP18]], [[SIZE]]
+; OPT-NEXT: br i1 [[TMP19]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p3.p3.i32(ptr addrspace(3) %dst, ptr addrspace(3) %src, i32 %size, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_private_align1_private_align1(ptr addrspace(5) %dst, ptr addrspace(5) %src) {
+; MAX1024-LABEL: @memmove_private_align1_private_align1(
+; MAX1024-NEXT: call void @llvm.memmove.p5.p5.i32(ptr addrspace(5) [[DST:%.*]], ptr addrspace(5) [[SRC:%.*]], i32 256, i1 false)
+; MAX1024-NEXT: ret void
+;
+; ALL-LABEL: @memmove_private_align1_private_align1(
+; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(5) [[SRC:%.*]], [[DST:%.*]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; ALL: memmove_copy_backwards:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_bwd_residual_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
+; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(5) [[TMP2]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(5) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; ALL: memmove_bwd_middle:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; ALL: memmove_bwd_main_loop:
+; ALL-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
+; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP5]], 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP7]], align 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; ALL: memmove_copy_forward:
+; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; ALL: memmove_fwd_main_loop:
+; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP9]], align 1
+; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(5) [[TMP10]], align 1
+; ALL-NEXT: [[TMP11]] = add i32 [[FWD_MAIN_INDEX]], 1
+; ALL-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 16
+; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; ALL: memmove_fwd_middle:
+; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; ALL: memmove_fwd_residual_loop:
+; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
+; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(5) [[TMP13]], align 1
+; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(5) [[TMP14]], align 1
+; ALL-NEXT: [[TMP15]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; ALL-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 256
+; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL: memmove_done:
+; ALL-NEXT: ret void
+;
+ call void @llvm.memmove.p5.p5.i32(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 256, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_private_align1_private_align1_unknown_size(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 %size) {
+; OPT-LABEL: @memmove_private_align1_private_align1_unknown_size(
+; OPT-NEXT: [[TMP1:%.*]] = lshr i32 [[SIZE:%.*]], 4
+; OPT-NEXT: [[TMP2:%.*]] = and i32 [[SIZE]], 15
+; OPT-NEXT: [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP1]], 0
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(5) [[SRC:%.*]], [[DST:%.*]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
+; OPT: memmove_copy_backwards:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_bwd_residual_loop:
+; OPT-NEXT: [[TMP4:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
+; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP4]], 1
+; OPT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(5) [[TMP5]], align 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr addrspace(5) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP3]]
+; OPT-NEXT: br i1 [[TMP7]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
+; OPT: memmove_bwd_middle:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
+; OPT: memmove_bwd_main_loop:
+; OPT-NEXT: [[TMP8:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP1]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP8]], 1
+; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP9]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP11]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
+; OPT: memmove_copy_forward:
+; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
+; OPT: memmove_fwd_main_loop:
+; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP14:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
+; OPT-NEXT: [[TMP12:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP12]], align 1
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(5) [[TMP13]], align 1
+; OPT-NEXT: [[TMP14]] = add i32 [[FWD_MAIN_INDEX]], 1
+; OPT-NEXT: [[TMP15:%.*]] = icmp eq i32 [[TMP14]], [[TMP1]]
+; OPT-NEXT: br i1 [[TMP15]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
+; OPT: memmove_fwd_middle:
+; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
+; OPT: memmove_fwd_residual_loop:
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP18:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP3]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(5) [[TMP16]], align 1
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr addrspace(5) [[TMP17]], align 1
+; OPT-NEXT: [[TMP18]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
+; OPT-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP18]], [[SIZE]]
+; OPT-NEXT: br i1 [[TMP19]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p5.p5.i32(ptr addrspace(5) %dst, ptr addrspace(5) %src, i32 %size, i1 false)
+ ret void
+}
+
define void @test_umin(i64 %0, i64 %idxprom, ptr %x, ptr %y) {
; OPT-LABEL: @test_umin(
; OPT-NEXT: entry:
>From b491cb637825798ac358189b490bfa7f3980fff4 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 24 Jul 2024 03:43:28 -0400
Subject: [PATCH 4/6] fixup! fixup! fixup! [LowerMemIntrinsics] Lower
llvm.memmove to wide memory accesses
Specify alignment for residual accesses.
---
.../Transforms/Utils/LowerMemIntrinsics.cpp | 40 ++++++++++++-------
1 file changed, 26 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index 2f8abd264135e..a6bf72789ed21 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -396,6 +396,9 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
// i8-accesses are required to move remaining bytes.
bool RequiresResidual = !LoopOpIsInt8;
+ Type *ResidualLoopOpType = Int8Type;
+ unsigned ResidualLoopOpSize = DL.getTypeStoreSize(ResidualLoopOpType);
+
// Calculate the loop trip count and remaining bytes to copy after the loop.
IntegerType *ILengthType = cast<IntegerType>(TypeOfCopyLen);
ConstantInt *CILoopOpSize = ConstantInt::get(ILengthType, LoopOpSize);
@@ -467,6 +470,11 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
Align PartSrcAlign(commonAlignment(SrcAlign, LoopOpSize));
Align PartDstAlign(commonAlignment(DstAlign, LoopOpSize));
+ // Accesses in the residual loops do not share the same alignment as those in
+ // the main loops.
+ Align ResidualSrcAlign(commonAlignment(PartSrcAlign, ResidualLoopOpSize));
+ Align ResidualDstAlign(commonAlignment(PartDstAlign, ResidualLoopOpSize));
+
// Copying backwards.
{
BasicBlock *MainLoopBB = BasicBlock::Create(
@@ -484,13 +492,15 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
PHINode *ResidualLoopPhi = ResidualLoopBuilder.CreatePHI(ILengthType, 0);
Value *ResidualIndex = ResidualLoopBuilder.CreateSub(
ResidualLoopPhi, One, "bwd_residual_index");
- Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
- ResidualIndex);
- Value *Element = ResidualLoopBuilder.CreateLoad(Int8Type, LoadGEP,
- SrcIsVolatile, "element");
- Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, DstAddr,
- ResidualIndex);
- ResidualLoopBuilder.CreateStore(Element, StoreGEP, DstIsVolatile);
+ Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(
+ ResidualLoopOpType, SrcAddr, ResidualIndex);
+ Value *Element = ResidualLoopBuilder.CreateAlignedLoad(
+ ResidualLoopOpType, LoadGEP, ResidualSrcAlign, SrcIsVolatile,
+ "element");
+ Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(
+ ResidualLoopOpType, DstAddr, ResidualIndex);
+ ResidualLoopBuilder.CreateAlignedStore(Element, StoreGEP,
+ ResidualDstAlign, DstIsVolatile);
// After the residual loop, go to an intermediate block.
BasicBlock *IntermediateBB = BasicBlock::Create(
@@ -587,13 +597,15 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
IRBuilder<> ResidualLoopBuilder(ResidualLoopBB);
PHINode *ResidualLoopPhi =
ResidualLoopBuilder.CreatePHI(ILengthType, 0, "fwd_residual_index");
- Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, SrcAddr,
- ResidualLoopPhi);
- Value *Element = ResidualLoopBuilder.CreateLoad(Int8Type, LoadGEP,
- SrcIsVolatile, "element");
- Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(Int8Type, DstAddr,
- ResidualLoopPhi);
- ResidualLoopBuilder.CreateStore(Element, StoreGEP, DstIsVolatile);
+ Value *LoadGEP = ResidualLoopBuilder.CreateInBoundsGEP(
+ ResidualLoopOpType, SrcAddr, ResidualLoopPhi);
+ Value *Element = ResidualLoopBuilder.CreateAlignedLoad(
+ ResidualLoopOpType, LoadGEP, ResidualSrcAlign, SrcIsVolatile,
+ "element");
+ Value *StoreGEP = ResidualLoopBuilder.CreateInBoundsGEP(
+ ResidualLoopOpType, DstAddr, ResidualLoopPhi);
+ ResidualLoopBuilder.CreateAlignedStore(Element, StoreGEP,
+ ResidualDstAlign, DstIsVolatile);
Value *ResidualIndex =
ResidualLoopBuilder.CreateAdd(ResidualLoopPhi, One);
ResidualLoopBuilder.CreateCondBr(
>From 8d8a1cae4f25426dbfef321dedfdc96787d4c27c Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Wed, 24 Jul 2024 10:19:22 -0400
Subject: [PATCH 5/6] fixup! fixup! fixup! fixup! [LowerMemIntrinsics] Lower
llvm.memmove to wide memory accesses
Implement separate lowering for const-sized memmoves to avoid generating unreachable code.
---
.../Transforms/Utils/LowerMemIntrinsics.cpp | 199 ++++-
.../CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll | 104 ++-
.../CodeGen/AMDGPU/lower-mem-intrinsics.ll | 810 +++++++-----------
3 files changed, 565 insertions(+), 548 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index a6bf72789ed21..c035fa7a2d4d6 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -373,11 +373,12 @@ void llvm::createMemCpyLoopUnknownSize(
// If the TargetTransformInfo specifies a wider MemcpyLoopLoweringType, it is
// used for the memory accesses in the loops. Then, additional loops with
// byte-wise accesses are added for the remaining bytes.
-static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
- Value *DstAddr, Value *CopyLen, Align SrcAlign,
- Align DstAlign, bool SrcIsVolatile,
- bool DstIsVolatile,
- const TargetTransformInfo &TTI) {
+static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
+ Value *SrcAddr, Value *DstAddr,
+ Value *CopyLen, Align SrcAlign,
+ Align DstAlign, bool SrcIsVolatile,
+ bool DstIsVolatile,
+ const TargetTransformInfo &TTI) {
Type *TypeOfCopyLen = CopyLen->getType();
BasicBlock *OrigBB = InsertBefore->getParent();
Function *F = OrigBB->getParent();
@@ -617,6 +618,182 @@ static void createMemMoveLoop(Instruction *InsertBefore, Value *SrcAddr,
}
}
+// Similar to createMemMoveLoopUnknownSize, only the trip counts are computed at
+// compile time, obsolete loops and branches are omitted, and the residual code
+// is straight-line code instead of a loop.
+static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
+ Value *SrcAddr, Value *DstAddr,
+ ConstantInt *CopyLen, Align SrcAlign,
+ Align DstAlign, bool SrcIsVolatile,
+ bool DstIsVolatile,
+ const TargetTransformInfo &TTI) {
+ // No need to expand zero length moves.
+ if (CopyLen->isZero())
+ return;
+
+ Type *TypeOfCopyLen = CopyLen->getType();
+ BasicBlock *OrigBB = InsertBefore->getParent();
+ Function *F = OrigBB->getParent();
+ const DataLayout &DL = F->getDataLayout();
+ LLVMContext &Ctx = OrigBB->getContext();
+ unsigned SrcAS = cast<PointerType>(SrcAddr->getType())->getAddressSpace();
+ unsigned DstAS = cast<PointerType>(DstAddr->getType())->getAddressSpace();
+
+ Type *LoopOpType = TTI.getMemcpyLoopLoweringType(
+ Ctx, CopyLen, SrcAS, DstAS, SrcAlign.value(), DstAlign.value());
+ unsigned LoopOpSize = DL.getTypeStoreSize(LoopOpType);
+
+ // Calculate the loop trip count and remaining bytes to copy after the loop.
+ uint64_t LoopEndCount = CopyLen->getZExtValue() / LoopOpSize;
+ uint64_t BytesCopiedInLoop = LoopEndCount * LoopOpSize;
+ uint64_t RemainingBytes = CopyLen->getZExtValue() - BytesCopiedInLoop;
+
+ IntegerType *ILengthType = cast<IntegerType>(TypeOfCopyLen);
+ ConstantInt *Zero = ConstantInt::get(ILengthType, 0);
+ ConstantInt *One = ConstantInt::get(ILengthType, 1);
+ ConstantInt *TripCount = ConstantInt::get(ILengthType, LoopEndCount);
+
+ IRBuilder<> PLBuilder(InsertBefore);
+
+ Value *PtrCompare =
+ PLBuilder.CreateICmpULT(SrcAddr, DstAddr, "compare_src_dst");
+ Instruction *ThenTerm, *ElseTerm;
+ SplitBlockAndInsertIfThenElse(PtrCompare, InsertBefore->getIterator(),
+ &ThenTerm, &ElseTerm);
+
+ BasicBlock *CopyBackwardsBB = ThenTerm->getParent();
+ BasicBlock *CopyForwardBB = ElseTerm->getParent();
+ BasicBlock *ExitBB = InsertBefore->getParent();
+ ExitBB->setName("memmove_done");
+
+ Align PartSrcAlign(commonAlignment(SrcAlign, LoopOpSize));
+ Align PartDstAlign(commonAlignment(DstAlign, LoopOpSize));
+
+ // Helper function to generate a load/store pair of a given type in the
+ // residual. Used in the forward and backward branches.
+ auto GenerateResidualLdStPair = [&](Type *OpTy, IRBuilder<> &Builder,
+ uint64_t &BytesCopied) {
+ Align ResSrcAlign(commonAlignment(SrcAlign, BytesCopied));
+ Align ResDstAlign(commonAlignment(DstAlign, BytesCopied));
+
+ // Calculate the new index
+ unsigned OperandSize = DL.getTypeStoreSize(OpTy);
+
+ uint64_t GepIndex = BytesCopied / OperandSize;
+ assert(GepIndex * OperandSize == BytesCopied &&
+ "Division should have no Remainder!");
+
+ Value *SrcGEP = Builder.CreateInBoundsGEP(
+ OpTy, SrcAddr, ConstantInt::get(TypeOfCopyLen, GepIndex));
+ LoadInst *Load =
+ Builder.CreateAlignedLoad(OpTy, SrcGEP, ResSrcAlign, SrcIsVolatile);
+ Value *DstGEP = Builder.CreateInBoundsGEP(
+ OpTy, DstAddr, ConstantInt::get(TypeOfCopyLen, GepIndex));
+ Builder.CreateAlignedStore(Load, DstGEP, ResDstAlign, DstIsVolatile);
+ BytesCopied += OperandSize;
+ };
+
+ // Copying backwards.
+ if (RemainingBytes != 0) {
+ CopyBackwardsBB->setName("memmove_bwd_residual");
+ uint64_t BytesCopied = BytesCopiedInLoop;
+
+ // Residual code is required to move the remaining bytes. We need the same
+ // instructions as in the forward case, only in reverse. So we generate code
+ // the same way, except that we change the IRBuilder insert point for each
+ // load/store pair so that each one is inserted before the previous one
+ // instead of after it.
+ IRBuilder<> BwdResBuilder(CopyBackwardsBB->getFirstNonPHI());
+ SmallVector<Type *, 5> RemainingOps;
+ TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
+ SrcAS, DstAS, PartSrcAlign.value(),
+ PartDstAlign.value());
+ for (auto *OpTy : RemainingOps) {
+ // reverse the order of the emitted operations
+ BwdResBuilder.SetInsertPoint(CopyBackwardsBB->getFirstNonPHI());
+ GenerateResidualLdStPair(OpTy, BwdResBuilder, BytesCopied);
+ }
+ }
+ if (LoopEndCount != 0) {
+ BasicBlock *LoopBB = CopyBackwardsBB;
+ BasicBlock *PredBB = OrigBB;
+ if (RemainingBytes != 0) {
+ // if we introduce residual code, it needs its separate BB
+ LoopBB = CopyBackwardsBB->splitBasicBlock(
+ CopyBackwardsBB->getTerminator(), "memmove_bwd_loop");
+ PredBB = CopyBackwardsBB;
+ } else {
+ CopyBackwardsBB->setName("memmove_bwd_loop");
+ }
+ IRBuilder<> LoopBuilder(LoopBB->getTerminator());
+ PHINode *LoopPhi = LoopBuilder.CreatePHI(ILengthType, 0);
+ Value *Index = LoopBuilder.CreateSub(LoopPhi, One, "bwd_index");
+ Value *LoadGEP = LoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, Index);
+ Value *Element = LoopBuilder.CreateAlignedLoad(
+ LoopOpType, LoadGEP, PartSrcAlign, SrcIsVolatile, "element");
+ Value *StoreGEP = LoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, Index);
+ LoopBuilder.CreateAlignedStore(Element, StoreGEP, PartDstAlign,
+ DstIsVolatile);
+
+ // Replace the unconditional branch introduced by
+ // SplitBlockAndInsertIfThenElse to turn LoopBB into a loop.
+ Instruction *UncondTerm = LoopBB->getTerminator();
+ LoopBuilder.CreateCondBr(LoopBuilder.CreateICmpEQ(Index, Zero), ExitBB,
+ LoopBB);
+ UncondTerm->eraseFromParent();
+
+ LoopPhi->addIncoming(Index, LoopBB);
+ LoopPhi->addIncoming(TripCount, PredBB);
+ }
+
+ // Copying forward.
+ BasicBlock *FwdResidualBB = CopyForwardBB;
+ if (LoopEndCount != 0) {
+ CopyForwardBB->setName("memmove_fwd_loop");
+ BasicBlock *LoopBB = CopyForwardBB;
+ BasicBlock *SuccBB = ExitBB;
+ if (RemainingBytes != 0) {
+ // if we introduce residual code, it needs its separate BB
+ SuccBB = CopyForwardBB->splitBasicBlock(CopyForwardBB->getTerminator(),
+ "memmove_fwd_residual");
+ FwdResidualBB = SuccBB;
+ }
+ IRBuilder<> LoopBuilder(LoopBB->getTerminator());
+ PHINode *LoopPhi = LoopBuilder.CreatePHI(ILengthType, 0, "fwd_index");
+ Value *LoadGEP =
+ LoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, LoopPhi);
+ Value *Element = LoopBuilder.CreateAlignedLoad(
+ LoopOpType, LoadGEP, PartSrcAlign, SrcIsVolatile, "element");
+ Value *StoreGEP =
+ LoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, LoopPhi);
+ LoopBuilder.CreateAlignedStore(Element, StoreGEP, PartDstAlign,
+ DstIsVolatile);
+ Value *Index = LoopBuilder.CreateAdd(LoopPhi, One);
+ LoopPhi->addIncoming(Index, LoopBB);
+ LoopPhi->addIncoming(Zero, OrigBB);
+
+ // Replace the unconditional branch to turn LoopBB into a loop.
+ Instruction *UncondTerm = LoopBB->getTerminator();
+ LoopBuilder.CreateCondBr(LoopBuilder.CreateICmpEQ(Index, TripCount), SuccBB,
+ LoopBB);
+ UncondTerm->eraseFromParent();
+ }
+
+ if (RemainingBytes != 0) {
+ uint64_t BytesCopied = BytesCopiedInLoop;
+
+ // Residual code is required to move the remaining bytes. In the forward
+ // case, we emit it in the normal order.
+ IRBuilder<> FwdResBuilder(FwdResidualBB->getTerminator());
+ SmallVector<Type *, 5> RemainingOps;
+ TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes,
+ SrcAS, DstAS, PartSrcAlign.value(),
+ PartDstAlign.value());
+ for (auto *OpTy : RemainingOps)
+ GenerateResidualLdStPair(OpTy, FwdResBuilder, BytesCopied);
+ }
+}
+
static void createMemSetLoop(Instruction *InsertBefore, Value *DstAddr,
Value *CopyLen, Value *SetValue, Align DstAlign,
bool IsVolatile) {
@@ -745,9 +922,15 @@ bool llvm::expandMemMoveAsLoop(MemMoveInst *Memmove,
}
}
- createMemMoveLoop(
- /*InsertBefore=*/Memmove, SrcAddr, DstAddr, CopyLen, SrcAlign, DstAlign,
- SrcIsVolatile, DstIsVolatile, TTI);
+ if (ConstantInt *CI = dyn_cast<ConstantInt>(CopyLen)) {
+ createMemMoveLoopKnownSize(
+ /*InsertBefore=*/Memmove, SrcAddr, DstAddr, CI, SrcAlign, DstAlign,
+ SrcIsVolatile, DstIsVolatile, TTI);
+ } else {
+ createMemMoveLoopUnknownSize(
+ /*InsertBefore=*/Memmove, SrcAddr, DstAddr, CopyLen, SrcAlign, DstAlign,
+ SrcIsVolatile, DstIsVolatile, TTI);
+ }
return true;
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
index f59b549c0f88a..de9af5209db16 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
@@ -9,57 +9,71 @@ define amdgpu_cs void @memmove_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src
; LOOP: ; %bb.0:
; LOOP-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[0:1]
; LOOP-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; LOOP-NEXT: s_xor_b64 s[4:5], exec, s[0:1]
-; LOOP-NEXT: s_cbranch_execz .LBB0_3
-; LOOP-NEXT: ; %bb.1: ; %memmove_fwd_middle
-; LOOP-NEXT: s_mov_b64 s[6:7], 0
+; LOOP-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; LOOP-NEXT: s_cbranch_execnz .LBB0_3
+; LOOP-NEXT: ; %bb.1: ; %Flow
+; LOOP-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; LOOP-NEXT: s_cbranch_execnz .LBB0_4
+; LOOP-NEXT: .LBB0_2: ; %memmove_done
+; LOOP-NEXT: s_endpgm
+; LOOP-NEXT: .LBB0_3:
+; LOOP-NEXT: s_mov_b32 s6, 0
+; LOOP-NEXT: s_mov_b32 s7, 0xf000
+; LOOP-NEXT: s_mov_b64 s[4:5], 0
+; LOOP-NEXT: buffer_load_ubyte v4, v[2:3], s[4:7], 0 addr64 offset:1
+; LOOP-NEXT: buffer_load_ubyte v5, v[2:3], s[4:7], 0 addr64 offset:3
+; LOOP-NEXT: buffer_load_ubyte v6, v[2:3], s[4:7], 0 addr64 offset:2
+; LOOP-NEXT: buffer_load_ubyte v2, v[2:3], s[4:7], 0 addr64
+; LOOP-NEXT: s_waitcnt vmcnt(3)
+; LOOP-NEXT: v_lshlrev_b32_e32 v3, 8, v4
+; LOOP-NEXT: s_waitcnt vmcnt(2)
+; LOOP-NEXT: v_lshlrev_b32_e32 v4, 24, v5
+; LOOP-NEXT: s_waitcnt vmcnt(1)
+; LOOP-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; LOOP-NEXT: s_waitcnt vmcnt(0)
+; LOOP-NEXT: v_or_b32_e32 v2, v3, v2
+; LOOP-NEXT: v_or_b32_e32 v3, v4, v5
+; LOOP-NEXT: v_or_b32_e32 v2, v3, v2
+; LOOP-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; LOOP-NEXT: v_bfe_u32 v4, v2, 8, 8
+; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[4:7], 0 addr64
+; LOOP-NEXT: s_waitcnt expcnt(0)
+; LOOP-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[4:7], 0 addr64 offset:1
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[4:7], 0 addr64 offset:2
+; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[4:7], 0 addr64 offset:3
+; LOOP-NEXT: ; implicit-def: $vgpr2_vgpr3
+; LOOP-NEXT: ; implicit-def: $vgpr0_vgpr1
+; LOOP-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; LOOP-NEXT: s_cbranch_execz .LBB0_2
+; LOOP-NEXT: .LBB0_4: ; %memmove_bwd_residual
; LOOP-NEXT: s_mov_b32 s2, 0
; LOOP-NEXT: s_mov_b32 s3, 0xf000
; LOOP-NEXT: s_mov_b64 s[0:1], 0
-; LOOP-NEXT: v_mov_b32_e32 v4, s6
-; LOOP-NEXT: v_mov_b32_e32 v5, s7
-; LOOP-NEXT: .LBB0_2: ; %memmove_fwd_residual_loop
-; LOOP-NEXT: ; =>This Inner Loop Header: Depth=1
-; LOOP-NEXT: v_add_i32_e32 v6, vcc, v2, v4
-; LOOP-NEXT: v_addc_u32_e32 v7, vcc, v3, v5, vcc
+; LOOP-NEXT: s_waitcnt expcnt(2)
+; LOOP-NEXT: buffer_load_ubyte v4, v[2:3], s[0:3], 0 addr64 offset:1
+; LOOP-NEXT: buffer_load_ubyte v5, v[2:3], s[0:3], 0 addr64 offset:3
+; LOOP-NEXT: buffer_load_ubyte v6, v[2:3], s[0:3], 0 addr64 offset:2
; LOOP-NEXT: s_waitcnt expcnt(0)
-; LOOP-NEXT: buffer_load_ubyte v8, v[6:7], s[0:3], 0 addr64
-; LOOP-NEXT: v_add_i32_e32 v6, vcc, v0, v4
-; LOOP-NEXT: v_addc_u32_e32 v7, vcc, v1, v5, vcc
-; LOOP-NEXT: v_add_i32_e32 v4, vcc, 1, v4
-; LOOP-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
-; LOOP-NEXT: v_cmp_ne_u32_e32 vcc, 4, v4
+; LOOP-NEXT: buffer_load_ubyte v2, v[2:3], s[0:3], 0 addr64
+; LOOP-NEXT: s_waitcnt vmcnt(3)
+; LOOP-NEXT: v_lshlrev_b32_e32 v3, 8, v4
+; LOOP-NEXT: s_waitcnt vmcnt(2)
+; LOOP-NEXT: v_lshlrev_b32_e32 v4, 24, v5
+; LOOP-NEXT: s_waitcnt vmcnt(1)
+; LOOP-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; LOOP-NEXT: s_waitcnt vmcnt(0)
-; LOOP-NEXT: buffer_store_byte v8, v[6:7], s[0:3], 0 addr64
-; LOOP-NEXT: s_cbranch_vccnz .LBB0_2
-; LOOP-NEXT: .LBB0_3: ; %Flow25
-; LOOP-NEXT: s_andn2_saveexec_b64 s[0:1], s[4:5]
-; LOOP-NEXT: s_cbranch_execz .LBB0_6
-; LOOP-NEXT: ; %bb.4: ; %memmove_copy_backwards
-; LOOP-NEXT: v_add_i32_e32 v0, vcc, 3, v0
-; LOOP-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; LOOP-NEXT: v_add_i32_e32 v2, vcc, 3, v2
-; LOOP-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; LOOP-NEXT: s_mov_b32 s0, -4
-; LOOP-NEXT: s_mov_b32 s6, 0
-; LOOP-NEXT: s_mov_b32 s7, 0xf000
-; LOOP-NEXT: s_mov_b64 s[4:5], 0
-; LOOP-NEXT: v_mov_b32_e32 v4, s0
-; LOOP-NEXT: .LBB0_5: ; %memmove_bwd_residual_loop
-; LOOP-NEXT: ; =>This Inner Loop Header: Depth=1
+; LOOP-NEXT: v_or_b32_e32 v2, v3, v2
+; LOOP-NEXT: v_or_b32_e32 v3, v4, v5
+; LOOP-NEXT: v_or_b32_e32 v2, v3, v2
+; LOOP-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; LOOP-NEXT: v_bfe_u32 v4, v2, 8, 8
+; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64
; LOOP-NEXT: s_waitcnt expcnt(0)
-; LOOP-NEXT: buffer_load_ubyte v5, v[2:3], s[4:7], 0 addr64
-; LOOP-NEXT: v_add_i32_e32 v4, vcc, 1, v4
-; LOOP-NEXT: s_xor_b64 s[0:1], vcc, -1
-; LOOP-NEXT: s_and_b64 vcc, s[0:1], exec
-; LOOP-NEXT: s_waitcnt vmcnt(0)
-; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[4:7], 0 addr64
-; LOOP-NEXT: v_add_i32_e64 v0, s[0:1], -1, v0
-; LOOP-NEXT: v_addc_u32_e64 v1, s[0:1], -1, v1, s[0:1]
-; LOOP-NEXT: v_add_i32_e64 v2, s[0:1], -1, v2
-; LOOP-NEXT: v_addc_u32_e64 v3, s[0:1], -1, v3, s[0:1]
-; LOOP-NEXT: s_cbranch_vccnz .LBB0_5
-; LOOP-NEXT: .LBB0_6: ; %memmove_done
+; LOOP-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:1
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:2
+; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:3
; LOOP-NEXT: s_endpgm
;
; UNROLL-LABEL: memmove_p1i8:
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index 5c6716cf5be16..d3bfa627fcb6e 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -90,51 +90,25 @@ define amdgpu_kernel void @max_size_small_static_memmove_caller0(ptr addrspace(1
;
; ALL-LABEL: @max_size_small_static_memmove_caller0(
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
-; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(1) [[TMP2]], align 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 1024
-; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 64, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
-; ALL-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 64
-; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(1) [[TMP13]], align 1
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
-; ALL-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 1024
-; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 64, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP2]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7]] = add i64 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 64
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -145,51 +119,37 @@ define amdgpu_kernel void @max_size_small_static_memmove_caller0(ptr addrspace(1
define amdgpu_kernel void @min_size_large_static_memmove_caller0(ptr addrspace(1) %dst, ptr addrspace(1) %src) #0 {
; OPT-LABEL: @min_size_large_static_memmove_caller0(
; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; OPT: memmove_copy_backwards:
-; OPT-NEXT: br i1 false, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; OPT: memmove_bwd_residual_loop:
-; OPT-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 1025, [[MEMMOVE_COPY_BACKWARDS]] ]
-; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
-; OPT-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
-; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(1) [[TMP2]], align 1
-; OPT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
-; OPT-NEXT: store i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; OPT-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 1024
-; OPT-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; OPT: memmove_bwd_middle:
-; OPT-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; OPT: memmove_bwd_main_loop:
-; OPT-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 64, [[MEMMOVE_BWD_MIDDLE]] ]
-; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
-; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
-; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
-; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
-; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
-; OPT-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
-; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; OPT: memmove_copy_forward:
-; OPT-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; OPT: memmove_fwd_main_loop:
-; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
-; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
-; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
-; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
-; OPT-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
-; OPT-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 64
-; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; OPT: memmove_fwd_middle:
-; OPT-NEXT: br i1 false, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; OPT: memmove_fwd_residual_loop:
-; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 1024, [[MEMMOVE_FWD_MIDDLE]] ]
-; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
-; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(1) [[TMP13]], align 1
-; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
-; OPT-NEXT: store i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
-; OPT-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
-; OPT-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 1025
-; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_RESIDUAL:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; OPT: memmove_bwd_residual:
+; OPT-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1024
+; OPT-NEXT: [[TMP2:%.*]] = load i8, ptr addrspace(1) [[TMP1]], align 1
+; OPT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1024
+; OPT-NEXT: store i8 [[TMP2]], ptr addrspace(1) [[TMP3]], align 1
+; OPT-NEXT: br label [[MEMMOVE_BWD_LOOP:%.*]]
+; OPT: memmove_bwd_loop:
+; OPT-NEXT: [[TMP4:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 64, [[MEMMOVE_BWD_RESIDUAL]] ]
+; OPT-NEXT: [[BWD_INDEX]] = sub i64 [[TMP4]], 1
+; OPT-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP7]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; OPT: memmove_fwd_loop:
+; OPT-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP10:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0:%.*]] ]
+; OPT-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP8]], align 1
+; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP9]], align 1
+; OPT-NEXT: [[TMP10]] = add i64 [[FWD_INDEX]], 1
+; OPT-NEXT: [[TMP11:%.*]] = icmp eq i64 [[TMP10]], 64
+; OPT-NEXT: br i1 [[TMP11]], label [[MEMMOVE_FWD_RESIDUAL:%.*]], label [[MEMMOVE_FWD_LOOP]]
+; OPT: memmove_fwd_residual:
+; OPT-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1024
+; OPT-NEXT: [[TMP13:%.*]] = load i8, ptr addrspace(1) [[TMP12]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1024
+; OPT-NEXT: store i8 [[TMP13]], ptr addrspace(1) [[TMP14]], align 1
+; OPT-NEXT: br label [[MEMMOVE_DONE]]
; OPT: memmove_done:
; OPT-NEXT: ret void
;
@@ -1378,51 +1338,25 @@ define amdgpu_kernel void @memmove_flat_align1_global_align1(ptr %dst, ptr addrs
; ALL-LABEL: @memmove_flat_align1_global_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[SRC:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
-; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
-; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
-; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
-; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
-; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
-; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
-; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8]] = add i64 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 16
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1438,51 +1372,25 @@ define amdgpu_kernel void @memmove_global_align1_flat_align1(ptr addrspace(1) %d
; ALL-LABEL: @memmove_global_align1_flat_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[DST:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
-; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
-; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
-; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
-; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
-; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
-; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
-; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8]] = add i64 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 16
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1498,51 +1406,25 @@ define amdgpu_kernel void @memmove_flat_align1_private_align1(ptr %dst, ptr addr
; ALL-LABEL: @memmove_flat_align1_private_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(5) [[SRC:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
-; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
-; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
-; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
-; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
-; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
-; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
-; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8]] = add i64 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 16
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1558,51 +1440,25 @@ define amdgpu_kernel void @memmove_private_align1_flat_align1(ptr addrspace(5) %
; ALL-LABEL: @memmove_private_align1_flat_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(5) [[DST:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 256
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP6:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP6]], 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
-; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
-; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
-; ALL-NEXT: [[TMP12]] = add i64 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP13:%.*]] = icmp eq i64 [[TMP12]], 16
-; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
-; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
-; ALL-NEXT: [[TMP16]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 256
-; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8]] = add i64 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 16
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1872,51 +1728,25 @@ define amdgpu_kernel void @memmove_flat_align1_local_align1(ptr addrspace(0) %ds
; ALL-LABEL: @memmove_flat_align1_local_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[SRC:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP6:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP6]], 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
-; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
-; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
-; ALL-NEXT: [[TMP12]] = add i32 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 16
-; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
-; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
-; ALL-NEXT: [[TMP16]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 256
-; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i32 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i32 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8]] = add i32 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 16
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -1993,51 +1823,25 @@ define amdgpu_kernel void @memmove_local_align1_flat_align1(ptr addrspace(3) %ds
; ALL-LABEL: @memmove_local_align1_flat_align1(
; ALL-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[DST:%.*]] to ptr
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr [[TMP4]], align 1
-; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
-; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP6:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP6]], 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP8]], align 1
-; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP12:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
-; ALL-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP11]], align 1
-; ALL-NEXT: [[TMP12]] = add i32 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP12]], 16
-; ALL-NEXT: br i1 [[TMP13]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP16:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP14]], align 1
-; ALL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr [[TMP15]], align 1
-; ALL-NEXT: [[TMP16]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 256
-; ALL-NEXT: br i1 [[TMP17]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i32 [[TMP2]], 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i32 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP8]] = add i32 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 16
+; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -2113,51 +1917,25 @@ define amdgpu_kernel void @memmove_local_align1_local_align1(ptr addrspace(3) %d
;
; ALL-LABEL: @memmove_local_align1_local_align1(
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(3) [[SRC:%.*]], [[DST:%.*]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP1:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP1]], 1
-; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(3) [[TMP2]], align 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(3) [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
-; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 32, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP5]], 1
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP9]], align 1
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <2 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP10]], align 1
-; ALL-NEXT: [[TMP11]] = add i32 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 32
-; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(3) [[TMP13]], align 1
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(3) [[TMP14]], align 1
-; ALL-NEXT: [[TMP15]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 256
-; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 32, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i32 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP2]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[BWD_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT]], ptr addrspace(3) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i32 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP5:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP5]], align 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[FWD_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7]] = add i32 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 32
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -2232,51 +2010,25 @@ define amdgpu_kernel void @memmove_private_align1_private_align1(ptr addrspace(5
;
; ALL-LABEL: @memmove_private_align1_private_align1(
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(5) [[SRC:%.*]], [[DST:%.*]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP1:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP1]], 1
-; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(5) [[TMP2]], align 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT]], ptr addrspace(5) [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], 256
-; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 16, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP5]], 1
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP9]], align 1
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT2]], ptr addrspace(5) [[TMP10]], align 1
-; ALL-NEXT: [[TMP11]] = add i32 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP11]], 16
-; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 256, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(5) [[TMP13]], align 1
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store i8 [[ELEMENT3]], ptr addrspace(5) [[TMP14]], align 1
-; ALL-NEXT: [[TMP15]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 256
-; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i32 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP2]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr addrspace(5) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i32 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i32 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP5]], align 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i32 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7]] = add i32 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 16
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
@@ -2344,6 +2096,100 @@ define amdgpu_kernel void @memmove_private_align1_private_align1_unknown_size(pt
ret void
}
+define amdgpu_kernel void @memmove_global_align4_static_residual_empty(ptr addrspace(1) %dst, ptr addrspace(1) %src) {
+; OPT-LABEL: @memmove_global_align4_static_residual_empty(
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; OPT: memmove_bwd_loop:
+; OPT-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 65, [[TMP0:%.*]] ]
+; OPT-NEXT: [[BWD_INDEX]] = sub i64 [[TMP1]], 1
+; OPT-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP2]], align 1
+; OPT-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
+; OPT-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; OPT: memmove_fwd_loop:
+; OPT-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; OPT-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP5]], align 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7]] = add i64 [[FWD_INDEX]], 1
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 65
+; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 1040, i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @memmove_global_align4_static_residual_full(ptr addrspace(1) %dst, ptr addrspace(1) %src) {
+; OPT-LABEL: @memmove_global_align4_static_residual_full(
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
+; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_RESIDUAL:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; OPT: memmove_bwd_residual:
+; OPT-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1038
+; OPT-NEXT: [[TMP2:%.*]] = load i8, ptr addrspace(1) [[TMP1]], align 1
+; OPT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1038
+; OPT-NEXT: store i8 [[TMP2]], ptr addrspace(1) [[TMP3]], align 1
+; OPT-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[SRC]], i64 518
+; OPT-NEXT: [[TMP5:%.*]] = load i16, ptr addrspace(1) [[TMP4]], align 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[DST]], i64 518
+; OPT-NEXT: store i16 [[TMP5]], ptr addrspace(1) [[TMP6]], align 1
+; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[SRC]], i64 258
+; OPT-NEXT: [[TMP8:%.*]] = load i32, ptr addrspace(1) [[TMP7]], align 1
+; OPT-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[DST]], i64 258
+; OPT-NEXT: store i32 [[TMP8]], ptr addrspace(1) [[TMP9]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[SRC]], i64 128
+; OPT-NEXT: [[TMP11:%.*]] = load i64, ptr addrspace(1) [[TMP10]], align 1
+; OPT-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[DST]], i64 128
+; OPT-NEXT: store i64 [[TMP11]], ptr addrspace(1) [[TMP12]], align 1
+; OPT-NEXT: br label [[MEMMOVE_BWD_LOOP:%.*]]
+; OPT: memmove_bwd_loop:
+; OPT-NEXT: [[TMP13:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 64, [[MEMMOVE_BWD_RESIDUAL]] ]
+; OPT-NEXT: [[BWD_INDEX]] = sub i64 [[TMP13]], 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP14]], align 1
+; OPT-NEXT: [[TMP15:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP15]], align 1
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; OPT: memmove_fwd_loop:
+; OPT-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP19:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0:%.*]] ]
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP17]], align 1
+; OPT-NEXT: [[TMP18:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
+; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP18]], align 1
+; OPT-NEXT: [[TMP19]] = add i64 [[FWD_INDEX]], 1
+; OPT-NEXT: [[TMP20:%.*]] = icmp eq i64 [[TMP19]], 64
+; OPT-NEXT: br i1 [[TMP20]], label [[MEMMOVE_FWD_RESIDUAL:%.*]], label [[MEMMOVE_FWD_LOOP]]
+; OPT: memmove_fwd_residual:
+; OPT-NEXT: [[TMP21:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[SRC]], i64 128
+; OPT-NEXT: [[TMP22:%.*]] = load i64, ptr addrspace(1) [[TMP21]], align 1
+; OPT-NEXT: [[TMP23:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[DST]], i64 128
+; OPT-NEXT: store i64 [[TMP22]], ptr addrspace(1) [[TMP23]], align 1
+; OPT-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[SRC]], i64 258
+; OPT-NEXT: [[TMP25:%.*]] = load i32, ptr addrspace(1) [[TMP24]], align 1
+; OPT-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[DST]], i64 258
+; OPT-NEXT: store i32 [[TMP25]], ptr addrspace(1) [[TMP26]], align 1
+; OPT-NEXT: [[TMP27:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[SRC]], i64 518
+; OPT-NEXT: [[TMP28:%.*]] = load i16, ptr addrspace(1) [[TMP27]], align 1
+; OPT-NEXT: [[TMP29:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[DST]], i64 518
+; OPT-NEXT: store i16 [[TMP28]], ptr addrspace(1) [[TMP29]], align 1
+; OPT-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 1038
+; OPT-NEXT: [[TMP31:%.*]] = load i8, ptr addrspace(1) [[TMP30]], align 1
+; OPT-NEXT: [[TMP32:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 1038
+; OPT-NEXT: store i8 [[TMP31]], ptr addrspace(1) [[TMP32]], align 1
+; OPT-NEXT: br label [[MEMMOVE_DONE]]
+; OPT: memmove_done:
+; OPT-NEXT: ret void
+;
+ call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) %dst, ptr addrspace(1) %src, i64 1039, i1 false)
+ ret void
+}
+
define void @test_umin(i64 %0, i64 %idxprom, ptr %x, ptr %y) {
; OPT-LABEL: @test_umin(
; OPT-NEXT: entry:
@@ -2393,51 +2239,25 @@ define amdgpu_kernel void @memmove_volatile(ptr addrspace(1) %dst, ptr addrspace
;
; ALL-LABEL: @memmove_volatile(
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr addrspace(1) [[SRC:%.*]], [[DST:%.*]]
-; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
-; ALL: memmove_copy_backwards:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_bwd_residual_loop:
-; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ 64, [[MEMMOVE_COPY_BACKWARDS]] ]
-; ALL-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i64 [[TMP1]], 1
-; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load volatile i8, ptr addrspace(1) [[TMP2]], align 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[BWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store volatile i8 [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_RESIDUAL_INDEX]], 64
-; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
-; ALL: memmove_bwd_middle:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
-; ALL: memmove_bwd_main_loop:
-; ALL-NEXT: [[TMP5:%.*]] = phi i64 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ 4, [[MEMMOVE_BWD_MIDDLE]] ]
-; ALL-NEXT: [[BWD_MAIN_INDEX]] = sub i64 [[TMP5]], 1
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_MAIN_INDEX]]
-; ALL-NEXT: store volatile <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
-; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[BWD_MAIN_INDEX]], 0
-; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
-; ALL: memmove_copy_forward:
-; ALL-NEXT: br i1 false, label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
-; ALL: memmove_fwd_main_loop:
-; ALL-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i64 [ [[TMP11:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; ALL-NEXT: [[TMP9:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: [[ELEMENT2:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP9]], align 1
-; ALL-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_MAIN_INDEX]]
-; ALL-NEXT: store volatile <4 x i32> [[ELEMENT2]], ptr addrspace(1) [[TMP10]], align 1
-; ALL-NEXT: [[TMP11]] = add i64 [[FWD_MAIN_INDEX]], 1
-; ALL-NEXT: [[TMP12:%.*]] = icmp eq i64 [[TMP11]], 4
-; ALL-NEXT: br i1 [[TMP12]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
-; ALL: memmove_fwd_middle:
-; ALL-NEXT: br i1 true, label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
-; ALL: memmove_fwd_residual_loop:
-; ALL-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i64 [ [[TMP15:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ 64, [[MEMMOVE_FWD_MIDDLE]] ]
-; ALL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[SRC]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: [[ELEMENT3:%.*]] = load volatile i8, ptr addrspace(1) [[TMP13]], align 1
-; ALL-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[DST]], i64 [[FWD_RESIDUAL_INDEX]]
-; ALL-NEXT: store volatile i8 [[ELEMENT3]], ptr addrspace(1) [[TMP14]], align 1
-; ALL-NEXT: [[TMP15]] = add i64 [[FWD_RESIDUAL_INDEX]], 1
-; ALL-NEXT: [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 64
-; ALL-NEXT: br i1 [[TMP16]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
+; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
+; ALL: memmove_bwd_loop:
+; ALL-NEXT: [[TMP1:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 4, [[TMP0:%.*]] ]
+; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP1]], 1
+; ALL-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP2]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
+; ALL-NEXT: store volatile <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
+; ALL-NEXT: br i1 [[TMP4]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
+; ALL: memmove_fwd_loop:
+; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP7:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
+; ALL-NEXT: [[TMP5:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load volatile <4 x i32>, ptr addrspace(1) [[TMP5]], align 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
+; ALL-NEXT: store volatile <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7]] = add i64 [[FWD_INDEX]], 1
+; ALL-NEXT: [[TMP8:%.*]] = icmp eq i64 [[TMP7]], 4
+; ALL-NEXT: br i1 [[TMP8]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
;
>From 47abbd709321a4c6ac0a764b103701a1b37ed446 Mon Sep 17 00:00:00 2001
From: Fabian Ritter <fabian.ritter at amd.com>
Date: Thu, 25 Jul 2024 05:26:23 -0400
Subject: [PATCH 6/6] fixup! fixup! fixup! fixup! fixup! [LowerMemIntrinsics]
Lower llvm.memmove to wide memory accesses
Use the original address spaces instead of those casted for the overlap check for accesses in the memmove lowering.
---
.../Transforms/Utils/LowerMemIntrinsics.cpp | 38 +++-
.../CodeGen/AMDGPU/lower-mem-intrinsics.ll | 164 +++++++++---------
2 files changed, 113 insertions(+), 89 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index c035fa7a2d4d6..0d0598235abdf 100644
--- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -431,8 +431,23 @@ static void createMemMoveLoopUnknownSize(Instruction *InsertBefore,
// SplitBlockAndInsertIfThenElse conveniently creates the basic if-then-else
// structure. Its block terminators (unconditional branches) are replaced by
// the appropriate conditional branches when the loop is built.
+ // If the pointers are in different address spaces, they need to be converted
+ // to a compatible one. Cases where memory ranges in the different address
+ // spaces cannot overlap are lowered as memcpy and not handled here.
+ Value *CmpSrcAddr = SrcAddr;
+ Value *CmpDstAddr = DstAddr;
+ if ((SrcAS != DstAS)) {
+ if (TTI.isValidAddrSpaceCast(DstAS, SrcAS)) {
+ CmpDstAddr = PLBuilder.CreateAddrSpaceCast(DstAddr, SrcAddr->getType());
+ } else if (TTI.isValidAddrSpaceCast(SrcAS, DstAS)) {
+ CmpSrcAddr = PLBuilder.CreateAddrSpaceCast(SrcAddr, DstAddr->getType());
+ } else {
+ llvm_unreachable("Can only lower memmove between address spaces if they "
+ "support addrspacecast");
+ }
+ }
Value *PtrCompare =
- PLBuilder.CreateICmpULT(SrcAddr, DstAddr, "compare_src_dst");
+ PLBuilder.CreateICmpULT(CmpSrcAddr, CmpDstAddr, "compare_src_dst");
Instruction *ThenTerm, *ElseTerm;
SplitBlockAndInsertIfThenElse(PtrCompare, InsertBefore->getIterator(),
&ThenTerm, &ElseTerm);
@@ -655,8 +670,20 @@ static void createMemMoveLoopKnownSize(Instruction *InsertBefore,
IRBuilder<> PLBuilder(InsertBefore);
+ Value *CmpSrcAddr = SrcAddr;
+ Value *CmpDstAddr = DstAddr;
+ if ((SrcAS != DstAS)) {
+ if (TTI.isValidAddrSpaceCast(DstAS, SrcAS)) {
+ CmpDstAddr = PLBuilder.CreateAddrSpaceCast(DstAddr, SrcAddr->getType());
+ } else if (TTI.isValidAddrSpaceCast(SrcAS, DstAS)) {
+ CmpSrcAddr = PLBuilder.CreateAddrSpaceCast(SrcAddr, DstAddr->getType());
+ } else {
+ llvm_unreachable("Can only lower memmove between address spaces if they "
+ "support addrspacecast");
+ }
+ }
Value *PtrCompare =
- PLBuilder.CreateICmpULT(SrcAddr, DstAddr, "compare_src_dst");
+ PLBuilder.CreateICmpULT(CmpSrcAddr, CmpDstAddr, "compare_src_dst");
Instruction *ThenTerm, *ElseTerm;
SplitBlockAndInsertIfThenElse(PtrCompare, InsertBefore->getIterator(),
&ThenTerm, &ElseTerm);
@@ -907,11 +934,8 @@ bool llvm::expandMemMoveAsLoop(MemMoveInst *Memmove,
return true;
}
- if (TTI.isValidAddrSpaceCast(DstAS, SrcAS))
- DstAddr = CastBuilder.CreateAddrSpaceCast(DstAddr, SrcAddr->getType());
- else if (TTI.isValidAddrSpaceCast(SrcAS, DstAS))
- SrcAddr = CastBuilder.CreateAddrSpaceCast(SrcAddr, DstAddr->getType());
- else {
+ if (!(TTI.isValidAddrSpaceCast(DstAS, SrcAS) ||
+ TTI.isValidAddrSpaceCast(SrcAS, DstAS))) {
// We don't know generically if it's legal to introduce an
// addrspacecast. We need to know either if it's legal to insert an
// addrspacecast, or if the address spaces cannot alias.
diff --git a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index d3bfa627fcb6e..9e2e37a886d1f 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -1342,16 +1342,16 @@ define amdgpu_kernel void @memmove_flat_align1_global_align1(ptr %dst, ptr addrs
; ALL: memmove_bwd_loop:
; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP3]], align 1
; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_INDEX]]
; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
; ALL: memmove_fwd_loop:
; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[SRC]], i64 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(1) [[TMP6]], align 1
; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_INDEX]]
; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
; ALL-NEXT: [[TMP8]] = add i64 [[FWD_INDEX]], 1
@@ -1378,16 +1378,16 @@ define amdgpu_kernel void @memmove_global_align1_flat_align1(ptr addrspace(1) %d
; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP2]], 1
; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr addrspace(1) [[TMP4]], align 1
; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
; ALL: memmove_fwd_loop:
; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_INDEX]]
; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(1) [[DST]], i64 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(1) [[TMP7]], align 1
; ALL-NEXT: [[TMP8]] = add i64 [[FWD_INDEX]], 1
; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 16
; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
@@ -1410,16 +1410,16 @@ define amdgpu_kernel void @memmove_flat_align1_private_align1(ptr %dst, ptr addr
; ALL: memmove_bwd_loop:
; ALL-NEXT: [[TMP2:%.*]] = phi i64 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i64 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP3]], align 1
; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[BWD_INDEX]]
; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
; ALL: memmove_fwd_loop:
; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[SRC]], i64 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr addrspace(5) [[TMP6]], align 1
; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i64 [[FWD_INDEX]]
; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
; ALL-NEXT: [[TMP8]] = add i64 [[FWD_INDEX]], 1
@@ -1446,16 +1446,16 @@ define amdgpu_kernel void @memmove_private_align1_flat_align1(ptr addrspace(5) %
; ALL-NEXT: [[BWD_INDEX]] = sub i64 [[TMP2]], 1
; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[BWD_INDEX]]
; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[BWD_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i64 [[BWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr addrspace(5) [[TMP4]], align 1
; ALL-NEXT: [[TMP5:%.*]] = icmp eq i64 [[BWD_INDEX]], 0
; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
; ALL: memmove_fwd_loop:
; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i64 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i64 [[FWD_INDEX]]
; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i64 [[FWD_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr addrspace(5) [[DST]], i64 [[FWD_INDEX]]
+; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr addrspace(5) [[TMP7]], align 1
; ALL-NEXT: [[TMP8]] = add i64 [[FWD_INDEX]], 1
; ALL-NEXT: [[TMP9:%.*]] = icmp eq i64 [[TMP8]], 16
; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
@@ -1730,22 +1730,22 @@ define amdgpu_kernel void @memmove_flat_align1_local_align1(ptr addrspace(0) %ds
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
; ALL: memmove_bwd_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 32, [[TMP0:%.*]] ]
; ALL-NEXT: [[BWD_INDEX]] = sub i32 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[BWD_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <2 x i32>, ptr [[DST]], i32 [[BWD_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
; ALL: memmove_fwd_loop:
; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i32 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[FWD_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, ptr [[DST]], i32 [[FWD_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
; ALL-NEXT: [[TMP8]] = add i32 [[FWD_INDEX]], 1
-; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 16
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 32
; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
@@ -1756,53 +1756,53 @@ define amdgpu_kernel void @memmove_flat_align1_local_align1(ptr addrspace(0) %ds
define amdgpu_kernel void @memmove_flat_align1_local_align1_unknown_size(ptr addrspace(0) %dst, ptr addrspace(3) %src, i32 %size) {
; OPT-LABEL: @memmove_flat_align1_local_align1_unknown_size(
-; OPT-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[SRC:%.*]] to ptr
-; OPT-NEXT: [[TMP2:%.*]] = lshr i32 [[SIZE:%.*]], 4
-; OPT-NEXT: [[TMP3:%.*]] = and i32 [[SIZE]], 15
-; OPT-NEXT: [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
-; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP3]], 0
-; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP2]], 0
-; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP1]], [[DST:%.*]]
+; OPT-NEXT: [[TMP1:%.*]] = lshr i32 [[SIZE:%.*]], 3
+; OPT-NEXT: [[TMP2:%.*]] = and i32 [[SIZE]], 7
+; OPT-NEXT: [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP1]], 0
+; OPT-NEXT: [[TMP4:%.*]] = addrspacecast ptr addrspace(3) [[SRC:%.*]] to ptr
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[TMP4]], [[DST:%.*]]
; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
; OPT: memmove_copy_backwards:
; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
; OPT: memmove_bwd_residual_loop:
; OPT-NEXT: [[TMP5:%.*]] = phi i32 [ [[BWD_RESIDUAL_INDEX:%.*]], [[MEMMOVE_BWD_RESIDUAL_LOOP]] ], [ [[SIZE]], [[MEMMOVE_COPY_BACKWARDS]] ]
; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP5]], 1
-; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
-; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP6]], align 1
+; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr addrspace(3) [[TMP6]], align 1
; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
; OPT-NEXT: store i8 [[ELEMENT]], ptr [[TMP7]], align 1
-; OPT-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP4]]
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP3]]
; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
; OPT: memmove_bwd_middle:
; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
; OPT: memmove_bwd_main_loop:
-; OPT-NEXT: [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP2]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP1]], [[MEMMOVE_BWD_MIDDLE]] ]
; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 1
-; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
-; OPT-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = getelementptr inbounds <2 x i32>, ptr [[DST]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <2 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
; OPT-NEXT: [[TMP12:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
; OPT: memmove_copy_forward:
; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
; OPT: memmove_fwd_main_loop:
; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 1
-; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <4 x i32>, ptr [[DST]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TMP13]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <2 x i32>, ptr [[DST]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <2 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
; OPT-NEXT: [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 1
-; OPT-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP2]]
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP1]]
; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
; OPT: memmove_fwd_middle:
; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
; OPT: memmove_fwd_residual_loop:
-; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP19:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP4]], [[MEMMOVE_FWD_MIDDLE]] ]
-; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
-; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP17]], align 1
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP19:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP3]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr addrspace(3) [[TMP17]], align 1
; OPT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
; OPT-NEXT: store i8 [[ELEMENT3]], ptr [[TMP18]], align 1
; OPT-NEXT: [[TMP19]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
@@ -1825,22 +1825,22 @@ define amdgpu_kernel void @memmove_local_align1_flat_align1(ptr addrspace(3) %ds
; ALL-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
; ALL-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_BWD_LOOP:%.*]], label [[MEMMOVE_FWD_LOOP:%.*]]
; ALL: memmove_bwd_loop:
-; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 16, [[TMP0:%.*]] ]
+; ALL-NEXT: [[TMP2:%.*]] = phi i32 [ [[BWD_INDEX:%.*]], [[MEMMOVE_BWD_LOOP]] ], [ 32, [[TMP0:%.*]] ]
; ALL-NEXT: [[BWD_INDEX]] = sub i32 [[TMP2]], 1
-; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[BWD_INDEX]]
-; ALL-NEXT: [[ELEMENT:%.*]] = load <4 x i32>, ptr [[TMP3]], align 1
-; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT]], ptr [[TMP4]], align 1
+; ALL-NEXT: [[TMP3:%.*]] = getelementptr inbounds <2 x i32>, ptr [[SRC]], i32 [[BWD_INDEX]]
+; ALL-NEXT: [[ELEMENT:%.*]] = load <2 x i32>, ptr [[TMP3]], align 1
+; ALL-NEXT: [[TMP4:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[BWD_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT]], ptr addrspace(3) [[TMP4]], align 1
; ALL-NEXT: [[TMP5:%.*]] = icmp eq i32 [[BWD_INDEX]], 0
; ALL-NEXT: br i1 [[TMP5]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_LOOP]]
; ALL: memmove_fwd_loop:
; ALL-NEXT: [[FWD_INDEX:%.*]] = phi i32 [ [[TMP8:%.*]], [[MEMMOVE_FWD_LOOP]] ], [ 0, [[TMP0]] ]
-; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[FWD_INDEX]]
-; ALL-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP6]], align 1
-; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_INDEX]]
-; ALL-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP7]], align 1
+; ALL-NEXT: [[TMP6:%.*]] = getelementptr inbounds <2 x i32>, ptr [[SRC]], i32 [[FWD_INDEX]]
+; ALL-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr [[TMP6]], align 1
+; ALL-NEXT: [[TMP7:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[FWD_INDEX]]
+; ALL-NEXT: store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP7]], align 1
; ALL-NEXT: [[TMP8]] = add i32 [[FWD_INDEX]], 1
-; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 16
+; ALL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 32
; ALL-NEXT: br i1 [[TMP9]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_LOOP]]
; ALL: memmove_done:
; ALL-NEXT: ret void
@@ -1851,13 +1851,13 @@ define amdgpu_kernel void @memmove_local_align1_flat_align1(ptr addrspace(3) %ds
define amdgpu_kernel void @memmove_local_align1_flat_align1_unknown_size(ptr addrspace(3) %dst, ptr addrspace(0) %src, i32 %size) {
; OPT-LABEL: @memmove_local_align1_flat_align1_unknown_size(
-; OPT-NEXT: [[TMP1:%.*]] = addrspacecast ptr addrspace(3) [[DST:%.*]] to ptr
-; OPT-NEXT: [[TMP2:%.*]] = lshr i32 [[SIZE:%.*]], 4
-; OPT-NEXT: [[TMP3:%.*]] = and i32 [[SIZE]], 15
-; OPT-NEXT: [[TMP4:%.*]] = sub i32 [[SIZE]], [[TMP3]]
-; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP3]], 0
-; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP2]], 0
-; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP1]]
+; OPT-NEXT: [[TMP1:%.*]] = lshr i32 [[SIZE:%.*]], 3
+; OPT-NEXT: [[TMP2:%.*]] = and i32 [[SIZE]], 7
+; OPT-NEXT: [[TMP3:%.*]] = sub i32 [[SIZE]], [[TMP2]]
+; OPT-NEXT: [[SKIP_RESIDUAL:%.*]] = icmp eq i32 [[TMP2]], 0
+; OPT-NEXT: [[SKIP_MAIN:%.*]] = icmp eq i32 [[TMP1]], 0
+; OPT-NEXT: [[TMP4:%.*]] = addrspacecast ptr addrspace(3) [[DST:%.*]] to ptr
+; OPT-NEXT: [[COMPARE_SRC_DST:%.*]] = icmp ult ptr [[SRC:%.*]], [[TMP4]]
; OPT-NEXT: br i1 [[COMPARE_SRC_DST]], label [[MEMMOVE_COPY_BACKWARDS:%.*]], label [[MEMMOVE_COPY_FORWARD:%.*]]
; OPT: memmove_copy_backwards:
; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_BWD_MIDDLE:%.*]], label [[MEMMOVE_BWD_RESIDUAL_LOOP:%.*]]
@@ -1866,40 +1866,40 @@ define amdgpu_kernel void @memmove_local_align1_flat_align1_unknown_size(ptr add
; OPT-NEXT: [[BWD_RESIDUAL_INDEX]] = sub i32 [[TMP5]], 1
; OPT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[BWD_RESIDUAL_INDEX]]
; OPT-NEXT: [[ELEMENT:%.*]] = load i8, ptr [[TMP6]], align 1
-; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[BWD_RESIDUAL_INDEX]]
-; OPT-NEXT: store i8 [[ELEMENT]], ptr [[TMP7]], align 1
-; OPT-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP4]]
+; OPT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[BWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT]], ptr addrspace(3) [[TMP7]], align 1
+; OPT-NEXT: [[TMP8:%.*]] = icmp eq i32 [[BWD_RESIDUAL_INDEX]], [[TMP3]]
; OPT-NEXT: br i1 [[TMP8]], label [[MEMMOVE_BWD_MIDDLE]], label [[MEMMOVE_BWD_RESIDUAL_LOOP]]
; OPT: memmove_bwd_middle:
; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_DONE:%.*]], label [[MEMMOVE_BWD_MAIN_LOOP:%.*]]
; OPT: memmove_bwd_main_loop:
-; OPT-NEXT: [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP2]], [[MEMMOVE_BWD_MIDDLE]] ]
+; OPT-NEXT: [[TMP9:%.*]] = phi i32 [ [[BWD_MAIN_INDEX:%.*]], [[MEMMOVE_BWD_MAIN_LOOP]] ], [ [[TMP1]], [[MEMMOVE_BWD_MIDDLE]] ]
; OPT-NEXT: [[BWD_MAIN_INDEX]] = sub i32 [[TMP9]], 1
-; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT: [[ELEMENT1:%.*]] = load <4 x i32>, ptr [[TMP10]], align 1
-; OPT-NEXT: [[TMP11:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[BWD_MAIN_INDEX]]
-; OPT-NEXT: store <4 x i32> [[ELEMENT1]], ptr [[TMP11]], align 1
+; OPT-NEXT: [[TMP10:%.*]] = getelementptr inbounds <2 x i32>, ptr [[SRC]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT1:%.*]] = load <2 x i32>, ptr [[TMP10]], align 1
+; OPT-NEXT: [[TMP11:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[BWD_MAIN_INDEX]]
+; OPT-NEXT: store <2 x i32> [[ELEMENT1]], ptr addrspace(3) [[TMP11]], align 1
; OPT-NEXT: [[TMP12:%.*]] = icmp eq i32 [[BWD_MAIN_INDEX]], 0
; OPT-NEXT: br i1 [[TMP12]], label [[MEMMOVE_DONE]], label [[MEMMOVE_BWD_MAIN_LOOP]]
; OPT: memmove_copy_forward:
; OPT-NEXT: br i1 [[SKIP_MAIN]], label [[MEMMOVE_FWD_MIDDLE:%.*]], label [[MEMMOVE_FWD_MAIN_LOOP:%.*]]
; OPT: memmove_fwd_main_loop:
; OPT-NEXT: [[FWD_MAIN_INDEX:%.*]] = phi i32 [ [[TMP15:%.*]], [[MEMMOVE_FWD_MAIN_LOOP]] ], [ 0, [[MEMMOVE_COPY_FORWARD]] ]
-; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <4 x i32>, ptr [[SRC]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT: [[ELEMENT2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 1
-; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 [[FWD_MAIN_INDEX]]
-; OPT-NEXT: store <4 x i32> [[ELEMENT2]], ptr [[TMP14]], align 1
+; OPT-NEXT: [[TMP13:%.*]] = getelementptr inbounds <2 x i32>, ptr [[SRC]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: [[ELEMENT2:%.*]] = load <2 x i32>, ptr [[TMP13]], align 1
+; OPT-NEXT: [[TMP14:%.*]] = getelementptr inbounds <2 x i32>, ptr addrspace(3) [[DST]], i32 [[FWD_MAIN_INDEX]]
+; OPT-NEXT: store <2 x i32> [[ELEMENT2]], ptr addrspace(3) [[TMP14]], align 1
; OPT-NEXT: [[TMP15]] = add i32 [[FWD_MAIN_INDEX]], 1
-; OPT-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP2]]
+; OPT-NEXT: [[TMP16:%.*]] = icmp eq i32 [[TMP15]], [[TMP1]]
; OPT-NEXT: br i1 [[TMP16]], label [[MEMMOVE_FWD_MIDDLE]], label [[MEMMOVE_FWD_MAIN_LOOP]]
; OPT: memmove_fwd_middle:
; OPT-NEXT: br i1 [[SKIP_RESIDUAL]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP:%.*]]
; OPT: memmove_fwd_residual_loop:
-; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP19:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP4]], [[MEMMOVE_FWD_MIDDLE]] ]
+; OPT-NEXT: [[FWD_RESIDUAL_INDEX:%.*]] = phi i32 [ [[TMP19:%.*]], [[MEMMOVE_FWD_RESIDUAL_LOOP]] ], [ [[TMP3]], [[MEMMOVE_FWD_MIDDLE]] ]
; OPT-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i32 [[FWD_RESIDUAL_INDEX]]
; OPT-NEXT: [[ELEMENT3:%.*]] = load i8, ptr [[TMP17]], align 1
-; OPT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 [[FWD_RESIDUAL_INDEX]]
-; OPT-NEXT: store i8 [[ELEMENT3]], ptr [[TMP18]], align 1
+; OPT-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[DST]], i32 [[FWD_RESIDUAL_INDEX]]
+; OPT-NEXT: store i8 [[ELEMENT3]], ptr addrspace(3) [[TMP18]], align 1
; OPT-NEXT: [[TMP19]] = add i32 [[FWD_RESIDUAL_INDEX]], 1
; OPT-NEXT: [[TMP20:%.*]] = icmp eq i32 [[TMP19]], [[SIZE]]
; OPT-NEXT: br i1 [[TMP20]], label [[MEMMOVE_DONE]], label [[MEMMOVE_FWD_RESIDUAL_LOOP]]
More information about the llvm-commits
mailing list