[llvm] [SystemZ] Improve handling of memmoves. (PR #196285)
Jonas Paulsson via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 07:07:28 PDT 2026
https://github.com/JonPsson1 updated https://github.com/llvm/llvm-project/pull/196285
>From 864d362b8e2f610bcc033cb994153da4f0e16711 Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1 at linux.ibm.com>
Date: Wed, 6 May 2026 10:14:12 +0200
Subject: [PATCH 1/4] [SystemZ] Improve handling of memmove.
Close to GCC behaviour (max 1 load/store, VLL/VSTL, MVC/MVCRL).
---
llvm/include/llvm/CodeGen/TargetLowering.h | 12 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 3 +-
.../Target/SystemZ/SystemZISelLowering.cpp | 100 +-
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 2 +
llvm/lib/Target/SystemZ/SystemZInstrInfo.td | 7 +
llvm/lib/Target/SystemZ/SystemZOperators.td | 3 +
.../SystemZ/SystemZSelectionDAGInfo.cpp | 17 +
.../Target/SystemZ/SystemZSelectionDAGInfo.h | 7 +
llvm/test/CodeGen/SystemZ/memmove-01.ll | 945 ++++++++----------
9 files changed, 579 insertions(+), 517 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 82c47cce0f522..dc7cde81d804d 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -131,12 +131,14 @@ struct MemOp {
// memcpy only
bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
// constant so it does not need to be loaded.
- Align SrcAlign; // Inferred alignment of the source or default value if the
- // memory operation does not need to load the value.
+ bool SrcDstMayOverlap; // True if the source and destination memory regions
+ // may overlap (memmove).
+ Align SrcAlign; // Inferred alignment of the source or default value if the
+ // memory operation does not need to load the value.
public:
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
- Align SrcAlign, bool IsVolatile,
- bool MemcpyStrSrc = false) {
+ Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc = false,
+ bool SrcDstMayOverlap = false) {
MemOp Op;
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
@@ -144,6 +146,7 @@ struct MemOp {
Op.AllowOverlap = !IsVolatile;
Op.IsMemset = false;
Op.ZeroMemset = false;
+ Op.SrcDstMayOverlap = SrcDstMayOverlap;
Op.MemcpyStrSrc = MemcpyStrSrc;
Op.SrcAlign = SrcAlign;
return Op;
@@ -171,6 +174,7 @@ struct MemOp {
bool allowOverlap() const { return AllowOverlap; }
bool isMemset() const { return IsMemset; }
bool isMemcpy() const { return !IsMemset; }
+ bool isMemmove() const { return isMemcpy() && SrcDstMayOverlap; }
bool isMemcpyWithFixedDstAlign() const {
return isMemcpy() && !DstAlignCanChange;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f91af9befb9ce..3f441e46726dd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -9557,7 +9557,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
if (!TLI.findOptimalMemOpLowering(
C, MemOps, Limit,
- MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign, isVol),
+ MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign, isVol,
+ /*MemcpyStrSrc*/ false, /*SrcDstMayOverlap*/ true),
DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
MF.getFunction().getAttributes(), nullptr))
return SDValue();
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 8723075d131b0..7b076b27c7960 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -43,6 +43,10 @@ static cl::opt<bool> EnableIntArgExtCheck(
cl::desc("Verify that narrow int args are properly extended per the "
"SystemZ ABI."));
+// EXPERIMENTAL
+static cl::opt<unsigned> MEMMOVESTORES("memmove-stores", cl::init(1));
+static cl::opt<bool> MEMMOVEVLL("memmove-vll", cl::init(true));
+
namespace {
// Represents information about a comparison.
struct Comparison {
@@ -815,7 +819,7 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
MaxStoresPerMemcpyOptSize = 0;
// Same with memmove.
- MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0;
+ MaxStoresPerMemmove = Subtarget.hasVector() ? MEMMOVESTORES : 0;
MaxStoresPerMemmoveOptSize = 0;
// The main memset sequence is a byte store followed by an MVC.
@@ -1466,6 +1470,14 @@ bool SystemZTargetLowering::findOptimalMemOpLowering(
assert(Limit != ~0U &&
"Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases.");
+ if (Op.isMemmove()) {
+ if (Op.size() >= 16 &&
+ (!Op.isAligned(Align(8)) || (Op.size() >= 25 && Op.size() <= 31)))
+ return false;
+ return TargetLowering::findOptimalMemOpLowering(
+ Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
+ }
+
if (Op.isZeroMemset())
return false; // Memset zero: Use XC.
@@ -10868,6 +10880,90 @@ SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
return MBB;
}
+MachineBasicBlock *SystemZTargetLowering::emitMemmoveImm(
+ MachineInstr &MI, MachineBasicBlock *MBB) const {
+ MachineFunction &MF = *MBB->getParent();
+ const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
+ MachineRegisterInfo &MRI = MF.getRegInfo();
+
+ DebugLoc DL = MI.getDebugLoc();
+ MachineOperand DstBase = earlyUseOperand(MI.getOperand(0));
+ uint64_t DstDisp = MI.getOperand(1).getImm();
+ MachineOperand SrcBase = earlyUseOperand(MI.getOperand(2));
+ uint64_t SrcDisp = MI.getOperand(3).getImm();
+ uint64_t Len = MI.getOperand(4).getImm();
+ assert(Len >= 1 && Len <=256 && "Memmove of of unsupported constant length.");
+ assert(isUInt<12>(DstDisp) && isUInt<12>(SrcDisp) &&
+ "Unexpected large displacement.");
+
+ // Fold any displacement (or frame index reference) into a new register.
+ auto foldAddressIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
+ if (Disp || Base.isFI()) {
+ Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
+ unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp);
+ BuildMI(*MBB, MI, DL, TII->get(Opcode), Reg)
+ .add(Base).addImm(Disp).addReg(0);
+ Base = MachineOperand::CreateReg(Reg, false);
+ Disp = 0;
+ }
+ };
+
+ if (Len <= 15 && MEMMOVEVLL) {
+ Register HighByteReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
+ BuildMI(*MBB, MI, DL, TII->get(SystemZ::LHI), HighByteReg)
+ .addImm(Len - 1);
+
+ Register VecReg = MRI.createVirtualRegister(&SystemZ::VR128BitRegClass);
+ BuildMI(*MBB, MI, DL, TII->get(SystemZ::VLL), VecReg)
+ .addReg(HighByteReg)
+ .add(SrcBase).addImm(SrcDisp);
+
+ BuildMI(*MBB, MI, DL, TII->get(SystemZ::VSTL))
+ .addReg(VecReg)
+ .addReg(HighByteReg)
+ .add(DstBase).addImm(DstDisp);
+
+ MI.eraseFromParent();
+ return MBB;
+ }
+
+ // Use MVC or MVCRL after comparing the addresses.
+ MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
+ MachineBasicBlock *MvcMBB = SystemZ::emitBlockAfter(MBB);
+ MachineBasicBlock *MvcrlMBB = SystemZ::emitBlockAfter(MvcMBB);
+ MBB->addSuccessor(MvcMBB);
+ MBB->addSuccessor(MvcrlMBB);
+ MvcMBB->addSuccessor(DoneMBB);
+ MvcrlMBB->addSuccessor(DoneMBB);
+
+ // Fold any displacements in order to do the compare.
+ foldAddressIfNeeded(SrcBase, SrcDisp);
+ foldAddressIfNeeded(DstBase, DstDisp);
+
+ BuildMI(MBB, DL, TII->get(SystemZ::CLGR))
+ .add(SrcBase)
+ .add(DstBase);
+ BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+ .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_LT)
+ .addMBB(MvcrlMBB);
+
+ BuildMI(MvcMBB, DL, TII->get(SystemZ::MVC))
+ .add(DstBase).addImm(DstDisp).addImm(Len)
+ .add(SrcBase).addImm(SrcDisp)
+ .setMemRefs(MI.memoperands());
+ BuildMI(MvcMBB, DL, TII->get(SystemZ::J)).addMBB(DoneMBB);
+
+ BuildMI(MvcrlMBB, DL, TII->get(SystemZ::LHI), SystemZ::R0L)
+ .addImm(Len - 1);
+ BuildMI(MvcrlMBB, DL, TII->get(SystemZ::MVCRL))
+ .add(DstBase).addImm(DstDisp)
+ .add(SrcBase).addImm(SrcDisp)
+ .setMemRefs(MI.memoperands());
+
+ MI.eraseFromParent();
+ return DoneMBB;
+}
+
// Decompose string pseudo-instruction MI into a loop that continually performs
// Opcode until CC != 3.
MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
@@ -11239,6 +11335,8 @@ MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
case SystemZ::MemsetRegImm:
case SystemZ::MemsetRegReg:
return emitMemMemWrapper(MI, MBB, SystemZ::MVC, true/*IsMemset*/);
+ case SystemZ::MemmoveImm:
+ return emitMemmoveImm(MI, MBB);
case SystemZ::CLSTLoop:
return emitStringWrapper(MI, MBB, SystemZ::CLST);
case SystemZ::MVSTLoop:
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 7facd3a27d97c..394530bbee9d8 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -464,6 +464,8 @@ class SystemZTargetLowering : public TargetLowering {
MachineBasicBlock *emitMemMemWrapper(MachineInstr &MI, MachineBasicBlock *BB,
unsigned Opcode,
bool IsMemset = false) const;
+ MachineBasicBlock *emitMemmoveImm(MachineInstr &MI,
+ MachineBasicBlock *BB) const;
MachineBasicBlock *emitStringWrapper(MachineInstr &MI, MachineBasicBlock *BB,
unsigned Opcode) const;
MachineBasicBlock *emitTransactionBegin(MachineInstr &MI,
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
index 269fa6ffda0b9..23eb0d51734f1 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -591,6 +591,13 @@ let Predicates = [FeatureMiscellaneousExtensions3],
mayLoad = 1, mayStore = 1, Uses = [R0L] in
def MVCRL : SideEffectBinarySSE<"mvcrl", 0xE50A>;
+let usesCustomInserter = 1, hasNoSchedulingInfo = 1, mayLoad = 1, mayStore = 1 in
+ def MemmoveImm : Pseudo<(outs),
+ (ins bdaddr12only:$dest, bdaddr12only:$src,
+ imm64:$length),
+ [(z_memmove bdaddr12only:$dest, bdaddr12only:$src,
+ imm64:$length)]>;
+
// String moves.
let mayLoad = 1, mayStore = 1, Defs = [CC] in
defm MVST : StringRRE<"mvst", 0xB255, z_stpcpy>;
diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td
index b69f84d78aaf1..fc1400653db71 100644
--- a/llvm/lib/Target/SystemZ/SystemZOperators.td
+++ b/llvm/lib/Target/SystemZ/SystemZOperators.td
@@ -679,6 +679,9 @@ def z_atomic_cmp_swap_128 : SDNode<"SystemZISD::ATOMIC_CMP_SWAP_128",
def z_mvc : SDNode<"SystemZISD::MVC", SDT_ZMemMemLength,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
+def z_memmove : SDNode<"SystemZISD::MEMMOVE", SDT_ZMemMemLength,
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
+
// Similar to MVC, but for logic operations (AND, OR, XOR).
def z_nc : SDNode<"SystemZISD::NC", SDT_ZMemMemLength,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index eec37a9df386f..2c804416e1cea 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -87,6 +87,23 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
return emitMemMemReg(DAG, DL, SystemZISD::MVC, Chain, Dst, Src, Size);
}
+SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemmove(
+ SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
+ SDValue Size, Align Alignment, bool IsVolatile,
+ MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
+ if (IsVolatile)
+ return SDValue();
+
+ // XXX VLL FeatureVector
+ // XXX MVCRL FeatureMiscellaneousExtensions3
+ if (auto *CSize = dyn_cast<ConstantSDNode>(Size))
+ if (CSize->getZExtValue() <= 256)
+ return DAG.getNode(SystemZISD::MEMMOVE, DL, MVT::Other,
+ { Chain, Dst, Src, Size });
+
+ return SDValue();
+}
+
// Handle a memset of 1, 2, 4 or 8 bytes with the operands given by
// Chain, Dst, ByteVal and Size. These cases are expected to use
// MVI, MVHHI, MVHI and MVGHI respectively.
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
index 8e6da4fe8b0ae..96286d0c192f5 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
@@ -51,6 +51,13 @@ class SystemZSelectionDAGInfo : public SelectionDAGGenTargetInfo {
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo) const override;
+ SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &DL,
+ SDValue Chain, SDValue Dst, SDValue Src,
+ SDValue Size, Align Alignment,
+ bool IsVolatile,
+ MachinePointerInfo DstPtrInfo,
+ MachinePointerInfo SrcPtrInfo) const override;
+
SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &DL,
SDValue Chain, SDValue Dst, SDValue Byte,
SDValue Size, Align Alignment,
diff --git a/llvm/test/CodeGen/SystemZ/memmove-01.ll b/llvm/test/CodeGen/SystemZ/memmove-01.ll
index 62459ccf487c8..653033c31a6bf 100644
--- a/llvm/test/CodeGen/SystemZ/memmove-01.ll
+++ b/llvm/test/CodeGen/SystemZ/memmove-01.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mcpu=z17 < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc -mcpu=z17 < %s -mtriple=s390x-linux-gnu -verify-machineinstrs \
+; RUN: | FileCheck %s
;
; Test non-volatile memmoves of small constant lengths in both aligned and
; unaligned cases.
@@ -9,14 +10,8 @@ declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) noun
define void @fun1(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun1:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 1
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lb %r0, 0(%r3)
+; CHECK-NEXT: stc %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 1, i1 false)
ret void
@@ -25,14 +20,8 @@ define void @fun1(ptr %Dst, ptr %Src) {
define void @fun1_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun1_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 1
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lb %r0, 0(%r3)
+; CHECK-NEXT: stc %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 1, i1 false)
ret void
@@ -41,14 +30,8 @@ define void @fun1_unaligned(ptr %Dst, ptr %Src) {
define void @fun2(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun2:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 2
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lh %r0, 0(%r3)
+; CHECK-NEXT: sth %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 2, i1 false)
ret void
@@ -57,14 +40,8 @@ define void @fun2(ptr %Dst, ptr %Src) {
define void @fun2_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun2_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 2
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lh %r0, 0(%r3)
+; CHECK-NEXT: sth %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 2, i1 false)
ret void
@@ -73,14 +50,9 @@ define void @fun2_unaligned(ptr %Dst, ptr %Src) {
define void @fun3(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun3:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 3
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 2
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 3, i1 false)
ret void
@@ -89,14 +61,9 @@ define void @fun3(ptr %Dst, ptr %Src) {
define void @fun3_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun3_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 3
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 2
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 3, i1 false)
ret void
@@ -105,14 +72,8 @@ define void @fun3_unaligned(ptr %Dst, ptr %Src) {
define void @fun4(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun4:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 4
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: l %r0, 0(%r3)
+; CHECK-NEXT: st %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 4, i1 false)
ret void
@@ -121,14 +82,8 @@ define void @fun4(ptr %Dst, ptr %Src) {
define void @fun4_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun4_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 4
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: l %r0, 0(%r3)
+; CHECK-NEXT: st %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 4, i1 false)
ret void
@@ -137,14 +92,9 @@ define void @fun4_unaligned(ptr %Dst, ptr %Src) {
define void @fun5(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun5:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 5
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 4
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 5, i1 false)
ret void
@@ -153,14 +103,9 @@ define void @fun5(ptr %Dst, ptr %Src) {
define void @fun5_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun5_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 5
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 4
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 5, i1 false)
ret void
@@ -169,14 +114,9 @@ define void @fun5_unaligned(ptr %Dst, ptr %Src) {
define void @fun6(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun6:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 6
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 5
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 6, i1 false)
ret void
@@ -185,14 +125,9 @@ define void @fun6(ptr %Dst, ptr %Src) {
define void @fun6_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun6_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 6
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 5
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 6, i1 false)
ret void
@@ -201,14 +136,9 @@ define void @fun6_unaligned(ptr %Dst, ptr %Src) {
define void @fun7(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun7:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 7
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 6
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 7, i1 false)
ret void
@@ -217,14 +147,9 @@ define void @fun7(ptr %Dst, ptr %Src) {
define void @fun7_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun7_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 7
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 6
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 7, i1 false)
ret void
@@ -233,14 +158,8 @@ define void @fun7_unaligned(ptr %Dst, ptr %Src) {
define void @fun8(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun8:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 8
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 8, i1 false)
ret void
@@ -249,14 +168,8 @@ define void @fun8(ptr %Dst, ptr %Src) {
define void @fun8_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun8_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 8
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 8, i1 false)
ret void
@@ -265,14 +178,9 @@ define void @fun8_unaligned(ptr %Dst, ptr %Src) {
define void @fun9(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun9:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 9
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 8
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 9, i1 false)
ret void
@@ -281,14 +189,9 @@ define void @fun9(ptr %Dst, ptr %Src) {
define void @fun9_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun9_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 9
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 8
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 9, i1 false)
ret void
@@ -297,14 +200,9 @@ define void @fun9_unaligned(ptr %Dst, ptr %Src) {
define void @fun10(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun10:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 10
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 9
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 10, i1 false)
ret void
@@ -313,14 +211,9 @@ define void @fun10(ptr %Dst, ptr %Src) {
define void @fun10_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun10_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 10
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 9
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 10, i1 false)
ret void
@@ -329,14 +222,9 @@ define void @fun10_unaligned(ptr %Dst, ptr %Src) {
define void @fun11(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun11:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 11
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 10
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 11, i1 false)
ret void
@@ -345,14 +233,9 @@ define void @fun11(ptr %Dst, ptr %Src) {
define void @fun11_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun11_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 11
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 10
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 11, i1 false)
ret void
@@ -361,14 +244,9 @@ define void @fun11_unaligned(ptr %Dst, ptr %Src) {
define void @fun12(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun12:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 12
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 11
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 12, i1 false)
ret void
@@ -377,14 +255,9 @@ define void @fun12(ptr %Dst, ptr %Src) {
define void @fun12_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun12_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 12
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 11
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 12, i1 false)
ret void
@@ -393,14 +266,9 @@ define void @fun12_unaligned(ptr %Dst, ptr %Src) {
define void @fun13(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun13:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 13
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 12
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 13, i1 false)
ret void
@@ -409,14 +277,9 @@ define void @fun13(ptr %Dst, ptr %Src) {
define void @fun13_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun13_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 13
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 12
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 13, i1 false)
ret void
@@ -425,14 +288,9 @@ define void @fun13_unaligned(ptr %Dst, ptr %Src) {
define void @fun14(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun14:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 14
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 13
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 14, i1 false)
ret void
@@ -441,14 +299,9 @@ define void @fun14(ptr %Dst, ptr %Src) {
define void @fun14_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun14_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 14
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 13
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 14, i1 false)
ret void
@@ -457,14 +310,9 @@ define void @fun14_unaligned(ptr %Dst, ptr %Src) {
define void @fun15(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun15:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 15
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 14
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 15, i1 false)
ret void
@@ -473,14 +321,9 @@ define void @fun15(ptr %Dst, ptr %Src) {
define void @fun15_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun15_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 15
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: lhi %r0, 14
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 15, i1 false)
ret void
@@ -489,14 +332,8 @@ define void @fun15_unaligned(ptr %Dst, ptr %Src) {
define void @fun16(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun16:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 16
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 16, i1 false)
ret void
@@ -505,14 +342,13 @@ define void @fun16(ptr %Dst, ptr %Src) {
define void @fun16_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun16_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 16
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB31_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(16,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB31_2:
+; CHECK-NEXT: lhi %r0, 15
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 16, i1 false)
ret void
@@ -521,10 +357,13 @@ define void @fun16_unaligned(ptr %Dst, ptr %Src) {
define void @fun17(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun17:
; CHECK: # %bb.0:
-; CHECK-NEXT: lb %r0, 16(%r3)
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: stc %r0, 16(%r2)
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB32_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(17,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB32_2:
+; CHECK-NEXT: lhi %r0, 16
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 17, i1 false)
ret void
@@ -533,14 +372,13 @@ define void @fun17(ptr %Dst, ptr %Src) {
define void @fun17_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun17_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 17
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB33_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(17,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB33_2:
+; CHECK-NEXT: lhi %r0, 16
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 17, i1 false)
ret void
@@ -549,14 +387,13 @@ define void @fun17_unaligned(ptr %Dst, ptr %Src) {
define void @fun17_unalignedSrc(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun17_unalignedSrc:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 17
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB34_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(17,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB34_2:
+; CHECK-NEXT: lhi %r0, 16
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 16 %Dst, ptr align 4 %Src, i64 17, i1 false)
ret void
@@ -565,10 +402,13 @@ define void @fun17_unalignedSrc(ptr %Dst, ptr %Src) {
define void @fun18(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun18:
; CHECK: # %bb.0:
-; CHECK-NEXT: lh %r0, 16(%r3)
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: sth %r0, 16(%r2)
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB35_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(18,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB35_2:
+; CHECK-NEXT: lhi %r0, 17
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 18, i1 false)
ret void
@@ -577,14 +417,13 @@ define void @fun18(ptr %Dst, ptr %Src) {
define void @fun18_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun18_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 18
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB36_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(18,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB36_2:
+; CHECK-NEXT: lhi %r0, 17
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 18, i1 false)
ret void
@@ -593,14 +432,13 @@ define void @fun18_unaligned(ptr %Dst, ptr %Src) {
define void @fun18_unalignedDst(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun18_unalignedDst:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 18
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB37_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(18,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB37_2:
+; CHECK-NEXT: lhi %r0, 17
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 2 %Dst, ptr align 16 %Src, i64 18, i1 false)
ret void
@@ -609,10 +447,13 @@ define void @fun18_unalignedDst(ptr %Dst, ptr %Src) {
define void @fun19(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun19:
; CHECK: # %bb.0:
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: l %r0, 15(%r3)
-; CHECK-NEXT: st %r0, 15(%r2)
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB38_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(19,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB38_2:
+; CHECK-NEXT: lhi %r0, 18
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 19, i1 false)
ret void
@@ -621,14 +462,13 @@ define void @fun19(ptr %Dst, ptr %Src) {
define void @fun19_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun19_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 19
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB39_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(19,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB39_2:
+; CHECK-NEXT: lhi %r0, 18
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 19, i1 false)
ret void
@@ -637,10 +477,13 @@ define void @fun19_unaligned(ptr %Dst, ptr %Src) {
define void @fun20(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun20:
; CHECK: # %bb.0:
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: l %r0, 16(%r3)
-; CHECK-NEXT: st %r0, 16(%r2)
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB40_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(20,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB40_2:
+; CHECK-NEXT: lhi %r0, 19
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 20, i1 false)
ret void
@@ -649,30 +492,37 @@ define void @fun20(ptr %Dst, ptr %Src) {
define void @fun20_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun20_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 20
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB41_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(20,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB41_2:
+; CHECK-NEXT: lhi %r0, 19
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 20, i1 false)
ret void
}
-define void @fun20_localDst(ptr %Src) {
-; CHECK-LABEL: fun20_localDst:
+define void @fun20_local() {
+; CHECK-LABEL: fun20_local:
; CHECK: # %bb.0:
-; CHECK-NEXT: aghi %r15, -184
-; CHECK-NEXT: .cfi_def_cfa_offset 344
-; CHECK-NEXT: vl %v0, 0(%r2), 4
-; CHECK-NEXT: vst %v0, 164(%r15), 4
-; CHECK-NEXT: mvc 180(4,%r15), 16(%r2)
-; CHECK-NEXT: aghi %r15, 184
+; CHECK-NEXT: aghi %r15, -200
+; CHECK-NEXT: .cfi_def_cfa_offset 360
+; CHECK-NEXT: la %r1, 160(%r15)
+; CHECK-NEXT: la %r2, 180(%r15)
+; CHECK-NEXT: clgrjl %r1, %r2, .LBB42_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(20,%r2), 0(%r1)
+; CHECK-NEXT: aghi %r15, 200
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB42_2:
+; CHECK-NEXT: lhi %r0, 19
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r1)
+; CHECK-NEXT: aghi %r15, 200
; CHECK-NEXT: br %r14
%Dst = alloca [20 x i8]
+ %Src = alloca [20 x i8]
call void @llvm.memmove.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 20, i1 false)
ret void
}
@@ -680,10 +530,13 @@ define void @fun20_localDst(ptr %Src) {
define void @fun21(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun21:
; CHECK: # %bb.0:
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: lg %r0, 13(%r3)
-; CHECK-NEXT: stg %r0, 13(%r2)
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB43_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(21,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB43_2:
+; CHECK-NEXT: lhi %r0, 20
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 21, i1 false)
ret void
@@ -692,14 +545,13 @@ define void @fun21(ptr %Dst, ptr %Src) {
define void @fun21_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun21_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 21
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB44_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(21,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB44_2:
+; CHECK-NEXT: lhi %r0, 20
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 21, i1 false)
ret void
@@ -708,10 +560,13 @@ define void @fun21_unaligned(ptr %Dst, ptr %Src) {
define void @fun22(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun22:
; CHECK: # %bb.0:
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: lg %r0, 14(%r3)
-; CHECK-NEXT: stg %r0, 14(%r2)
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB45_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(22,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB45_2:
+; CHECK-NEXT: lhi %r0, 21
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 22, i1 false)
ret void
@@ -720,14 +575,13 @@ define void @fun22(ptr %Dst, ptr %Src) {
define void @fun22_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun22_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 22
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB46_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(22,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB46_2:
+; CHECK-NEXT: lhi %r0, 21
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 22, i1 false)
ret void
@@ -736,10 +590,13 @@ define void @fun22_unaligned(ptr %Dst, ptr %Src) {
define void @fun23(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun23:
; CHECK: # %bb.0:
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: lg %r0, 15(%r3)
-; CHECK-NEXT: stg %r0, 15(%r2)
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB47_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(23,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB47_2:
+; CHECK-NEXT: lhi %r0, 22
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 23, i1 false)
ret void
@@ -748,14 +605,13 @@ define void @fun23(ptr %Dst, ptr %Src) {
define void @fun23_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun23_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 23
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB48_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(23,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB48_2:
+; CHECK-NEXT: lhi %r0, 22
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 23, i1 false)
ret void
@@ -764,10 +620,13 @@ define void @fun23_unaligned(ptr %Dst, ptr %Src) {
define void @fun24(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun24:
; CHECK: # %bb.0:
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: lg %r0, 16(%r3)
-; CHECK-NEXT: stg %r0, 16(%r2)
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB49_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(24,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB49_2:
+; CHECK-NEXT: lhi %r0, 23
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 24, i1 false)
ret void
@@ -776,14 +635,13 @@ define void @fun24(ptr %Dst, ptr %Src) {
define void @fun24_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun24_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 24
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB50_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(24,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB50_2:
+; CHECK-NEXT: lhi %r0, 23
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 24, i1 false)
ret void
@@ -792,14 +650,13 @@ define void @fun24_unaligned(ptr %Dst, ptr %Src) {
define void @fun25(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun25:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 25
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB51_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(25,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB51_2:
+; CHECK-NEXT: lhi %r0, 24
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 25, i1 false)
ret void
@@ -808,14 +665,13 @@ define void @fun25(ptr %Dst, ptr %Src) {
define void @fun25_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun25_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 25
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB52_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(25,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB52_2:
+; CHECK-NEXT: lhi %r0, 24
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 25, i1 false)
ret void
@@ -824,14 +680,13 @@ define void @fun25_unaligned(ptr %Dst, ptr %Src) {
define void @fun26(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun26:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 26
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB53_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(26,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB53_2:
+; CHECK-NEXT: lhi %r0, 25
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 26, i1 false)
ret void
@@ -840,14 +695,13 @@ define void @fun26(ptr %Dst, ptr %Src) {
define void @fun26_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun26_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 26
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB54_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(26,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB54_2:
+; CHECK-NEXT: lhi %r0, 25
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 26, i1 false)
ret void
@@ -856,14 +710,13 @@ define void @fun26_unaligned(ptr %Dst, ptr %Src) {
define void @fun27(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun27:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 27
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB55_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(27,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB55_2:
+; CHECK-NEXT: lhi %r0, 26
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 27, i1 false)
ret void
@@ -872,14 +725,13 @@ define void @fun27(ptr %Dst, ptr %Src) {
define void @fun27_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun27_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 27
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB56_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(27,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB56_2:
+; CHECK-NEXT: lhi %r0, 26
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 27, i1 false)
ret void
@@ -888,14 +740,13 @@ define void @fun27_unaligned(ptr %Dst, ptr %Src) {
define void @fun28(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun28:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 28
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB57_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(28,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB57_2:
+; CHECK-NEXT: lhi %r0, 27
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 28, i1 false)
ret void
@@ -904,14 +755,13 @@ define void @fun28(ptr %Dst, ptr %Src) {
define void @fun28_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun28_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 28
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB58_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(28,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB58_2:
+; CHECK-NEXT: lhi %r0, 27
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 28, i1 false)
ret void
@@ -920,14 +770,13 @@ define void @fun28_unaligned(ptr %Dst, ptr %Src) {
define void @fun29(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun29:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 29
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB59_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(29,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB59_2:
+; CHECK-NEXT: lhi %r0, 28
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 29, i1 false)
ret void
@@ -936,14 +785,13 @@ define void @fun29(ptr %Dst, ptr %Src) {
define void @fun29_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun29_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 29
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB60_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(29,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB60_2:
+; CHECK-NEXT: lhi %r0, 28
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 29, i1 false)
ret void
@@ -952,14 +800,13 @@ define void @fun29_unaligned(ptr %Dst, ptr %Src) {
define void @fun30(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun30:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 30
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB61_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(30,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB61_2:
+; CHECK-NEXT: lhi %r0, 29
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 30, i1 false)
ret void
@@ -968,14 +815,13 @@ define void @fun30(ptr %Dst, ptr %Src) {
define void @fun30_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun30_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 30
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB62_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(30,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB62_2:
+; CHECK-NEXT: lhi %r0, 29
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 30, i1 false)
ret void
@@ -984,14 +830,13 @@ define void @fun30_unaligned(ptr %Dst, ptr %Src) {
define void @fun31(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun31:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 31
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB63_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(31,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB63_2:
+; CHECK-NEXT: lhi %r0, 30
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 31, i1 false)
ret void
@@ -1000,14 +845,13 @@ define void @fun31(ptr %Dst, ptr %Src) {
define void @fun31_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun31_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 31
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB64_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(31,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB64_2:
+; CHECK-NEXT: lhi %r0, 30
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 31, i1 false)
ret void
@@ -1016,10 +860,13 @@ define void @fun31_unaligned(ptr %Dst, ptr %Src) {
define void @fun32(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun32:
; CHECK: # %bb.0:
-; CHECK-NEXT: vl %v0, 0(%r3), 3
-; CHECK-NEXT: vl %v1, 16(%r3), 3
-; CHECK-NEXT: vst %v1, 16(%r2), 3
-; CHECK-NEXT: vst %v0, 0(%r2), 3
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB65_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(32,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB65_2:
+; CHECK-NEXT: lhi %r0, 31
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 32, i1 false)
ret void
@@ -1028,14 +875,13 @@ define void @fun32(ptr %Dst, ptr %Src) {
define void @fun32_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun32_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 32
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB66_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(32,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB66_2:
+; CHECK-NEXT: lhi %r0, 31
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 32, i1 false)
ret void
@@ -1044,14 +890,13 @@ define void @fun32_unaligned(ptr %Dst, ptr %Src) {
define void @fun33(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun33:
; CHECK: # %bb.0:
-; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
-; CHECK-NEXT: .cfi_offset %r14, -48
-; CHECK-NEXT: .cfi_offset %r15, -40
-; CHECK-NEXT: aghi %r15, -160
-; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 33
-; CHECK-NEXT: brasl %r14, memmove at PLT
-; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB67_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(33,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB67_2:
+; CHECK-NEXT: lhi %r0, 32
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 33, i1 false)
ret void
@@ -1060,15 +905,93 @@ define void @fun33(ptr %Dst, ptr %Src) {
define void @fun33_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun33_unaligned:
; CHECK: # %bb.0:
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB68_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(33,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB68_2:
+; CHECK-NEXT: lhi %r0, 32
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 33, i1 false)
+ ret void
+}
+
+; Try the longest length.
+define void @fun256(ptr %Dst, ptr %Src) {
+; CHECK-LABEL: fun256:
+; CHECK: # %bb.0:
+; CHECK-NEXT: clgrjl %r3, %r2, .LBB69_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(256,%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB69_2:
+; CHECK-NEXT: lhi %r0, 255
+; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: br %r14
+ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 256, i1 false)
+ ret void
+}
+
+; And the next past it.
+define void @fun257(ptr %Dst, ptr %Src) {
+; CHECK-LABEL: fun257:
+; CHECK: # %bb.0:
; CHECK-NEXT: stmg %r14, %r15, 112(%r15)
; CHECK-NEXT: .cfi_offset %r14, -48
; CHECK-NEXT: .cfi_offset %r15, -40
; CHECK-NEXT: aghi %r15, -160
; CHECK-NEXT: .cfi_def_cfa_offset 320
-; CHECK-NEXT: lghi %r4, 33
+; CHECK-NEXT: lghi %r4, 257
; CHECK-NEXT: brasl %r14, memmove at PLT
; CHECK-NEXT: lmg %r14, %r15, 272(%r15)
; CHECK-NEXT: br %r14
- call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 33, i1 false)
+ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 257, i1 false)
+ ret void
+}
+
+; Large displacements.
+define void @displ0(ptr %Dst, ptr %Src) {
+; CHECK-LABEL: displ0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lhi %r0, 2
+; CHECK-NEXT: vll %v0, %r0, 4095(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 48(%r2)
+; CHECK-NEXT: br %r14
+ %Dst.fld = getelementptr inbounds nuw i8, ptr %Dst, i64 48
+ %Src.fld = getelementptr inbounds nuw i8, ptr %Src, i64 4095
+ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst.fld, ptr align 8 %Src.fld, i64 3,
+ i1 false)
+ ret void
+}
+
+define void @displ1(ptr %Dst, ptr %Src) {
+; CHECK-LABEL: displ1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: aghi %r3, 4096
+; CHECK-NEXT: lhi %r0, 2
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: vstl %v0, %r0, 48(%r2)
+; CHECK-NEXT: br %r14
+ %Dst.fld = getelementptr inbounds nuw i8, ptr %Dst, i64 48
+ %Src.fld = getelementptr inbounds nuw i8, ptr %Src, i64 4096
+ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst.fld, ptr align 8 %Src.fld, i64 3,
+ i1 false)
+ ret void
+}
+
+define void @displ2(ptr %Dst, ptr %Src) {
+; CHECK-LABEL: displ2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: aghi %r3, 4096
+; CHECK-NEXT: lhi %r0, 2
+; CHECK-NEXT: vll %v0, %r0, 0(%r3)
+; CHECK-NEXT: aghi %r2, 5000
+; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: br %r14
+ %Dst.fld = getelementptr inbounds nuw i8, ptr %Dst, i64 5000
+ %Src.fld = getelementptr inbounds nuw i8, ptr %Src, i64 4096
+ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst.fld, ptr align 8 %Src.fld, i64 3,
+ i1 false)
ret void
}
>From 2e498685aba56c3f7026cc431211e978bc89cf9c Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <Jonas.Paulsson2 at ibm.com>
Date: Thu, 7 May 2026 12:46:26 +0200
Subject: [PATCH 2/4] clang-format
---
llvm/include/llvm/CodeGen/TargetLowering.h | 4 +-
.../Target/SystemZ/SystemZISelLowering.cpp | 49 +++++++++----------
.../SystemZ/SystemZSelectionDAGInfo.cpp | 2 +-
3 files changed, 27 insertions(+), 28 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index dc7cde81d804d..0c092d14eb699 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -133,8 +133,8 @@ struct MemOp {
// constant so it does not need to be loaded.
bool SrcDstMayOverlap; // True if the source and destination memory regions
// may overlap (memmove).
- Align SrcAlign; // Inferred alignment of the source or default value if the
- // memory operation does not need to load the value.
+ Align SrcAlign; // Inferred alignment of the source or default value if the
+ // memory operation does not need to load the value.
public:
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc = false,
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 7b076b27c7960..4ce94ca76feaa 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1475,7 +1475,7 @@ bool SystemZTargetLowering::findOptimalMemOpLowering(
(!Op.isAligned(Align(8)) || (Op.size() >= 25 && Op.size() <= 31)))
return false;
return TargetLowering::findOptimalMemOpLowering(
- Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
+ Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
}
if (Op.isZeroMemset())
@@ -10880,8 +10880,9 @@ SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
return MBB;
}
-MachineBasicBlock *SystemZTargetLowering::emitMemmoveImm(
- MachineInstr &MI, MachineBasicBlock *MBB) const {
+MachineBasicBlock *
+SystemZTargetLowering::emitMemmoveImm(MachineInstr &MI,
+ MachineBasicBlock *MBB) const {
MachineFunction &MF = *MBB->getParent();
const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
MachineRegisterInfo &MRI = MF.getRegInfo();
@@ -10892,7 +10893,8 @@ MachineBasicBlock *SystemZTargetLowering::emitMemmoveImm(
MachineOperand SrcBase = earlyUseOperand(MI.getOperand(2));
uint64_t SrcDisp = MI.getOperand(3).getImm();
uint64_t Len = MI.getOperand(4).getImm();
- assert(Len >= 1 && Len <=256 && "Memmove of of unsupported constant length.");
+ assert(Len >= 1 && Len <= 256 &&
+ "Memmove of of unsupported constant length.");
assert(isUInt<12>(DstDisp) && isUInt<12>(SrcDisp) &&
"Unexpected large displacement.");
@@ -10902,7 +10904,7 @@ MachineBasicBlock *SystemZTargetLowering::emitMemmoveImm(
Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp);
BuildMI(*MBB, MI, DL, TII->get(Opcode), Reg)
- .add(Base).addImm(Disp).addReg(0);
+ .add(Base).addImm(Disp).addReg(0);
Base = MachineOperand::CreateReg(Reg, false);
Disp = 0;
}
@@ -10910,18 +10912,17 @@ MachineBasicBlock *SystemZTargetLowering::emitMemmoveImm(
if (Len <= 15 && MEMMOVEVLL) {
Register HighByteReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
- BuildMI(*MBB, MI, DL, TII->get(SystemZ::LHI), HighByteReg)
- .addImm(Len - 1);
+ BuildMI(*MBB, MI, DL, TII->get(SystemZ::LHI), HighByteReg).addImm(Len - 1);
Register VecReg = MRI.createVirtualRegister(&SystemZ::VR128BitRegClass);
BuildMI(*MBB, MI, DL, TII->get(SystemZ::VLL), VecReg)
- .addReg(HighByteReg)
- .add(SrcBase).addImm(SrcDisp);
+ .addReg(HighByteReg)
+ .add(SrcBase).addImm(SrcDisp);
BuildMI(*MBB, MI, DL, TII->get(SystemZ::VSTL))
- .addReg(VecReg)
- .addReg(HighByteReg)
- .add(DstBase).addImm(DstDisp);
+ .addReg(VecReg)
+ .addReg(HighByteReg)
+ .add(DstBase).addImm(DstDisp);
MI.eraseFromParent();
return MBB;
@@ -10940,25 +10941,23 @@ MachineBasicBlock *SystemZTargetLowering::emitMemmoveImm(
foldAddressIfNeeded(SrcBase, SrcDisp);
foldAddressIfNeeded(DstBase, DstDisp);
- BuildMI(MBB, DL, TII->get(SystemZ::CLGR))
- .add(SrcBase)
- .add(DstBase);
+ BuildMI(MBB, DL, TII->get(SystemZ::CLGR)).add(SrcBase).add(DstBase);
BuildMI(MBB, DL, TII->get(SystemZ::BRC))
- .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_LT)
- .addMBB(MvcrlMBB);
+ .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_LT)
+ .addMBB(MvcrlMBB);
BuildMI(MvcMBB, DL, TII->get(SystemZ::MVC))
- .add(DstBase).addImm(DstDisp).addImm(Len)
- .add(SrcBase).addImm(SrcDisp)
- .setMemRefs(MI.memoperands());
+ .add(DstBase).addImm(DstDisp)
+ .addImm(Len)
+ .add(SrcBase).addImm(SrcDisp)
+ .setMemRefs(MI.memoperands());
BuildMI(MvcMBB, DL, TII->get(SystemZ::J)).addMBB(DoneMBB);
- BuildMI(MvcrlMBB, DL, TII->get(SystemZ::LHI), SystemZ::R0L)
- .addImm(Len - 1);
+ BuildMI(MvcrlMBB, DL, TII->get(SystemZ::LHI), SystemZ::R0L).addImm(Len - 1);
BuildMI(MvcrlMBB, DL, TII->get(SystemZ::MVCRL))
- .add(DstBase).addImm(DstDisp)
- .add(SrcBase).addImm(SrcDisp)
- .setMemRefs(MI.memoperands());
+ .add(DstBase).addImm(DstDisp)
+ .add(SrcBase).addImm(SrcDisp)
+ .setMemRefs(MI.memoperands());
MI.eraseFromParent();
return DoneMBB;
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index 2c804416e1cea..01bbd5024e409 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -99,7 +99,7 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemmove(
if (auto *CSize = dyn_cast<ConstantSDNode>(Size))
if (CSize->getZExtValue() <= 256)
return DAG.getNode(SystemZISD::MEMMOVE, DL, MVT::Other,
- { Chain, Dst, Src, Size });
+ {Chain, Dst, Src, Size});
return SDValue();
}
>From 75688e5fd54d61981ef8ee7679675f1549f5bbee Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1 at linux.ibm.com>
Date: Mon, 11 May 2026 15:02:04 +0200
Subject: [PATCH 3/4] - Use MaxStoresPerMemmove = 2, and remove VLL/VSTL
lowering. - Simplify address folding as VLL is no longer used. - Try to
constant-fold some cases with combineMEMMOVE() - Add def operands on pseudo
definition.
---
.../Target/SystemZ/SystemZISelLowering.cpp | 110 ++--
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 1 +
llvm/lib/Target/SystemZ/SystemZInstrInfo.td | 13 +-
llvm/lib/Target/SystemZ/SystemZOperators.td | 6 +-
.../SystemZ/SystemZSelectionDAGInfo.cpp | 8 +-
llvm/test/CodeGen/SystemZ/memmove-01.ll | 468 ++++++++++++------
6 files changed, 391 insertions(+), 215 deletions(-)
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 4ce94ca76feaa..31e89db2b2224 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -43,10 +43,6 @@ static cl::opt<bool> EnableIntArgExtCheck(
cl::desc("Verify that narrow int args are properly extended per the "
"SystemZ ABI."));
-// EXPERIMENTAL
-static cl::opt<unsigned> MEMMOVESTORES("memmove-stores", cl::init(1));
-static cl::opt<bool> MEMMOVEVLL("memmove-vll", cl::init(true));
-
namespace {
// Represents information about a comparison.
struct Comparison {
@@ -819,7 +815,7 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
MaxStoresPerMemcpyOptSize = 0;
// Same with memmove.
- MaxStoresPerMemmove = Subtarget.hasVector() ? MEMMOVESTORES : 0;
+ MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0;
MaxStoresPerMemmoveOptSize = 0;
// The main memset sequence is a byte store followed by an MVC.
@@ -9409,6 +9405,51 @@ SDValue SystemZTargetLowering::combineINTRINSIC(
return SDValue();
}
+SDValue SystemZTargetLowering::combineMEMMOVE(
+ SDNode *N, DAGCombinerInfo &DCI) const {
+ SelectionDAG &DAG = DCI.DAG;
+
+ SDValue Chain = N->getOperand(0);
+ SDValue Dst = N->getOperand(1);
+ SDValue Src = N->getOperand(2);
+ unsigned Len = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue();
+
+ struct Address {
+ SDValue Addr;
+ Address(SDValue V) : Addr(V) {}
+ SDValue Base() {
+ if (Addr->getOpcode() == ISD::ADD &&
+ isa<ConstantSDNode>(Addr->getOperand(1)))
+ return Addr->getOperand(0);
+ return Addr;
+ }
+ uint64_t Offset() {
+ if (Addr->getOpcode() == ISD::ADD)
+ if (auto *Const = dyn_cast<ConstantSDNode>(Addr->getOperand(1)))
+ return Const->getZExtValue();
+ return 0;
+ }
+ };
+
+ Address DstAddr(Dst), SrcAddr(Src);
+ if (DstAddr.Base() == SrcAddr.Base()) {
+ assert(Len >= 16 && Len <= 256 &&
+ "Memmove of of unsupported constant length.");
+ if (DstAddr.Offset() <= SrcAddr.Offset()) {
+ SDValue LenAdj = DAG.getConstant(Len - 1, SDLoc(N), MVT::i64);
+ return DAG.getNode(SystemZISD::MVC, SDLoc(N), MVT::Other,
+ { Chain, Dst, Src, LenAdj });
+ } else {
+ SDValue LenAdj = DAG.getConstant(Len - 1, SDLoc(N), MVT::i32);
+ Chain = DAG.getCopyToReg(Chain, SDLoc(N), SystemZ::R0L, LenAdj);
+ return DAG.getNode(SystemZISD::MVCRL, SDLoc(N), MVT::Other,
+ { Chain, Dst, Src });
+ }
+ }
+
+ return SDValue();
+}
+
SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
return N->getOperand(0);
@@ -9450,6 +9491,7 @@ SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
case ISD::UREM: return combineIntDIVREM(N, DCI);
case ISD::INTRINSIC_W_CHAIN:
case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
+ case SystemZISD::MEMMOVE: return combineMEMMOVE(N, DCI);
}
return SDValue();
@@ -10883,50 +10925,14 @@ SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
MachineBasicBlock *
SystemZTargetLowering::emitMemmoveImm(MachineInstr &MI,
MachineBasicBlock *MBB) const {
- MachineFunction &MF = *MBB->getParent();
const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
- MachineRegisterInfo &MRI = MF.getRegInfo();
DebugLoc DL = MI.getDebugLoc();
- MachineOperand DstBase = earlyUseOperand(MI.getOperand(0));
- uint64_t DstDisp = MI.getOperand(1).getImm();
- MachineOperand SrcBase = earlyUseOperand(MI.getOperand(2));
- uint64_t SrcDisp = MI.getOperand(3).getImm();
- uint64_t Len = MI.getOperand(4).getImm();
- assert(Len >= 1 && Len <= 256 &&
+ MachineOperand DstAddr = earlyUseOperand(MI.getOperand(0));
+ MachineOperand SrcAddr = earlyUseOperand(MI.getOperand(1));
+ uint64_t Len = MI.getOperand(2).getImm();
+ assert(Len >= 16 && Len <= 256 &&
"Memmove of of unsupported constant length.");
- assert(isUInt<12>(DstDisp) && isUInt<12>(SrcDisp) &&
- "Unexpected large displacement.");
-
- // Fold any displacement (or frame index reference) into a new register.
- auto foldAddressIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
- if (Disp || Base.isFI()) {
- Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
- unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp);
- BuildMI(*MBB, MI, DL, TII->get(Opcode), Reg)
- .add(Base).addImm(Disp).addReg(0);
- Base = MachineOperand::CreateReg(Reg, false);
- Disp = 0;
- }
- };
-
- if (Len <= 15 && MEMMOVEVLL) {
- Register HighByteReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);
- BuildMI(*MBB, MI, DL, TII->get(SystemZ::LHI), HighByteReg).addImm(Len - 1);
-
- Register VecReg = MRI.createVirtualRegister(&SystemZ::VR128BitRegClass);
- BuildMI(*MBB, MI, DL, TII->get(SystemZ::VLL), VecReg)
- .addReg(HighByteReg)
- .add(SrcBase).addImm(SrcDisp);
-
- BuildMI(*MBB, MI, DL, TII->get(SystemZ::VSTL))
- .addReg(VecReg)
- .addReg(HighByteReg)
- .add(DstBase).addImm(DstDisp);
-
- MI.eraseFromParent();
- return MBB;
- }
// Use MVC or MVCRL after comparing the addresses.
MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
@@ -10937,26 +10943,22 @@ SystemZTargetLowering::emitMemmoveImm(MachineInstr &MI,
MvcMBB->addSuccessor(DoneMBB);
MvcrlMBB->addSuccessor(DoneMBB);
- // Fold any displacements in order to do the compare.
- foldAddressIfNeeded(SrcBase, SrcDisp);
- foldAddressIfNeeded(DstBase, DstDisp);
-
- BuildMI(MBB, DL, TII->get(SystemZ::CLGR)).add(SrcBase).add(DstBase);
+ BuildMI(MBB, DL, TII->get(SystemZ::CLGR)).add(SrcAddr).add(DstAddr);
BuildMI(MBB, DL, TII->get(SystemZ::BRC))
.addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_LT)
.addMBB(MvcrlMBB);
BuildMI(MvcMBB, DL, TII->get(SystemZ::MVC))
- .add(DstBase).addImm(DstDisp)
+ .add(DstAddr).addImm(0)
.addImm(Len)
- .add(SrcBase).addImm(SrcDisp)
+ .add(SrcAddr).addImm(0)
.setMemRefs(MI.memoperands());
BuildMI(MvcMBB, DL, TII->get(SystemZ::J)).addMBB(DoneMBB);
BuildMI(MvcrlMBB, DL, TII->get(SystemZ::LHI), SystemZ::R0L).addImm(Len - 1);
BuildMI(MvcrlMBB, DL, TII->get(SystemZ::MVCRL))
- .add(DstBase).addImm(DstDisp)
- .add(SrcBase).addImm(SrcDisp)
+ .add(DstAddr).addImm(0)
+ .add(SrcAddr).addImm(0)
.setMemRefs(MI.memoperands());
MI.eraseFromParent();
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 394530bbee9d8..4bb76df09d75f 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -426,6 +426,7 @@ class SystemZTargetLowering : public TargetLowering {
SDValue combineMUL(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue combineIntDIVREM(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue combineINTRINSIC(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue combineMEMMOVE(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue unwrapAddress(SDValue N) const override;
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
index 23eb0d51734f1..837784558b880 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -586,17 +586,18 @@ def MemsetImmReg : MemsetPseudo<imm64, GR32>;
def MemsetRegImm : MemsetPseudo<ADDR64, imm32zx8trunc>;
def MemsetRegReg : MemsetPseudo<ADDR64, GR32>;
-// Move right.
+// Move right to left.
let Predicates = [FeatureMiscellaneousExtensions3],
mayLoad = 1, mayStore = 1, Uses = [R0L] in
def MVCRL : SideEffectBinarySSE<"mvcrl", 0xE50A>;
+def : Pat<(z_mvcrl bdaddr12only:$dest, bdaddr12only:$src),
+ (MVCRL bdaddr12only:$dest, bdaddr12only:$src)>;
-let usesCustomInserter = 1, hasNoSchedulingInfo = 1, mayLoad = 1, mayStore = 1 in
+let usesCustomInserter = 1, hasNoSchedulingInfo = 1, mayLoad = 1, mayStore = 1,
+ Defs = [CC, R0L] in
def MemmoveImm : Pseudo<(outs),
- (ins bdaddr12only:$dest, bdaddr12only:$src,
- imm64:$length),
- [(z_memmove bdaddr12only:$dest, bdaddr12only:$src,
- imm64:$length)]>;
+ (ins ADDR64:$dest, ADDR64:$src, imm64:$length),
+ [(z_memmove ADDR64:$dest, ADDR64:$src, imm64:$length)]>;
// String moves.
let mayLoad = 1, mayStore = 1, Defs = [CC] in
diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td
index fc1400653db71..6787e24bb58fc 100644
--- a/llvm/lib/Target/SystemZ/SystemZOperators.td
+++ b/llvm/lib/Target/SystemZ/SystemZOperators.td
@@ -116,6 +116,9 @@ def SDT_ZMemMemLengthCC : SDTypeProfile<1, 3,
SDTCisPtrTy<1>,
SDTCisPtrTy<2>,
SDTCisVT<3, i64>]>;
+def SDT_ZMemMem : SDTypeProfile<0, 2,
+ [SDTCisPtrTy<0>,
+ SDTCisPtrTy<1>]>;
def SDT_ZMemsetMVC : SDTypeProfile<0, 3,
[SDTCisPtrTy<0>,
SDTCisVT<1, i64>,
@@ -678,7 +681,8 @@ def z_atomic_cmp_swap_128 : SDNode<"SystemZISD::ATOMIC_CMP_SWAP_128",
// MachineMemOperands rather than one.
def z_mvc : SDNode<"SystemZISD::MVC", SDT_ZMemMemLength,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
-
+def z_mvcrl : SDNode<"SystemZISD::MVCRL", SDT_ZMemMem,
+ [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
def z_memmove : SDNode<"SystemZISD::MEMMOVE", SDT_ZMemMemLength,
[SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index 01bbd5024e409..c49a3bc7324c0 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -94,10 +94,12 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemmove(
if (IsVolatile)
return SDValue();
- // XXX VLL FeatureVector
- // XXX MVCRL FeatureMiscellaneousExtensions3
+ const SystemZSubtarget &Subtarget =
+ DAG.getMachineFunction().getSubtarget<SystemZSubtarget>();
+
if (auto *CSize = dyn_cast<ConstantSDNode>(Size))
- if (CSize->getZExtValue() <= 256)
+ if (Subtarget.hasMiscellaneousExtensions3() &&
+ CSize->getZExtValue() > 0 && CSize->getZExtValue() <= 256)
return DAG.getNode(SystemZISD::MEMMOVE, DL, MVT::Other,
{Chain, Dst, Src, Size});
diff --git a/llvm/test/CodeGen/SystemZ/memmove-01.ll b/llvm/test/CodeGen/SystemZ/memmove-01.ll
index 653033c31a6bf..983afa7398f77 100644
--- a/llvm/test/CodeGen/SystemZ/memmove-01.ll
+++ b/llvm/test/CodeGen/SystemZ/memmove-01.ll
@@ -50,9 +50,10 @@ define void @fun2_unaligned(ptr %Dst, ptr %Src) {
define void @fun3(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun3:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 2
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lh %r0, 0(%r3)
+; CHECK-NEXT: lb %r1, 2(%r3)
+; CHECK-NEXT: stc %r1, 2(%r2)
+; CHECK-NEXT: sth %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 3, i1 false)
ret void
@@ -61,9 +62,10 @@ define void @fun3(ptr %Dst, ptr %Src) {
define void @fun3_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun3_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 2
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lh %r0, 0(%r3)
+; CHECK-NEXT: lb %r1, 2(%r3)
+; CHECK-NEXT: stc %r1, 2(%r2)
+; CHECK-NEXT: sth %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 3, i1 false)
ret void
@@ -92,9 +94,10 @@ define void @fun4_unaligned(ptr %Dst, ptr %Src) {
define void @fun5(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun5:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 4
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lb %r1, 4(%r3)
+; CHECK-NEXT: l %r0, 0(%r3)
+; CHECK-NEXT: stc %r1, 4(%r2)
+; CHECK-NEXT: st %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 5, i1 false)
ret void
@@ -103,9 +106,10 @@ define void @fun5(ptr %Dst, ptr %Src) {
define void @fun5_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun5_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 4
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lb %r1, 4(%r3)
+; CHECK-NEXT: l %r0, 0(%r3)
+; CHECK-NEXT: stc %r1, 4(%r2)
+; CHECK-NEXT: st %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 5, i1 false)
ret void
@@ -114,9 +118,10 @@ define void @fun5_unaligned(ptr %Dst, ptr %Src) {
define void @fun6(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun6:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 5
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lh %r1, 4(%r3)
+; CHECK-NEXT: l %r0, 0(%r3)
+; CHECK-NEXT: sth %r1, 4(%r2)
+; CHECK-NEXT: st %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 6, i1 false)
ret void
@@ -125,9 +130,10 @@ define void @fun6(ptr %Dst, ptr %Src) {
define void @fun6_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun6_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 5
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lh %r1, 4(%r3)
+; CHECK-NEXT: l %r0, 0(%r3)
+; CHECK-NEXT: sth %r1, 4(%r2)
+; CHECK-NEXT: st %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 6, i1 false)
ret void
@@ -136,9 +142,10 @@ define void @fun6_unaligned(ptr %Dst, ptr %Src) {
define void @fun7(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun7:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 6
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: l %r0, 0(%r3)
+; CHECK-NEXT: l %r1, 3(%r3)
+; CHECK-NEXT: st %r1, 3(%r2)
+; CHECK-NEXT: st %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 7, i1 false)
ret void
@@ -147,9 +154,10 @@ define void @fun7(ptr %Dst, ptr %Src) {
define void @fun7_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun7_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 6
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: l %r0, 0(%r3)
+; CHECK-NEXT: l %r1, 3(%r3)
+; CHECK-NEXT: st %r1, 3(%r2)
+; CHECK-NEXT: st %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 7, i1 false)
ret void
@@ -178,9 +186,10 @@ define void @fun8_unaligned(ptr %Dst, ptr %Src) {
define void @fun9(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun9:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 8
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lb %r1, 8(%r3)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: stc %r1, 8(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 9, i1 false)
ret void
@@ -189,9 +198,10 @@ define void @fun9(ptr %Dst, ptr %Src) {
define void @fun9_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun9_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 8
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lb %r1, 8(%r3)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: stc %r1, 8(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 9, i1 false)
ret void
@@ -200,9 +210,10 @@ define void @fun9_unaligned(ptr %Dst, ptr %Src) {
define void @fun10(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun10:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 9
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lh %r1, 8(%r3)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: sth %r1, 8(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 10, i1 false)
ret void
@@ -211,9 +222,10 @@ define void @fun10(ptr %Dst, ptr %Src) {
define void @fun10_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun10_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 9
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lh %r1, 8(%r3)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: sth %r1, 8(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 10, i1 false)
ret void
@@ -222,9 +234,10 @@ define void @fun10_unaligned(ptr %Dst, ptr %Src) {
define void @fun11(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun11:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 10
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: l %r1, 7(%r3)
+; CHECK-NEXT: st %r1, 7(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 11, i1 false)
ret void
@@ -233,9 +246,10 @@ define void @fun11(ptr %Dst, ptr %Src) {
define void @fun11_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun11_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 10
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: l %r1, 7(%r3)
+; CHECK-NEXT: st %r1, 7(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 11, i1 false)
ret void
@@ -244,9 +258,10 @@ define void @fun11_unaligned(ptr %Dst, ptr %Src) {
define void @fun12(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun12:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 11
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: l %r1, 8(%r3)
+; CHECK-NEXT: st %r1, 8(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 12, i1 false)
ret void
@@ -255,9 +270,10 @@ define void @fun12(ptr %Dst, ptr %Src) {
define void @fun12_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun12_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 11
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: l %r1, 8(%r3)
+; CHECK-NEXT: st %r1, 8(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 12, i1 false)
ret void
@@ -266,9 +282,10 @@ define void @fun12_unaligned(ptr %Dst, ptr %Src) {
define void @fun13(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun13:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 12
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: lg %r1, 5(%r3)
+; CHECK-NEXT: stg %r1, 5(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 13, i1 false)
ret void
@@ -277,9 +294,10 @@ define void @fun13(ptr %Dst, ptr %Src) {
define void @fun13_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun13_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 12
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: lg %r1, 5(%r3)
+; CHECK-NEXT: stg %r1, 5(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 13, i1 false)
ret void
@@ -288,9 +306,10 @@ define void @fun13_unaligned(ptr %Dst, ptr %Src) {
define void @fun14(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun14:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 13
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: lg %r1, 6(%r3)
+; CHECK-NEXT: stg %r1, 6(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 14, i1 false)
ret void
@@ -299,9 +318,10 @@ define void @fun14(ptr %Dst, ptr %Src) {
define void @fun14_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun14_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 13
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: lg %r1, 6(%r3)
+; CHECK-NEXT: stg %r1, 6(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 14, i1 false)
ret void
@@ -310,9 +330,10 @@ define void @fun14_unaligned(ptr %Dst, ptr %Src) {
define void @fun15(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun15:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 14
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: lg %r1, 7(%r3)
+; CHECK-NEXT: stg %r1, 7(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 15, i1 false)
ret void
@@ -321,9 +342,10 @@ define void @fun15(ptr %Dst, ptr %Src) {
define void @fun15_unaligned(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun15_unaligned:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 14
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lg %r0, 0(%r3)
+; CHECK-NEXT: lg %r1, 7(%r3)
+; CHECK-NEXT: stg %r1, 7(%r2)
+; CHECK-NEXT: stg %r0, 0(%r2)
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 15, i1 false)
ret void
@@ -357,13 +379,10 @@ define void @fun16_unaligned(ptr %Dst, ptr %Src) {
define void @fun17(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun17:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB32_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(17,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB32_2:
-; CHECK-NEXT: lhi %r0, 16
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: lb %r0, 16(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: stc %r0, 16(%r2)
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 17, i1 false)
ret void
@@ -402,13 +421,10 @@ define void @fun17_unalignedSrc(ptr %Dst, ptr %Src) {
define void @fun18(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun18:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB35_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(18,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB35_2:
-; CHECK-NEXT: lhi %r0, 17
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: lh %r0, 16(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: sth %r0, 16(%r2)
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 18, i1 false)
ret void
@@ -447,13 +463,10 @@ define void @fun18_unalignedDst(ptr %Dst, ptr %Src) {
define void @fun19(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun19:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB38_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(19,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB38_2:
-; CHECK-NEXT: lhi %r0, 18
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: l %r0, 15(%r3)
+; CHECK-NEXT: st %r0, 15(%r2)
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 19, i1 false)
ret void
@@ -477,13 +490,10 @@ define void @fun19_unaligned(ptr %Dst, ptr %Src) {
define void @fun20(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun20:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB40_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(20,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB40_2:
-; CHECK-NEXT: lhi %r0, 19
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: l %r0, 16(%r3)
+; CHECK-NEXT: st %r0, 16(%r2)
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 20, i1 false)
ret void
@@ -509,16 +519,9 @@ define void @fun20_local() {
; CHECK: # %bb.0:
; CHECK-NEXT: aghi %r15, -200
; CHECK-NEXT: .cfi_def_cfa_offset 360
-; CHECK-NEXT: la %r1, 160(%r15)
-; CHECK-NEXT: la %r2, 180(%r15)
-; CHECK-NEXT: clgrjl %r1, %r2, .LBB42_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(20,%r2), 0(%r1)
-; CHECK-NEXT: aghi %r15, 200
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB42_2:
-; CHECK-NEXT: lhi %r0, 19
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r1)
+; CHECK-NEXT: vl %v0, 160(%r15), 4
+; CHECK-NEXT: vst %v0, 180(%r15), 4
+; CHECK-NEXT: mvc 196(4,%r15), 176(%r15)
; CHECK-NEXT: aghi %r15, 200
; CHECK-NEXT: br %r14
%Dst = alloca [20 x i8]
@@ -530,13 +533,10 @@ define void @fun20_local() {
define void @fun21(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun21:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB43_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(21,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB43_2:
-; CHECK-NEXT: lhi %r0, 20
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: lg %r0, 13(%r3)
+; CHECK-NEXT: stg %r0, 13(%r2)
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 21, i1 false)
ret void
@@ -560,13 +560,10 @@ define void @fun21_unaligned(ptr %Dst, ptr %Src) {
define void @fun22(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun22:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB45_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(22,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB45_2:
-; CHECK-NEXT: lhi %r0, 21
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: lg %r0, 14(%r3)
+; CHECK-NEXT: stg %r0, 14(%r2)
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 22, i1 false)
ret void
@@ -590,13 +587,10 @@ define void @fun22_unaligned(ptr %Dst, ptr %Src) {
define void @fun23(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun23:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB47_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(23,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB47_2:
-; CHECK-NEXT: lhi %r0, 22
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: lg %r0, 15(%r3)
+; CHECK-NEXT: stg %r0, 15(%r2)
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 23, i1 false)
ret void
@@ -620,13 +614,10 @@ define void @fun23_unaligned(ptr %Dst, ptr %Src) {
define void @fun24(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun24:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB49_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(24,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB49_2:
-; CHECK-NEXT: lhi %r0, 23
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: lg %r0, 16(%r3)
+; CHECK-NEXT: stg %r0, 16(%r2)
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 24, i1 false)
ret void
@@ -860,13 +851,10 @@ define void @fun31_unaligned(ptr %Dst, ptr %Src) {
define void @fun32(ptr %Dst, ptr %Src) {
; CHECK-LABEL: fun32:
; CHECK: # %bb.0:
-; CHECK-NEXT: clgrjl %r3, %r2, .LBB65_2
-; CHECK-NEXT: # %bb.1:
-; CHECK-NEXT: mvc 0(32,%r2), 0(%r3)
-; CHECK-NEXT: br %r14
-; CHECK-NEXT: .LBB65_2:
-; CHECK-NEXT: lhi %r0, 31
-; CHECK-NEXT: mvcrl 0(%r2), 0(%r3)
+; CHECK-NEXT: vl %v0, 0(%r3), 3
+; CHECK-NEXT: vl %v1, 16(%r3), 3
+; CHECK-NEXT: vst %v1, 16(%r2), 3
+; CHECK-NEXT: vst %v0, 0(%r2), 3
; CHECK-NEXT: br %r14
call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 32, i1 false)
ret void
@@ -954,9 +942,10 @@ define void @fun257(ptr %Dst, ptr %Src) {
define void @displ0(ptr %Dst, ptr %Src) {
; CHECK-LABEL: displ0:
; CHECK: # %bb.0:
-; CHECK-NEXT: lhi %r0, 2
-; CHECK-NEXT: vll %v0, %r0, 4095(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 48(%r2)
+; CHECK-NEXT: lh %r0, 4095(%r3)
+; CHECK-NEXT: lb %r1, 4097(%r3)
+; CHECK-NEXT: stc %r1, 50(%r2)
+; CHECK-NEXT: sth %r0, 48(%r2)
; CHECK-NEXT: br %r14
%Dst.fld = getelementptr inbounds nuw i8, ptr %Dst, i64 48
%Src.fld = getelementptr inbounds nuw i8, ptr %Src, i64 4095
@@ -968,10 +957,10 @@ define void @displ0(ptr %Dst, ptr %Src) {
define void @displ1(ptr %Dst, ptr %Src) {
; CHECK-LABEL: displ1:
; CHECK: # %bb.0:
-; CHECK-NEXT: aghi %r3, 4096
-; CHECK-NEXT: lhi %r0, 2
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: vstl %v0, %r0, 48(%r2)
+; CHECK-NEXT: lhy %r0, 4096(%r3)
+; CHECK-NEXT: lb %r1, 4098(%r3)
+; CHECK-NEXT: stc %r1, 50(%r2)
+; CHECK-NEXT: sth %r0, 48(%r2)
; CHECK-NEXT: br %r14
%Dst.fld = getelementptr inbounds nuw i8, ptr %Dst, i64 48
%Src.fld = getelementptr inbounds nuw i8, ptr %Src, i64 4096
@@ -983,11 +972,10 @@ define void @displ1(ptr %Dst, ptr %Src) {
define void @displ2(ptr %Dst, ptr %Src) {
; CHECK-LABEL: displ2:
; CHECK: # %bb.0:
-; CHECK-NEXT: aghi %r3, 4096
-; CHECK-NEXT: lhi %r0, 2
-; CHECK-NEXT: vll %v0, %r0, 0(%r3)
-; CHECK-NEXT: aghi %r2, 5000
-; CHECK-NEXT: vstl %v0, %r0, 0(%r2)
+; CHECK-NEXT: lhy %r0, 4096(%r3)
+; CHECK-NEXT: lb %r1, 4098(%r3)
+; CHECK-NEXT: stcy %r1, 5002(%r2)
+; CHECK-NEXT: sthy %r0, 5000(%r2)
; CHECK-NEXT: br %r14
%Dst.fld = getelementptr inbounds nuw i8, ptr %Dst, i64 5000
%Src.fld = getelementptr inbounds nuw i8, ptr %Src, i64 4096
@@ -995,3 +983,181 @@ define void @displ2(ptr %Dst, ptr %Src) {
i1 false)
ret void
}
+
+; In some cases constant offsets known at compile time can eliminate the
+; comparison before MVC or MVCRL.
+
+define void @constOffs0(ptr %Arr) {
+; CHECK-LABEL: constOffs0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: mvc 0(48,%r2), 32(%r2)
+; CHECK-NEXT: br %r14
+ %Src = getelementptr inbounds nuw i8, ptr %Arr, i64 32
+ call void @llvm.memmove.p0.p0.i64(ptr %Arr, ptr %Src, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs1(ptr %Arr) {
+; CHECK-LABEL: constOffs1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lhi %r0, 47
+; CHECK-NEXT: mvcrl 32(%r2), 0(%r2)
+; CHECK-NEXT: br %r14
+ %Dst = getelementptr inbounds nuw i8, ptr %Arr, i64 32
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Arr, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs2(ptr %Arr) {
+; CHECK-LABEL: constOffs2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lay %r1, 10032(%r2)
+; CHECK-NEXT: lay %r2, 10064(%r2)
+; CHECK-NEXT: mvc 0(48,%r1), 0(%r2)
+; CHECK-NEXT: br %r14
+ %Dst = getelementptr inbounds nuw i8, ptr %Arr, i64 10032
+ %Src = getelementptr inbounds nuw i8, ptr %Arr, i64 10064
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Src, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs3(ptr %Arr) {
+; CHECK-LABEL: constOffs3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lhi %r0, 47
+; CHECK-NEXT: mvcrl 64(%r2), 32(%r2)
+; CHECK-NEXT: br %r14
+ %Dst = getelementptr inbounds nuw i8, ptr %Arr, i64 64
+ %Src = getelementptr inbounds nuw i8, ptr %Arr, i64 32
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Src, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs4() {
+; CHECK-LABEL: constOffs4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: aghi %r15, -20176
+; CHECK-NEXT: .cfi_def_cfa_offset 20336
+; CHECK-NEXT: lay %r1, 10208(%r15)
+; CHECK-NEXT: mvc 176(48,%r15), 0(%r1)
+; CHECK-NEXT: aghi %r15, 20176
+; CHECK-NEXT: br %r14
+ %Alloc = alloca [20000 x i8]
+ %Src = getelementptr inbounds nuw i8, ptr %Alloc, i64 10032
+ call void @llvm.memmove.p0.p0.i64(ptr %Alloc, ptr %Src, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs5() {
+; CHECK-LABEL: constOffs5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: aghi %r15, -672
+; CHECK-NEXT: .cfi_def_cfa_offset 832
+; CHECK-NEXT: lhi %r0, 47
+; CHECK-NEXT: mvcrl 192(%r15), 160(%r15)
+; CHECK-NEXT: aghi %r15, 672
+; CHECK-NEXT: br %r14
+ %Alloc = alloca [512 x i8]
+ %Dst = getelementptr inbounds nuw i8, ptr %Alloc, i64 32
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Alloc, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs6() {
+; CHECK-LABEL: constOffs6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: aghi %r15, -20176
+; CHECK-NEXT: .cfi_def_cfa_offset 20336
+; CHECK-NEXT: lay %r1, 10240(%r15)
+; CHECK-NEXT: mvc 208(48,%r15), 0(%r1)
+; CHECK-NEXT: aghi %r15, 20176
+; CHECK-NEXT: br %r14
+ %Alloc = alloca [20000 x i8]
+ %Dst = getelementptr inbounds nuw i8, ptr %Alloc, i64 32
+ %Src = getelementptr inbounds nuw i8, ptr %Alloc, i64 10064
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Src, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs7() {
+; CHECK-LABEL: constOffs7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: aghi %r15, -672
+; CHECK-NEXT: .cfi_def_cfa_offset 832
+; CHECK-NEXT: lhi %r0, 47
+; CHECK-NEXT: mvcrl 224(%r15), 192(%r15)
+; CHECK-NEXT: aghi %r15, 672
+; CHECK-NEXT: br %r14
+ %Alloc = alloca [512 x i8]
+ %Dst = getelementptr inbounds nuw i8, ptr %Alloc, i64 64
+ %Src = getelementptr inbounds nuw i8, ptr %Alloc, i64 32
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Src, i64 48, i1 false)
+ ret void
+}
+
+; Not in cases with different base addresses:
+define void @constOffs8(ptr %Arr) {
+; CHECK-LABEL: constOffs8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: aghi %r15, -20176
+; CHECK-NEXT: .cfi_def_cfa_offset 20336
+; CHECK-NEXT: lay %r1, 10208(%r15)
+; CHECK-NEXT: clgrjl %r2, %r1, .LBB82_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(48,%r1), 0(%r2)
+; CHECK-NEXT: aghi %r15, 20176
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB82_2:
+; CHECK-NEXT: lhi %r0, 47
+; CHECK-NEXT: mvcrl 0(%r1), 0(%r2)
+; CHECK-NEXT: aghi %r15, 20176
+; CHECK-NEXT: br %r14
+ %Alloc = alloca [20000 x i8]
+ %Dst = getelementptr inbounds nuw i8, ptr %Alloc, i64 10032
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Arr, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs9(ptr %Arr, ptr %Arr2) {
+; CHECK-LABEL: constOffs9:
+; CHECK: # %bb.0:
+; CHECK-NEXT: la %r1, 64(%r3)
+; CHECK-NEXT: la %r2, 32(%r2)
+; CHECK-NEXT: clgrjl %r2, %r1, .LBB83_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(48,%r1), 0(%r2)
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB83_2:
+; CHECK-NEXT: lhi %r0, 47
+; CHECK-NEXT: mvcrl 0(%r1), 0(%r2)
+; CHECK-NEXT: br %r14
+ %Dst = getelementptr inbounds nuw i8, ptr %Arr2, i64 64
+ %Src = getelementptr inbounds nuw i8, ptr %Arr, i64 32
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Src, i64 48, i1 false)
+ ret void
+}
+
+define void @constOffs10() {
+; CHECK-LABEL: constOffs10:
+; CHECK: # %bb.0:
+; CHECK-NEXT: aghi %r15, -1184
+; CHECK-NEXT: .cfi_def_cfa_offset 1344
+; CHECK-NEXT: la %r1, 192(%r15)
+; CHECK-NEXT: la %r2, 736(%r15)
+; CHECK-NEXT: clgrjl %r2, %r1, .LBB84_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: mvc 0(48,%r1), 0(%r2)
+; CHECK-NEXT: aghi %r15, 1184
+; CHECK-NEXT: br %r14
+; CHECK-NEXT: .LBB84_2:
+; CHECK-NEXT: lhi %r0, 47
+; CHECK-NEXT: mvcrl 0(%r1), 0(%r2)
+; CHECK-NEXT: aghi %r15, 1184
+; CHECK-NEXT: br %r14
+ %Alloc = alloca [512 x i8]
+ %Alloc2 = alloca [512 x i8]
+ %Dst = getelementptr inbounds nuw i8, ptr %Alloc2, i64 32
+ %Src = getelementptr inbounds nuw i8, ptr %Alloc, i64 64
+ call void @llvm.memmove.p0.p0.i64(ptr %Dst, ptr %Src, i64 48, i1 false)
+ ret void
+}
>From a8c25216567cdf4e41593a1f3bd78f3916919dc6 Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1 at linux.ibm.com>
Date: Thu, 11 Jun 2026 15:55:49 +0200
Subject: [PATCH 4/4] Experiment with stripAndAccumulateConstantOffsets().
---
.../Target/SystemZ/SystemZISelLowering.cpp | 48 +------------------
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 1 -
.../SystemZ/SystemZSelectionDAGInfo.cpp | 24 +++++++++-
3 files changed, 24 insertions(+), 49 deletions(-)
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 31e89db2b2224..ba5f1e1eddfb2 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -9405,51 +9405,6 @@ SDValue SystemZTargetLowering::combineINTRINSIC(
return SDValue();
}
-SDValue SystemZTargetLowering::combineMEMMOVE(
- SDNode *N, DAGCombinerInfo &DCI) const {
- SelectionDAG &DAG = DCI.DAG;
-
- SDValue Chain = N->getOperand(0);
- SDValue Dst = N->getOperand(1);
- SDValue Src = N->getOperand(2);
- unsigned Len = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue();
-
- struct Address {
- SDValue Addr;
- Address(SDValue V) : Addr(V) {}
- SDValue Base() {
- if (Addr->getOpcode() == ISD::ADD &&
- isa<ConstantSDNode>(Addr->getOperand(1)))
- return Addr->getOperand(0);
- return Addr;
- }
- uint64_t Offset() {
- if (Addr->getOpcode() == ISD::ADD)
- if (auto *Const = dyn_cast<ConstantSDNode>(Addr->getOperand(1)))
- return Const->getZExtValue();
- return 0;
- }
- };
-
- Address DstAddr(Dst), SrcAddr(Src);
- if (DstAddr.Base() == SrcAddr.Base()) {
- assert(Len >= 16 && Len <= 256 &&
- "Memmove of of unsupported constant length.");
- if (DstAddr.Offset() <= SrcAddr.Offset()) {
- SDValue LenAdj = DAG.getConstant(Len - 1, SDLoc(N), MVT::i64);
- return DAG.getNode(SystemZISD::MVC, SDLoc(N), MVT::Other,
- { Chain, Dst, Src, LenAdj });
- } else {
- SDValue LenAdj = DAG.getConstant(Len - 1, SDLoc(N), MVT::i32);
- Chain = DAG.getCopyToReg(Chain, SDLoc(N), SystemZ::R0L, LenAdj);
- return DAG.getNode(SystemZISD::MVCRL, SDLoc(N), MVT::Other,
- { Chain, Dst, Src });
- }
- }
-
- return SDValue();
-}
-
SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
return N->getOperand(0);
@@ -9491,7 +9446,6 @@ SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
case ISD::UREM: return combineIntDIVREM(N, DCI);
case ISD::INTRINSIC_W_CHAIN:
case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
- case SystemZISD::MEMMOVE: return combineMEMMOVE(N, DCI);
}
return SDValue();
@@ -11571,4 +11525,4 @@ void SystemZTargetLowering::insertSSPDeclarations(
// Otherwise (in the global case), insert the appropriate global variable.
TargetLowering::insertSSPDeclarations(M, Libcalls);
-}
\ No newline at end of file
+}
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 4bb76df09d75f..394530bbee9d8 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -426,7 +426,6 @@ class SystemZTargetLowering : public TargetLowering {
SDValue combineMUL(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue combineIntDIVREM(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue combineINTRINSIC(SDNode *N, DAGCombinerInfo &DCI) const;
- SDValue combineMEMMOVE(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue unwrapAddress(SDValue N) const override;
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index c49a3bc7324c0..6138e394cfab2 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -99,9 +99,31 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemmove(
if (auto *CSize = dyn_cast<ConstantSDNode>(Size))
if (Subtarget.hasMiscellaneousExtensions3() &&
- CSize->getZExtValue() > 0 && CSize->getZExtValue() <= 256)
+ CSize->getZExtValue() > 0 && CSize->getZExtValue() <= 256) {
+
+ // EXPERIMENT: On SPEC, only 1 case with common base (with or w/out
+ // LookThroughIntToPtr) :-/
+ const DataLayout &DataLt = DAG.getDataLayout();
+ const Value *DstV = dyn_cast_if_present<const Value *>(DstPtrInfo.V);
+ const Value *SrcV = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);
+ if (DstV && SrcV) {
+ APInt DstOffset(64, 0);
+ const Value *DstBase =
+ DstV->stripAndAccumulateConstantOffsets(DataLt, DstOffset,
+ /* AllowNonInbounds */ true, /*AllowInvariantGroup=*/false,
+ /*ExternalAnalysis=*/nullptr, /*LookThroughIntToPtr=*/true);
+ APInt SrcOffset(64, 0);
+ const Value *SrcBase =
+ SrcV->stripAndAccumulateConstantOffsets(DataLt, SrcOffset,
+ /* AllowNonInbounds */ true, /*AllowInvariantGroup=*/false,
+ /*ExternalAnalysis=*/nullptr, /*LookThroughIntToPtr=*/true);
+ if (DstBase == SrcBase)
+ dbgs() << "COMMON BASE: " << DstOffset << " : " << SrcOffset << "\n";
+ }
+
return DAG.getNode(SystemZISD::MEMMOVE, DL, MVT::Other,
{Chain, Dst, Src, Size});
+ }
return SDValue();
}
More information about the llvm-commits
mailing list