[llvm] [X86] For inline memset and memcpy with minsize, use size for alignment, rather than actual alignment (PR #87003)
Rose Silicon via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 5 22:36:16 PDT 2024
https://github.com/RSilicon updated https://github.com/llvm/llvm-project/pull/87003
>From 5849809d3e2195d98e4d5522b37d4b1807da86b3 Mon Sep 17 00:00:00 2001
From: Rose <gfunni234 at gmail.com>
Date: Fri, 5 Apr 2024 14:16:40 -0400
Subject: [PATCH 1/2] [X86] Pre-commit test (NFC)
---
.../CodeGen/X86/memset-vs-memset-inline.ll | 137 ++++++++++++++++++
1 file changed, 137 insertions(+)
diff --git a/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll b/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll
index b8fdd936b43895..90a3f8e785e906 100644
--- a/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll
+++ b/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll
@@ -163,3 +163,140 @@ define void @inlined_set_doesnt_call_external_function(ptr %a, i8 %value) nounwi
tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 %value, i64 1024, i1 0)
ret void
}
+
+define void @memset_inlined_insize(ptr %a) nounwind minsize {
+; CHECK-LABEL: memset_inlined_insize:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movabsq $3038287259199220266, %rax # imm = 0x2A2A2A2A2A2A2A2A
+; CHECK-NEXT: movq %rax, 1016(%rdi)
+; CHECK-NEXT: movq %rax, 1008(%rdi)
+; CHECK-NEXT: movq %rax, 1000(%rdi)
+; CHECK-NEXT: movq %rax, 992(%rdi)
+; CHECK-NEXT: movq %rax, 984(%rdi)
+; CHECK-NEXT: movq %rax, 976(%rdi)
+; CHECK-NEXT: movq %rax, 968(%rdi)
+; CHECK-NEXT: movq %rax, 960(%rdi)
+; CHECK-NEXT: movq %rax, 952(%rdi)
+; CHECK-NEXT: movq %rax, 944(%rdi)
+; CHECK-NEXT: movq %rax, 936(%rdi)
+; CHECK-NEXT: movq %rax, 928(%rdi)
+; CHECK-NEXT: movq %rax, 920(%rdi)
+; CHECK-NEXT: movq %rax, 912(%rdi)
+; CHECK-NEXT: movq %rax, 904(%rdi)
+; CHECK-NEXT: movq %rax, 896(%rdi)
+; CHECK-NEXT: movq %rax, 888(%rdi)
+; CHECK-NEXT: movq %rax, 880(%rdi)
+; CHECK-NEXT: movq %rax, 872(%rdi)
+; CHECK-NEXT: movq %rax, 864(%rdi)
+; CHECK-NEXT: movq %rax, 856(%rdi)
+; CHECK-NEXT: movq %rax, 848(%rdi)
+; CHECK-NEXT: movq %rax, 840(%rdi)
+; CHECK-NEXT: movq %rax, 832(%rdi)
+; CHECK-NEXT: movq %rax, 824(%rdi)
+; CHECK-NEXT: movq %rax, 816(%rdi)
+; CHECK-NEXT: movq %rax, 808(%rdi)
+; CHECK-NEXT: movq %rax, 800(%rdi)
+; CHECK-NEXT: movq %rax, 792(%rdi)
+; CHECK-NEXT: movq %rax, 784(%rdi)
+; CHECK-NEXT: movq %rax, 776(%rdi)
+; CHECK-NEXT: movq %rax, 768(%rdi)
+; CHECK-NEXT: movq %rax, 760(%rdi)
+; CHECK-NEXT: movq %rax, 752(%rdi)
+; CHECK-NEXT: movq %rax, 744(%rdi)
+; CHECK-NEXT: movq %rax, 736(%rdi)
+; CHECK-NEXT: movq %rax, 728(%rdi)
+; CHECK-NEXT: movq %rax, 720(%rdi)
+; CHECK-NEXT: movq %rax, 712(%rdi)
+; CHECK-NEXT: movq %rax, 704(%rdi)
+; CHECK-NEXT: movq %rax, 696(%rdi)
+; CHECK-NEXT: movq %rax, 688(%rdi)
+; CHECK-NEXT: movq %rax, 680(%rdi)
+; CHECK-NEXT: movq %rax, 672(%rdi)
+; CHECK-NEXT: movq %rax, 664(%rdi)
+; CHECK-NEXT: movq %rax, 656(%rdi)
+; CHECK-NEXT: movq %rax, 648(%rdi)
+; CHECK-NEXT: movq %rax, 640(%rdi)
+; CHECK-NEXT: movq %rax, 632(%rdi)
+; CHECK-NEXT: movq %rax, 624(%rdi)
+; CHECK-NEXT: movq %rax, 616(%rdi)
+; CHECK-NEXT: movq %rax, 608(%rdi)
+; CHECK-NEXT: movq %rax, 600(%rdi)
+; CHECK-NEXT: movq %rax, 592(%rdi)
+; CHECK-NEXT: movq %rax, 584(%rdi)
+; CHECK-NEXT: movq %rax, 576(%rdi)
+; CHECK-NEXT: movq %rax, 568(%rdi)
+; CHECK-NEXT: movq %rax, 560(%rdi)
+; CHECK-NEXT: movq %rax, 552(%rdi)
+; CHECK-NEXT: movq %rax, 544(%rdi)
+; CHECK-NEXT: movq %rax, 536(%rdi)
+; CHECK-NEXT: movq %rax, 528(%rdi)
+; CHECK-NEXT: movq %rax, 520(%rdi)
+; CHECK-NEXT: movq %rax, 512(%rdi)
+; CHECK-NEXT: movq %rax, 504(%rdi)
+; CHECK-NEXT: movq %rax, 496(%rdi)
+; CHECK-NEXT: movq %rax, 488(%rdi)
+; CHECK-NEXT: movq %rax, 480(%rdi)
+; CHECK-NEXT: movq %rax, 472(%rdi)
+; CHECK-NEXT: movq %rax, 464(%rdi)
+; CHECK-NEXT: movq %rax, 456(%rdi)
+; CHECK-NEXT: movq %rax, 448(%rdi)
+; CHECK-NEXT: movq %rax, 440(%rdi)
+; CHECK-NEXT: movq %rax, 432(%rdi)
+; CHECK-NEXT: movq %rax, 424(%rdi)
+; CHECK-NEXT: movq %rax, 416(%rdi)
+; CHECK-NEXT: movq %rax, 408(%rdi)
+; CHECK-NEXT: movq %rax, 400(%rdi)
+; CHECK-NEXT: movq %rax, 392(%rdi)
+; CHECK-NEXT: movq %rax, 384(%rdi)
+; CHECK-NEXT: movq %rax, 376(%rdi)
+; CHECK-NEXT: movq %rax, 368(%rdi)
+; CHECK-NEXT: movq %rax, 360(%rdi)
+; CHECK-NEXT: movq %rax, 352(%rdi)
+; CHECK-NEXT: movq %rax, 344(%rdi)
+; CHECK-NEXT: movq %rax, 336(%rdi)
+; CHECK-NEXT: movq %rax, 328(%rdi)
+; CHECK-NEXT: movq %rax, 320(%rdi)
+; CHECK-NEXT: movq %rax, 312(%rdi)
+; CHECK-NEXT: movq %rax, 304(%rdi)
+; CHECK-NEXT: movq %rax, 296(%rdi)
+; CHECK-NEXT: movq %rax, 288(%rdi)
+; CHECK-NEXT: movq %rax, 280(%rdi)
+; CHECK-NEXT: movq %rax, 272(%rdi)
+; CHECK-NEXT: movq %rax, 264(%rdi)
+; CHECK-NEXT: movq %rax, 256(%rdi)
+; CHECK-NEXT: movq %rax, 248(%rdi)
+; CHECK-NEXT: movq %rax, 240(%rdi)
+; CHECK-NEXT: movq %rax, 232(%rdi)
+; CHECK-NEXT: movq %rax, 224(%rdi)
+; CHECK-NEXT: movq %rax, 216(%rdi)
+; CHECK-NEXT: movq %rax, 208(%rdi)
+; CHECK-NEXT: movq %rax, 200(%rdi)
+; CHECK-NEXT: movq %rax, 192(%rdi)
+; CHECK-NEXT: movq %rax, 184(%rdi)
+; CHECK-NEXT: movq %rax, 176(%rdi)
+; CHECK-NEXT: movq %rax, 168(%rdi)
+; CHECK-NEXT: movq %rax, 160(%rdi)
+; CHECK-NEXT: movq %rax, 152(%rdi)
+; CHECK-NEXT: movq %rax, 144(%rdi)
+; CHECK-NEXT: movq %rax, 136(%rdi)
+; CHECK-NEXT: movq %rax, 128(%rdi)
+; CHECK-NEXT: movq %rax, 120(%rdi)
+; CHECK-NEXT: movq %rax, 112(%rdi)
+; CHECK-NEXT: movq %rax, 104(%rdi)
+; CHECK-NEXT: movq %rax, 96(%rdi)
+; CHECK-NEXT: movq %rax, 88(%rdi)
+; CHECK-NEXT: movq %rax, 80(%rdi)
+; CHECK-NEXT: movq %rax, 72(%rdi)
+; CHECK-NEXT: movq %rax, 64(%rdi)
+; CHECK-NEXT: movq %rax, 56(%rdi)
+; CHECK-NEXT: movq %rax, 48(%rdi)
+; CHECK-NEXT: movq %rax, 40(%rdi)
+; CHECK-NEXT: movq %rax, 32(%rdi)
+; CHECK-NEXT: movq %rax, 24(%rdi)
+; CHECK-NEXT: movq %rax, 16(%rdi)
+; CHECK-NEXT: movq %rax, 8(%rdi)
+; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: retq
+ tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 42, i64 1024, i1 0)
+ ret void
+}
>From 329f42989abb307ba5347841d1f79a400f959222 Mon Sep 17 00:00:00 2001
From: Rose <gfunni234 at gmail.com>
Date: Thu, 28 Mar 2024 16:19:34 -0400
Subject: [PATCH 2/2] [X86] For inline memset with minsize, use stosb
This is the way to do this in the smallest encoding possible.
---
llvm/lib/Target/X86/X86SelectionDAGInfo.cpp | 53 ++--
.../CodeGen/X86/memcpy-struct-by-value.ll | 8 +-
llvm/test/CodeGen/X86/memcpy.ll | 20 +-
llvm/test/CodeGen/X86/memset-minsize.ll | 74 ++---
.../CodeGen/X86/memset-vs-memset-inline.ll | 267 +-----------------
5 files changed, 82 insertions(+), 340 deletions(-)
diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
index 055466ac660ccc..c064af143156f9 100644
--- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -59,21 +59,31 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
return SDValue();
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
+ if (!ConstantSize)
+ return SDValue();
+
+ // If we have minsize, then just set alignment to 1
+ // to ensure we end up using repmovsb
+ if (DAG.getMachineFunction().getFunction().hasMinSize()) {
+ Alignment = Align(1);
+ AlwaysInline = true;
+ }
+
const X86Subtarget &Subtarget =
DAG.getMachineFunction().getSubtarget<X86Subtarget>();
// If not DWORD aligned or size is more than the threshold, call the library.
// The libc version is likely to be faster for these cases. It can use the
// address value and run time information about the CPU.
- if (Alignment < Align(4) || !ConstantSize ||
- ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold())
+ uint64_t SizeVal = ConstantSize->getZExtValue();
+ if (!AlwaysInline &&
+ (Alignment < Align(4) || SizeVal > Subtarget.getMaxInlineSizeThreshold()))
return SDValue();
- uint64_t SizeVal = ConstantSize->getZExtValue();
SDValue InGlue;
EVT AVT;
SDValue Count;
- unsigned BytesLeft = 0;
+ uint64_t BytesLeft = 0;
if (auto *ValC = dyn_cast<ConstantSDNode>(Val)) {
unsigned ValReg;
uint64_t Val = ValC->getZExtValue() & 255;
@@ -99,14 +109,12 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
// Byte aligned
AVT = MVT::i8;
ValReg = X86::AL;
- Count = DAG.getIntPtrConstant(SizeVal, dl);
}
- if (AVT.bitsGT(MVT::i8)) {
- unsigned UBytes = AVT.getSizeInBits() / 8;
- Count = DAG.getIntPtrConstant(SizeVal / UBytes, dl);
- BytesLeft = SizeVal % UBytes;
- }
+ const uint64_t UBytes = AVT.getSizeInBits() / 8;
+ const uint64_t BlockCount = SizeVal / UBytes;
+ BytesLeft = SizeVal % UBytes;
+ Count = DAG.getIntPtrConstant(BlockCount, dl);
Chain = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, dl, AVT),
InGlue);
@@ -119,11 +127,11 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
}
bool Use64BitRegs = Subtarget.isTarget64BitLP64();
- Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RCX : X86::ECX,
- Count, InGlue);
+ Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RCX : X86::ECX, Count,
+ InGlue);
InGlue = Chain.getValue(1);
- Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RDI : X86::EDI,
- Dst, InGlue);
+ Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RDI : X86::EDI, Dst,
+ InGlue);
InGlue = Chain.getValue(1);
SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
@@ -146,8 +154,8 @@ SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
DAG.getConstant(Offset, dl, AddrVT)),
Val, DAG.getConstant(BytesLeft, dl, SizeVT), Alignment,
- isVolatile, AlwaysInline,
- /* CI */ nullptr, DstPtrInfo.getWithOffset(Offset)));
+ isVolatile, /*AlwaysInline*/ true, /* CI */ nullptr,
+ DstPtrInfo.getWithOffset(Offset)));
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
@@ -220,9 +228,15 @@ static SDValue emitConstantSizeRepmov(
return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
assert(!Subtarget.hasERMSB() && "No efficient RepMovs");
+
+ /// In case we optimize for size, we use repmovsb even if it's less efficient
+ /// so we can save the loads/stores of the leftover.
+ if (DAG.getMachineFunction().getFunction().hasMinSize())
+ return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
+
/// We assume runtime memcpy will do a better job for unaligned copies when
/// ERMS is not present.
- if (!AlwaysInline && (Alignment.value() & 3) != 0)
+ if (!AlwaysInline && (Alignment < Align(4)))
return SDValue();
const MVT BlockType = getOptimalRepmovsType(Subtarget, Alignment);
@@ -239,11 +253,6 @@ static SDValue emitConstantSizeRepmov(
assert(BytesLeft && "We have leftover at this point");
- /// In case we optimize for size we use repmovsb even if it's less efficient
- /// so we can save the loads/stores of the leftover.
- if (DAG.getMachineFunction().getFunction().hasMinSize())
- return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
-
// Handle the last 1 - 7 bytes.
SmallVector<SDValue, 4> Results;
Results.push_back(RepMovs);
diff --git a/llvm/test/CodeGen/X86/memcpy-struct-by-value.ll b/llvm/test/CodeGen/X86/memcpy-struct-by-value.ll
index 8bc4098b0f7c60..f6b1e487000976 100644
--- a/llvm/test/CodeGen/X86/memcpy-struct-by-value.ll
+++ b/llvm/test/CodeGen/X86/memcpy-struct-by-value.ll
@@ -78,9 +78,9 @@ define void @test2(ptr nocapture %x) nounwind minsize {
; NOFAST32-NEXT: pushl %esi
; NOFAST32-NEXT: subl $4100, %esp # imm = 0x1004
; NOFAST32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; NOFAST32-NEXT: movl $1024, %ecx # imm = 0x400
+; NOFAST32-NEXT: movl $4096, %ecx # imm = 0x1000
; NOFAST32-NEXT: movl %esp, %edi
-; NOFAST32-NEXT: rep;movsl (%esi), %es:(%edi)
+; NOFAST32-NEXT: rep;movsb (%esi), %es:(%edi)
; NOFAST32-NEXT: calll foo at PLT
; NOFAST32-NEXT: addl $4100, %esp # imm = 0x1004
; NOFAST32-NEXT: popl %esi
@@ -106,9 +106,9 @@ define void @test2(ptr nocapture %x) nounwind minsize {
; NOFAST: # %bb.0:
; NOFAST-NEXT: subq $4104, %rsp # imm = 0x1008
; NOFAST-NEXT: movq %rdi, %rsi
-; NOFAST-NEXT: movl $512, %ecx # imm = 0x200
+; NOFAST-NEXT: movl $4096, %ecx # imm = 0x1000
; NOFAST-NEXT: movq %rsp, %rdi
-; NOFAST-NEXT: rep;movsq (%rsi), %es:(%rdi)
+; NOFAST-NEXT: rep;movsb (%rsi), %es:(%rdi)
; NOFAST-NEXT: callq foo at PLT
; NOFAST-NEXT: addq $4104, %rsp # imm = 0x1008
; NOFAST-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/memcpy.ll b/llvm/test/CodeGen/X86/memcpy.ll
index 6ec9b20163051b..ff026b142ecf3c 100644
--- a/llvm/test/CodeGen/X86/memcpy.ll
+++ b/llvm/test/CodeGen/X86/memcpy.ll
@@ -202,14 +202,16 @@ define void @test3_minsize(ptr nocapture %A, ptr nocapture %B) nounwind minsize
; DARWIN-LABEL: test3_minsize:
; DARWIN: ## %bb.0:
; DARWIN-NEXT: pushq $64
-; DARWIN-NEXT: popq %rdx
-; DARWIN-NEXT: jmp _memcpy ## TAILCALL
+; DARWIN-NEXT: popq %rcx
+; DARWIN-NEXT: rep;movsb (%rsi), %es:(%rdi)
+; DARWIN-NEXT: retq
;
; LINUX-LABEL: test3_minsize:
; LINUX: # %bb.0:
; LINUX-NEXT: pushq $64
-; LINUX-NEXT: popq %rdx
-; LINUX-NEXT: jmp memcpy at PLT # TAILCALL
+; LINUX-NEXT: popq %rcx
+; LINUX-NEXT: rep;movsb (%rsi), %es:(%rdi)
+; LINUX-NEXT: retq
;
; LINUX-SKL-LABEL: test3_minsize:
; LINUX-SKL: # %bb.0:
@@ -249,14 +251,16 @@ define void @test3_minsize_optsize(ptr nocapture %A, ptr nocapture %B) nounwind
; DARWIN-LABEL: test3_minsize_optsize:
; DARWIN: ## %bb.0:
; DARWIN-NEXT: pushq $64
-; DARWIN-NEXT: popq %rdx
-; DARWIN-NEXT: jmp _memcpy ## TAILCALL
+; DARWIN-NEXT: popq %rcx
+; DARWIN-NEXT: rep;movsb (%rsi), %es:(%rdi)
+; DARWIN-NEXT: retq
;
; LINUX-LABEL: test3_minsize_optsize:
; LINUX: # %bb.0:
; LINUX-NEXT: pushq $64
-; LINUX-NEXT: popq %rdx
-; LINUX-NEXT: jmp memcpy at PLT # TAILCALL
+; LINUX-NEXT: popq %rcx
+; LINUX-NEXT: rep;movsb (%rsi), %es:(%rdi)
+; LINUX-NEXT: retq
;
; LINUX-SKL-LABEL: test3_minsize_optsize:
; LINUX-SKL: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/memset-minsize.ll b/llvm/test/CodeGen/X86/memset-minsize.ll
index cc0f2156262bba..23399b07acfbd6 100644
--- a/llvm/test/CodeGen/X86/memset-minsize.ll
+++ b/llvm/test/CodeGen/X86/memset-minsize.ll
@@ -14,10 +14,9 @@ entry:
define void @small_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: small_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq $32
-; CHECK-NEXT: popq %rcx
+; CHECK-NEXT: movl $128, %ecx
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: rep;stosl %eax, %es:(%rdi)
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 128, i1 false)
@@ -27,11 +26,9 @@ entry:
define void @medium_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: medium_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $512, %edx # imm = 0x200
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: callq memset at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl $512, %ecx # imm = 0x200
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 512, i1 false)
@@ -41,11 +38,9 @@ entry:
define void @large_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: large_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $4096, %edx # imm = 0x1000
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: callq memset at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl $4096, %ecx # imm = 0x1000
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 4096, i1 false)
@@ -55,11 +50,9 @@ entry:
define void @huge_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: huge_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $8192, %edx # imm = 0x2000
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: callq memset at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl $8192, %ecx # imm = 0x2000
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 8192, i1 false)
@@ -69,11 +62,9 @@ entry:
define void @odd_length_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: odd_length_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $255, %edx
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: callq memset at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl $255, %ecx
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 255, i1 false)
@@ -83,11 +74,9 @@ entry:
define void @align_1_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: align_1_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $256, %edx # imm = 0x100
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: callq memset at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl $256, %ecx # imm = 0x100
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 1 %ptr, i8 0, i32 256, i1 false)
@@ -97,11 +86,9 @@ entry:
define void @align_2_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: align_2_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $256, %edx # imm = 0x100
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: callq memset at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl $256, %ecx # imm = 0x100
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 2 %ptr, i8 0, i32 256, i1 false)
@@ -111,11 +98,9 @@ entry:
define void @align_4_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: align_4_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $256, %edx # imm = 0x100
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: callq memset at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl $256, %ecx # imm = 0x100
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 4 %ptr, i8 0, i32 256, i1 false)
@@ -125,11 +110,9 @@ entry:
define void @align_8_memset_to_rep_stos(ptr %ptr) minsize nounwind {
; CHECK-LABEL: align_8_memset_to_rep_stos:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $256, %edx # imm = 0x100
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: callq memset at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: movl $256, %ecx # imm = 0x100
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i32(ptr align 8 %ptr, i8 0, i32 256, i1 false)
@@ -139,10 +122,9 @@ entry:
define void @small_memset_to_rep_stos_64(ptr %ptr) minsize nounwind {
; CHECK-LABEL: small_memset_to_rep_stos_64:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq $16
-; CHECK-NEXT: popq %rcx
+; CHECK-NEXT: movl $128, %ecx
; CHECK-NEXT: xorl %eax, %eax
-; CHECK-NEXT: rep;stosq %rax, %es:(%rdi)
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
entry:
call void @llvm.memset.p0.i64(ptr align 8 %ptr, i8 0, i64 128, i1 false)
diff --git a/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll b/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll
index 90a3f8e785e906..7786d380d7fb4a 100644
--- a/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll
+++ b/llvm/test/CodeGen/X86/memset-vs-memset-inline.ll
@@ -28,137 +28,10 @@ define void @regular_memset_calls_external_function(ptr %a, i8 %value) nounwind
define void @inlined_set_doesnt_call_external_function(ptr %a, i8 %value) nounwind {
; CHECK-LABEL: inlined_set_doesnt_call_external_function:
; CHECK: # %bb.0:
-; CHECK-NEXT: movzbl %sil, %ecx
-; CHECK-NEXT: movabsq $72340172838076673, %rax # imm = 0x101010101010101
-; CHECK-NEXT: imulq %rcx, %rax
-; CHECK-NEXT: movq %rax, 1016(%rdi)
-; CHECK-NEXT: movq %rax, 1008(%rdi)
-; CHECK-NEXT: movq %rax, 1000(%rdi)
-; CHECK-NEXT: movq %rax, 992(%rdi)
-; CHECK-NEXT: movq %rax, 984(%rdi)
-; CHECK-NEXT: movq %rax, 976(%rdi)
-; CHECK-NEXT: movq %rax, 968(%rdi)
-; CHECK-NEXT: movq %rax, 960(%rdi)
-; CHECK-NEXT: movq %rax, 952(%rdi)
-; CHECK-NEXT: movq %rax, 944(%rdi)
-; CHECK-NEXT: movq %rax, 936(%rdi)
-; CHECK-NEXT: movq %rax, 928(%rdi)
-; CHECK-NEXT: movq %rax, 920(%rdi)
-; CHECK-NEXT: movq %rax, 912(%rdi)
-; CHECK-NEXT: movq %rax, 904(%rdi)
-; CHECK-NEXT: movq %rax, 896(%rdi)
-; CHECK-NEXT: movq %rax, 888(%rdi)
-; CHECK-NEXT: movq %rax, 880(%rdi)
-; CHECK-NEXT: movq %rax, 872(%rdi)
-; CHECK-NEXT: movq %rax, 864(%rdi)
-; CHECK-NEXT: movq %rax, 856(%rdi)
-; CHECK-NEXT: movq %rax, 848(%rdi)
-; CHECK-NEXT: movq %rax, 840(%rdi)
-; CHECK-NEXT: movq %rax, 832(%rdi)
-; CHECK-NEXT: movq %rax, 824(%rdi)
-; CHECK-NEXT: movq %rax, 816(%rdi)
-; CHECK-NEXT: movq %rax, 808(%rdi)
-; CHECK-NEXT: movq %rax, 800(%rdi)
-; CHECK-NEXT: movq %rax, 792(%rdi)
-; CHECK-NEXT: movq %rax, 784(%rdi)
-; CHECK-NEXT: movq %rax, 776(%rdi)
-; CHECK-NEXT: movq %rax, 768(%rdi)
-; CHECK-NEXT: movq %rax, 760(%rdi)
-; CHECK-NEXT: movq %rax, 752(%rdi)
-; CHECK-NEXT: movq %rax, 744(%rdi)
-; CHECK-NEXT: movq %rax, 736(%rdi)
-; CHECK-NEXT: movq %rax, 728(%rdi)
-; CHECK-NEXT: movq %rax, 720(%rdi)
-; CHECK-NEXT: movq %rax, 712(%rdi)
-; CHECK-NEXT: movq %rax, 704(%rdi)
-; CHECK-NEXT: movq %rax, 696(%rdi)
-; CHECK-NEXT: movq %rax, 688(%rdi)
-; CHECK-NEXT: movq %rax, 680(%rdi)
-; CHECK-NEXT: movq %rax, 672(%rdi)
-; CHECK-NEXT: movq %rax, 664(%rdi)
-; CHECK-NEXT: movq %rax, 656(%rdi)
-; CHECK-NEXT: movq %rax, 648(%rdi)
-; CHECK-NEXT: movq %rax, 640(%rdi)
-; CHECK-NEXT: movq %rax, 632(%rdi)
-; CHECK-NEXT: movq %rax, 624(%rdi)
-; CHECK-NEXT: movq %rax, 616(%rdi)
-; CHECK-NEXT: movq %rax, 608(%rdi)
-; CHECK-NEXT: movq %rax, 600(%rdi)
-; CHECK-NEXT: movq %rax, 592(%rdi)
-; CHECK-NEXT: movq %rax, 584(%rdi)
-; CHECK-NEXT: movq %rax, 576(%rdi)
-; CHECK-NEXT: movq %rax, 568(%rdi)
-; CHECK-NEXT: movq %rax, 560(%rdi)
-; CHECK-NEXT: movq %rax, 552(%rdi)
-; CHECK-NEXT: movq %rax, 544(%rdi)
-; CHECK-NEXT: movq %rax, 536(%rdi)
-; CHECK-NEXT: movq %rax, 528(%rdi)
-; CHECK-NEXT: movq %rax, 520(%rdi)
-; CHECK-NEXT: movq %rax, 512(%rdi)
-; CHECK-NEXT: movq %rax, 504(%rdi)
-; CHECK-NEXT: movq %rax, 496(%rdi)
-; CHECK-NEXT: movq %rax, 488(%rdi)
-; CHECK-NEXT: movq %rax, 480(%rdi)
-; CHECK-NEXT: movq %rax, 472(%rdi)
-; CHECK-NEXT: movq %rax, 464(%rdi)
-; CHECK-NEXT: movq %rax, 456(%rdi)
-; CHECK-NEXT: movq %rax, 448(%rdi)
-; CHECK-NEXT: movq %rax, 440(%rdi)
-; CHECK-NEXT: movq %rax, 432(%rdi)
-; CHECK-NEXT: movq %rax, 424(%rdi)
-; CHECK-NEXT: movq %rax, 416(%rdi)
-; CHECK-NEXT: movq %rax, 408(%rdi)
-; CHECK-NEXT: movq %rax, 400(%rdi)
-; CHECK-NEXT: movq %rax, 392(%rdi)
-; CHECK-NEXT: movq %rax, 384(%rdi)
-; CHECK-NEXT: movq %rax, 376(%rdi)
-; CHECK-NEXT: movq %rax, 368(%rdi)
-; CHECK-NEXT: movq %rax, 360(%rdi)
-; CHECK-NEXT: movq %rax, 352(%rdi)
-; CHECK-NEXT: movq %rax, 344(%rdi)
-; CHECK-NEXT: movq %rax, 336(%rdi)
-; CHECK-NEXT: movq %rax, 328(%rdi)
-; CHECK-NEXT: movq %rax, 320(%rdi)
-; CHECK-NEXT: movq %rax, 312(%rdi)
-; CHECK-NEXT: movq %rax, 304(%rdi)
-; CHECK-NEXT: movq %rax, 296(%rdi)
-; CHECK-NEXT: movq %rax, 288(%rdi)
-; CHECK-NEXT: movq %rax, 280(%rdi)
-; CHECK-NEXT: movq %rax, 272(%rdi)
-; CHECK-NEXT: movq %rax, 264(%rdi)
-; CHECK-NEXT: movq %rax, 256(%rdi)
-; CHECK-NEXT: movq %rax, 248(%rdi)
-; CHECK-NEXT: movq %rax, 240(%rdi)
-; CHECK-NEXT: movq %rax, 232(%rdi)
-; CHECK-NEXT: movq %rax, 224(%rdi)
-; CHECK-NEXT: movq %rax, 216(%rdi)
-; CHECK-NEXT: movq %rax, 208(%rdi)
-; CHECK-NEXT: movq %rax, 200(%rdi)
-; CHECK-NEXT: movq %rax, 192(%rdi)
-; CHECK-NEXT: movq %rax, 184(%rdi)
-; CHECK-NEXT: movq %rax, 176(%rdi)
-; CHECK-NEXT: movq %rax, 168(%rdi)
-; CHECK-NEXT: movq %rax, 160(%rdi)
-; CHECK-NEXT: movq %rax, 152(%rdi)
-; CHECK-NEXT: movq %rax, 144(%rdi)
-; CHECK-NEXT: movq %rax, 136(%rdi)
-; CHECK-NEXT: movq %rax, 128(%rdi)
-; CHECK-NEXT: movq %rax, 120(%rdi)
-; CHECK-NEXT: movq %rax, 112(%rdi)
-; CHECK-NEXT: movq %rax, 104(%rdi)
-; CHECK-NEXT: movq %rax, 96(%rdi)
-; CHECK-NEXT: movq %rax, 88(%rdi)
-; CHECK-NEXT: movq %rax, 80(%rdi)
-; CHECK-NEXT: movq %rax, 72(%rdi)
-; CHECK-NEXT: movq %rax, 64(%rdi)
-; CHECK-NEXT: movq %rax, 56(%rdi)
-; CHECK-NEXT: movq %rax, 48(%rdi)
-; CHECK-NEXT: movq %rax, 40(%rdi)
-; CHECK-NEXT: movq %rax, 32(%rdi)
-; CHECK-NEXT: movq %rax, 24(%rdi)
-; CHECK-NEXT: movq %rax, 16(%rdi)
-; CHECK-NEXT: movq %rax, 8(%rdi)
-; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: movl $1024, %ecx # imm = 0x400
+; CHECK-NEXT: # kill: def $al killed $al killed $eax
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 %value, i64 1024, i1 0)
ret void
@@ -167,135 +40,9 @@ define void @inlined_set_doesnt_call_external_function(ptr %a, i8 %value) nounwi
define void @memset_inlined_insize(ptr %a) nounwind minsize {
; CHECK-LABEL: memset_inlined_insize:
; CHECK: # %bb.0:
-; CHECK-NEXT: movabsq $3038287259199220266, %rax # imm = 0x2A2A2A2A2A2A2A2A
-; CHECK-NEXT: movq %rax, 1016(%rdi)
-; CHECK-NEXT: movq %rax, 1008(%rdi)
-; CHECK-NEXT: movq %rax, 1000(%rdi)
-; CHECK-NEXT: movq %rax, 992(%rdi)
-; CHECK-NEXT: movq %rax, 984(%rdi)
-; CHECK-NEXT: movq %rax, 976(%rdi)
-; CHECK-NEXT: movq %rax, 968(%rdi)
-; CHECK-NEXT: movq %rax, 960(%rdi)
-; CHECK-NEXT: movq %rax, 952(%rdi)
-; CHECK-NEXT: movq %rax, 944(%rdi)
-; CHECK-NEXT: movq %rax, 936(%rdi)
-; CHECK-NEXT: movq %rax, 928(%rdi)
-; CHECK-NEXT: movq %rax, 920(%rdi)
-; CHECK-NEXT: movq %rax, 912(%rdi)
-; CHECK-NEXT: movq %rax, 904(%rdi)
-; CHECK-NEXT: movq %rax, 896(%rdi)
-; CHECK-NEXT: movq %rax, 888(%rdi)
-; CHECK-NEXT: movq %rax, 880(%rdi)
-; CHECK-NEXT: movq %rax, 872(%rdi)
-; CHECK-NEXT: movq %rax, 864(%rdi)
-; CHECK-NEXT: movq %rax, 856(%rdi)
-; CHECK-NEXT: movq %rax, 848(%rdi)
-; CHECK-NEXT: movq %rax, 840(%rdi)
-; CHECK-NEXT: movq %rax, 832(%rdi)
-; CHECK-NEXT: movq %rax, 824(%rdi)
-; CHECK-NEXT: movq %rax, 816(%rdi)
-; CHECK-NEXT: movq %rax, 808(%rdi)
-; CHECK-NEXT: movq %rax, 800(%rdi)
-; CHECK-NEXT: movq %rax, 792(%rdi)
-; CHECK-NEXT: movq %rax, 784(%rdi)
-; CHECK-NEXT: movq %rax, 776(%rdi)
-; CHECK-NEXT: movq %rax, 768(%rdi)
-; CHECK-NEXT: movq %rax, 760(%rdi)
-; CHECK-NEXT: movq %rax, 752(%rdi)
-; CHECK-NEXT: movq %rax, 744(%rdi)
-; CHECK-NEXT: movq %rax, 736(%rdi)
-; CHECK-NEXT: movq %rax, 728(%rdi)
-; CHECK-NEXT: movq %rax, 720(%rdi)
-; CHECK-NEXT: movq %rax, 712(%rdi)
-; CHECK-NEXT: movq %rax, 704(%rdi)
-; CHECK-NEXT: movq %rax, 696(%rdi)
-; CHECK-NEXT: movq %rax, 688(%rdi)
-; CHECK-NEXT: movq %rax, 680(%rdi)
-; CHECK-NEXT: movq %rax, 672(%rdi)
-; CHECK-NEXT: movq %rax, 664(%rdi)
-; CHECK-NEXT: movq %rax, 656(%rdi)
-; CHECK-NEXT: movq %rax, 648(%rdi)
-; CHECK-NEXT: movq %rax, 640(%rdi)
-; CHECK-NEXT: movq %rax, 632(%rdi)
-; CHECK-NEXT: movq %rax, 624(%rdi)
-; CHECK-NEXT: movq %rax, 616(%rdi)
-; CHECK-NEXT: movq %rax, 608(%rdi)
-; CHECK-NEXT: movq %rax, 600(%rdi)
-; CHECK-NEXT: movq %rax, 592(%rdi)
-; CHECK-NEXT: movq %rax, 584(%rdi)
-; CHECK-NEXT: movq %rax, 576(%rdi)
-; CHECK-NEXT: movq %rax, 568(%rdi)
-; CHECK-NEXT: movq %rax, 560(%rdi)
-; CHECK-NEXT: movq %rax, 552(%rdi)
-; CHECK-NEXT: movq %rax, 544(%rdi)
-; CHECK-NEXT: movq %rax, 536(%rdi)
-; CHECK-NEXT: movq %rax, 528(%rdi)
-; CHECK-NEXT: movq %rax, 520(%rdi)
-; CHECK-NEXT: movq %rax, 512(%rdi)
-; CHECK-NEXT: movq %rax, 504(%rdi)
-; CHECK-NEXT: movq %rax, 496(%rdi)
-; CHECK-NEXT: movq %rax, 488(%rdi)
-; CHECK-NEXT: movq %rax, 480(%rdi)
-; CHECK-NEXT: movq %rax, 472(%rdi)
-; CHECK-NEXT: movq %rax, 464(%rdi)
-; CHECK-NEXT: movq %rax, 456(%rdi)
-; CHECK-NEXT: movq %rax, 448(%rdi)
-; CHECK-NEXT: movq %rax, 440(%rdi)
-; CHECK-NEXT: movq %rax, 432(%rdi)
-; CHECK-NEXT: movq %rax, 424(%rdi)
-; CHECK-NEXT: movq %rax, 416(%rdi)
-; CHECK-NEXT: movq %rax, 408(%rdi)
-; CHECK-NEXT: movq %rax, 400(%rdi)
-; CHECK-NEXT: movq %rax, 392(%rdi)
-; CHECK-NEXT: movq %rax, 384(%rdi)
-; CHECK-NEXT: movq %rax, 376(%rdi)
-; CHECK-NEXT: movq %rax, 368(%rdi)
-; CHECK-NEXT: movq %rax, 360(%rdi)
-; CHECK-NEXT: movq %rax, 352(%rdi)
-; CHECK-NEXT: movq %rax, 344(%rdi)
-; CHECK-NEXT: movq %rax, 336(%rdi)
-; CHECK-NEXT: movq %rax, 328(%rdi)
-; CHECK-NEXT: movq %rax, 320(%rdi)
-; CHECK-NEXT: movq %rax, 312(%rdi)
-; CHECK-NEXT: movq %rax, 304(%rdi)
-; CHECK-NEXT: movq %rax, 296(%rdi)
-; CHECK-NEXT: movq %rax, 288(%rdi)
-; CHECK-NEXT: movq %rax, 280(%rdi)
-; CHECK-NEXT: movq %rax, 272(%rdi)
-; CHECK-NEXT: movq %rax, 264(%rdi)
-; CHECK-NEXT: movq %rax, 256(%rdi)
-; CHECK-NEXT: movq %rax, 248(%rdi)
-; CHECK-NEXT: movq %rax, 240(%rdi)
-; CHECK-NEXT: movq %rax, 232(%rdi)
-; CHECK-NEXT: movq %rax, 224(%rdi)
-; CHECK-NEXT: movq %rax, 216(%rdi)
-; CHECK-NEXT: movq %rax, 208(%rdi)
-; CHECK-NEXT: movq %rax, 200(%rdi)
-; CHECK-NEXT: movq %rax, 192(%rdi)
-; CHECK-NEXT: movq %rax, 184(%rdi)
-; CHECK-NEXT: movq %rax, 176(%rdi)
-; CHECK-NEXT: movq %rax, 168(%rdi)
-; CHECK-NEXT: movq %rax, 160(%rdi)
-; CHECK-NEXT: movq %rax, 152(%rdi)
-; CHECK-NEXT: movq %rax, 144(%rdi)
-; CHECK-NEXT: movq %rax, 136(%rdi)
-; CHECK-NEXT: movq %rax, 128(%rdi)
-; CHECK-NEXT: movq %rax, 120(%rdi)
-; CHECK-NEXT: movq %rax, 112(%rdi)
-; CHECK-NEXT: movq %rax, 104(%rdi)
-; CHECK-NEXT: movq %rax, 96(%rdi)
-; CHECK-NEXT: movq %rax, 88(%rdi)
-; CHECK-NEXT: movq %rax, 80(%rdi)
-; CHECK-NEXT: movq %rax, 72(%rdi)
-; CHECK-NEXT: movq %rax, 64(%rdi)
-; CHECK-NEXT: movq %rax, 56(%rdi)
-; CHECK-NEXT: movq %rax, 48(%rdi)
-; CHECK-NEXT: movq %rax, 40(%rdi)
-; CHECK-NEXT: movq %rax, 32(%rdi)
-; CHECK-NEXT: movq %rax, 24(%rdi)
-; CHECK-NEXT: movq %rax, 16(%rdi)
-; CHECK-NEXT: movq %rax, 8(%rdi)
-; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: movl $1024, %ecx # imm = 0x400
+; CHECK-NEXT: movb $42, %al
+; CHECK-NEXT: rep;stosb %al, %es:(%rdi)
; CHECK-NEXT: retq
tail call void @llvm.memset.inline.p0.i64(ptr %a, i8 42, i64 1024, i1 0)
ret void
More information about the llvm-commits
mailing list