[llvm-branch-commits] [llvm] [X86] Remove extra MOV after widening atomic store (PR #197619)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue May 19 06:28:48 PDT 2026
https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/197619
>From 7b098913b7a3608b5b142f7c9c22a23ad2f026f1 Mon Sep 17 00:00:00 2001
From: jofrn <jo7frn1 at gmail.com>
Date: Wed, 13 May 2026 22:27:20 -0700
Subject: [PATCH] [X86] Remove extra MOV after widening atomic store
This change adds patterns to optimize out an extra MOV present after
widening the atomic store. Covers <2 x i8> (SSE4.1+), <2 x i16>,
<4 x i8>, <2 x i32>, <2 x float>, <4 x i16>, <2 x ptr addrspace(270)>.
---
.../include/llvm/Target/TargetSelectionDAG.td | 35 ++++++
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
llvm/lib/Target/X86/X86InstrAVX512.td | 12 +-
llvm/lib/Target/X86/X86InstrSSE.td | 20 ++--
llvm/test/CodeGen/X86/atomic-load-store.ll | 111 ++++++++----------
llvm/test/CodeGen/X86/atomic-unordered.ll | 54 +++++----
6 files changed, 129 insertions(+), 105 deletions(-)
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 35848f76897b3..c73bb6bf3de52 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -2393,6 +2393,41 @@ def atomic_store_128 :
let MemoryVT = i128;
}
+// Matches store or atomic_store, no alignment requirement.
+def any_store : PatFrags<(ops node:$val, node:$ptr),
+ [(store node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)]>;
+
+def any_store_8 : PatFrags<(ops node:$val, node:$ptr),
+ [(store node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)]> {
+ let MemoryVT = i8;
+}
+
+def any_store_16 : PatFrags<(ops node:$val, node:$ptr),
+ [(store node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)]> {
+ let MemoryVT = i16;
+}
+
+def any_store_32 : PatFrags<(ops node:$val, node:$ptr),
+ [(store node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)]> {
+ let MemoryVT = i32;
+}
+
+def any_store_64 : PatFrags<(ops node:$val, node:$ptr),
+ [(store node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)]> {
+ let MemoryVT = i64;
+}
+
+def any_store_128 : PatFrags<(ops node:$val, node:$ptr),
+ [(store node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)]> {
+ let MemoryVT = i128;
+}
+
//===----------------------------------------------------------------------===//
// Selection DAG Pattern Support.
//
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index fea1caf0854f5..fe11137b86e53 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2949,7 +2949,7 @@ bool X86::mayFoldIntoStore(SDValue Op) {
return false;
User = *User->user_begin();
}
- return ISD::isNormalStore(User);
+ return ISD::isNormalStore(User) || User->getOpcode() == ISD::ATOMIC_STORE;
}
bool X86::mayFoldIntoZeroExtend(SDValue Op) {
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 95c75165ed4eb..d8367412be587 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -3869,8 +3869,8 @@ def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$s
def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),
(ins i32mem:$dst, VR128X:$src),
"vmovd\t{$src, $dst|$dst, $src}",
- [(store (i32 (extractelt (v4i32 VR128X:$src),
- (iPTR 0))), addr:$dst)]>,
+ [(any_store_32 (i32 (extractelt (v4i32 VR128X:$src),
+ (iPTR 0))), addr:$dst)]>,
EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecStore]>;
} // ExeDomain = SSEPackedInt
@@ -3893,8 +3893,8 @@ def VMOVPQIto64Zmr : I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src),
def VMOVPQI2QIZmr : I<0xD6, MRMDestMem, (outs),
(ins i64mem:$dst, VR128X:$src),
"vmovq\t{$src, $dst|$dst, $src}",
- [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)),
- addr:$dst)]>,
+ [(any_store_64 (extractelt (v2i64 VR128X:$src), (iPTR 0)),
+ addr:$dst)]>,
EVEX, TB, PD, REX_W, EVEX_CD8<64, CD8VT1>,
Sched<[WriteVecStore]>, Requires<[HasAVX512]>;
@@ -11476,8 +11476,8 @@ multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
def mri : AVX512Ii8<opc, MRMDestMem, (outs),
(ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),
OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
- [(store (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), timm:$src2))),
- addr:$dst)]>,
+ [(any_store (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), timm:$src2))),
+ addr:$dst)]>,
EVEX, EVEX_CD8<_.EltSize, CD8VT1>, Sched<[WriteVecExtractSt]>;
}
diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index 203344cd81aab..e93b3c91b7861 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -4206,8 +4206,8 @@ def VMOVPDI2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),
def VMOVPDI2DImr : VS2I<0x7E, MRMDestMem, (outs),
(ins i32mem:$dst, VR128:$src),
"movd\t{$src, $dst|$dst, $src}",
- [(store (i32 (extractelt (v4i32 VR128:$src),
- (iPTR 0))), addr:$dst)]>,
+ [(any_store_32 (i32 (extractelt (v4i32 VR128:$src),
+ (iPTR 0))), addr:$dst)]>,
VEX, Sched<[WriteVecStore]>;
def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),
"movd\t{$src, $dst|$dst, $src}",
@@ -4216,8 +4216,8 @@ def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),
Sched<[WriteVecMoveToGpr]>;
def MOVPDI2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src),
"movd\t{$src, $dst|$dst, $src}",
- [(store (i32 (extractelt (v4i32 VR128:$src),
- (iPTR 0))), addr:$dst)]>,
+ [(any_store_32 (i32 (extractelt (v4i32 VR128:$src),
+ (iPTR 0))), addr:$dst)]>,
Sched<[WriteVecStore]>;
} // ExeDomain = SSEPackedInt
@@ -4346,13 +4346,13 @@ def MOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
let ExeDomain = SSEPackedInt, SchedRW = [WriteVecStore] in {
def VMOVPQI2QImr : VS2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),
"movq\t{$src, $dst|$dst, $src}",
- [(store (i64 (extractelt (v2i64 VR128:$src),
- (iPTR 0))), addr:$dst)]>,
+ [(any_store_64 (i64 (extractelt (v2i64 VR128:$src),
+ (iPTR 0))), addr:$dst)]>,
VEX, WIG;
def MOVPQI2QImr : S2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),
"movq\t{$src, $dst|$dst, $src}",
- [(store (i64 (extractelt (v2i64 VR128:$src),
- (iPTR 0))), addr:$dst)]>;
+ [(any_store_64 (i64 (extractelt (v2i64 VR128:$src),
+ (iPTR 0))), addr:$dst)]>;
} // ExeDomain, SchedRW
// For disassembler only
@@ -5280,8 +5280,8 @@ multiclass SS41I_extract16<bits<8> opc, string OpcodeStr> {
(ins i16mem:$dst, VR128:$src1, u8imm:$src2),
!strconcat(OpcodeStr,
"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
- [(store (i16 (trunc (X86pextrw (v8i16 VR128:$src1), timm:$src2))),
- addr:$dst)]>, Sched<[WriteVecExtractSt]>;
+ [(any_store_16 (i16 (trunc (X86pextrw (v8i16 VR128:$src1), timm:$src2))),
+ addr:$dst)]>, Sched<[WriteVecExtractSt]>;
}
let Predicates = [HasAVX, NoBWI] in
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll
index 659cdec91d3e7..91c4d0a3d8c1c 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -353,30 +353,37 @@ define void @store_atomic_vec1_double_align(ptr %x, <1 x double> %v) nounwind {
}
define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) {
-; CHECK-SSE-O3-LABEL: store_atomic_vec2_i8:
-; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O3-NEXT: movw %ax, (%rdi)
-; CHECK-SSE-O3-NEXT: retq
+; CHECK-SSE2-O3-LABEL: store_atomic_vec2_i8:
+; CHECK-SSE2-O3: # %bb.0:
+; CHECK-SSE2-O3-NEXT: movd %xmm0, %eax
+; CHECK-SSE2-O3-NEXT: movw %ax, (%rdi)
+; CHECK-SSE2-O3-NEXT: retq
+;
+; CHECK-SSE4-O3-LABEL: store_atomic_vec2_i8:
+; CHECK-SSE4-O3: # %bb.0:
+; CHECK-SSE4-O3-NEXT: pextrw $0, %xmm0, (%rdi)
+; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O3-NEXT: movw %ax, (%rdi)
+; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
-; CHECK-SSE-O0-LABEL: store_atomic_vec2_i8:
-; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE-O0-NEXT: movw %ax, (%rdi)
-; CHECK-SSE-O0-NEXT: retq
+; CHECK-SSE2-O0-LABEL: store_atomic_vec2_i8:
+; CHECK-SSE2-O0: # %bb.0:
+; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
+; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE2-O0-NEXT: movw %ax, (%rdi)
+; CHECK-SSE2-O0-NEXT: retq
+;
+; CHECK-SSE4-O0-LABEL: store_atomic_vec2_i8:
+; CHECK-SSE4-O0: # %bb.0:
+; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, (%rdi)
+; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-AVX-O0-NEXT: movw %ax, (%rdi)
+; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i8> %v, ptr %x release, align 4
ret void
@@ -385,26 +392,22 @@ define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) {
define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O3-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O3-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O0-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i16> %v, ptr %x release, align 4
ret void
@@ -413,26 +416,22 @@ define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) {
define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O3-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O3-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x ptr addrspace(270)> %v, ptr %x release, align 8
ret void
@@ -441,26 +440,22 @@ define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) {
define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O3-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O3-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i32> %v, ptr %x release, align 8
ret void
@@ -469,26 +464,22 @@ define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) {
define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O3-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O3-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x float> %v, ptr %x release, align 8
ret void
@@ -497,26 +488,22 @@ define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) {
define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O3-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O3-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O0-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x i8> %v, ptr %x release, align 4
ret void
@@ -525,26 +512,22 @@ define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind {
define void @store_atomic_vec4_i16(ptr %x, <4 x i16> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O3-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O3-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x i16> %v, ptr %x release, align 8
ret void
diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll
index 7946204865ab6..edfdec37150c2 100644
--- a/llvm/test/CodeGen/X86/atomic-unordered.ll
+++ b/llvm/test/CodeGen/X86/atomic-unordered.ll
@@ -348,18 +348,16 @@ define void @store_i256(ptr %ptr, i256 %v) {
define void @vec_store(ptr %p0, <2 x i32> %vec) {
; CHECK-O0-LABEL: vec_store:
; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: vmovd %xmm0, %ecx
; CHECK-O0-NEXT: vpextrd $1, %xmm0, %eax
-; CHECK-O0-NEXT: movl %ecx, (%rdi)
+; CHECK-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-O0-NEXT: movl %eax, 4(%rdi)
; CHECK-O0-NEXT: retq
;
; CHECK-O3-LABEL: vec_store:
; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-O3-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-O3-NEXT: movl %eax, (%rdi)
-; CHECK-O3-NEXT: movl %ecx, 4(%rdi)
+; CHECK-O3-NEXT: vextractps $1, %xmm0, %eax
+; CHECK-O3-NEXT: vmovss %xmm0, (%rdi)
+; CHECK-O3-NEXT: movl %eax, 4(%rdi)
; CHECK-O3-NEXT: retq
%v1 = extractelement <2 x i32> %vec, i32 0
%v2 = extractelement <2 x i32> %vec, i32 1
@@ -373,18 +371,16 @@ define void @vec_store(ptr %p0, <2 x i32> %vec) {
define void @vec_store_unaligned(ptr %p0, <2 x i32> %vec) {
; CHECK-O0-LABEL: vec_store_unaligned:
; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: vmovd %xmm0, %ecx
; CHECK-O0-NEXT: vpextrd $1, %xmm0, %eax
-; CHECK-O0-NEXT: movl %ecx, (%rdi)
+; CHECK-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-O0-NEXT: movl %eax, 4(%rdi)
; CHECK-O0-NEXT: retq
;
; CHECK-O3-LABEL: vec_store_unaligned:
; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-O3-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-O3-NEXT: movl %eax, (%rdi)
-; CHECK-O3-NEXT: movl %ecx, 4(%rdi)
+; CHECK-O3-NEXT: vextractps $1, %xmm0, %eax
+; CHECK-O3-NEXT: vmovss %xmm0, (%rdi)
+; CHECK-O3-NEXT: movl %eax, 4(%rdi)
; CHECK-O3-NEXT: retq
%v1 = extractelement <2 x i32> %vec, i32 0
%v2 = extractelement <2 x i32> %vec, i32 1
@@ -399,12 +395,17 @@ define void @vec_store_unaligned(ptr %p0, <2 x i32> %vec) {
; Legal if wider type is also atomic (TODO)
; Also, can avoid register move from xmm to eax (TODO)
define void @widen_broadcast2(ptr %p0, <2 x i32> %vec) {
-; CHECK-LABEL: widen_broadcast2:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: movl %eax, (%rdi)
-; CHECK-NEXT: movl %eax, 4(%rdi)
-; CHECK-NEXT: retq
+; CHECK-O0-LABEL: widen_broadcast2:
+; CHECK-O0: # %bb.0:
+; CHECK-O0-NEXT: vmovd %xmm0, (%rdi)
+; CHECK-O0-NEXT: vmovd %xmm0, 4(%rdi)
+; CHECK-O0-NEXT: retq
+;
+; CHECK-O3-LABEL: widen_broadcast2:
+; CHECK-O3: # %bb.0:
+; CHECK-O3-NEXT: vmovss %xmm0, (%rdi)
+; CHECK-O3-NEXT: vmovss %xmm0, 4(%rdi)
+; CHECK-O3-NEXT: retq
%v1 = extractelement <2 x i32> %vec, i32 0
%p1 = getelementptr i32, ptr %p0, i64 1
store atomic i32 %v1, ptr %p0 unordered, align 8
@@ -414,12 +415,17 @@ define void @widen_broadcast2(ptr %p0, <2 x i32> %vec) {
; Not legal to widen due to alignment restriction
define void @widen_broadcast2_unaligned(ptr %p0, <2 x i32> %vec) {
-; CHECK-LABEL: widen_broadcast2_unaligned:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: movl %eax, (%rdi)
-; CHECK-NEXT: movl %eax, 4(%rdi)
-; CHECK-NEXT: retq
+; CHECK-O0-LABEL: widen_broadcast2_unaligned:
+; CHECK-O0: # %bb.0:
+; CHECK-O0-NEXT: vmovd %xmm0, (%rdi)
+; CHECK-O0-NEXT: vmovd %xmm0, 4(%rdi)
+; CHECK-O0-NEXT: retq
+;
+; CHECK-O3-LABEL: widen_broadcast2_unaligned:
+; CHECK-O3: # %bb.0:
+; CHECK-O3-NEXT: vmovss %xmm0, (%rdi)
+; CHECK-O3-NEXT: vmovss %xmm0, 4(%rdi)
+; CHECK-O3-NEXT: retq
%v1 = extractelement <2 x i32> %vec, i32 0
%p1 = getelementptr i32, ptr %p0, i64 1
store atomic i32 %v1, ptr %p0 unordered, align 4
More information about the llvm-branch-commits
mailing list