[llvm-branch-commits] [llvm] [X86] Remove extra MOV after widening atomic store (PR #197619)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat May 16 06:00:27 PDT 2026
https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/197619
>From a730eaf771448ace34e2ea5ca8ff82a5b8b60637 Mon Sep 17 00:00:00 2001
From: jofrn <165626406+jofrn at users.noreply.github.com>
Date: Wed, 13 May 2026 22:27:20 -0700
Subject: [PATCH] [X86] Remove extra MOV after widening atomic store
This change adds patterns to optimize out an extra MOV present after
widening the atomic store. Covers <2 x i8> (SSE4.1+), <2 x i16>,
<4 x i8>, <2 x i32>, <2 x float>, <4 x i16>, <2 x ptr addrspace(270)>.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
llvm/lib/Target/X86/X86InstrAVX512.td | 12 +-
llvm/lib/Target/X86/X86InstrFragmentsSIMD.td | 5 +
llvm/lib/Target/X86/X86InstrSSE.td | 20 ++--
llvm/test/CodeGen/X86/atomic-load-store.ll | 111 ++++++++-----------
llvm/test/CodeGen/X86/atomic-unordered.ll | 54 +++++----
6 files changed, 99 insertions(+), 105 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index fea1caf0854f5..fe11137b86e53 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2949,7 +2949,7 @@ bool X86::mayFoldIntoStore(SDValue Op) {
return false;
User = *User->user_begin();
}
- return ISD::isNormalStore(User);
+ return ISD::isNormalStore(User) || User->getOpcode() == ISD::ATOMIC_STORE;
}
bool X86::mayFoldIntoZeroExtend(SDValue Op) {
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 95c75165ed4eb..7e0054ef9e5f6 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -3869,8 +3869,8 @@ def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$s
def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),
(ins i32mem:$dst, VR128X:$src),
"vmovd\t{$src, $dst|$dst, $src}",
- [(store (i32 (extractelt (v4i32 VR128X:$src),
- (iPTR 0))), addr:$dst)]>,
+ [(memstore (i32 (extractelt (v4i32 VR128X:$src),
+ (iPTR 0))), addr:$dst)]>,
EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecStore]>;
} // ExeDomain = SSEPackedInt
@@ -3893,8 +3893,8 @@ def VMOVPQIto64Zmr : I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src),
def VMOVPQI2QIZmr : I<0xD6, MRMDestMem, (outs),
(ins i64mem:$dst, VR128X:$src),
"vmovq\t{$src, $dst|$dst, $src}",
- [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)),
- addr:$dst)]>,
+ [(memstore (extractelt (v2i64 VR128X:$src), (iPTR 0)),
+ addr:$dst)]>,
EVEX, TB, PD, REX_W, EVEX_CD8<64, CD8VT1>,
Sched<[WriteVecStore]>, Requires<[HasAVX512]>;
@@ -11476,8 +11476,8 @@ multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
def mri : AVX512Ii8<opc, MRMDestMem, (outs),
(ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),
OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
- [(store (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), timm:$src2))),
- addr:$dst)]>,
+ [(memstore (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), timm:$src2))),
+ addr:$dst)]>,
EVEX, EVEX_CD8<_.EltSize, CD8VT1>, Sched<[WriteVecExtractSt]>;
}
diff --git a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
index c3e11fd1d5e42..ea06b34e52cdf 100644
--- a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -1254,6 +1254,11 @@ def extloadv4f16 : PatFrag<(ops node:$ptr), (extloadvf16 node:$ptr)>;
def extloadv8f16 : PatFrag<(ops node:$ptr), (extloadvf16 node:$ptr)>;
def extloadv16f16 : PatFrag<(ops node:$ptr), (extloadvf16 node:$ptr)>;
+// Matches either 'store' or 'atomic_store' (any alignment, any ordering).
+def memstore : PatFrags<(ops node:$val, node:$ptr),
+ [(store node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)]>;
+
// Like 'store', but always requires vector size alignment.
def alignedstore : PatFrag<(ops node:$val, node:$ptr),
(store node:$val, node:$ptr), [{
diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index 203344cd81aab..de1293b969968 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -4206,8 +4206,8 @@ def VMOVPDI2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),
def VMOVPDI2DImr : VS2I<0x7E, MRMDestMem, (outs),
(ins i32mem:$dst, VR128:$src),
"movd\t{$src, $dst|$dst, $src}",
- [(store (i32 (extractelt (v4i32 VR128:$src),
- (iPTR 0))), addr:$dst)]>,
+ [(memstore (i32 (extractelt (v4i32 VR128:$src),
+ (iPTR 0))), addr:$dst)]>,
VEX, Sched<[WriteVecStore]>;
def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),
"movd\t{$src, $dst|$dst, $src}",
@@ -4216,8 +4216,8 @@ def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src),
Sched<[WriteVecMoveToGpr]>;
def MOVPDI2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src),
"movd\t{$src, $dst|$dst, $src}",
- [(store (i32 (extractelt (v4i32 VR128:$src),
- (iPTR 0))), addr:$dst)]>,
+ [(memstore (i32 (extractelt (v4i32 VR128:$src),
+ (iPTR 0))), addr:$dst)]>,
Sched<[WriteVecStore]>;
} // ExeDomain = SSEPackedInt
@@ -4346,13 +4346,13 @@ def MOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
let ExeDomain = SSEPackedInt, SchedRW = [WriteVecStore] in {
def VMOVPQI2QImr : VS2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),
"movq\t{$src, $dst|$dst, $src}",
- [(store (i64 (extractelt (v2i64 VR128:$src),
- (iPTR 0))), addr:$dst)]>,
+ [(memstore (i64 (extractelt (v2i64 VR128:$src),
+ (iPTR 0))), addr:$dst)]>,
VEX, WIG;
def MOVPQI2QImr : S2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src),
"movq\t{$src, $dst|$dst, $src}",
- [(store (i64 (extractelt (v2i64 VR128:$src),
- (iPTR 0))), addr:$dst)]>;
+ [(memstore (i64 (extractelt (v2i64 VR128:$src),
+ (iPTR 0))), addr:$dst)]>;
} // ExeDomain, SchedRW
// For disassembler only
@@ -5280,8 +5280,8 @@ multiclass SS41I_extract16<bits<8> opc, string OpcodeStr> {
(ins i16mem:$dst, VR128:$src1, u8imm:$src2),
!strconcat(OpcodeStr,
"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
- [(store (i16 (trunc (X86pextrw (v8i16 VR128:$src1), timm:$src2))),
- addr:$dst)]>, Sched<[WriteVecExtractSt]>;
+ [(memstore (i16 (trunc (X86pextrw (v8i16 VR128:$src1), timm:$src2))),
+ addr:$dst)]>, Sched<[WriteVecExtractSt]>;
}
let Predicates = [HasAVX, NoBWI] in
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll
index 659cdec91d3e7..91c4d0a3d8c1c 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -353,30 +353,37 @@ define void @store_atomic_vec1_double_align(ptr %x, <1 x double> %v) nounwind {
}
define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) {
-; CHECK-SSE-O3-LABEL: store_atomic_vec2_i8:
-; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O3-NEXT: movw %ax, (%rdi)
-; CHECK-SSE-O3-NEXT: retq
+; CHECK-SSE2-O3-LABEL: store_atomic_vec2_i8:
+; CHECK-SSE2-O3: # %bb.0:
+; CHECK-SSE2-O3-NEXT: movd %xmm0, %eax
+; CHECK-SSE2-O3-NEXT: movw %ax, (%rdi)
+; CHECK-SSE2-O3-NEXT: retq
+;
+; CHECK-SSE4-O3-LABEL: store_atomic_vec2_i8:
+; CHECK-SSE4-O3: # %bb.0:
+; CHECK-SSE4-O3-NEXT: pextrw $0, %xmm0, (%rdi)
+; CHECK-SSE4-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O3-NEXT: movw %ax, (%rdi)
+; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
-; CHECK-SSE-O0-LABEL: store_atomic_vec2_i8:
-; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-SSE-O0-NEXT: movw %ax, (%rdi)
-; CHECK-SSE-O0-NEXT: retq
+; CHECK-SSE2-O0-LABEL: store_atomic_vec2_i8:
+; CHECK-SSE2-O0: # %bb.0:
+; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
+; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax
+; CHECK-SSE2-O0-NEXT: movw %ax, (%rdi)
+; CHECK-SSE2-O0-NEXT: retq
+;
+; CHECK-SSE4-O0-LABEL: store_atomic_vec2_i8:
+; CHECK-SSE4-O0: # %bb.0:
+; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, (%rdi)
+; CHECK-SSE4-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax
-; CHECK-AVX-O0-NEXT: movw %ax, (%rdi)
+; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i8> %v, ptr %x release, align 4
ret void
@@ -385,26 +392,22 @@ define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) {
define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O3-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O3-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O0-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i16> %v, ptr %x release, align 4
ret void
@@ -413,26 +416,22 @@ define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) {
define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O3-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O3-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x ptr addrspace(270)> %v, ptr %x release, align 8
ret void
@@ -441,26 +440,22 @@ define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) {
define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O3-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O3-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x i32> %v, ptr %x release, align 8
ret void
@@ -469,26 +464,22 @@ define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) {
define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O3-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O3-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <2 x float> %v, ptr %x release, align 8
ret void
@@ -497,26 +488,22 @@ define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) {
define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O3-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O3-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE-O0-NEXT: movl %eax, (%rdi)
+; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x i8> %v, ptr %x release, align 4
ret void
@@ -525,26 +512,22 @@ define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind {
define void @store_atomic_vec4_i16(ptr %x, <4 x i16> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O3-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O3-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq %xmm0, %rax
-; CHECK-SSE-O0-NEXT: movq %rax, (%rdi)
+; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT: retq
store atomic <4 x i16> %v, ptr %x release, align 8
ret void
diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll
index 7946204865ab6..edfdec37150c2 100644
--- a/llvm/test/CodeGen/X86/atomic-unordered.ll
+++ b/llvm/test/CodeGen/X86/atomic-unordered.ll
@@ -348,18 +348,16 @@ define void @store_i256(ptr %ptr, i256 %v) {
define void @vec_store(ptr %p0, <2 x i32> %vec) {
; CHECK-O0-LABEL: vec_store:
; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: vmovd %xmm0, %ecx
; CHECK-O0-NEXT: vpextrd $1, %xmm0, %eax
-; CHECK-O0-NEXT: movl %ecx, (%rdi)
+; CHECK-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-O0-NEXT: movl %eax, 4(%rdi)
; CHECK-O0-NEXT: retq
;
; CHECK-O3-LABEL: vec_store:
; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-O3-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-O3-NEXT: movl %eax, (%rdi)
-; CHECK-O3-NEXT: movl %ecx, 4(%rdi)
+; CHECK-O3-NEXT: vextractps $1, %xmm0, %eax
+; CHECK-O3-NEXT: vmovss %xmm0, (%rdi)
+; CHECK-O3-NEXT: movl %eax, 4(%rdi)
; CHECK-O3-NEXT: retq
%v1 = extractelement <2 x i32> %vec, i32 0
%v2 = extractelement <2 x i32> %vec, i32 1
@@ -373,18 +371,16 @@ define void @vec_store(ptr %p0, <2 x i32> %vec) {
define void @vec_store_unaligned(ptr %p0, <2 x i32> %vec) {
; CHECK-O0-LABEL: vec_store_unaligned:
; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: vmovd %xmm0, %ecx
; CHECK-O0-NEXT: vpextrd $1, %xmm0, %eax
-; CHECK-O0-NEXT: movl %ecx, (%rdi)
+; CHECK-O0-NEXT: vmovd %xmm0, (%rdi)
; CHECK-O0-NEXT: movl %eax, 4(%rdi)
; CHECK-O0-NEXT: retq
;
; CHECK-O3-LABEL: vec_store_unaligned:
; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: vmovd %xmm0, %eax
-; CHECK-O3-NEXT: vpextrd $1, %xmm0, %ecx
-; CHECK-O3-NEXT: movl %eax, (%rdi)
-; CHECK-O3-NEXT: movl %ecx, 4(%rdi)
+; CHECK-O3-NEXT: vextractps $1, %xmm0, %eax
+; CHECK-O3-NEXT: vmovss %xmm0, (%rdi)
+; CHECK-O3-NEXT: movl %eax, 4(%rdi)
; CHECK-O3-NEXT: retq
%v1 = extractelement <2 x i32> %vec, i32 0
%v2 = extractelement <2 x i32> %vec, i32 1
@@ -399,12 +395,17 @@ define void @vec_store_unaligned(ptr %p0, <2 x i32> %vec) {
; Legal if wider type is also atomic (TODO)
; Also, can avoid register move from xmm to eax (TODO)
define void @widen_broadcast2(ptr %p0, <2 x i32> %vec) {
-; CHECK-LABEL: widen_broadcast2:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: movl %eax, (%rdi)
-; CHECK-NEXT: movl %eax, 4(%rdi)
-; CHECK-NEXT: retq
+; CHECK-O0-LABEL: widen_broadcast2:
+; CHECK-O0: # %bb.0:
+; CHECK-O0-NEXT: vmovd %xmm0, (%rdi)
+; CHECK-O0-NEXT: vmovd %xmm0, 4(%rdi)
+; CHECK-O0-NEXT: retq
+;
+; CHECK-O3-LABEL: widen_broadcast2:
+; CHECK-O3: # %bb.0:
+; CHECK-O3-NEXT: vmovss %xmm0, (%rdi)
+; CHECK-O3-NEXT: vmovss %xmm0, 4(%rdi)
+; CHECK-O3-NEXT: retq
%v1 = extractelement <2 x i32> %vec, i32 0
%p1 = getelementptr i32, ptr %p0, i64 1
store atomic i32 %v1, ptr %p0 unordered, align 8
@@ -414,12 +415,17 @@ define void @widen_broadcast2(ptr %p0, <2 x i32> %vec) {
; Not legal to widen due to alignment restriction
define void @widen_broadcast2_unaligned(ptr %p0, <2 x i32> %vec) {
-; CHECK-LABEL: widen_broadcast2_unaligned:
-; CHECK: # %bb.0:
-; CHECK-NEXT: vmovd %xmm0, %eax
-; CHECK-NEXT: movl %eax, (%rdi)
-; CHECK-NEXT: movl %eax, 4(%rdi)
-; CHECK-NEXT: retq
+; CHECK-O0-LABEL: widen_broadcast2_unaligned:
+; CHECK-O0: # %bb.0:
+; CHECK-O0-NEXT: vmovd %xmm0, (%rdi)
+; CHECK-O0-NEXT: vmovd %xmm0, 4(%rdi)
+; CHECK-O0-NEXT: retq
+;
+; CHECK-O3-LABEL: widen_broadcast2_unaligned:
+; CHECK-O3: # %bb.0:
+; CHECK-O3-NEXT: vmovss %xmm0, (%rdi)
+; CHECK-O3-NEXT: vmovss %xmm0, 4(%rdi)
+; CHECK-O3-NEXT: retq
%v1 = extractelement <2 x i32> %vec, i32 0
%p1 = getelementptr i32, ptr %p0, i64 1
store atomic i32 %v1, ptr %p0 unordered, align 4
More information about the llvm-branch-commits
mailing list