[llvm] [X86] Extend alignedstore PatFrag to cover atomic_store (PR #197861)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 4 05:08:50 PDT 2026
https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/197861
>From ae93fed86d47d5d27d92927039837245073fd61a Mon Sep 17 00:00:00 2001
From: jofrn <165626406+jofrn at users.noreply.github.com>
Date: Thu, 4 Jun 2026 04:57:45 -0700
Subject: [PATCH] [X86] Extend alignedstore PatFrag to cover atomic_store
---
.../include/llvm/Target/TargetSelectionDAG.td | 2 +-
llvm/lib/Target/X86/X86InstrAVX512.td | 5 +-
llvm/lib/Target/X86/X86InstrFragmentsSIMD.td | 9 +-
llvm/test/CodeGen/X86/atomic-load-store.ll | 86 +++++++++++++++++++
4 files changed, 95 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index ea152bd130e9a..7919f43c86880 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -346,7 +346,7 @@ def SDTFPAtomic2 : SDTypeProfile<1, 2, [
]>;
def SDTAtomicStore : SDTypeProfile<0, 2, [
- SDTCisInt<0>, SDTCisPtrTy<1>
+ SDTCisPtrTy<1>
]>;
def SDTAtomicLoad : SDTypeProfile<1, 1, [
SDTCisPtrTy<1>
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 34ea27a55d77e..fcc8595aeafb8 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -3356,8 +3356,9 @@ multiclass avx512_load_vl<bits<8> opc, string OpcodeStr,
}
multiclass avx512_store<bits<8> opc, string OpcodeStr, string BaseName,
- X86VectorVTInfo _, PatFrag st_frag, PatFrag mstore,
- X86SchedWriteMoveLS Sched, bit NoMRPattern = 0> {
+ X86VectorVTInfo _, SDPatternOperator st_frag,
+ PatFrag mstore, X86SchedWriteMoveLS Sched,
+ bit NoMRPattern = 0> {
let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {
let isMoveReg = 1 in
def rr_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src),
diff --git a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
index 3df4fc2cb34d8..95a548dd2b30b 100644
--- a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -1254,10 +1254,11 @@ def extloadv4f16 : PatFrag<(ops node:$ptr), (extloadvf16 node:$ptr)>;
def extloadv8f16 : PatFrag<(ops node:$ptr), (extloadvf16 node:$ptr)>;
def extloadv16f16 : PatFrag<(ops node:$ptr), (extloadvf16 node:$ptr)>;
-// Like 'store', but always requires vector size alignment.
-def alignedstore : PatFrag<(ops node:$val, node:$ptr),
- (store node:$val, node:$ptr), [{
- auto *St = cast<StoreSDNode>(N);
+// Like 'store' or 'atomic_store', but always requires vector size alignment.
+def alignedstore : PatFrags<(ops node:$val, node:$ptr),
+ [(store node:$val, node:$ptr),
+ (atomic_store node:$val, node:$ptr)], [{
+ auto *St = cast<MemSDNode>(N);
return St->getAlign() >= St->getMemoryVT().getStoreSize();
}]> {
let GISelPredicateCode = [{
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll
index b2d2eab1e8a47..5be8d4d47fa9e 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -1191,6 +1191,92 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
ret void
}
+define void @store_atomic_vec4_float_align(ptr %x, <4 x float> %v) nounwind {
+; CHECK-SSE2-O3-LABEL: store_atomic_vec4_float_align:
+; CHECK-SSE2-O3: # %bb.0:
+; CHECK-SSE2-O3-NEXT: pushq %rax
+; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi
+; CHECK-SSE2-O3-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx
+; CHECK-SSE2-O3-NEXT: movl $3, %ecx
+; CHECK-SSE2-O3-NEXT: callq __atomic_store_16 at PLT
+; CHECK-SSE2-O3-NEXT: popq %rax
+; CHECK-SSE2-O3-NEXT: retq
+;
+; CHECK-SSE4-O3-LABEL: store_atomic_vec4_float_align:
+; CHECK-SSE4-O3: # %bb.0:
+; CHECK-SSE4-O3-NEXT: pushq %rbx
+; CHECK-SSE4-O3-NEXT: movdqa (%rdi), %xmm1
+; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm0, %rcx
+; CHECK-SSE4-O3-NEXT: movq %xmm0, %rbx
+; CHECK-SSE4-O3-NEXT: .p2align 4
+; CHECK-SSE4-O3-NEXT: .LBB39_1: # %atomicrmw.start
+; CHECK-SSE4-O3-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-SSE4-O3-NEXT: movq %xmm1, %rax
+; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm1, %rdx
+; CHECK-SSE4-O3-NEXT: lock cmpxchg16b (%rdi)
+; CHECK-SSE4-O3-NEXT: movq %rdx, %xmm0
+; CHECK-SSE4-O3-NEXT: movq %rax, %xmm1
+; CHECK-SSE4-O3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; CHECK-SSE4-O3-NEXT: jne .LBB39_1
+; CHECK-SSE4-O3-NEXT: # %bb.2: # %atomicrmw.end
+; CHECK-SSE4-O3-NEXT: popq %rbx
+; CHECK-SSE4-O3-NEXT: retq
+;
+; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_align:
+; CHECK-AVX-O3: # %bb.0:
+; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi)
+; CHECK-AVX-O3-NEXT: retq
+;
+; CHECK-SSE2-O0-LABEL: store_atomic_vec4_float_align:
+; CHECK-SSE2-O0: # %bb.0:
+; CHECK-SSE2-O0-NEXT: pushq %rax
+; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi
+; CHECK-SSE2-O0-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx
+; CHECK-SSE2-O0-NEXT: movl $3, %ecx
+; CHECK-SSE2-O0-NEXT: callq __atomic_store_16 at PLT
+; CHECK-SSE2-O0-NEXT: popq %rax
+; CHECK-SSE2-O0-NEXT: retq
+;
+; CHECK-SSE4-O0-LABEL: store_atomic_vec4_float_align:
+; CHECK-SSE4-O0: # %bb.0:
+; CHECK-SSE4-O0-NEXT: pushq %rbx
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-SSE4-O0-NEXT: movaps (%rdi), %xmm0
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-SSE4-O0-NEXT: .LBB39_1: # %atomicrmw.start
+; CHECK-SSE4-O0-NEXT: # =>This Inner Loop Header: Depth=1
+; CHECK-SSE4-O0-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; CHECK-SSE4-O0-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm1, %rcx
+; CHECK-SSE4-O0-NEXT: movq %xmm1, %rbx
+; CHECK-SSE4-O0-NEXT: movq %xmm0, %rax
+; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm0, %rdx
+; CHECK-SSE4-O0-NEXT: lock cmpxchg16b (%rsi)
+; CHECK-SSE4-O0-NEXT: movq %rax, %rcx
+; CHECK-SSE4-O0-NEXT: sete %al
+; CHECK-SSE4-O0-NEXT: movq %rdx, %xmm1
+; CHECK-SSE4-O0-NEXT: movq %rcx, %xmm0
+; CHECK-SSE4-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-SSE4-O0-NEXT: testb $1, %al
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-SSE4-O0-NEXT: jne .LBB39_2
+; CHECK-SSE4-O0-NEXT: jmp .LBB39_1
+; CHECK-SSE4-O0-NEXT: .LBB39_2: # %atomicrmw.end
+; CHECK-SSE4-O0-NEXT: popq %rbx
+; CHECK-SSE4-O0-NEXT: retq
+;
+; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_align:
+; CHECK-AVX-O0: # %bb.0:
+; CHECK-AVX-O0-NEXT: vmovaps %xmm0, (%rdi)
+; CHECK-AVX-O0-NEXT: retq
+ store atomic <4 x float> %v, ptr %x release, align 16
+ ret void
+}
+
define <2 x half> @atomic_vec2_half(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_half:
; CHECK-SSE-O3: # %bb.0:
More information about the llvm-commits
mailing list