[llvm] [X86] Lower bf16->f32/f64 fpext without a GPR round-trip (PR #218359)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 03:26:23 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: tfzee (tfzee)
<details>
<summary>Changes</summary>
`fpext bfloat to float/double` currently lowers `ISD::BF16_TO_FP` via the
generic target-independent expansion (extend to i32, shl 16, bitcast to
f32), which always forces the value out of its XMM register into a GPR
and back. This would fix #<!-- -->155270
Add a custom `BF16_TO_FP` lowering (mirroring the existing `FP_TO_BF16` function) for f32/f64 results that does
the shift in-register instead.
Only on targets with SSE2 or higher otherwise still falls back to the existing generic expansion.
The updated lowering then for the example
```llvm
define float @<!-- -->src32(bfloat %a0) {
%res = fpext bfloat %a0 to float
ret float %res
}
define double @<!-- -->src64(bfloat %a0) {
%res = fpext bfloat %a0 to double
ret double %res
}
```
generates
```asm
src32:
pslld $16, %xmm0
retq
src64:
pslld $16, %xmm0
cvtss2sd %xmm0, %xmm0
retq
```
instead of the previous roundtrip
```asm
src32:
pextrw $0, %xmm0, %eax
shll $16, %eax
movd %eax, %xmm0
retq
src64:
pextrw $0, %xmm0, %eax
shll $16, %eax
movd %eax, %xmm0
cvtss2sd %xmm0, %xmm0
retq
```
---
Patch is 341.30 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/218359.diff
16 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+40)
- (modified) llvm/lib/Target/X86/X86ISelLowering.h (+1)
- (modified) llvm/test/CodeGen/X86/atomic-load-store.ll (+507-583)
- (modified) llvm/test/CodeGen/X86/avx10_2bf16-arith-scalar.ll (+36-88)
- (modified) llvm/test/CodeGen/X86/avx10_2bf16-fma.ll (+36-272)
- (modified) llvm/test/CodeGen/X86/avx10_2bf16-select-minmax.ll (+7-11)
- (modified) llvm/test/CodeGen/X86/bf16-fast-isel.ll (+11-25)
- (modified) llvm/test/CodeGen/X86/bfloat-calling-conv.ll (+318-363)
- (modified) llvm/test/CodeGen/X86/bfloat-int-cvt.ll (+90-186)
- (modified) llvm/test/CodeGen/X86/bfloat.ll (+261-389)
- (modified) llvm/test/CodeGen/X86/float-to-arbitrary-fp.ll (+34-41)
- (modified) llvm/test/CodeGen/X86/fminimum-fmaximum.ll (+332-502)
- (modified) llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll (+434-584)
- (modified) llvm/test/CodeGen/X86/llvm.frexp.ll (+1-3)
- (modified) llvm/test/CodeGen/X86/pr86305.ll (+2-2)
- (modified) llvm/test/CodeGen/X86/select-phi-s16-fp.ll (+1-1)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a0c92a22b7e2f..8561b1f9e8f8f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -462,6 +462,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::BF16_TO_FP, VT, Expand);
setOperationAction(ISD::FP_TO_BF16, VT, Custom);
}
+ // The vector-based lowering below needs SSE2 registers.
+ if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
+ setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
+ setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
+ }
setOperationAction(ISD::PARITY, MVT::i8, Custom);
setOperationAction(ISD::PARITY, MVT::i16, Custom);
@@ -22973,6 +22978,40 @@ static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
return Res;
}
+SDValue X86TargetLowering::LowerBF16_TO_FP(SDValue Op,
+ SelectionDAG &DAG) const {
+ SDLoc DL(Op);
+ SDValue Src = Op.getOperand(0);
+ // Operand is usually already softened to i16 by type legalization.
+ if (Src.getValueType() == MVT::bf16)
+ Src = DAG.getBitcast(MVT::i16, Src);
+ else
+ Src = DAG.getAnyExtOrTrunc(Src, DL, MVT::i16);
+
+ // Peek through to bf16's underlying f16 register
+ MVT VecVT = MVT::v8i16;
+ SDValue VecSrc = Src;
+ if (Src.getOpcode() == ISD::BITCAST &&
+ Src.getOperand(0).getValueType() == MVT::f16) {
+ VecSrc = Src.getOperand(0);
+ VecVT = MVT::v8f16;
+ }
+
+ // Shift the bf16 bits into the high half of the f32.
+ SDValue Vec = DAG.getBitcast(
+ MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, VecSrc));
+ Vec = DAG.getBitcast(MVT::v4i32, Vec);
+ Vec = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, MVT::v4i32, Vec, 16, DAG);
+ Vec = DAG.getBitcast(MVT::v4f32, Vec);
+ SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Vec,
+ DAG.getVectorIdxConstant(0, DL));
+
+ EVT DstVT = Op.getValueType();
+ if (DstVT != MVT::f32)
+ Res = DAG.getNode(ISD::FP_EXTEND, DL, DstVT, Res);
+ return Res;
+}
+
SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
@@ -34541,6 +34580,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::STRICT_FP16_TO_FP: return LowerFP16_TO_FP(Op, DAG);
case ISD::FP_TO_FP16:
case ISD::STRICT_FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
+ case ISD::BF16_TO_FP: return LowerBF16_TO_FP(Op, DAG);
case ISD::FP_TO_BF16: return LowerFP_TO_BF16(Op, DAG);
case ISD::LOAD: return LowerLoad(Op, Subtarget, DAG);
case ISD::STORE: return LowerStore(Op, Subtarget, DAG);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 836997e1a45e8..b220c1bbe7bc7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -851,6 +851,7 @@ namespace llvm {
SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerBF16_TO_FP(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
SDValue
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll
index 7cfe7af47748a..41131b28d9b23 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -239,12 +239,7 @@ define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
; CHECK-SSE-O0: # %bb.0:
; CHECK-SSE-O0-NEXT: pushq %rax
; CHECK-SSE-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill
-; CHECK-SSE-O0-NEXT: pextrw $0, %xmm0, %eax
-; CHECK-SSE-O0-NEXT: movw %ax, %cx
-; CHECK-SSE-O0-NEXT: # implicit-def: $eax
-; CHECK-SSE-O0-NEXT: movw %cx, %ax
-; CHECK-SSE-O0-NEXT: shll $16, %eax
-; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload
; CHECK-SSE-O0-NEXT: pextrw $0, %xmm0, %eax
@@ -257,12 +252,7 @@ define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
; CHECK-AVX-O0: # %bb.0:
; CHECK-AVX-O0-NEXT: pushq %rax
; CHECK-AVX-O0-NEXT: movq %rdi, (%rsp) # 8-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: movw %ax, %cx
-; CHECK-AVX-O0-NEXT: # implicit-def: $eax
-; CHECK-AVX-O0-NEXT: movw %cx, %ax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
+; CHECK-AVX-O0-NEXT: vpslld $16, %xmm0, %xmm0
; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-AVX-O0-NEXT: movq (%rsp), %rdi # 8-byte Reload
; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, %eax
@@ -886,14 +876,14 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: subq $24, %rsp
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -915,13 +905,11 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: subq $24, %rsp
; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE4-O0-NEXT: xorps %xmm2, %xmm2
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -935,29 +923,47 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE4-O0-NEXT: addq $24, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
-; CHECK-AVX-O0-LABEL: store_atomic_vec2_bfloat:
-; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: subq $24, %rsp
-; CHECK-AVX-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; CHECK-AVX-O0-NEXT: movl %eax, (%rdi)
-; CHECK-AVX-O0-NEXT: addq $24, %rsp
-; CHECK-AVX-O0-NEXT: retq
+; CHECK-AVX2-O0-LABEL: store_atomic_vec2_bfloat:
+; CHECK-AVX2-O0: # %bb.0:
+; CHECK-AVX2-O0-NEXT: subq $24, %rsp
+; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX2-O0-NEXT: addq $24, %rsp
+; CHECK-AVX2-O0-NEXT: retq
+;
+; CHECK-AVX512-O0-LABEL: store_atomic_vec2_bfloat:
+; CHECK-AVX512-O0: # %bb.0:
+; CHECK-AVX512-O0-NEXT: subq $24, %rsp
+; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-AVX512-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-O0-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-O0-NEXT: movl %eax, (%rdi)
+; CHECK-AVX512-O0-NEXT: addq $24, %rsp
+; CHECK-AVX512-O0-NEXT: retq
store atomic <2 x bfloat> %v, ptr %x release, align 4
ret void
}
@@ -1024,22 +1030,22 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE2-O0: # %bb.0:
; CHECK-SSE2-O0-NEXT: subq $40, %rsp
+; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
+; CHECK-SSE2-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE2-O0-NEXT: pextrw $3, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm3
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm3
+; CHECK-SSE2-O0-NEXT: pextrw $2, %xmm1, %eax
+; CHECK-SSE2-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE2-O0-NEXT: pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE2-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE2-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE2-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE2-O0-NEXT: shll $16, %eax
-; CHECK-SSE2-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE2-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -1074,22 +1080,22 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE4-O0: # %bb.0:
; CHECK-SSE4-O0-NEXT: subq $40, %rsp
+; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
-; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: pextrw $1, %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
+; CHECK-SSE4-O0-NEXT: xorps %xmm0, %xmm0
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm3
+; CHECK-SSE4-O0-NEXT: pblendw {{.*#+}} xmm3 = xmm0[0],xmm3[1],xmm0[2],xmm3[3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
+; CHECK-SSE4-O0-NEXT: movaps %xmm1, %xmm0
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm0
+; CHECK-SSE4-O0-NEXT: pextrw $3, %xmm1, %eax
+; CHECK-SSE4-O0-NEXT: movd %eax, %xmm2
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm2
+; CHECK-SSE4-O0-NEXT: pextrw $2, %xmm1, %eax
; CHECK-SSE4-O0-NEXT: movd %eax, %xmm1
+; CHECK-SSE4-O0-NEXT: pslld $16, %xmm1
+; CHECK-SSE4-O0-NEXT: movss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-SSE4-O0-NEXT: movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT: movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-SSE4-O0-NEXT: movd %xmm0, %eax
-; CHECK-SSE4-O0-NEXT: shll $16, %eax
-; CHECK-SSE4-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT: callq __truncsfbf2 at PLT
; CHECK-SSE4-O0-NEXT: movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -1113,47 +1119,79 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE4-O0-NEXT: addq $40, %rsp
; CHECK-SSE4-O0-NEXT: retq
;
-; CHECK-AVX-O0-LABEL: store_atomic_vec4_bfloat:
-; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: subq $40, %rsp
-; CHECK-AVX-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $3, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $2, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX-O0-NEXT: vpextrw $1, %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax
-; CHECK-AVX-O0-NEXT: shll $16, %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: vmovaps %xmm0, %xmm1
-; CHECK-AVX-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
-; CHECK-AVX-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: callq __truncsfbf2 at PLT
-; CHECK-AVX-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
-; CHECK-AVX-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-AVX-O0-NEXT: movq %rax, (%rdi)
-; CHECK-AVX-O0-NEXT: addq $40, %rsp
-; CHECK-AVX-O0-NEXT: retq
+; CHECK-AVX2-O0-LABEL: store_atomic_vec4_bfloat:
+; CHECK-AVX2-O0: # %bb.0:
+; CHECK-AVX2-O0-NEXT: subq $40, %rsp
+; CHECK-AVX2-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm3
+; CHECK-AVX2-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX2-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm2
+; CHECK-AVX2-O0-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-AVX2-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX2-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX2-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX2-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX2-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX2-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-AVX2-O0-NEXT: vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-O0-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX2-O0-NEXT: movq %rax, (%rdi)
+; CHECK-AVX2-O0-NEXT: addq $40, %rsp
+; CHECK-AVX2-O0-NEXT: retq
+;
+; CHECK-AVX512-O0-LABEL: store_atomic_vec4_bfloat:
+; CHECK-AVX512-O0: # %bb.0:
+; CHECK-AVX512-O0-NEXT: subq $40, %rsp
+; CHECK-AVX512-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm3
+; CHECK-AVX512-O0-NEXT: vmovdqa {{.*#+}} xmm1 = [128,128,4,5,u,u,u,u,u,u,u,u,u,u,u,u]
+; CHECK-AVX512-O0-NEXT: vpshufb %xmm1, %xmm0, %xmm2
+; CHECK-AVX512-O0-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX512-O0-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; CHECK-AVX512-O0-NEXT: vpslld $16, %xmm0, %xmm0
+; CHECK-AVX512-O0-NEXT: vmovss %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-AVX512-O0-NEXT: callq __truncsfbf2 at PLT
+; CHECK-AVX512-O0-NEXT: vmovaps %xmm0, %xmm1
+; CHECK-AVX512-O0-NEXT: vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
+; CHECK-AVX512-O0-NEXT: # xmm0 = mem[0],zero,zero,zero
+; CHECK-A...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/218359
More information about the llvm-commits
mailing list