[llvm] [X86] Keep scalar bf16/f16 selects in vector registers (PR #224218)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 17 01:56:27 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: tfzee (tfzee)
<details>
<summary>Changes</summary>
Scalar `select` on a soft-promoted `bf16`/`f16` currently lowers to a CMOV,
which extracts both operands out of XMM into GPRs and moves the result back.
Instead, blend them in vector register.
I do blending in `v8i16` rather than `v8f16` because a `v8f16` VSELECT can
fail to select on subtargets that fall back to BLENDV when there is no VBLENDVPH.
Right now I skip the transform when the operands are constants(no extra cost being in GPR), a compare feeds the condition (CMOV reuses the flags so hard to tell if worth it), both operands bit cast from GPR or f16 is legal and a mask register select can be selected.
---
Patch is 37.12 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/224218.diff
4 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+55)
- (modified) llvm/test/CodeGen/X86/bfloat.ll (+253)
- (modified) llvm/test/CodeGen/X86/combine-storetomstore.ll (+62-70)
- (modified) llvm/test/CodeGen/X86/half.ll (+95-64)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 64a4a191e5881..de73e7ac75b58 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48981,6 +48981,61 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
bool CondConstantVector = ISD::isBuildVectorOfConstantSDNodes(Cond.getNode());
unsigned EltBitWidth = VT.getScalarSizeInBits();
+ // Soft bf16/f16 scalar selects do a VSELECT in vector registers instead
+ // of a scalar CMOV, to avoid a GPR round-trip. Skip constant operands
+ // (cheaper as immediates) and compare-driven conds (CMOV already reuses
+ // the flags).
+ if (N->getOpcode() == ISD::SELECT && !CondVT.isVector() &&
+ Subtarget.hasSSE2() && !isIntOrFPConstant(LHS) &&
+ !isIntOrFPConstant(RHS)) {
+ // Only worth it if both operands already live in a vector register
+ auto IsBitcastFromGPR = [](SDValue Op) {
+ return Op.getOpcode() == ISD::BITCAST &&
+ Op.getOperand(0).getValueType().isScalarInteger();
+ };
+ SDValue F16LHS, F16RHS;
+ if (!VT.isVector() && isSoftF16(VT, Subtarget)) {
+ if (!IsBitcastFromGPR(LHS) || !IsBitcastFromGPR(RHS)) {
+ F16LHS = DAG.getBitcast(MVT::f16, LHS);
+ F16RHS = DAG.getBitcast(MVT::f16, RHS);
+ }
+ } else if (VT == MVT::i16 && LHS.getOpcode() == ISD::BITCAST &&
+ RHS.getOpcode() == ISD::BITCAST) {
+ MVT SVT = LHS.getOperand(0).getSimpleValueType();
+ if ((SVT == MVT::f16 || SVT == MVT::bf16) &&
+ SVT == RHS.getOperand(0).getSimpleValueType()) {
+ F16LHS = DAG.getBitcast(MVT::f16, LHS.getOperand(0));
+ F16RHS = DAG.getBitcast(MVT::f16, RHS.getOperand(0));
+ }
+ }
+
+ SDValue CondRoot = Cond;
+ while (CondRoot.getOpcode() == ISD::AND ||
+ CondRoot.getOpcode() == ISD::ANY_EXTEND ||
+ CondRoot.getOpcode() == ISD::ZERO_EXTEND ||
+ CondRoot.getOpcode() == ISD::TRUNCATE)
+ CondRoot = CondRoot.getOperand(0);
+
+ if (F16LHS && CondRoot.getOpcode() != ISD::SETCC &&
+ CondRoot.getOpcode() != X86ISD::SETCC) {
+ // Currently blend in v8i16 (not v8f16) since a v8f16 VSELECT can fail to
+ // select on some subtargets
+ SDValue Mask =
+ DAG.getNode(ISD::SUB, DL, MVT::i16, DAG.getConstant(0, DL, MVT::i16),
+ DAG.getZExtOrTrunc(Cond, DL, MVT::i16));
+ SDValue VLHS =
+ DAG.getBitcast(MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL,
+ MVT::v8f16, F16LHS));
+ SDValue VRHS =
+ DAG.getBitcast(MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL,
+ MVT::v8f16, F16RHS));
+ SDValue VMask = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8i16, Mask);
+ SDValue VSel = DAG.getSelect(DL, MVT::v8i16, VMask, VLHS, VRHS);
+ SDValue Res = DAG.getExtractVectorElt(DL, MVT::i16, VSel, 0);
+ return DAG.getBitcast(VT, Res);
+ }
+ }
+
// Attempt to combine (select M, (sub 0, X), X) -> (sub (xor X, M), M).
// Limit this to cases of non-constant masks that createShuffleMaskFromVSELECT
// can't catch, plus vXi8 cases where we'd likely end up with BLENDV.
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 4a234a5e80608..b0fb7a147c170 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -2076,3 +2076,256 @@ define bfloat @PR115710(fp128 %0) nounwind {
%2 = fptrunc fp128 %0 to bfloat
ret bfloat %2
}
+
+define bfloat @select_bf16(i1 %cond, bfloat %a, bfloat %b) nounwind {
+; X86-LABEL: select_bf16:
+; X86: # %bb.0:
+; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: vmovw %eax, %xmm2
+; X86-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; X86-NEXT: retl
+;
+; SSE2-LABEL: select_bf16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: andl $1, %edi
+; SSE2-NEXT: negl %edi
+; SSE2-NEXT: movd %edi, %xmm2
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: select_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: andl $1, %edi
+; AVX512BF16-NEXT: negl %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: select_bf16:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: andl $1, %edi
+; AVX512FP16-NEXT: negl %edi
+; AVX512FP16-NEXT: vmovw %edi, %xmm2
+; AVX512FP16-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: select_bf16:
+; AVXNC: # %bb.0:
+; AVXNC-NEXT: andl $1, %edi
+; AVXNC-NEXT: negl %edi
+; AVXNC-NEXT: vmovd %edi, %xmm2
+; AVXNC-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVXNC-NEXT: retq
+ %sel = select i1 %cond, bfloat %a, bfloat %b
+ ret bfloat %sel
+}
+
+define bfloat @select_bf16_constants(i1 %cond) nounwind {
+; X86-LABEL: select_bf16_constants:
+; X86: # %bb.0:
+; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $16256, %ecx # imm = 0x3F80
+; X86-NEXT: cmovel %eax, %ecx
+; X86-NEXT: vmovw %ecx, %xmm0
+; X86-NEXT: retl
+;
+; SSE2-LABEL: select_bf16_constants:
+; SSE2: # %bb.0:
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $16256, %ecx # imm = 0x3F80
+; SSE2-NEXT: cmovel %eax, %ecx
+; SSE2-NEXT: pinsrw $0, %ecx, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: select_bf16_constants:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: xorl %eax, %eax
+; AVX512BF16-NEXT: testb $1, %dil
+; AVX512BF16-NEXT: movl $16256, %ecx # imm = 0x3F80
+; AVX512BF16-NEXT: cmovel %eax, %ecx
+; AVX512BF16-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: select_bf16_constants:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: xorl %eax, %eax
+; AVX512FP16-NEXT: testb $1, %dil
+; AVX512FP16-NEXT: movl $16256, %ecx # imm = 0x3F80
+; AVX512FP16-NEXT: cmovel %eax, %ecx
+; AVX512FP16-NEXT: vmovw %ecx, %xmm0
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: select_bf16_constants:
+; AVXNC: # %bb.0:
+; AVXNC-NEXT: xorl %eax, %eax
+; AVXNC-NEXT: testb $1, %dil
+; AVXNC-NEXT: movl $16256, %ecx # imm = 0x3F80
+; AVXNC-NEXT: cmovel %eax, %ecx
+; AVXNC-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm0
+; AVXNC-NEXT: retq
+ %sel = select i1 %cond, bfloat 1.0, bfloat 0.0
+ ret bfloat %sel
+}
+
+define bfloat @select_ogt_bf16(bfloat %a, bfloat %b) nounwind {
+; X86-LABEL: select_ogt_bf16:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: vmovw %ecx, %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vmovw %eax, %xmm1
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
+; X86-NEXT: vucomiss %xmm0, %xmm1
+; X86-NEXT: cmoval %eax, %ecx
+; X86-NEXT: vmovw %ecx, %xmm0
+; X86-NEXT: retl
+;
+; SSE2-LABEL: select_ogt_bf16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pextrw $0, %xmm1, %eax
+; SSE2-NEXT: pextrw $0, %xmm0, %ecx
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: ucomiss %xmm1, %xmm0
+; SSE2-NEXT: cmoval %ecx, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: select_ogt_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %ecx
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vucomiss %xmm1, %xmm0
+; AVX512BF16-NEXT: cmoval %ecx, %eax
+; AVX512BF16-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: select_ogt_bf16:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovw %xmm1, %eax
+; AVX512FP16-NEXT: vmovw %xmm0, %ecx
+; AVX512FP16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512FP16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512FP16-NEXT: vucomiss %xmm1, %xmm0
+; AVX512FP16-NEXT: cmoval %ecx, %eax
+; AVX512FP16-NEXT: vmovw %eax, %xmm0
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: select_ogt_bf16:
+; AVXNC: # %bb.0:
+; AVXNC-NEXT: vpextrw $0, %xmm1, %eax
+; AVXNC-NEXT: vpextrw $0, %xmm0, %ecx
+; AVXNC-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNC-NEXT: vucomiss %xmm1, %xmm0
+; AVXNC-NEXT: cmoval %ecx, %eax
+; AVXNC-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; AVXNC-NEXT: retq
+ %cmp = fcmp ogt bfloat %a, %b
+ %sel = select i1 %cmp, bfloat %a, bfloat %b
+ ret bfloat %sel
+}
+
+; Both operands come out of GPRs, so the CMOV is cheaper than moving them into
+; vector registers and the result back out.
+define i16 @select_bf16_from_gpr(i1 %cond, i16 %a, i16 %b) nounwind {
+; X86-LABEL: select_bf16_from_gpr:
+; X86: # %bb.0:
+; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: vmovw %eax, %xmm2
+; X86-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT: vmovw %xmm0, %eax
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: retl
+;
+; X64-LABEL: select_bf16_from_gpr:
+; X64: # %bb.0:
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovel %edx, %eax
+; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: retq
+ %fa = bitcast i16 %a to bfloat
+ %fb = bitcast i16 %b to bfloat
+ %sel = select i1 %cond, bfloat %fa, bfloat %fb
+ %res = bitcast bfloat %sel to i16
+ ret i16 %res
+}
+
+; The operands already live in vector registers, so blend there even though the
+; select itself is on i16.
+define i16 @select_i16_of_bf16(i1 %cond, bfloat %a, bfloat %b) nounwind {
+; X86-LABEL: select_i16_of_bf16:
+; X86: # %bb.0:
+; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: vmovw %eax, %xmm2
+; X86-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; X86-NEXT: vmovw %xmm0, %eax
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: retl
+;
+; SSE2-LABEL: select_i16_of_bf16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: andl $1, %edi
+; SSE2-NEXT: negl %edi
+; SSE2-NEXT: movd %edi, %xmm2
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: select_i16_of_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: andl $1, %edi
+; AVX512BF16-NEXT: negl %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: select_i16_of_bf16:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: andl $1, %edi
+; AVX512FP16-NEXT: negl %edi
+; AVX512FP16-NEXT: vmovw %edi, %xmm2
+; AVX512FP16-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vmovw %xmm0, %eax
+; AVX512FP16-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: select_i16_of_bf16:
+; AVXNC: # %bb.0:
+; AVXNC-NEXT: andl $1, %edi
+; AVXNC-NEXT: negl %edi
+; AVXNC-NEXT: vmovd %edi, %xmm2
+; AVXNC-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVXNC-NEXT: vmovd %xmm0, %eax
+; AVXNC-NEXT: # kill: def $ax killed $ax killed $eax
+; AVXNC-NEXT: retq
+ %ai = bitcast bfloat %a to i16
+ %bi = bitcast bfloat %b to i16
+ %sel = select i1 %cond, i16 %ai, i16 %bi
+ ret i16 %sel
+}
diff --git a/llvm/test/CodeGen/X86/combine-storetomstore.ll b/llvm/test/CodeGen/X86/combine-storetomstore.ll
index a87047761b691..b99722970a476 100644
--- a/llvm/test/CodeGen/X86/combine-storetomstore.ll
+++ b/llvm/test/CodeGen/X86/combine-storetomstore.ll
@@ -132,80 +132,72 @@ define void @test_masked_store_success_v4i64(<4 x i64> %x, ptr %ptr, <4 x i1> %m
define void @test_masked_store_success_v4f16(<4 x half> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4f16:
; AVX: # %bb.0:
-; AVX-NEXT: vpsrlq $48, %xmm0, %xmm2
-; AVX-NEXT: vpextrw $0, %xmm2, %edx
-; AVX-NEXT: vpsrld $16, %xmm0, %xmm2
-; AVX-NEXT: vpextrw $0, %xmm2, %ecx
-; AVX-NEXT: movzwl 2(%rdi), %eax
-; AVX-NEXT: vpextrb $4, %xmm1, %esi
-; AVX-NEXT: testb $1, %sil
-; AVX-NEXT: cmovnel %ecx, %eax
-; AVX-NEXT: vpextrb $8, %xmm1, %ecx
-; AVX-NEXT: testb $1, %cl
-; AVX-NEXT: jne .LBB4_1
-; AVX-NEXT: # %bb.2:
-; AVX-NEXT: movl 4(%rdi), %ecx
-; AVX-NEXT: jmp .LBB4_3
-; AVX-NEXT: .LBB4_1:
-; AVX-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; AVX-NEXT: vpextrw $0, %xmm2, %ecx
-; AVX-NEXT: .LBB4_3:
-; AVX-NEXT: movzwl 6(%rdi), %esi
-; AVX-NEXT: vpextrb $12, %xmm1, %r8d
-; AVX-NEXT: testb $1, %r8b
-; AVX-NEXT: cmovnel %edx, %esi
-; AVX-NEXT: vmovd %xmm1, %edx
-; AVX-NEXT: testb $1, %dl
-; AVX-NEXT: jne .LBB4_4
-; AVX-NEXT: # %bb.5:
-; AVX-NEXT: movl (%rdi), %edx
-; AVX-NEXT: jmp .LBB4_6
-; AVX-NEXT: .LBB4_4:
-; AVX-NEXT: vpextrw $0, %xmm0, %edx
-; AVX-NEXT: .LBB4_6:
-; AVX-NEXT: movw %dx, (%rdi)
-; AVX-NEXT: movw %si, 6(%rdi)
-; AVX-NEXT: movw %cx, 4(%rdi)
-; AVX-NEXT: movw %ax, 2(%rdi)
+; AVX-NEXT: vpinsrw $0, 6(%rdi), %xmm0, %xmm2
+; AVX-NEXT: vpinsrw $0, 4(%rdi), %xmm0, %xmm3
+; AVX-NEXT: vpinsrw $0, 2(%rdi), %xmm0, %xmm4
+; AVX-NEXT: vpinsrw $0, (%rdi), %xmm0, %xmm5
+; AVX-NEXT: vmovd %xmm1, %eax
+; AVX-NEXT: andl $1, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: vmovd %eax, %xmm6
+; AVX-NEXT: vpblendvb %xmm6, %xmm0, %xmm5, %xmm5
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm6
+; AVX-NEXT: vpextrb $4, %xmm1, %eax
+; AVX-NEXT: andl $1, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: vmovd %eax, %xmm7
+; AVX-NEXT: vpblendvb %xmm7, %xmm6, %xmm4, %xmm4
+; AVX-NEXT: vmovshdup {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; AVX-NEXT: vpextrb $8, %xmm1, %eax
+; AVX-NEXT: andl $1, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: vmovd %eax, %xmm7
+; AVX-NEXT: vpblendvb %xmm7, %xmm6, %xmm3, %xmm3
+; AVX-NEXT: vpsrlq $48, %xmm0, %xmm0
+; AVX-NEXT: vpextrb $12, %xmm1, %eax
+; AVX-NEXT: andl $1, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: vmovd %eax, %xmm1
+; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
+; AVX-NEXT: vpextrw $0, %xmm0, 6(%rdi)
+; AVX-NEXT: vpextrw $0, %xmm3, 4(%rdi)
+; AVX-NEXT: vpextrw $0, %xmm4, 2(%rdi)
+; AVX-NEXT: vpextrw $0, %xmm5, (%rdi)
; AVX-NEXT: retq
;
; AVX2-LABEL: test_masked_store_success_v4f16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm2
-; AVX2-NEXT: vpextrw $0, %xmm2, %edx
-; AVX2-NEXT: vpsrld $16, %xmm0, %xmm2
-; AVX2-NEXT: vpextrw $0, %xmm2, %ecx
-; AVX2-NEXT: movzwl 2(%rdi), %eax
-; AVX2-NEXT: vpextrb $4, %xmm1, %esi
-; AVX2-NEXT: testb $1, %sil
-; AVX2-NEXT: cmovnel %ecx, %eax
-; AVX2-NEXT: vpextrb $8, %xmm1, %ecx
-; AVX2-NEXT: testb $1, %cl
-; AVX2-NEXT: jne .LBB4_1
-; AVX2-NEXT: # %bb.2:
-; AVX2-NEXT: movl 4(%rdi), %ecx
-; AVX2-NEXT: jmp .LBB4_3
-; AVX2-NEXT: .LBB4_1:
-; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; AVX2-NEXT: vpextrw $0, %xmm2, %ecx
-; AVX2-NEXT: .LBB4_3:
-; AVX2-NEXT: movzwl 6(%rdi), %esi
-; AVX2-NEXT: vpextrb $12, %xmm1, %r8d
-; AVX2-NEXT: testb $1, %r8b
-; AVX2-NEXT: cmovnel %edx, %esi
-; AVX2-NEXT: vmovd %xmm1, %edx
-; AVX2-NEXT: testb $1, %dl
-; AVX2-NEXT: jne .LBB4_4
-; AVX2-NEXT: # %bb.5:
-; AVX2-NEXT: movl (%rdi), %edx
-; AVX2-NEXT: jmp .LBB4_6
-; AVX2-NEXT: .LBB4_4:
-; AVX2-NEXT: vpextrw $0, %xmm0, %edx
-; AVX2-NEXT: .LBB4_6:
-; AVX2-NEXT: movw %dx, (%rdi)
-; AVX2-NEXT: movw %si, 6(%rdi)
-; AVX2-NEXT: movw %cx, 4(%rdi)
-; AVX2-NEXT: movw %ax, 2(%rdi)
+; AVX2-NEXT: vpinsrw $0, 6(%rdi), %xmm0, %xmm2
+; AVX2-NEXT: vpinsrw $0, 4(%rdi), %xmm0, %xmm3
+; AVX2-NEXT: vpinsrw $0, 2(%rdi), %xmm0, %xmm4
+; AVX2-NEXT: vpinsrw $0, (%rdi), %xmm0, %xmm5
+; AVX2-NEXT: vmovd %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: vmovd %eax, %xmm6
+; AVX2-NEXT: vpblendvb %xmm6, %xmm0, %xmm5, %xmm5
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm6
+; AVX2-NEXT: vpextrb $4, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: vmovd %eax, %xmm7
+; AVX2-NEXT: vpblendvb %xmm7, %xmm6, %xmm4, %xmm4
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpextrb $8, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: vmovd %eax, %xmm7
+; AVX2-NEXT: vpblendvb %xmm7, %xmm6, %xmm3, %xmm3
+; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $12, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: vmovd %eax, %xmm1
+; AVX2-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: vpextrw $0, %xmm0, 6(%rdi)
+; AVX2-NEXT: vpextrw $0, %xmm3, 4(%rdi)
+; AVX2-NEXT: vpextrw $0, %xmm4, 2(%rdi)
+; AVX2-NEXT: vpextrw $0, %xmm5, (%rdi)
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_masked_store_success_v4f16:
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index b6a4a12eb0fac..67ddb936b7416 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -1248,6 +1248,37 @@ define <8 x half> @select(i1 %c, <8 x half> %x, <8 x half> %y) {
ret <8 x half> %s
}
+define half @select_scalar(i1 %c, half %x, half %y) {
+; CHECK-LIBCALL-LABEL: select_scalar:
+; CHECK-LIBCALL: # %bb.0:
+; CHECK-LIBCALL-NEXT: andl $1, %edi
+; CHECK-LIBCALL-NEXT: negl %edi
+; CHECK-LIBCALL-NEXT: movd %edi, %xmm2
+; CHECK-LIBCALL-NEXT: pand %xmm2, %xmm0
+; CHECK-LIBCALL-NEXT: pandn %xmm1, %xmm2
+; CHECK-LIBCALL-NEXT: por %xmm2, %xmm0
+; CHECK-LIBCALL-NEXT: retq
+;
+; BWON-F16C-LABEL: select_scalar:
+; BWON-F16C: # %bb.0:
+; BWON-F16C-NEXT: andl $1, %edi
+; BWON-F16C-NEXT: negl %edi
+; BWON-F16C-NEXT: vmovd %edi, %xmm2
+; BWON-F16C-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; BWON-F16C-NEXT: retq
+;
+; CHECK-I686-LABEL: select_scalar:
+; CHECK-I686: # %bb.0:
+; CHECK-I686-NEXT: testb $1, {{[0-9]+}}(%esp)
+; CHECK-I686-NEXT: leal {{[0-9]+}}(%esp), %eax
+; CHECK-I686-NEXT: leal {{[0-9]+}}(%esp), %ecx
+; CHECK-I686-NEXT: cmovnel %eax, %ecx
+; CHECK-I686-NEXT: pinsrw $0, (%ecx), %xmm0
+; CHECK-I686-NEXT: retl
+ %s = select i1 %c, half %x, half %y
+ ret half %s
+}
+
define <8 x half> @shuffle(ptr %p) {
; CHECK-LIBCALL-LABEL: shuffle:
; CHECK-LIBCALL: # %bb.0:
@@ -1350,11 +1381,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: ja .LBB26_2
+; CHECK-LIBCALL-NEXT: ja .LBB27_2
; CHECK-LIBCALL-NEXT: # %bb.1:
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_2:
+; CHECK-LIBCALL-NEXT: .LBB27_2:
; CHECK-LIBCALL-NEXT: callq __truncsfhf2 at PLT
; CHECK-LIBCALL-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-LIBCALL-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -1375,11 +1406,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
; CH...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/224218
More information about the llvm-commits
mailing list