[llvm] [X86] Keep scalar bf16/f16 selects in vector registers (PR #224218)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 00:05:48 PDT 2026
https://github.com/tfzee updated https://github.com/llvm/llvm-project/pull/224218
>From 3c281843a795b7f69d6ff8b4e9940784c42c7f28 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Mon, 24 Aug 2026 15:41:31 +0200
Subject: [PATCH 1/7] Stay in vector domain when selecting bf16
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 52 ++++++
llvm/test/CodeGen/X86/bfloat.ll | 160 ++++++++++++++++++
.../test/CodeGen/X86/combine-storetomstore.ll | 132 +++++++--------
llvm/test/CodeGen/X86/half.ll | 159 ++++++++++-------
4 files changed, 369 insertions(+), 134 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a0c92a22b7e2f..f9757336af038 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48681,6 +48681,58 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
bool CondConstantVector = ISD::isBuildVectorOfConstantSDNodes(Cond.getNode());
unsigned EltBitWidth = VT.getScalarSizeInBits();
+ // Soft bf16/f16 scalar selects: do a VSELECT in vector registers instead
+ // of a scalar CMOV, to avoid a GPR round-trip. Skip constant operands
+ // (cheaper as immediates) and compare-driven conds (CMOV already reuses
+ // the flags). bf16 has no register class, so it can only appear as a
+ // BITCAST operand (the one case the type legalizer's soft-promotion
+ // knows how to handle) -- bitcast to f16 (always legal) before building
+ // any vector node, never leave a bare scalar bf16 as an operand.
+ if (N->getOpcode() == ISD::SELECT && !CondVT.isVector() &&
+ Subtarget.hasSSE2() && !isIntOrFPConstant(LHS) &&
+ !isIntOrFPConstant(RHS)) {
+ SDValue CondRoot = Cond;
+ while (CondRoot.getOpcode() == ISD::AND ||
+ CondRoot.getOpcode() == ISD::ANY_EXTEND ||
+ CondRoot.getOpcode() == ISD::ZERO_EXTEND ||
+ CondRoot.getOpcode() == ISD::TRUNCATE)
+ CondRoot = CondRoot.getOperand(0);
+ bool CondIsCompare = CondRoot.getOpcode() == ISD::SETCC ||
+ CondRoot.getOpcode() == X86ISD::SETCC;
+
+ SDValue F16LHS, F16RHS;
+ if (!CondIsCompare && (VT == MVT::f16 || VT == MVT::bf16) &&
+ isSoftF16(VT, Subtarget)) {
+ F16LHS = DAG.getBitcast(MVT::f16, LHS);
+ F16RHS = DAG.getBitcast(MVT::f16, RHS);
+ } else if (!CondIsCompare && VT == MVT::i16 &&
+ LHS.getOpcode() == ISD::BITCAST &&
+ RHS.getOpcode() == ISD::BITCAST) {
+ MVT SVT = LHS.getOperand(0).getSimpleValueType();
+ if ((SVT == MVT::f16 || SVT == MVT::bf16) &&
+ SVT == RHS.getOperand(0).getSimpleValueType()) {
+ F16LHS = DAG.getBitcast(MVT::f16, LHS.getOperand(0));
+ F16RHS = DAG.getBitcast(MVT::f16, RHS.getOperand(0));
+ }
+ }
+ if (F16LHS) {
+ // Blend in v8i16 (not v8f16): there's no VBLENDVPH, so a v8f16
+ // VSELECT can fail to select on subtargets that fall back to
+ // BLENDV instead of a mask-register select.
+ SDValue Mask = DAG.getNode(ISD::SUB, DL, MVT::i16,
+ DAG.getConstant(0, DL, MVT::i16),
+ DAG.getZExtOrTrunc(Cond, DL, MVT::i16));
+ SDValue VLHS = DAG.getBitcast(
+ MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, F16LHS));
+ SDValue VRHS = DAG.getBitcast(
+ MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, F16RHS));
+ SDValue VMask = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8i16, Mask);
+ SDValue VSel = DAG.getSelect(DL, MVT::v8i16, VMask, VLHS, VRHS);
+ SDValue Res = DAG.getExtractVectorElt(DL, MVT::i16, VSel, 0);
+ return DAG.getBitcast(VT, Res);
+ }
+ }
+
// Attempt to combine (select M, (sub 0, X), X) -> (sub (xor X, M), M).
// Limit this to cases of non-constant masks that createShuffleMaskFromVSELECT
// can't catch, plus vXi8 cases where we'd likely end up with BLENDV.
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 7bccd6ba088ac..003716bbfc075 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -2264,3 +2264,163 @@ define bfloat @PR115710(fp128 %0) nounwind {
%2 = fptrunc fp128 %0 to bfloat
ret bfloat %2
}
+
+define bfloat @select_bf16(i1 %cond, bfloat %a, bfloat %b) nounwind {
+; X86-LABEL: select_bf16:
+; X86: # %bb.0:
+; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: vmovw %eax, %xmm2
+; X86-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; X86-NEXT: retl
+;
+; SSE2-LABEL: select_bf16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: andl $1, %edi
+; SSE2-NEXT: negl %edi
+; SSE2-NEXT: movd %edi, %xmm2
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: select_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: andl $1, %edi
+; AVX512BF16-NEXT: negl %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: select_bf16:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: andl $1, %edi
+; AVX512FP16-NEXT: negl %edi
+; AVX512FP16-NEXT: vmovw %edi, %xmm2
+; AVX512FP16-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: select_bf16:
+; AVXNC: # %bb.0:
+; AVXNC-NEXT: andl $1, %edi
+; AVXNC-NEXT: negl %edi
+; AVXNC-NEXT: vmovd %edi, %xmm2
+; AVXNC-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVXNC-NEXT: retq
+ %sel = select i1 %cond, bfloat %a, bfloat %b
+ ret bfloat %sel
+}
+
+define bfloat @select_bf16_constants(i1 %cond) nounwind {
+; X86-LABEL: select_bf16_constants:
+; X86: # %bb.0:
+; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X86-NEXT: movl $16256, %ecx # imm = 0x3F80
+; X86-NEXT: cmovel %eax, %ecx
+; X86-NEXT: vmovw %ecx, %xmm0
+; X86-NEXT: retl
+;
+; SSE2-LABEL: select_bf16_constants:
+; SSE2: # %bb.0:
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $16256, %ecx # imm = 0x3F80
+; SSE2-NEXT: cmovel %eax, %ecx
+; SSE2-NEXT: pinsrw $0, %ecx, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: select_bf16_constants:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: xorl %eax, %eax
+; AVX512BF16-NEXT: testb $1, %dil
+; AVX512BF16-NEXT: movl $16256, %ecx # imm = 0x3F80
+; AVX512BF16-NEXT: cmovel %eax, %ecx
+; AVX512BF16-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: select_bf16_constants:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: xorl %eax, %eax
+; AVX512FP16-NEXT: testb $1, %dil
+; AVX512FP16-NEXT: movl $16256, %ecx # imm = 0x3F80
+; AVX512FP16-NEXT: cmovel %eax, %ecx
+; AVX512FP16-NEXT: vmovw %ecx, %xmm0
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: select_bf16_constants:
+; AVXNC: # %bb.0:
+; AVXNC-NEXT: xorl %eax, %eax
+; AVXNC-NEXT: testb $1, %dil
+; AVXNC-NEXT: movl $16256, %ecx # imm = 0x3F80
+; AVXNC-NEXT: cmovel %eax, %ecx
+; AVXNC-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm0
+; AVXNC-NEXT: retq
+ %sel = select i1 %cond, bfloat 1.0, bfloat 0.0
+ ret bfloat %sel
+}
+
+define bfloat @select_ogt_bf16(bfloat %a, bfloat %b) nounwind {
+; X86-LABEL: select_ogt_bf16:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: vmovw %ecx, %xmm0
+; X86-NEXT: vpslld $16, %xmm0, %xmm0
+; X86-NEXT: vmovw %eax, %xmm1
+; X86-NEXT: vpslld $16, %xmm1, %xmm1
+; X86-NEXT: vucomiss %xmm0, %xmm1
+; X86-NEXT: cmoval %eax, %ecx
+; X86-NEXT: vmovw %ecx, %xmm0
+; X86-NEXT: retl
+;
+; SSE2-LABEL: select_ogt_bf16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pextrw $0, %xmm1, %eax
+; SSE2-NEXT: pextrw $0, %xmm0, %ecx
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: ucomiss %xmm1, %xmm0
+; SSE2-NEXT: cmoval %ecx, %eax
+; SSE2-NEXT: pinsrw $0, %eax, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: select_ogt_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: vpextrw $0, %xmm1, %eax
+; AVX512BF16-NEXT: vpextrw $0, %xmm0, %ecx
+; AVX512BF16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512BF16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512BF16-NEXT: vucomiss %xmm1, %xmm0
+; AVX512BF16-NEXT: cmoval %ecx, %eax
+; AVX512BF16-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: select_ogt_bf16:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: vmovw %xmm1, %eax
+; AVX512FP16-NEXT: vmovw %xmm0, %ecx
+; AVX512FP16-NEXT: vpslld $16, %xmm1, %xmm1
+; AVX512FP16-NEXT: vpslld $16, %xmm0, %xmm0
+; AVX512FP16-NEXT: vucomiss %xmm1, %xmm0
+; AVX512FP16-NEXT: cmoval %ecx, %eax
+; AVX512FP16-NEXT: vmovw %eax, %xmm0
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: select_ogt_bf16:
+; AVXNC: # %bb.0:
+; AVXNC-NEXT: vpextrw $0, %xmm1, %eax
+; AVXNC-NEXT: vpextrw $0, %xmm0, %ecx
+; AVXNC-NEXT: vpslld $16, %xmm1, %xmm1
+; AVXNC-NEXT: vpslld $16, %xmm0, %xmm0
+; AVXNC-NEXT: vucomiss %xmm1, %xmm0
+; AVXNC-NEXT: cmoval %ecx, %eax
+; AVXNC-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; AVXNC-NEXT: retq
+ %cmp = fcmp ogt bfloat %a, %b
+ %sel = select i1 %cmp, bfloat %a, bfloat %b
+ ret bfloat %sel
+}
diff --git a/llvm/test/CodeGen/X86/combine-storetomstore.ll b/llvm/test/CodeGen/X86/combine-storetomstore.ll
index a87047761b691..b99722970a476 100644
--- a/llvm/test/CodeGen/X86/combine-storetomstore.ll
+++ b/llvm/test/CodeGen/X86/combine-storetomstore.ll
@@ -132,80 +132,72 @@ define void @test_masked_store_success_v4i64(<4 x i64> %x, ptr %ptr, <4 x i1> %m
define void @test_masked_store_success_v4f16(<4 x half> %x, ptr %ptr, <4 x i1> %mask) {
; AVX-LABEL: test_masked_store_success_v4f16:
; AVX: # %bb.0:
-; AVX-NEXT: vpsrlq $48, %xmm0, %xmm2
-; AVX-NEXT: vpextrw $0, %xmm2, %edx
-; AVX-NEXT: vpsrld $16, %xmm0, %xmm2
-; AVX-NEXT: vpextrw $0, %xmm2, %ecx
-; AVX-NEXT: movzwl 2(%rdi), %eax
-; AVX-NEXT: vpextrb $4, %xmm1, %esi
-; AVX-NEXT: testb $1, %sil
-; AVX-NEXT: cmovnel %ecx, %eax
-; AVX-NEXT: vpextrb $8, %xmm1, %ecx
-; AVX-NEXT: testb $1, %cl
-; AVX-NEXT: jne .LBB4_1
-; AVX-NEXT: # %bb.2:
-; AVX-NEXT: movl 4(%rdi), %ecx
-; AVX-NEXT: jmp .LBB4_3
-; AVX-NEXT: .LBB4_1:
-; AVX-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; AVX-NEXT: vpextrw $0, %xmm2, %ecx
-; AVX-NEXT: .LBB4_3:
-; AVX-NEXT: movzwl 6(%rdi), %esi
-; AVX-NEXT: vpextrb $12, %xmm1, %r8d
-; AVX-NEXT: testb $1, %r8b
-; AVX-NEXT: cmovnel %edx, %esi
-; AVX-NEXT: vmovd %xmm1, %edx
-; AVX-NEXT: testb $1, %dl
-; AVX-NEXT: jne .LBB4_4
-; AVX-NEXT: # %bb.5:
-; AVX-NEXT: movl (%rdi), %edx
-; AVX-NEXT: jmp .LBB4_6
-; AVX-NEXT: .LBB4_4:
-; AVX-NEXT: vpextrw $0, %xmm0, %edx
-; AVX-NEXT: .LBB4_6:
-; AVX-NEXT: movw %dx, (%rdi)
-; AVX-NEXT: movw %si, 6(%rdi)
-; AVX-NEXT: movw %cx, 4(%rdi)
-; AVX-NEXT: movw %ax, 2(%rdi)
+; AVX-NEXT: vpinsrw $0, 6(%rdi), %xmm0, %xmm2
+; AVX-NEXT: vpinsrw $0, 4(%rdi), %xmm0, %xmm3
+; AVX-NEXT: vpinsrw $0, 2(%rdi), %xmm0, %xmm4
+; AVX-NEXT: vpinsrw $0, (%rdi), %xmm0, %xmm5
+; AVX-NEXT: vmovd %xmm1, %eax
+; AVX-NEXT: andl $1, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: vmovd %eax, %xmm6
+; AVX-NEXT: vpblendvb %xmm6, %xmm0, %xmm5, %xmm5
+; AVX-NEXT: vpsrld $16, %xmm0, %xmm6
+; AVX-NEXT: vpextrb $4, %xmm1, %eax
+; AVX-NEXT: andl $1, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: vmovd %eax, %xmm7
+; AVX-NEXT: vpblendvb %xmm7, %xmm6, %xmm4, %xmm4
+; AVX-NEXT: vmovshdup {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; AVX-NEXT: vpextrb $8, %xmm1, %eax
+; AVX-NEXT: andl $1, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: vmovd %eax, %xmm7
+; AVX-NEXT: vpblendvb %xmm7, %xmm6, %xmm3, %xmm3
+; AVX-NEXT: vpsrlq $48, %xmm0, %xmm0
+; AVX-NEXT: vpextrb $12, %xmm1, %eax
+; AVX-NEXT: andl $1, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: vmovd %eax, %xmm1
+; AVX-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
+; AVX-NEXT: vpextrw $0, %xmm0, 6(%rdi)
+; AVX-NEXT: vpextrw $0, %xmm3, 4(%rdi)
+; AVX-NEXT: vpextrw $0, %xmm4, 2(%rdi)
+; AVX-NEXT: vpextrw $0, %xmm5, (%rdi)
; AVX-NEXT: retq
;
; AVX2-LABEL: test_masked_store_success_v4f16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm2
-; AVX2-NEXT: vpextrw $0, %xmm2, %edx
-; AVX2-NEXT: vpsrld $16, %xmm0, %xmm2
-; AVX2-NEXT: vpextrw $0, %xmm2, %ecx
-; AVX2-NEXT: movzwl 2(%rdi), %eax
-; AVX2-NEXT: vpextrb $4, %xmm1, %esi
-; AVX2-NEXT: testb $1, %sil
-; AVX2-NEXT: cmovnel %ecx, %eax
-; AVX2-NEXT: vpextrb $8, %xmm1, %ecx
-; AVX2-NEXT: testb $1, %cl
-; AVX2-NEXT: jne .LBB4_1
-; AVX2-NEXT: # %bb.2:
-; AVX2-NEXT: movl 4(%rdi), %ecx
-; AVX2-NEXT: jmp .LBB4_3
-; AVX2-NEXT: .LBB4_1:
-; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; AVX2-NEXT: vpextrw $0, %xmm2, %ecx
-; AVX2-NEXT: .LBB4_3:
-; AVX2-NEXT: movzwl 6(%rdi), %esi
-; AVX2-NEXT: vpextrb $12, %xmm1, %r8d
-; AVX2-NEXT: testb $1, %r8b
-; AVX2-NEXT: cmovnel %edx, %esi
-; AVX2-NEXT: vmovd %xmm1, %edx
-; AVX2-NEXT: testb $1, %dl
-; AVX2-NEXT: jne .LBB4_4
-; AVX2-NEXT: # %bb.5:
-; AVX2-NEXT: movl (%rdi), %edx
-; AVX2-NEXT: jmp .LBB4_6
-; AVX2-NEXT: .LBB4_4:
-; AVX2-NEXT: vpextrw $0, %xmm0, %edx
-; AVX2-NEXT: .LBB4_6:
-; AVX2-NEXT: movw %dx, (%rdi)
-; AVX2-NEXT: movw %si, 6(%rdi)
-; AVX2-NEXT: movw %cx, 4(%rdi)
-; AVX2-NEXT: movw %ax, 2(%rdi)
+; AVX2-NEXT: vpinsrw $0, 6(%rdi), %xmm0, %xmm2
+; AVX2-NEXT: vpinsrw $0, 4(%rdi), %xmm0, %xmm3
+; AVX2-NEXT: vpinsrw $0, 2(%rdi), %xmm0, %xmm4
+; AVX2-NEXT: vpinsrw $0, (%rdi), %xmm0, %xmm5
+; AVX2-NEXT: vmovd %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: vmovd %eax, %xmm6
+; AVX2-NEXT: vpblendvb %xmm6, %xmm0, %xmm5, %xmm5
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm6
+; AVX2-NEXT: vpextrb $4, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: vmovd %eax, %xmm7
+; AVX2-NEXT: vpblendvb %xmm7, %xmm6, %xmm4, %xmm4
+; AVX2-NEXT: vmovshdup {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; AVX2-NEXT: vpextrb $8, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: vmovd %eax, %xmm7
+; AVX2-NEXT: vpblendvb %xmm7, %xmm6, %xmm3, %xmm3
+; AVX2-NEXT: vpsrlq $48, %xmm0, %xmm0
+; AVX2-NEXT: vpextrb $12, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: vmovd %eax, %xmm1
+; AVX2-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: vpextrw $0, %xmm0, 6(%rdi)
+; AVX2-NEXT: vpextrw $0, %xmm3, 4(%rdi)
+; AVX2-NEXT: vpextrw $0, %xmm4, 2(%rdi)
+; AVX2-NEXT: vpextrw $0, %xmm5, (%rdi)
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_masked_store_success_v4f16:
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index b6a4a12eb0fac..67ddb936b7416 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -1248,6 +1248,37 @@ define <8 x half> @select(i1 %c, <8 x half> %x, <8 x half> %y) {
ret <8 x half> %s
}
+define half @select_scalar(i1 %c, half %x, half %y) {
+; CHECK-LIBCALL-LABEL: select_scalar:
+; CHECK-LIBCALL: # %bb.0:
+; CHECK-LIBCALL-NEXT: andl $1, %edi
+; CHECK-LIBCALL-NEXT: negl %edi
+; CHECK-LIBCALL-NEXT: movd %edi, %xmm2
+; CHECK-LIBCALL-NEXT: pand %xmm2, %xmm0
+; CHECK-LIBCALL-NEXT: pandn %xmm1, %xmm2
+; CHECK-LIBCALL-NEXT: por %xmm2, %xmm0
+; CHECK-LIBCALL-NEXT: retq
+;
+; BWON-F16C-LABEL: select_scalar:
+; BWON-F16C: # %bb.0:
+; BWON-F16C-NEXT: andl $1, %edi
+; BWON-F16C-NEXT: negl %edi
+; BWON-F16C-NEXT: vmovd %edi, %xmm2
+; BWON-F16C-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; BWON-F16C-NEXT: retq
+;
+; CHECK-I686-LABEL: select_scalar:
+; CHECK-I686: # %bb.0:
+; CHECK-I686-NEXT: testb $1, {{[0-9]+}}(%esp)
+; CHECK-I686-NEXT: leal {{[0-9]+}}(%esp), %eax
+; CHECK-I686-NEXT: leal {{[0-9]+}}(%esp), %ecx
+; CHECK-I686-NEXT: cmovnel %eax, %ecx
+; CHECK-I686-NEXT: pinsrw $0, (%ecx), %xmm0
+; CHECK-I686-NEXT: retl
+ %s = select i1 %c, half %x, half %y
+ ret half %s
+}
+
define <8 x half> @shuffle(ptr %p) {
; CHECK-LIBCALL-LABEL: shuffle:
; CHECK-LIBCALL: # %bb.0:
@@ -1350,11 +1381,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: ja .LBB26_2
+; CHECK-LIBCALL-NEXT: ja .LBB27_2
; CHECK-LIBCALL-NEXT: # %bb.1:
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_2:
+; CHECK-LIBCALL-NEXT: .LBB27_2:
; CHECK-LIBCALL-NEXT: callq __truncsfhf2 at PLT
; CHECK-LIBCALL-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-LIBCALL-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -1375,11 +1406,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: ja .LBB26_4
+; CHECK-LIBCALL-NEXT: ja .LBB27_4
; CHECK-LIBCALL-NEXT: # %bb.3:
; CHECK-LIBCALL-NEXT: movss (%rsp), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_4:
+; CHECK-LIBCALL-NEXT: .LBB27_4:
; CHECK-LIBCALL-NEXT: callq __truncsfhf2 at PLT
; CHECK-LIBCALL-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-LIBCALL-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -1400,11 +1431,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: ucomiss (%rsp), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: ja .LBB26_6
+; CHECK-LIBCALL-NEXT: ja .LBB27_6
; CHECK-LIBCALL-NEXT: # %bb.5:
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_6:
+; CHECK-LIBCALL-NEXT: .LBB27_6:
; CHECK-LIBCALL-NEXT: callq __truncsfhf2 at PLT
; CHECK-LIBCALL-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-LIBCALL-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -1425,11 +1456,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: ja .LBB26_8
+; CHECK-LIBCALL-NEXT: ja .LBB27_8
; CHECK-LIBCALL-NEXT: # %bb.7:
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_8:
+; CHECK-LIBCALL-NEXT: .LBB27_8:
; CHECK-LIBCALL-NEXT: callq __truncsfhf2 at PLT
; CHECK-LIBCALL-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-LIBCALL-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -1450,11 +1481,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: movss (%rsp), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: ja .LBB26_10
+; CHECK-LIBCALL-NEXT: ja .LBB27_10
; CHECK-LIBCALL-NEXT: # %bb.9:
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_10:
+; CHECK-LIBCALL-NEXT: .LBB27_10:
; CHECK-LIBCALL-NEXT: callq __truncsfhf2 at PLT
; CHECK-LIBCALL-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill
; CHECK-LIBCALL-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
@@ -1475,11 +1506,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm1 = mem[0],zero,zero,zero
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; CHECK-LIBCALL-NEXT: ja .LBB26_12
+; CHECK-LIBCALL-NEXT: ja .LBB27_12
; CHECK-LIBCALL-NEXT: # %bb.11:
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm1 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_12:
+; CHECK-LIBCALL-NEXT: .LBB27_12:
; CHECK-LIBCALL-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-LIBCALL-NEXT: punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; CHECK-LIBCALL-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
@@ -1505,11 +1536,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm1 = mem[0],zero,zero,zero
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; CHECK-LIBCALL-NEXT: ja .LBB26_14
+; CHECK-LIBCALL-NEXT: ja .LBB27_14
; CHECK-LIBCALL-NEXT: # %bb.13:
; CHECK-LIBCALL-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 4-byte Reload
; CHECK-LIBCALL-NEXT: # xmm1 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_14:
+; CHECK-LIBCALL-NEXT: .LBB27_14:
; CHECK-LIBCALL-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-LIBCALL-NEXT: unpcklps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; CHECK-LIBCALL-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
@@ -1539,11 +1570,11 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-LIBCALL-NEXT: ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: ja .LBB26_16
+; CHECK-LIBCALL-NEXT: ja .LBB27_16
; CHECK-LIBCALL-NEXT: # %bb.15:
; CHECK-LIBCALL-NEXT: movd (%rsp), %xmm0 # 4-byte Folded Reload
; CHECK-LIBCALL-NEXT: # xmm0 = mem[0],zero,zero,zero
-; CHECK-LIBCALL-NEXT: .LBB26_16:
+; CHECK-LIBCALL-NEXT: .LBB27_16:
; CHECK-LIBCALL-NEXT: callq __truncsfhf2 at PLT
; CHECK-LIBCALL-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; CHECK-LIBCALL-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
@@ -1562,46 +1593,46 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; BWON-F16C-NEXT: vpsrldq {{.*#+}} xmm2 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; BWON-F16C-NEXT: vcvtph2ps %xmm2, %xmm2
; BWON-F16C-NEXT: vucomiss %xmm3, %xmm2
-; BWON-F16C-NEXT: ja .LBB26_2
+; BWON-F16C-NEXT: ja .LBB27_2
; BWON-F16C-NEXT: # %bb.1:
; BWON-F16C-NEXT: vmovaps %xmm3, %xmm2
-; BWON-F16C-NEXT: .LBB26_2:
+; BWON-F16C-NEXT: .LBB27_2:
; BWON-F16C-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
; BWON-F16C-NEXT: vcvtph2ps %xmm3, %xmm4
; BWON-F16C-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
; BWON-F16C-NEXT: vcvtph2ps %xmm3, %xmm3
; BWON-F16C-NEXT: vucomiss %xmm4, %xmm3
-; BWON-F16C-NEXT: ja .LBB26_4
+; BWON-F16C-NEXT: ja .LBB27_4
; BWON-F16C-NEXT: # %bb.3:
; BWON-F16C-NEXT: vmovaps %xmm4, %xmm3
-; BWON-F16C-NEXT: .LBB26_4:
+; BWON-F16C-NEXT: .LBB27_4:
; BWON-F16C-NEXT: vpsrldq {{.*#+}} xmm4 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; BWON-F16C-NEXT: vcvtph2ps %xmm4, %xmm5
; BWON-F16C-NEXT: vpsrldq {{.*#+}} xmm4 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; BWON-F16C-NEXT: vcvtph2ps %xmm4, %xmm4
; BWON-F16C-NEXT: vucomiss %xmm5, %xmm4
-; BWON-F16C-NEXT: ja .LBB26_6
+; BWON-F16C-NEXT: ja .LBB27_6
; BWON-F16C-NEXT: # %bb.5:
; BWON-F16C-NEXT: vmovaps %xmm5, %xmm4
-; BWON-F16C-NEXT: .LBB26_6:
+; BWON-F16C-NEXT: .LBB27_6:
; BWON-F16C-NEXT: vshufpd {{.*#+}} xmm5 = xmm1[1,0]
; BWON-F16C-NEXT: vcvtph2ps %xmm5, %xmm5
; BWON-F16C-NEXT: vshufpd {{.*#+}} xmm6 = xmm0[1,0]
; BWON-F16C-NEXT: vcvtph2ps %xmm6, %xmm6
; BWON-F16C-NEXT: vucomiss %xmm5, %xmm6
-; BWON-F16C-NEXT: ja .LBB26_8
+; BWON-F16C-NEXT: ja .LBB27_8
; BWON-F16C-NEXT: # %bb.7:
; BWON-F16C-NEXT: vmovaps %xmm5, %xmm6
-; BWON-F16C-NEXT: .LBB26_8:
+; BWON-F16C-NEXT: .LBB27_8:
; BWON-F16C-NEXT: vpsrlq $48, %xmm1, %xmm5
; BWON-F16C-NEXT: vcvtph2ps %xmm5, %xmm7
; BWON-F16C-NEXT: vpsrlq $48, %xmm0, %xmm5
; BWON-F16C-NEXT: vcvtph2ps %xmm5, %xmm5
; BWON-F16C-NEXT: vucomiss %xmm7, %xmm5
-; BWON-F16C-NEXT: ja .LBB26_10
+; BWON-F16C-NEXT: ja .LBB27_10
; BWON-F16C-NEXT: # %bb.9:
; BWON-F16C-NEXT: vmovaps %xmm7, %xmm5
-; BWON-F16C-NEXT: .LBB26_10:
+; BWON-F16C-NEXT: .LBB27_10:
; BWON-F16C-NEXT: vcvtps2ph $4, %xmm2, %xmm2
; BWON-F16C-NEXT: vcvtps2ph $4, %xmm3, %xmm3
; BWON-F16C-NEXT: vcvtps2ph $4, %xmm4, %xmm4
@@ -1611,10 +1642,10 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; BWON-F16C-NEXT: vmovshdup {{.*#+}} xmm7 = xmm0[1,1,3,3]
; BWON-F16C-NEXT: vcvtph2ps %xmm7, %xmm7
; BWON-F16C-NEXT: vucomiss %xmm8, %xmm7
-; BWON-F16C-NEXT: ja .LBB26_12
+; BWON-F16C-NEXT: ja .LBB27_12
; BWON-F16C-NEXT: # %bb.11:
; BWON-F16C-NEXT: vmovaps %xmm8, %xmm7
-; BWON-F16C-NEXT: .LBB26_12:
+; BWON-F16C-NEXT: .LBB27_12:
; BWON-F16C-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
; BWON-F16C-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
; BWON-F16C-NEXT: vcvtps2ph $4, %xmm5, %xmm4
@@ -1622,10 +1653,10 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; BWON-F16C-NEXT: vcvtph2ps %xmm1, %xmm7
; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm6
; BWON-F16C-NEXT: vucomiss %xmm7, %xmm6
-; BWON-F16C-NEXT: ja .LBB26_14
+; BWON-F16C-NEXT: ja .LBB27_14
; BWON-F16C-NEXT: # %bb.13:
; BWON-F16C-NEXT: vmovaps %xmm7, %xmm6
-; BWON-F16C-NEXT: .LBB26_14:
+; BWON-F16C-NEXT: .LBB27_14:
; BWON-F16C-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
; BWON-F16C-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; BWON-F16C-NEXT: vcvtps2ph $4, %xmm6, %xmm4
@@ -1634,10 +1665,10 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; BWON-F16C-NEXT: vpsrld $16, %xmm0, %xmm0
; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0
; BWON-F16C-NEXT: vucomiss %xmm1, %xmm0
-; BWON-F16C-NEXT: ja .LBB26_16
+; BWON-F16C-NEXT: ja .LBB27_16
; BWON-F16C-NEXT: # %bb.15:
; BWON-F16C-NEXT: vmovaps %xmm1, %xmm0
-; BWON-F16C-NEXT: .LBB26_16:
+; BWON-F16C-NEXT: .LBB27_16:
; BWON-F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm0
; BWON-F16C-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
; BWON-F16C-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
@@ -1692,13 +1723,13 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-I686-NEXT: fstps {{[0-9]+}}(%esp)
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-I686-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0
-; CHECK-I686-NEXT: ja .LBB26_1
+; CHECK-I686-NEXT: ja .LBB27_1
; CHECK-I686-NEXT: # %bb.2:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: jmp .LBB26_3
-; CHECK-I686-NEXT: .LBB26_1:
+; CHECK-I686-NEXT: jmp .LBB27_3
+; CHECK-I686-NEXT: .LBB27_1:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: .LBB26_3:
+; CHECK-I686-NEXT: .LBB27_3:
; CHECK-I686-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-I686-NEXT: calll __extendhfsf2
; CHECK-I686-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -1713,13 +1744,13 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-I686-NEXT: fstps {{[0-9]+}}(%esp)
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-I686-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0
-; CHECK-I686-NEXT: ja .LBB26_4
+; CHECK-I686-NEXT: ja .LBB27_4
; CHECK-I686-NEXT: # %bb.5:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: jmp .LBB26_6
-; CHECK-I686-NEXT: .LBB26_4:
+; CHECK-I686-NEXT: jmp .LBB27_6
+; CHECK-I686-NEXT: .LBB27_4:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: .LBB26_6:
+; CHECK-I686-NEXT: .LBB27_6:
; CHECK-I686-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-I686-NEXT: calll __truncsfhf2
; CHECK-I686-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -1770,13 +1801,13 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-I686-NEXT: fstps {{[0-9]+}}(%esp)
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-I686-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0
-; CHECK-I686-NEXT: ja .LBB26_7
+; CHECK-I686-NEXT: ja .LBB27_7
; CHECK-I686-NEXT: # %bb.8:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: jmp .LBB26_9
-; CHECK-I686-NEXT: .LBB26_7:
+; CHECK-I686-NEXT: jmp .LBB27_9
+; CHECK-I686-NEXT: .LBB27_7:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: .LBB26_9:
+; CHECK-I686-NEXT: .LBB27_9:
; CHECK-I686-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-I686-NEXT: calll __extendhfsf2
; CHECK-I686-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -1791,13 +1822,13 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-I686-NEXT: fstps {{[0-9]+}}(%esp)
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-I686-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0
-; CHECK-I686-NEXT: ja .LBB26_10
+; CHECK-I686-NEXT: ja .LBB27_10
; CHECK-I686-NEXT: # %bb.11:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: jmp .LBB26_12
-; CHECK-I686-NEXT: .LBB26_10:
+; CHECK-I686-NEXT: jmp .LBB27_12
+; CHECK-I686-NEXT: .LBB27_10:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: .LBB26_12:
+; CHECK-I686-NEXT: .LBB27_12:
; CHECK-I686-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-I686-NEXT: calll __truncsfhf2
; CHECK-I686-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -1848,13 +1879,13 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-I686-NEXT: fstps {{[0-9]+}}(%esp)
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-I686-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0
-; CHECK-I686-NEXT: ja .LBB26_13
+; CHECK-I686-NEXT: ja .LBB27_13
; CHECK-I686-NEXT: # %bb.14:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: jmp .LBB26_15
-; CHECK-I686-NEXT: .LBB26_13:
+; CHECK-I686-NEXT: jmp .LBB27_15
+; CHECK-I686-NEXT: .LBB27_13:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: .LBB26_15:
+; CHECK-I686-NEXT: .LBB27_15:
; CHECK-I686-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-I686-NEXT: calll __extendhfsf2
; CHECK-I686-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -1869,13 +1900,13 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-I686-NEXT: fstps {{[0-9]+}}(%esp)
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-I686-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0
-; CHECK-I686-NEXT: ja .LBB26_16
+; CHECK-I686-NEXT: ja .LBB27_16
; CHECK-I686-NEXT: # %bb.17:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: jmp .LBB26_18
-; CHECK-I686-NEXT: .LBB26_16:
+; CHECK-I686-NEXT: jmp .LBB27_18
+; CHECK-I686-NEXT: .LBB27_16:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: .LBB26_18:
+; CHECK-I686-NEXT: .LBB27_18:
; CHECK-I686-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-I686-NEXT: calll __truncsfhf2
; CHECK-I686-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -1926,13 +1957,13 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-I686-NEXT: fstps {{[0-9]+}}(%esp)
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-I686-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0
-; CHECK-I686-NEXT: ja .LBB26_19
+; CHECK-I686-NEXT: ja .LBB27_19
; CHECK-I686-NEXT: # %bb.20:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: jmp .LBB26_21
-; CHECK-I686-NEXT: .LBB26_19:
+; CHECK-I686-NEXT: jmp .LBB27_21
+; CHECK-I686-NEXT: .LBB27_19:
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: .LBB26_21:
+; CHECK-I686-NEXT: .LBB27_21:
; CHECK-I686-NEXT: movss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-I686-NEXT: calll __extendhfsf2
; CHECK-I686-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Reload
@@ -1947,13 +1978,13 @@ define <8 x half> @maxnum_v8f16(<8 x half> %0, <8 x half> %1) #0 {
; CHECK-I686-NEXT: fstps {{[0-9]+}}(%esp)
; CHECK-I686-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-I686-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0
-; CHECK-I686-NEXT: ja .LBB26_22
+; CHECK-I686-NEXT: ja .LBB27_22
; CHECK-I686-NEXT: # %bb.23:
; CHECK-I686-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: jmp .LBB26_24
-; CHECK-I686-NEXT: .LBB26_22:
+; CHECK-I686-NEXT: jmp .LBB27_24
+; CHECK-I686-NEXT: .LBB27_22:
; CHECK-I686-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-I686-NEXT: .LBB26_24:
+; CHECK-I686-NEXT: .LBB27_24:
; CHECK-I686-NEXT: movd %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; CHECK-I686-NEXT: calll __truncsfhf2
; CHECK-I686-NEXT: movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 4-byte Reload
>From 7be5f2ca250bfe541c5ddc564b60d21440444b8d Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Mon, 24 Aug 2026 15:47:31 +0200
Subject: [PATCH 2/7] Shorten comments
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index f9757336af038..a7a7c05eb7888 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48684,10 +48684,7 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
// Soft bf16/f16 scalar selects: do a VSELECT in vector registers instead
// of a scalar CMOV, to avoid a GPR round-trip. Skip constant operands
// (cheaper as immediates) and compare-driven conds (CMOV already reuses
- // the flags). bf16 has no register class, so it can only appear as a
- // BITCAST operand (the one case the type legalizer's soft-promotion
- // knows how to handle) -- bitcast to f16 (always legal) before building
- // any vector node, never leave a bare scalar bf16 as an operand.
+ // the flags). Use f16 since bf16 not a legal register type.
if (N->getOpcode() == ISD::SELECT && !CondVT.isVector() &&
Subtarget.hasSSE2() && !isIntOrFPConstant(LHS) &&
!isIntOrFPConstant(RHS)) {
@@ -48716,9 +48713,9 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
}
}
if (F16LHS) {
- // Blend in v8i16 (not v8f16): there's no VBLENDVPH, so a v8f16
+ // Current blend in v8i16 (not v8f16) sincea v8f16
// VSELECT can fail to select on subtargets that fall back to
- // BLENDV instead of a mask-register select.
+ // BLENDV(since there's no VBLENDVPH) instead of a mask-register select.
SDValue Mask = DAG.getNode(ISD::SUB, DL, MVT::i16,
DAG.getConstant(0, DL, MVT::i16),
DAG.getZExtOrTrunc(Cond, DL, MVT::i16));
>From 5f75a30a3c783346b2daacf8cd0307cf441887b4 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Thu, 17 Sep 2026 10:28:19 +0200
Subject: [PATCH 3/7] Fix formatting
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 2fb96ca01fd23..1d22db47ccf84 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -49016,13 +49016,15 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
// Current blend in v8i16 (not v8f16) sincea v8f16
// VSELECT can fail to select on subtargets that fall back to
// BLENDV(since there's no VBLENDVPH) instead of a mask-register select.
- SDValue Mask = DAG.getNode(ISD::SUB, DL, MVT::i16,
- DAG.getConstant(0, DL, MVT::i16),
- DAG.getZExtOrTrunc(Cond, DL, MVT::i16));
- SDValue VLHS = DAG.getBitcast(
- MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, F16LHS));
- SDValue VRHS = DAG.getBitcast(
- MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, F16RHS));
+ SDValue Mask =
+ DAG.getNode(ISD::SUB, DL, MVT::i16, DAG.getConstant(0, DL, MVT::i16),
+ DAG.getZExtOrTrunc(Cond, DL, MVT::i16));
+ SDValue VLHS =
+ DAG.getBitcast(MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL,
+ MVT::v8f16, F16LHS));
+ SDValue VRHS =
+ DAG.getBitcast(MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL,
+ MVT::v8f16, F16RHS));
SDValue VMask = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8i16, Mask);
SDValue VSel = DAG.getSelect(DL, MVT::v8i16, VMask, VLHS, VRHS);
SDValue Res = DAG.getExtractVectorElt(DL, MVT::i16, VSel, 0);
>From 54779947d59152939aebbbdd9bd23d12c0fdd5b8 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Thu, 17 Sep 2026 10:43:24 +0200
Subject: [PATCH 4/7] Enable bitcast check and cleanup
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 48 +++++++++++++++----------
1 file changed, 29 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1d22db47ccf84..509c8b64f4f7b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48988,22 +48988,22 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
if (N->getOpcode() == ISD::SELECT && !CondVT.isVector() &&
Subtarget.hasSSE2() && !isIntOrFPConstant(LHS) &&
!isIntOrFPConstant(RHS)) {
- SDValue CondRoot = Cond;
- while (CondRoot.getOpcode() == ISD::AND ||
- CondRoot.getOpcode() == ISD::ANY_EXTEND ||
- CondRoot.getOpcode() == ISD::ZERO_EXTEND ||
- CondRoot.getOpcode() == ISD::TRUNCATE)
- CondRoot = CondRoot.getOperand(0);
- bool CondIsCompare = CondRoot.getOpcode() == ISD::SETCC ||
- CondRoot.getOpcode() == X86ISD::SETCC;
-
+ // Only worth it if both operands already live in a vector register: either
+ // the select is on the 16-bit float itself, or it is the equivalent i16
+ // select of bitcast 16-bit floats. Bitcasts from a GPR mean the values are
+ // there instead, and moving them in and the result back out costs more than
+ // the CMOV.
+ auto IsBitcastFromGPR = [](SDValue Op) {
+ return Op.getOpcode() == ISD::BITCAST &&
+ Op.getOperand(0).getValueType().isScalarInteger();
+ };
SDValue F16LHS, F16RHS;
- if (!CondIsCompare && (VT == MVT::f16 || VT == MVT::bf16) &&
- isSoftF16(VT, Subtarget)) {
- F16LHS = DAG.getBitcast(MVT::f16, LHS);
- F16RHS = DAG.getBitcast(MVT::f16, RHS);
- } else if (!CondIsCompare && VT == MVT::i16 &&
- LHS.getOpcode() == ISD::BITCAST &&
+ if (!VT.isVector() && isSoftF16(VT, Subtarget)) {
+ if (!IsBitcastFromGPR(LHS) || !IsBitcastFromGPR(RHS)) {
+ F16LHS = DAG.getBitcast(MVT::f16, LHS);
+ F16RHS = DAG.getBitcast(MVT::f16, RHS);
+ }
+ } else if (VT == MVT::i16 && LHS.getOpcode() == ISD::BITCAST &&
RHS.getOpcode() == ISD::BITCAST) {
MVT SVT = LHS.getOperand(0).getSimpleValueType();
if ((SVT == MVT::f16 || SVT == MVT::bf16) &&
@@ -49012,10 +49012,20 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
F16RHS = DAG.getBitcast(MVT::f16, RHS.getOperand(0));
}
}
- if (F16LHS) {
- // Current blend in v8i16 (not v8f16) sincea v8f16
- // VSELECT can fail to select on subtargets that fall back to
- // BLENDV(since there's no VBLENDVPH) instead of a mask-register select.
+
+ // Peek past the boolean plumbing to see what produced the condition.
+ SDValue CondRoot = Cond;
+ while (CondRoot.getOpcode() == ISD::AND ||
+ CondRoot.getOpcode() == ISD::ANY_EXTEND ||
+ CondRoot.getOpcode() == ISD::ZERO_EXTEND ||
+ CondRoot.getOpcode() == ISD::TRUNCATE)
+ CondRoot = CondRoot.getOperand(0);
+
+ if (F16LHS && CondRoot.getOpcode() != ISD::SETCC &&
+ CondRoot.getOpcode() != X86ISD::SETCC) {
+ // Currently blend in v8i16 (not v8f16) since a v8f16 VSELECT can fail to
+ // select on subtargets that fall back to BLENDV (since there's no
+ // VBLENDVPH) instead of a mask-register select.
SDValue Mask =
DAG.getNode(ISD::SUB, DL, MVT::i16, DAG.getConstant(0, DL, MVT::i16),
DAG.getZExtOrTrunc(Cond, DL, MVT::i16));
>From 9817aabe990f6dde88da57ac18c01c63b7f61646 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Thu, 17 Sep 2026 10:43:42 +0200
Subject: [PATCH 5/7] Additional tests
---
llvm/test/CodeGen/X86/bfloat.ll | 93 +++++++++++++++++++++++++++++++++
1 file changed, 93 insertions(+)
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index be83bb557ecb1..b0fb7a147c170 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -2236,3 +2236,96 @@ define bfloat @select_ogt_bf16(bfloat %a, bfloat %b) nounwind {
%sel = select i1 %cmp, bfloat %a, bfloat %b
ret bfloat %sel
}
+
+; Both operands come out of GPRs, so the CMOV is cheaper than moving them into
+; vector registers and the result back out.
+define i16 @select_bf16_from_gpr(i1 %cond, i16 %a, i16 %b) nounwind {
+; X86-LABEL: select_bf16_from_gpr:
+; X86: # %bb.0:
+; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: vmovw %eax, %xmm2
+; X86-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT: vmovw %xmm0, %eax
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: retl
+;
+; X64-LABEL: select_bf16_from_gpr:
+; X64: # %bb.0:
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovel %edx, %eax
+; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: retq
+ %fa = bitcast i16 %a to bfloat
+ %fb = bitcast i16 %b to bfloat
+ %sel = select i1 %cond, bfloat %fa, bfloat %fb
+ %res = bitcast bfloat %sel to i16
+ ret i16 %res
+}
+
+; The operands already live in vector registers, so blend there even though the
+; select itself is on i16.
+define i16 @select_i16_of_bf16(i1 %cond, bfloat %a, bfloat %b) nounwind {
+; X86-LABEL: select_i16_of_bf16:
+; X86: # %bb.0:
+; X86-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: vmovsh {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: vmovw %eax, %xmm2
+; X86-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; X86-NEXT: vmovw %xmm0, %eax
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: retl
+;
+; SSE2-LABEL: select_i16_of_bf16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: andl $1, %edi
+; SSE2-NEXT: negl %edi
+; SSE2-NEXT: movd %edi, %xmm2
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: retq
+;
+; AVX512BF16-LABEL: select_i16_of_bf16:
+; AVX512BF16: # %bb.0:
+; AVX512BF16-NEXT: andl $1, %edi
+; AVX512BF16-NEXT: negl %edi
+; AVX512BF16-NEXT: vmovd %edi, %xmm2
+; AVX512BF16-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512BF16-NEXT: vmovd %xmm0, %eax
+; AVX512BF16-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BF16-NEXT: retq
+;
+; AVX512FP16-LABEL: select_i16_of_bf16:
+; AVX512FP16: # %bb.0:
+; AVX512FP16-NEXT: andl $1, %edi
+; AVX512FP16-NEXT: negl %edi
+; AVX512FP16-NEXT: vmovw %edi, %xmm2
+; AVX512FP16-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512FP16-NEXT: vmovw %xmm0, %eax
+; AVX512FP16-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512FP16-NEXT: retq
+;
+; AVXNC-LABEL: select_i16_of_bf16:
+; AVXNC: # %bb.0:
+; AVXNC-NEXT: andl $1, %edi
+; AVXNC-NEXT: negl %edi
+; AVXNC-NEXT: vmovd %edi, %xmm2
+; AVXNC-NEXT: vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVXNC-NEXT: vmovd %xmm0, %eax
+; AVXNC-NEXT: # kill: def $ax killed $ax killed $eax
+; AVXNC-NEXT: retq
+ %ai = bitcast bfloat %a to i16
+ %bi = bitcast bfloat %b to i16
+ %sel = select i1 %cond, i16 %ai, i16 %bi
+ ret i16 %sel
+}
>From b036126cefc2b6a58db4026805c3ca2fe9617c85 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Thu, 17 Sep 2026 10:46:22 +0200
Subject: [PATCH 6/7] Clenaup comments
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 509c8b64f4f7b..de73e7ac75b58 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -48981,18 +48981,14 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
bool CondConstantVector = ISD::isBuildVectorOfConstantSDNodes(Cond.getNode());
unsigned EltBitWidth = VT.getScalarSizeInBits();
- // Soft bf16/f16 scalar selects: do a VSELECT in vector registers instead
+ // Soft bf16/f16 scalar selects do a VSELECT in vector registers instead
// of a scalar CMOV, to avoid a GPR round-trip. Skip constant operands
// (cheaper as immediates) and compare-driven conds (CMOV already reuses
- // the flags). Use f16 since bf16 not a legal register type.
+ // the flags).
if (N->getOpcode() == ISD::SELECT && !CondVT.isVector() &&
Subtarget.hasSSE2() && !isIntOrFPConstant(LHS) &&
!isIntOrFPConstant(RHS)) {
- // Only worth it if both operands already live in a vector register: either
- // the select is on the 16-bit float itself, or it is the equivalent i16
- // select of bitcast 16-bit floats. Bitcasts from a GPR mean the values are
- // there instead, and moving them in and the result back out costs more than
- // the CMOV.
+ // Only worth it if both operands already live in a vector register
auto IsBitcastFromGPR = [](SDValue Op) {
return Op.getOpcode() == ISD::BITCAST &&
Op.getOperand(0).getValueType().isScalarInteger();
@@ -49013,7 +49009,6 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
}
}
- // Peek past the boolean plumbing to see what produced the condition.
SDValue CondRoot = Cond;
while (CondRoot.getOpcode() == ISD::AND ||
CondRoot.getOpcode() == ISD::ANY_EXTEND ||
@@ -49024,8 +49019,7 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
if (F16LHS && CondRoot.getOpcode() != ISD::SETCC &&
CondRoot.getOpcode() != X86ISD::SETCC) {
// Currently blend in v8i16 (not v8f16) since a v8f16 VSELECT can fail to
- // select on subtargets that fall back to BLENDV (since there's no
- // VBLENDVPH) instead of a mask-register select.
+ // select on some subtargets
SDValue Mask =
DAG.getNode(ISD::SUB, DL, MVT::i16, DAG.getConstant(0, DL, MVT::i16),
DAG.getZExtOrTrunc(Cond, DL, MVT::i16));
>From bd19edfd0887c60abec6a3ae99e47e4d0f33d855 Mon Sep 17 00:00:00 2001
From: "Ziegler, Tim" <tim.ziegler at intel.com>
Date: Mon, 21 Sep 2026 09:05:29 +0200
Subject: [PATCH 7/7] Use getNegative to get the actual negative :)
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index de73e7ac75b58..58e2eee10f1c0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -49021,8 +49021,7 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
// Currently blend in v8i16 (not v8f16) since a v8f16 VSELECT can fail to
// select on some subtargets
SDValue Mask =
- DAG.getNode(ISD::SUB, DL, MVT::i16, DAG.getConstant(0, DL, MVT::i16),
- DAG.getZExtOrTrunc(Cond, DL, MVT::i16));
+ DAG.getNegative(DAG.getZExtOrTrunc(Cond, DL, MVT::i16), DL, MVT::i16);
SDValue VLHS =
DAG.getBitcast(MVT::v8i16, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL,
MVT::v8f16, F16LHS));
More information about the llvm-commits
mailing list