[llvm] [X86] LowerBUILD_VECTORvXi1 - scalarize the bool masks if we insert a single non-const value (PR #199523)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon May 25 06:18:30 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/199523
>From b9ee3b98c7bac46ad475ccf8ce4a76dc842cbbc0 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 25 May 2026 13:25:27 +0100
Subject: [PATCH 1/3] [X86] LowerBUILD_VECTORvXi1 - scalarize the bool masks if
we inserting a single non-const value
Minor generalization of the existing fold for splat bool masks - if only a single value is used in insertion(s), as well as any immediate/undefs values - then fold to a scalar select (val, insert|immediate, immediate)
Yak shaving for #198162
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 53 ++++++++++++------
llvm/test/CodeGen/X86/avx512-ext.ll | 40 +++----------
.../test/CodeGen/X86/avx512-insert-extract.ll | 32 +++--------
llvm/test/CodeGen/X86/avx512-mask-op.ll | 56 +++++++++----------
4 files changed, 75 insertions(+), 106 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 349d4fa4c0947..2fdc4625b5b27 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -8503,51 +8503,68 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl,
ISD::isBuildVectorAllOnes(Op.getNode()))
return Op;
+ uint64_t Undefs = 0;
uint64_t Immediate = 0;
SmallVector<unsigned, 16> NonConstIdx;
- bool IsSplat = true;
+ SmallSet<SDValue, 16> NonConstElts;
bool HasConstElts = false;
- int SplatIdx = -1;
for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
SDValue In = Op.getOperand(idx);
- if (In.isUndef())
+ if (In.isUndef()) {
+ Undefs |= 1ULL << idx;
continue;
+ }
if (auto *InC = dyn_cast<ConstantSDNode>(In)) {
Immediate |= (InC->getZExtValue() & 0x1) << idx;
HasConstElts = true;
} else {
NonConstIdx.push_back(idx);
+ NonConstElts.insert(In);
}
- if (SplatIdx < 0)
- SplatIdx = idx;
- else if (In != Op.getOperand(SplatIdx))
- IsSplat = false;
}
- // for splat use " (select i1 splat_elt, all-ones, all-zeroes)"
- if (IsSplat) {
+ // for single non-const use " (select i1 elt, imm | elt_mask, imm)"
+ if (NonConstElts.size() == 1) {
// The build_vector allows the scalar element to be larger than the vector
// element type. We need to mask it to use as a condition unless we know
// the upper bits are zero.
// FIXME: Use computeKnownBits instead of checking specific opcode?
- SDValue Cond = Op.getOperand(SplatIdx);
+ SDValue Cond = *NonConstElts.begin();
assert(Cond.getValueType() == MVT::i8 && "Unexpected VT!");
if (Cond.getOpcode() != ISD::SETCC)
Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond,
DAG.getConstant(1, dl, MVT::i8));
+ uint64_t NonConstMask = 0;
+ for (unsigned InsertIdx : NonConstIdx)
+ NonConstMask |= 1ULL << InsertIdx;
+
+ uint64_t TrueImm = NonConstMask | Immediate;
+ uint64_t FalseImm = Immediate;
+
// Perform the select in the scalar domain so we can use cmov.
if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
- SDValue Select = DAG.getSelect(dl, MVT::i32, Cond,
- DAG.getAllOnesConstant(dl, MVT::i32),
- DAG.getConstant(0, dl, MVT::i32));
- Select = DAG.getBitcast(MVT::v32i1, Select);
- return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Select, Select);
+ uint64_t TrueLo = (unsigned)TrueImm;
+ uint64_t TrueHi = TrueImm >> 32;
+ uint64_t FalseLo = (unsigned)FalseImm;
+ uint64_t FalseHi = FalseImm >> 32;
+ SDValue Lo = DAG.getSelect(dl, MVT::i32, Cond,
+ DAG.getConstant(TrueLo, dl, MVT::i32),
+ DAG.getConstant(FalseLo, dl, MVT::i32));
+ SDValue Hi = DAG.getSelect(dl, MVT::i32, Cond,
+ DAG.getConstant(TrueHi, dl, MVT::i32),
+ DAG.getConstant(FalseHi, dl, MVT::i32));
+ Lo = DAG.getBitcast(MVT::v32i1, Lo);
+ Hi = DAG.getBitcast(MVT::v32i1, Hi);
+ return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
} else {
MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
- SDValue Select = DAG.getSelect(dl, ImmVT, Cond,
- DAG.getAllOnesConstant(dl, ImmVT),
- DAG.getConstant(0, dl, ImmVT));
+ // Adjust extended value to -1 as it will improve folding.
+ if (llvm::popcount(TrueImm | Undefs) == VT.getSizeInBits())
+ TrueImm = ~0ULL >> (64 - ImmVT.getSizeInBits());
+ SDValue Select =
+ DAG.getSelect(dl, ImmVT, Cond, DAG.getConstant(TrueImm, dl, ImmVT),
+ DAG.getConstant(FalseImm, dl, ImmVT));
MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
Select = DAG.getBitcast(VecVT, Select);
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select,
diff --git a/llvm/test/CodeGen/X86/avx512-ext.ll b/llvm/test/CodeGen/X86/avx512-ext.ll
index 0e0a9f6546529..5aad782c7134e 100644
--- a/llvm/test/CodeGen/X86/avx512-ext.ll
+++ b/llvm/test/CodeGen/X86/avx512-ext.ll
@@ -1771,38 +1771,14 @@ define <8 x i32> @sext_8i1_8i32(<8 x i32> %a1, <8 x i32> %a2) nounwind {
define i16 @trunc_i32_to_i1(i32 %a) {
-; KNL-LABEL: trunc_i32_to_i1:
-; KNL: # %bb.0:
-; KNL-NEXT: andl $1, %edi
-; KNL-NEXT: kmovw %edi, %k0
-; KNL-NEXT: movw $-4, %ax
-; KNL-NEXT: kmovw %eax, %k1
-; KNL-NEXT: korw %k1, %k0, %k0
-; KNL-NEXT: kmovw %k0, %eax
-; KNL-NEXT: # kill: def $ax killed $ax killed $eax
-; KNL-NEXT: retq
-;
-; SKX-LABEL: trunc_i32_to_i1:
-; SKX: # %bb.0:
-; SKX-NEXT: andl $1, %edi
-; SKX-NEXT: kmovw %edi, %k0
-; SKX-NEXT: movw $-4, %ax
-; SKX-NEXT: kmovd %eax, %k1
-; SKX-NEXT: korw %k1, %k0, %k0
-; SKX-NEXT: kmovd %k0, %eax
-; SKX-NEXT: # kill: def $ax killed $ax killed $eax
-; SKX-NEXT: retq
-;
-; AVX512DQNOBW-LABEL: trunc_i32_to_i1:
-; AVX512DQNOBW: # %bb.0:
-; AVX512DQNOBW-NEXT: andl $1, %edi
-; AVX512DQNOBW-NEXT: kmovw %edi, %k0
-; AVX512DQNOBW-NEXT: movw $-4, %ax
-; AVX512DQNOBW-NEXT: kmovw %eax, %k1
-; AVX512DQNOBW-NEXT: korw %k1, %k0, %k0
-; AVX512DQNOBW-NEXT: kmovw %k0, %eax
-; AVX512DQNOBW-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512DQNOBW-NEXT: retq
+; ALL-LABEL: trunc_i32_to_i1:
+; ALL: # %bb.0:
+; ALL-NEXT: andb $1, %dil
+; ALL-NEXT: cmpb $1, %dil
+; ALL-NEXT: movl $65532, %eax # imm = 0xFFFC
+; ALL-NEXT: sbbl $-1, %eax
+; ALL-NEXT: # kill: def $ax killed $ax killed $eax
+; ALL-NEXT: retq
%a_i = trunc i32 %a to i1
%maskv = insertelement <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
%res = bitcast <16 x i1> %maskv to i16
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index df0d5989bc1e7..4b44afb33d103 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -234,31 +234,13 @@ define i64 @test12(<16 x i64>%a, <16 x i64>%b, i64 %a1, i64 %b1) nounwind {
}
define i16 @test13(i32 %a, i32 %b) nounwind {
-; KNL-LABEL: test13:
-; KNL: ## %bb.0:
-; KNL-NEXT: cmpl %esi, %edi
-; KNL-NEXT: setb %al
-; KNL-NEXT: andl $1, %eax
-; KNL-NEXT: kmovw %eax, %k0
-; KNL-NEXT: movw $-4, %ax
-; KNL-NEXT: kmovw %eax, %k1
-; KNL-NEXT: korw %k1, %k0, %k0
-; KNL-NEXT: kmovw %k0, %eax
-; KNL-NEXT: ## kill: def $ax killed $ax killed $eax
-; KNL-NEXT: retq
-;
-; SKX-LABEL: test13:
-; SKX: ## %bb.0:
-; SKX-NEXT: cmpl %esi, %edi
-; SKX-NEXT: setb %al
-; SKX-NEXT: andl $1, %eax
-; SKX-NEXT: kmovw %eax, %k0
-; SKX-NEXT: movw $-4, %ax
-; SKX-NEXT: kmovd %eax, %k1
-; SKX-NEXT: korw %k1, %k0, %k0
-; SKX-NEXT: kmovd %k0, %eax
-; SKX-NEXT: ## kill: def $ax killed $ax killed $eax
-; SKX-NEXT: retq
+; CHECK-LABEL: test13:
+; CHECK: ## %bb.0:
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: adcl $65532, %eax ## imm = 0xFFFC
+; CHECK-NEXT: ## kill: def $ax killed $ax killed $eax
+; CHECK-NEXT: retq
%cmp_res = icmp ult i32 %a, %b
%maskv = insertelement <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %cmp_res, i32 0
%res = bitcast <16 x i1> %maskv to i16
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index 4bd67a05d9c7d..83fbb1d90aa66 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -4337,64 +4337,58 @@ define <64 x i1> @mask64_insert(i32 %a) {
; KNL-LABEL: mask64_insert:
; KNL: ## %bb.0:
; KNL-NEXT: movq %rdi, %rax
-; KNL-NEXT: andl $1, %esi
-; KNL-NEXT: kmovw %esi, %k0
-; KNL-NEXT: movw $-4, %cx
-; KNL-NEXT: kmovw %ecx, %k1
-; KNL-NEXT: korw %k1, %k0, %k0
-; KNL-NEXT: kmovw %k0, (%rdi)
+; KNL-NEXT: andb $1, %sil
+; KNL-NEXT: cmpb $1, %sil
+; KNL-NEXT: movl $65532, %ecx ## imm = 0xFFFC
+; KNL-NEXT: sbbl $-1, %ecx
+; KNL-NEXT: movw %cx, (%rdi)
; KNL-NEXT: movw $-3, 6(%rdi)
; KNL-NEXT: movl $-131075, 2(%rdi) ## imm = 0xFFFDFFFD
; KNL-NEXT: retq
;
; SKX-LABEL: mask64_insert:
; SKX: ## %bb.0:
-; SKX-NEXT: kmovd %edi, %k0
-; SKX-NEXT: kshiftlq $63, %k0, %k0
-; SKX-NEXT: kshiftrq $63, %k0, %k0
+; SKX-NEXT: andb $1, %dil
+; SKX-NEXT: cmpb $1, %dil
; SKX-NEXT: movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
-; SKX-NEXT: kmovq %rax, %k1
-; SKX-NEXT: korq %k1, %k0, %k0
+; SKX-NEXT: sbbq $-1, %rax
+; SKX-NEXT: kmovq %rax, %k0
; SKX-NEXT: vpmovm2b %k0, %zmm0
; SKX-NEXT: retq
;
; AVX512BW-LABEL: mask64_insert:
; AVX512BW: ## %bb.0:
-; AVX512BW-NEXT: kmovd %edi, %k0
-; AVX512BW-NEXT: kshiftlq $63, %k0, %k0
-; AVX512BW-NEXT: kshiftrq $63, %k0, %k0
+; AVX512BW-NEXT: andb $1, %dil
+; AVX512BW-NEXT: cmpb $1, %dil
; AVX512BW-NEXT: movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
-; AVX512BW-NEXT: kmovq %rax, %k1
-; AVX512BW-NEXT: korq %k1, %k0, %k0
+; AVX512BW-NEXT: sbbq $-1, %rax
+; AVX512BW-NEXT: kmovq %rax, %k0
; AVX512BW-NEXT: vpmovm2b %k0, %zmm0
; AVX512BW-NEXT: retq
;
; AVX512DQ-LABEL: mask64_insert:
; AVX512DQ: ## %bb.0:
; AVX512DQ-NEXT: movq %rdi, %rax
-; AVX512DQ-NEXT: andl $1, %esi
-; AVX512DQ-NEXT: kmovw %esi, %k0
-; AVX512DQ-NEXT: movw $-4, %cx
-; AVX512DQ-NEXT: kmovw %ecx, %k1
-; AVX512DQ-NEXT: korw %k1, %k0, %k0
-; AVX512DQ-NEXT: kmovw %k0, (%rdi)
+; AVX512DQ-NEXT: andb $1, %sil
+; AVX512DQ-NEXT: cmpb $1, %sil
+; AVX512DQ-NEXT: movl $65532, %ecx ## imm = 0xFFFC
+; AVX512DQ-NEXT: sbbl $-1, %ecx
+; AVX512DQ-NEXT: movw %cx, (%rdi)
; AVX512DQ-NEXT: movw $-3, 6(%rdi)
; AVX512DQ-NEXT: movl $-131075, 2(%rdi) ## imm = 0xFFFDFFFD
; AVX512DQ-NEXT: retq
;
; X86-LABEL: mask64_insert:
; X86: ## %bb.0:
-; X86-NEXT: kmovb {{[0-9]+}}(%esp), %k0
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: andb $1, %al
+; X86-NEXT: cmpb $1, %al
; X86-NEXT: movl $-131076, %eax ## imm = 0xFFFDFFFC
-; X86-NEXT: kmovd %eax, %k1
+; X86-NEXT: sbbl $-1, %eax
+; X86-NEXT: kmovd %eax, %k0
; X86-NEXT: movl $-131075, %eax ## imm = 0xFFFDFFFD
-; X86-NEXT: kmovd %eax, %k2
-; X86-NEXT: kunpckdq %k1, %k2, %k1
-; X86-NEXT: kshiftrq $1, %k1, %k1
-; X86-NEXT: kshiftlq $1, %k1, %k1
-; X86-NEXT: kshiftlq $63, %k0, %k0
-; X86-NEXT: kshiftrq $63, %k0, %k0
-; X86-NEXT: korq %k0, %k1, %k0
+; X86-NEXT: kmovd %eax, %k1
+; X86-NEXT: kunpckdq %k0, %k1, %k0
; X86-NEXT: vpmovm2b %k0, %zmm0
; X86-NEXT: retl
%a_i = trunc i32 %a to i1
>From 9d865351d24048c8a3e77bd6494ffe6963792622 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 25 May 2026 14:15:55 +0100
Subject: [PATCH 2/3] Fix comparison type mismatch
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 2fdc4625b5b27..dc782497552d0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -8560,7 +8560,7 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl,
} else {
MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
// Adjust extended value to -1 as it will improve folding.
- if (llvm::popcount(TrueImm | Undefs) == VT.getSizeInBits())
+ if (llvm::popcount(TrueImm | Undefs) == (int)VT.getSizeInBits())
TrueImm = ~0ULL >> (64 - ImmVT.getSizeInBits());
SDValue Select =
DAG.getSelect(dl, ImmVT, Cond, DAG.getConstant(TrueImm, dl, ImmVT),
>From 99d9ce0501b05b7dd697578ccf471caafd4d2929 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 25 May 2026 14:18:10 +0100
Subject: [PATCH 3/3] clang-format
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index dc782497552d0..c85536f6195c2 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -8561,7 +8561,7 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl,
MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
// Adjust extended value to -1 as it will improve folding.
if (llvm::popcount(TrueImm | Undefs) == (int)VT.getSizeInBits())
- TrueImm = ~0ULL >> (64 - ImmVT.getSizeInBits());
+ TrueImm = ~0ULL >> (64 - ImmVT.getSizeInBits());
SDValue Select =
DAG.getSelect(dl, ImmVT, Cond, DAG.getConstant(TrueImm, dl, ImmVT),
DAG.getConstant(FalseImm, dl, ImmVT));
More information about the llvm-commits
mailing list