[llvm] [X86] LowerCTTZ - prefer legal CTPOP expansion vs GFNI to avoid constant pool load (PR #191623)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 12 03:16:27 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/191623
>From 2bf237bd2f681e999fab7a8f6a2791c38bd70275 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Sat, 11 Apr 2026 13:01:22 +0100
Subject: [PATCH] [X86] LowerCTTZ - prefer legal CTPOP expansion vs GFNI to
avoid constant pool load
Fixes #191520
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 5 +++
llvm/test/CodeGen/X86/gfni-lzcnt.ll | 56 ++++++++++++++++++++++++-
llvm/test/CodeGen/X86/gfni-tzcnt.ll | 50 +++++++++++++++++++++-
3 files changed, 109 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index cb701814154ff..7ccfc412ff184 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -29606,6 +29606,11 @@ static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
SDLoc dl(Op);
bool NonZeroSrc = DAG.isKnownNeverZero(N0);
+ // Default to expansion if CTPOP is legal.
+ if (VT.isVector() &&
+ DAG.getTargetLoweringInfo().isOperationLegal(ISD::CTPOP, VT))
+ return SDValue();
+
// GFNI - isolate LSB and perform GF2P8AFFINEQB lookup.
if (Subtarget.hasGFNI() && VT.isVector() &&
VT.getVectorElementType() == MVT::i8) {
diff --git a/llvm/test/CodeGen/X86/gfni-lzcnt.ll b/llvm/test/CodeGen/X86/gfni-lzcnt.ll
index 031d35cbc3907..3ca86fd86b388 100644
--- a/llvm/test/CodeGen/X86/gfni-lzcnt.ll
+++ b/llvm/test/CodeGen/X86/gfni-lzcnt.ll
@@ -4,7 +4,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX1OR2,GFNIAVX2
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512VL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+gfni,+avx512bitalg | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+gfni,+avx512bitalg | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512BITALG
define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
; GFNISSE-LABEL: testv16i8:
@@ -33,6 +33,15 @@ define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
; GFNIAVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; GFNIAVX512-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
; GFNIAVX512-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv16i8:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; GFNIAVX512BITALG-NEXT: vpaddb %xmm1, %xmm0, %xmm1
+; GFNIAVX512BITALG-NEXT: vpandn %xmm1, %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 0)
ret <16 x i8> %out
}
@@ -64,6 +73,15 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
; GFNIAVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; GFNIAVX512-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
; GFNIAVX512-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv16i8u:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; GFNIAVX512BITALG-NEXT: vpaddb %xmm1, %xmm0, %xmm1
+; GFNIAVX512BITALG-NEXT: vpandn %xmm1, %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %in, i1 -1)
ret <16 x i8> %out
}
@@ -118,6 +136,15 @@ define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
; GFNIAVX512-NEXT: vpand %ymm1, %ymm0, %ymm0
; GFNIAVX512-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
; GFNIAVX512-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv32i8:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; GFNIAVX512BITALG-NEXT: vpaddb %ymm1, %ymm0, %ymm1
+; GFNIAVX512BITALG-NEXT: vpandn %ymm1, %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 0)
ret <32 x i8> %out
}
@@ -172,6 +199,15 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
; GFNIAVX512-NEXT: vpand %ymm1, %ymm0, %ymm0
; GFNIAVX512-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
; GFNIAVX512-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv32i8u:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; GFNIAVX512BITALG-NEXT: vpaddb %ymm1, %ymm0, %ymm1
+; GFNIAVX512BITALG-NEXT: vpandn %ymm1, %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %in, i1 -1)
ret <32 x i8> %out
}
@@ -267,6 +303,15 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
; GFNIAVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv64i8:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; GFNIAVX512BITALG-NEXT: vpaddb %zmm1, %zmm0, %zmm1
+; GFNIAVX512BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 0)
ret <64 x i8> %out
}
@@ -362,6 +407,15 @@ define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
; GFNIAVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv64i8u:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; GFNIAVX512BITALG-NEXT: vpaddb %zmm1, %zmm0, %zmm1
+; GFNIAVX512BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %in, i1 -1)
ret <64 x i8> %out
}
diff --git a/llvm/test/CodeGen/X86/gfni-tzcnt.ll b/llvm/test/CodeGen/X86/gfni-tzcnt.ll
index 291bef2def2ca..ce9ae7bd97385 100644
--- a/llvm/test/CodeGen/X86/gfni-tzcnt.ll
+++ b/llvm/test/CodeGen/X86/gfni-tzcnt.ll
@@ -4,7 +4,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX1OR2,GFNIAVX2
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512VL
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+gfni | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+gfni,+avx512bitalg | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512,GFNIAVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+gfni,+avx512bitalg | FileCheck %s --check-prefixes=GFNIAVX,GFNIAVX512BITALG
define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
; GFNISSE-LABEL: testv16i8:
@@ -30,6 +30,14 @@ define <16 x i8> @testv16i8(<16 x i8> %in) nounwind {
; GFNIAVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; GFNIAVX512-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
; GFNIAVX512-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv16i8:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; GFNIAVX512BITALG-NEXT: vpaddb %xmm1, %xmm0, %xmm1
+; GFNIAVX512BITALG-NEXT: vpandn %xmm1, %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 0)
ret <16 x i8> %out
}
@@ -58,6 +66,14 @@ define <16 x i8> @testv16i8u(<16 x i8> %in) nounwind {
; GFNIAVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; GFNIAVX512-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
; GFNIAVX512-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv16i8u:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; GFNIAVX512BITALG-NEXT: vpaddb %xmm1, %xmm0, %xmm1
+; GFNIAVX512BITALG-NEXT: vpandn %xmm1, %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %xmm0, %xmm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %in, i1 -1)
ret <16 x i8> %out
}
@@ -102,6 +118,14 @@ define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
; GFNIAVX512-NEXT: vpand %ymm1, %ymm0, %ymm0
; GFNIAVX512-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
; GFNIAVX512-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv32i8:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; GFNIAVX512BITALG-NEXT: vpaddb %ymm1, %ymm0, %ymm1
+; GFNIAVX512BITALG-NEXT: vpandn %ymm1, %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 0)
ret <32 x i8> %out
}
@@ -146,6 +170,14 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
; GFNIAVX512-NEXT: vpand %ymm1, %ymm0, %ymm0
; GFNIAVX512-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
; GFNIAVX512-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv32i8u:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; GFNIAVX512BITALG-NEXT: vpaddb %ymm1, %ymm0, %ymm1
+; GFNIAVX512BITALG-NEXT: vpandn %ymm1, %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %ymm0, %ymm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %in, i1 -1)
ret <32 x i8> %out
}
@@ -220,6 +252,14 @@ define <64 x i8> @testv64i8(<64 x i8> %in) nounwind {
; GFNIAVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv64i8:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; GFNIAVX512BITALG-NEXT: vpaddb %zmm1, %zmm0, %zmm1
+; GFNIAVX512BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 0)
ret <64 x i8> %out
}
@@ -294,6 +334,14 @@ define <64 x i8> @testv64i8u(<64 x i8> %in) nounwind {
; GFNIAVX512BW-NEXT: vpandq %zmm1, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: vgf2p8affineqb $8, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0
; GFNIAVX512BW-NEXT: retq
+;
+; GFNIAVX512BITALG-LABEL: testv64i8u:
+; GFNIAVX512BITALG: # %bb.0:
+; GFNIAVX512BITALG-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; GFNIAVX512BITALG-NEXT: vpaddb %zmm1, %zmm0, %zmm1
+; GFNIAVX512BITALG-NEXT: vpandnq %zmm1, %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: vpopcntb %zmm0, %zmm0
+; GFNIAVX512BITALG-NEXT: retq
%out = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %in, i1 -1)
ret <64 x i8> %out
}
More information about the llvm-commits
mailing list