[llvm] [X86] Avoid shl->mul lowbit-isolate fold when BMI1/BMI2 are available (PR #217776)

[PJ Dailey via llvm-commits]

https://github.com/PjDailey11 updated https://github.com/llvm/llvm-project/pull/217776

>From 1c8c8142e028e7f92e188e12af4b36ccfd86ad13 Mon Sep 17 00:00:00 2001
From: pjdailey <pjdailey13 at gmail.com>
Date: Thu, 20 Aug 2026 17:05:08 -0500
Subject: [PATCH] [X86] Avoid shl->mul lowbit-isolate fold when BMI1/BMI2 are
 available

214517 folds shl X, cttz(Y) into mul(and(neg(Y), Y), X), which is
usually profitable but is worse when BMI1/BMI2 are available: the mul
form lowers to blsi+imul while the shl form lowers to tzcnt+shlx, which
has better latency/throughput despite matching instruction count.
Reverse the fold when those features are available and Y is provably
nonzero. Also test that vector types are correctly rejected by the
type-limit guard.

Fixes #216550 (x86 half only; AArch64 left for a follow-up)
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    | 21 +++++++
 llvm/test/CodeGen/X86/shl-cttz-blsi-mul.ll | 72 ++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 llvm/test/CodeGen/X86/shl-cttz-blsi-mul.ll

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index bf435b0c2f849..873b168f1e017 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50780,6 +50780,7 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
 static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
                           TargetLowering::DAGCombinerInfo &DCI,
                           const X86Subtarget &Subtarget) {
+  using namespace SDPatternMatch;
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
 
@@ -50792,6 +50793,26 @@ static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
   if (SDValue V = combineMulToPMADD52(N, DL, DAG, Subtarget))
     return V;
 
+  // InstCombine folds shl X, cttz(Y) into mul(and(neg(Y), Y), X), which is
+  // usually profitable but is worse when BMI1/BMI2 are available: the mul
+  // form lowers to blsi+imul while the shl form lowers to tzcnt+shlx, which
+  // has better latency/throughput despite matching instruction count
+  // (https://github.com/llvm/llvm-project/issues/216550). Reverse the fold
+  // when those features are available. Only valid when Y is provably
+  // nonzero: CTTZ_ZERO_POISON is poison at Y == 0, while the mul form is
+  // well-defined (0) at Y == 0 - so the rewrite would be a miscompile for
+  // Y == 0 without that guard.
+  if (Subtarget.hasBMI() && Subtarget.hasBMI2() && VT.isScalarInteger() &&
+      (VT == MVT::i32 || VT == MVT::i64)) {
+    SDValue X, Y;
+    if (sd_match(N, m_Mul(m_Value(X),
+                          m_OneUse(m_And(m_Neg(m_Value(Y)), m_Deferred(Y))))) &&
+        DAG.isKnownNeverZero(Y)) {
+      SDValue Cttz = DAG.getNode(ISD::CTTZ_ZERO_POISON, DL, VT, Y);
+      return DAG.getNode(ISD::SHL, DL, VT, X, Cttz);
+    }
+  }
+
   if (DCI.isBeforeLegalize() && VT.isVector())
     return reduceVMULWidth(N, DL, DAG, Subtarget);
 
diff --git a/llvm/test/CodeGen/X86/shl-cttz-blsi-mul.ll b/llvm/test/CodeGen/X86/shl-cttz-blsi-mul.ll
new file mode 100644
index 0000000000000..1da46b8f58c5d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shl-cttz-blsi-mul.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v2 | FileCheck %s --check-prefix=NOBMI
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 | FileCheck %s --check-prefix=BMI
+
+define i32 @mul_unconstrained(i32 %x, i32 %y){
+; NOBMI-LABEL: mul_unconstrained:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movl %esi, %eax
+; NOBMI-NEXT:    negl %eax
+; NOBMI-NEXT:    andl %esi, %eax
+; NOBMI-NEXT:    imull %edi, %eax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: mul_unconstrained:
+; BMI:       # %bb.0:
+; BMI-NEXT:    blsil %esi, %eax
+; BMI-NEXT:    imull %edi, %eax
+; BMI-NEXT:    retq
+    %neg = sub i32 0, %y
+    %lsb = and i32 %neg, %y
+    %mul = mul i32 %lsb, %x
+    ret i32 %mul
+}
+
+define i32 @mul_nonzero(i32 %x, i32 %y){
+; NOBMI-LABEL: mul_nonzero:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    orl $2, %esi
+; NOBMI-NEXT:    movl %esi, %eax
+; NOBMI-NEXT:    negl %eax
+; NOBMI-NEXT:    andl %esi, %eax
+; NOBMI-NEXT:    imull %edi, %eax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: mul_nonzero:
+; BMI:       # %bb.0:
+; BMI-NEXT:    orl $2, %esi
+; BMI-NEXT:    tzcntl %esi, %eax
+; BMI-NEXT:    shlxl %eax, %edi, %eax
+; BMI-NEXT:    retq
+    %y2 = or i32 %y, 2
+    %neg = sub i32 0, %y2
+    %lsb = and i32 %neg, %y2
+    %mul = mul i32 %lsb, %x
+    ret i32 %mul
+}
+
+define <4 x i32> @mul_nonzero_vector(<4 x i32> %x, <4 x i32> %y) {
+; NOBMI-LABEL: mul_nonzero_vector:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; NOBMI-NEXT:    pxor %xmm2, %xmm2
+; NOBMI-NEXT:    psubd %xmm1, %xmm2
+; NOBMI-NEXT:    pand %xmm1, %xmm2
+; NOBMI-NEXT:    pmulld %xmm2, %xmm0
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: mul_nonzero_vector:
+; BMI:       # %bb.0:
+; BMI-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2,2,2,2]
+; BMI-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; BMI-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; BMI-NEXT:    vpsubd %xmm1, %xmm2, %xmm2
+; BMI-NEXT:    vpand %xmm1, %xmm2, %xmm1
+; BMI-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; BMI-NEXT:    retq
+    %y2 = or <4 x i32> %y, splat(i32 2)
+    %neg = sub <4 x i32> zeroinitializer, %y2
+    %lsb = and <4 x i32> %neg, %y2
+    %mul = mul <4 x i32> %lsb, %x
+    ret <4 x i32> %mul
+}