[llvm] [DAGCombiner] Widen zero-extended low-bits masks to select and-not (PR #228910)

Chris Kennelly via llvm-commits llvm-commits at lists.llvm.org
Sun Oct 4 08:39:42 PDT 2026


https://github.com/ckennelly created https://github.com/llvm/llvm-project/pull/228910

(and X, (zext (not (shl -1, Y)))) is the same low-bits mask built in the wide type: for shift amounts within the narrow width the values are equal, and beyond it the narrow shift is poison. Build the mask in the wide type when the target has an and-not instruction, so the and selects as one: on AArch64, lsl + bic instead of lsl + mvn + and. On X86 the same shape now selects the 64-bit bzhi or bextr directly instead of the 32-bit form followed by a zero-extension.

This is the canonical form of a 32-bit low-bits mask applied to a 64-bit value, whether written as (1u << n) - 1 in source, as in brotli's hashers and zlib's inflate, or produced from a 32-bit mask table by AggressiveInstCombine's fold (https://github.com/llvm/llvm-project/commit/394c8dd1493308cb9add77b18987aee2219a65c1), as for zstd's BIT_mask on AArch64.

Measured on Neoverse-V2 against main, interleaved rounds pinned to one core: Fleetbench's compression family is 0.5% +/- 0.2% faster in geomean (zstd compress +0.5%, zstd decompress +1.0%, snappy +0.1% to +0.4%); zstd 1.5.7 on enwik8 compresses 0.6% faster at level 1 and 0.5% at level 3, with level 9 and decompression unchanged within half a percent; brotli and snappy are unchanged within noise (brotli's hashers lose the mvn but it is not measurable; snappy has no such masks). On
Zen 3 the only difference in the generated code is bzhil becoming bzhiq, and the measured deltas are within noise.

Speed change with the fold, mean +/- standard deviation of the per-round ratio over 10 interleaved rounds pinned to one core:

```
                                      Neoverse-V2         Zen 3
  Fleetbench compression (geomean)   +0.49% +/- 0.17%   +0.22% +/- 1.69%
    ZSTD compress                    +0.46% +/- 0.21%   -0.84% +/- 1.68%
    ZSTD decompress                  +1.02% +/- 0.78%   -0.62% +/- 1.66%
    Snappy compress                  +0.36% +/- 0.28%   +1.68% +/- 4.62%
    Snappy decompress                +0.14% +/- 0.45%   +0.81% +/- 3.50%
  zstd 1.5.7, enwik8
    compress   L1                    +0.59% +/- 0.02%   -0.33% +/- 0.39%
    compress   L3                    +0.54% +/- 0.59%   -0.31% +/- 1.59%
    compress   L9                    -0.05% +/- 0.41%   -1.64% +/- 5.04%
    decompress L1                    +0.31% +/- 0.12%   +0.00% +/- 0.23%
    decompress L3                    -0.09% +/- 0.16%   +1.53% +/- 3.68%
    decompress L9                    +0.21% +/- 0.20%   -0.77% +/- 2.66%
  brotli 1.1.0, enwik8
    compress   q5                    -0.15% +/- 0.32%   +0.55% +/- 0.96%
    compress   q9                    +0.06% +/- 0.21%   +1.72% +/- 3.34%
    decompress q5                    +1.08% +/- 3.17%   +2.14% +/- 5.11%
    decompress q9                    +0.94% +/- 2.94%   +4.02% +/- 8.30%
  snappy 1.2.2, 77 benchmarks        -0.04% +/- 0.15%   -0.11% +/- 0.57%
```

Assisted-by: Claude Code

>From 87b6c011671e1041f4699e06291c7e67b607c712 Mon Sep 17 00:00:00 2001
From: Chris Kennelly <ckennelly at ckennelly.com>
Date: Sat, 3 Oct 2026 21:20:37 +0000
Subject: [PATCH 1/2] [DAGCombiner] Add tests for zero-extended low-bits masks
 (NFC)

A low-bits mask built in i32 (or i16) and zero-extended into a 64-bit
and, with and without other uses of the mask, on AArch64 and X86.

Baseline output, to show the change in the following commit.

Assisted-by: Claude Code
---
 llvm/test/CodeGen/AArch64/extract-lowbits.ll | 156 +++++
 llvm/test/CodeGen/X86/andnot-patterns.ll     | 212 +++++++
 llvm/test/CodeGen/X86/extract-lowbits.ll     | 567 +++++++++++++++++++
 3 files changed, 935 insertions(+)

diff --git a/llvm/test/CodeGen/AArch64/extract-lowbits.ll b/llvm/test/CodeGen/AArch64/extract-lowbits.ll
index 368440c65df84..a199a5fe0191c 100644
--- a/llvm/test/CodeGen/AArch64/extract-lowbits.ll
+++ b/llvm/test/CodeGen/AArch64/extract-lowbits.ll
@@ -765,3 +765,159 @@ define i64 @bzhi64_constant_mask8_load(ptr %val) nounwind {
   %masked = and i64 %val1, 127
   ret i64 %masked
 }
+
+; ---------------------------------------------------------------------------- ;
+; Pattern e. 32-bit mask zero-extended into a 64-bit and
+; ---------------------------------------------------------------------------- ;
+
+; The mask is built in i32, as a 32-bit mask table or (1u << n) - 1 would be,
+; and applied to a 64-bit value. The same mask built directly in i64 selects
+; as a single and-not.
+
+define i64 @bzhi64_32_e0(i64 %val, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bzhi64_32_e0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    and x0, x8, x0
+; CHECK-NEXT:    ret
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+define i64 @bzhi64_32_e1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
+; CHECK-LABEL: bzhi64_32_e1_indexzext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    and x0, x8, x0
+; CHECK-NEXT:    ret
+  %conv = zext i8 %numlowbits to i32
+  %notmask = shl i32 -1, %conv
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+define i64 @bzhi64_32_e2_load(ptr %w, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bzhi64_32_e2_load:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    ldr x9, [x0]
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    and x0, x8, x9
+; CHECK-NEXT:    ret
+  %val = load i64, ptr %w
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+define i64 @bzhi64_32_e3_commutative(i64 %val, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bzhi64_32_e3_commutative:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    and x0, x0, x8
+; CHECK-NEXT:    ret
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %val, %zmask ; swapped order
+  ret i64 %masked
+}
+
+; The value is a shifted operand: the and could absorb the shift, an and-not
+; cannot, so the instruction count does not change here.
+define i64 @bzhi64_32_e4_shifted_val(i64 %val, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bzhi64_32_e4_shifted_val:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    and x0, x8, x0, lsr #40
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, 40
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %shifted
+  ret i64 %masked
+}
+
+define i64 @bzhi64_16_e0(i64 %val, i16 %numlowbits) nounwind {
+; CHECK-LABEL: bzhi64_16_e0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    bic w8, w0, w8
+; CHECK-NEXT:    and x0, x8, #0xffff
+; CHECK-NEXT:    ret
+  %notmask = shl i16 -1, %numlowbits
+  %mask = xor i16 %notmask, -1
+  %zmask = zext i16 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+; Negative: the narrow mask has another use.
+define i64 @bzhi64_32_e5_multiuse_mask(i64 %val, i32 %numlowbits, ptr %escape) nounwind {
+; CHECK-LABEL: bzhi64_32_e5_multiuse_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    and x0, x8, x0
+; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    ret
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  store i32 %mask, ptr %escape
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+; Negative: the zero-extended mask has another use.
+define i64 @bzhi64_32_e6_multiuse_zext(i64 %val, i32 %numlowbits, ptr %escape) nounwind {
+; CHECK-LABEL: bzhi64_32_e6_multiuse_zext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    and x0, x8, x0
+; CHECK-NEXT:    str x8, [x2]
+; CHECK-NEXT:    ret
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  store i64 %zmask, ptr %escape
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+; Not a low-bits mask.
+define i64 @bzhi64_32_e7_not_allones(i64 %val, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bzhi64_32_e7_not_allones:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #3 // =0x3
+; CHECK-NEXT:    lsl w8, w8, w1
+; CHECK-NEXT:    mvn w8, w8
+; CHECK-NEXT:    and x0, x8, x0
+; CHECK-NEXT:    ret
+  %notmask = shl i32 3, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
diff --git a/llvm/test/CodeGen/X86/andnot-patterns.ll b/llvm/test/CodeGen/X86/andnot-patterns.ll
index b4029ed1a63c5..710cce093798d 100644
--- a/llvm/test/CodeGen/X86/andnot-patterns.ll
+++ b/llvm/test/CodeGen/X86/andnot-patterns.ll
@@ -1200,3 +1200,215 @@ define i8 @andnot_load_i8(ptr %p, i8 %x) nounwind {
   %and = and i8 %not, %load
   ret i8 %and
 }
+
+;
+; Zero-extended low-bits mask: (and X, (zext (not (shl -1, Y))))
+;
+
+define i64 @andnot_zext_lowmask_i64_i32(i64 %a0, i32 %a1) nounwind {
+; X86-NOBMI-LABEL: andnot_zext_lowmask_i64_i32:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI-LABEL: andnot_zext_lowmask_i64_i32:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    shll $8, %eax
+; X86-BMI-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    xorl %edx, %edx
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_zext_lowmask_i64_i32:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_zext_lowmask_i64_i32:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    shll $8, %esi
+; X64-BMI-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI-NEXT:    retq
+  %notmask = shl i32 -1, %a1
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %and = and i64 %a0, %zmask
+  ret i64 %and
+}
+
+define i64 @andnot_zext_lowmask_i64_i16(i64 %a0, i16 %a1) nounwind {
+; X86-NOBMI-LABEL: andnot_zext_lowmask_i64_i16:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %edx, %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI-LABEL: andnot_zext_lowmask_i64_i16:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    shll $8, %ecx
+; X86-BMI-NEXT:    bextrl %ecx, %eax, %eax
+; X86-BMI-NEXT:    xorl %edx, %edx
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_zext_lowmask_i64_i16:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %eax, %edi
+; X64-NOBMI-NEXT:    movzwl %di, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_zext_lowmask_i64_i16:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    shll $8, %esi
+; X64-BMI-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI-NEXT:    movzwl %ax, %eax
+; X64-BMI-NEXT:    retq
+  %notmask = shl i16 -1, %a1
+  %mask = xor i16 %notmask, -1
+  %zmask = zext i16 %mask to i64
+  %and = and i64 %a0, %zmask
+  ret i64 %and
+}
+
+define i32 @andnot_zext_lowmask_i32_i16(i32 %a0, i16 %a1) nounwind {
+; X86-NOBMI-LABEL: andnot_zext_lowmask_i32_i16:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %edx, %eax
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI-LABEL: andnot_zext_lowmask_i32_i16:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    shll $8, %ecx
+; X86-BMI-NEXT:    bextrl %ecx, %eax, %eax
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_zext_lowmask_i32_i16:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    movzwl %ax, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_zext_lowmask_i32_i16:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    shll $8, %esi
+; X64-BMI-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI-NEXT:    movzwl %ax, %eax
+; X64-BMI-NEXT:    retq
+  %notmask = shl i16 -1, %a1
+  %mask = xor i16 %notmask, -1
+  %zmask = zext i16 %mask to i32
+  %and = and i32 %a0, %zmask
+  ret i32 %and
+}
+
+define i64 @andnot_zext_lowmask_i64_i32_multiuse_mask(i64 %a0, i32 %a1) nounwind {
+; X86-NOBMI-LABEL: andnot_zext_lowmask_i64_i32_multiuse_mask:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    shll %cl, %esi
+; X86-NOBMI-NEXT:    notl %esi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    calll use_i32 at PLT
+; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI-LABEL: andnot_zext_lowmask_i64_i32_multiuse_mask:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    pushl %esi
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    movl $-1, %esi
+; X86-BMI-NEXT:    shll %cl, %esi
+; X86-BMI-NEXT:    movl %esi, %eax
+; X86-BMI-NEXT:    notl %eax
+; X86-BMI-NEXT:    pushl %eax
+; X86-BMI-NEXT:    calll use_i32 at PLT
+; X86-BMI-NEXT:    addl $4, %esp
+; X86-BMI-NEXT:    andnl {{[0-9]+}}(%esp), %esi, %eax
+; X86-BMI-NEXT:    xorl %edx, %edx
+; X86-BMI-NEXT:    popl %esi
+; X86-BMI-NEXT:    retl
+;
+; X64-NOBMI-LABEL: andnot_zext_lowmask_i64_i32_multiuse_mask:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %rbp
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    pushq %rax
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movq %rdi, %rbx
+; X64-NOBMI-NEXT:    movl $-1, %ebp
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %ebp
+; X64-NOBMI-NEXT:    notl %ebp
+; X64-NOBMI-NEXT:    movl %ebp, %edi
+; X64-NOBMI-NEXT:    callq use_i32 at PLT
+; X64-NOBMI-NEXT:    andl %ebp, %ebx
+; X64-NOBMI-NEXT:    movq %rbx, %rax
+; X64-NOBMI-NEXT:    addq $8, %rsp
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    popq %rbp
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI-LABEL: andnot_zext_lowmask_i64_i32_multiuse_mask:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    pushq %rbp
+; X64-BMI-NEXT:    pushq %rbx
+; X64-BMI-NEXT:    pushq %rax
+; X64-BMI-NEXT:    movl %esi, %ecx
+; X64-BMI-NEXT:    movq %rdi, %rbx
+; X64-BMI-NEXT:    movl $-1, %ebp
+; X64-BMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI-NEXT:    shll %cl, %ebp
+; X64-BMI-NEXT:    movl %ebp, %edi
+; X64-BMI-NEXT:    notl %edi
+; X64-BMI-NEXT:    callq use_i32 at PLT
+; X64-BMI-NEXT:    andnl %ebx, %ebp, %eax
+; X64-BMI-NEXT:    addq $8, %rsp
+; X64-BMI-NEXT:    popq %rbx
+; X64-BMI-NEXT:    popq %rbp
+; X64-BMI-NEXT:    retq
+  %notmask = shl i32 -1, %a1
+  %mask = xor i32 %notmask, -1
+  call void @use_i32(i32 %mask)
+  %zmask = zext i32 %mask to i64
+  %and = and i64 %a0, %zmask
+  ret i64 %and
+}
diff --git a/llvm/test/CodeGen/X86/extract-lowbits.ll b/llvm/test/CodeGen/X86/extract-lowbits.ll
index aff5ac7437a29..29f5ce8e19687 100644
--- a/llvm/test/CodeGen/X86/extract-lowbits.ll
+++ b/llvm/test/CodeGen/X86/extract-lowbits.ll
@@ -4612,3 +4612,570 @@ if.end:
 return:
   ret void
 }
+
+; ---------------------------------------------------------------------------- ;
+; Pattern e. 32-bit mask zero-extended into a 64-bit and
+; ---------------------------------------------------------------------------- ;
+
+; The mask is built in i32, as a 32-bit mask table or (1u << n) - 1 would be,
+; and applied to a 64-bit value. The same mask built directly in i64 selects
+; as a single and-not.
+
+define i64 @bzhi64_32_e0(i64 %val, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_32_e0:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_32_e0:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll $8, %eax
+; X86-BMI1-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_32_e0:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    bzhil %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_32_e0:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_32_e0:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    shll $8, %esi
+; X64-BMI1-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_32_e0:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    retq
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+define i64 @bzhi64_32_e1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_32_e1_indexzext:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_32_e1_indexzext:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll $8, %eax
+; X86-BMI1-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_32_e1_indexzext:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    bzhil %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_32_e1_indexzext:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_32_e1_indexzext:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    shll $8, %esi
+; X64-BMI1-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_32_e1_indexzext:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    retq
+  %conv = zext i8 %numlowbits to i32
+  %notmask = shl i32 -1, %conv
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+define i64 @bzhi64_32_e2_load(ptr %w, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_32_e2_load:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl (%edx), %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_32_e2_load:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1-NEXT:    shll $8, %ecx
+; X86-BMI1-NEXT:    bextrl %ecx, (%eax), %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_32_e2_load:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    bzhil %ecx, (%eax), %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_32_e2_load:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl (%rdi), %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_32_e2_load:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    shll $8, %esi
+; X64-BMI1-NEXT:    bextrl %esi, (%rdi), %eax
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_32_e2_load:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    bzhil %esi, (%rdi), %eax
+; X64-BMI2-NEXT:    retq
+  %val = load i64, ptr %w
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+define i64 @bzhi64_32_e3_commutative(i64 %val, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_32_e3_commutative:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_32_e3_commutative:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll $8, %eax
+; X86-BMI1-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_32_e3_commutative:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    bzhil %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_32_e3_commutative:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_32_e3_commutative:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    shll $8, %esi
+; X64-BMI1-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_32_e3_commutative:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    retq
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %val, %zmask ; swapped order
+  ret i64 %masked
+}
+
+; The value is a shifted operand: the and could absorb the shift, an and-not
+; cannot, so the instruction count does not change here.
+define i64 @bzhi64_32_e4_shifted_val(i64 %val, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_32_e4_shifted_val:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    shrl $8, %edx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %edx, %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_32_e4_shifted_val:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll $8, %eax
+; X86-BMI1-NEXT:    orl $8, %eax
+; X86-BMI1-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_32_e4_shifted_val:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    shrl $8, %ecx
+; X86-BMI2-NEXT:    bzhil %eax, %ecx, %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_32_e4_shifted_val:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    shrq $40, %rdi
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_32_e4_shifted_val:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI1-NEXT:    shll $8, %esi
+; X64-BMI1-NEXT:    orl $40, %esi
+; X64-BMI1-NEXT:    bextrq %rsi, %rdi, %rax
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_32_e4_shifted_val:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    shrq $40, %rdi
+; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    retq
+  %shifted = lshr i64 %val, 40
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %shifted
+  ret i64 %masked
+}
+
+define i64 @bzhi64_16_e0(i64 %val, i16 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_16_e0:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %edx, %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_16_e0:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1-NEXT:    shll $8, %ecx
+; X86-BMI1-NEXT:    bextrl %ecx, %eax, %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_16_e0:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    bzhil %ecx, %eax, %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_16_e0:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %eax, %edi
+; X64-NOBMI-NEXT:    movzwl %di, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_16_e0:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    shll $8, %esi
+; X64-BMI1-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI1-NEXT:    movzwl %ax, %eax
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_16_e0:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    movzwl %ax, %eax
+; X64-BMI2-NEXT:    retq
+  %notmask = shl i16 -1, %numlowbits
+  %mask = xor i16 %notmask, -1
+  %zmask = zext i16 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+; Negative: the narrow mask has another use.
+define i64 @bzhi64_32_e5_multiuse_mask(i64 %val, i32 %numlowbits, ptr %escape) nounwind {
+; X86-NOBMI-LABEL: bzhi64_32_e5_multiuse_mask:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    movl %eax, (%edx)
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_32_e5_multiuse_mask:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    pushl %esi
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1-NEXT:    movl $-1, %esi
+; X86-BMI1-NEXT:    shll %cl, %esi
+; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %esi, %eax
+; X86-BMI1-NEXT:    notl %esi
+; X86-BMI1-NEXT:    movl %esi, (%edx)
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    popl %esi
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_32_e5_multiuse_mask:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    movl $-1, %edx
+; X86-BMI2-NEXT:    bzhil %ecx, %edx, %edx
+; X86-BMI2-NEXT:    movl %edx, (%eax)
+; X86-BMI2-NEXT:    bzhil %ecx, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_32_e5_multiuse_mask:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    movl %eax, (%rdx)
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_32_e5_multiuse_mask:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    movl %esi, %ecx
+; X64-BMI1-NEXT:    movl $-1, %esi
+; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI1-NEXT:    shll %cl, %esi
+; X64-BMI1-NEXT:    andnl %edi, %esi, %eax
+; X64-BMI1-NEXT:    notl %esi
+; X64-BMI1-NEXT:    movl %esi, (%rdx)
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_32_e5_multiuse_mask:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %esi, %eax, %eax
+; X64-BMI2-NEXT:    movl %eax, (%rdx)
+; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    retq
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  store i32 %mask, ptr %escape
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+; Negative: the zero-extended mask has another use.
+define i64 @bzhi64_32_e6_multiuse_zext(i64 %val, i32 %numlowbits, ptr %escape) nounwind {
+; X86-NOBMI-LABEL: bzhi64_32_e6_multiuse_zext:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    movl %eax, (%edx)
+; X86-NOBMI-NEXT:    movl $0, 4(%edx)
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_32_e6_multiuse_zext:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    pushl %esi
+; X86-BMI1-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1-NEXT:    movl $-1, %esi
+; X86-BMI1-NEXT:    shll %cl, %esi
+; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %esi, %eax
+; X86-BMI1-NEXT:    notl %esi
+; X86-BMI1-NEXT:    movl %esi, (%edx)
+; X86-BMI1-NEXT:    movl $0, 4(%edx)
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    popl %esi
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_32_e6_multiuse_zext:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT:    movl $-1, %edx
+; X86-BMI2-NEXT:    bzhil %ecx, %edx, %edx
+; X86-BMI2-NEXT:    movl %edx, (%eax)
+; X86-BMI2-NEXT:    movl $0, 4(%eax)
+; X86-BMI2-NEXT:    bzhil %ecx, {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_32_e6_multiuse_zext:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    movq %rax, (%rdx)
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_32_e6_multiuse_zext:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    movl %esi, %ecx
+; X64-BMI1-NEXT:    movl $-1, %esi
+; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI1-NEXT:    shll %cl, %esi
+; X64-BMI1-NEXT:    andnl %edi, %esi, %eax
+; X64-BMI1-NEXT:    movl %esi, %ecx
+; X64-BMI1-NEXT:    notl %ecx
+; X64-BMI1-NEXT:    movq %rcx, (%rdx)
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_32_e6_multiuse_zext:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movl $-1, %eax
+; X64-BMI2-NEXT:    bzhil %esi, %eax, %eax
+; X64-BMI2-NEXT:    movq %rax, (%rdx)
+; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    retq
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  store i64 %zmask, ptr %escape
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}
+
+; Not a low-bits mask.
+define i64 @bzhi64_32_e7_not_allones(i64 %val, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bzhi64_32_e7_not_allones:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    movl $3, %eax
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1-LABEL: bzhi64_32_e7_not_allones:
+; X86-BMI1:       # %bb.0:
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1-NEXT:    movl $3, %eax
+; X86-BMI1-NEXT:    shll %cl, %eax
+; X86-BMI1-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI1-NEXT:    xorl %edx, %edx
+; X86-BMI1-NEXT:    retl
+;
+; X86-BMI2-LABEL: bzhi64_32_e7_not_allones:
+; X86-BMI2:       # %bb.0:
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    movl $3, %ecx
+; X86-BMI2-NEXT:    shlxl %eax, %ecx, %eax
+; X86-BMI2-NEXT:    andnl {{[0-9]+}}(%esp), %eax, %eax
+; X86-BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bzhi64_32_e7_not_allones:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl $3, %eax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1-LABEL: bzhi64_32_e7_not_allones:
+; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    movl %esi, %ecx
+; X64-BMI1-NEXT:    movl $3, %eax
+; X64-BMI1-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI1-NEXT:    shll %cl, %eax
+; X64-BMI1-NEXT:    andnl %edi, %eax, %eax
+; X64-BMI1-NEXT:    retq
+;
+; X64-BMI2-LABEL: bzhi64_32_e7_not_allones:
+; X64-BMI2:       # %bb.0:
+; X64-BMI2-NEXT:    movl $3, %eax
+; X64-BMI2-NEXT:    shlxl %esi, %eax, %eax
+; X64-BMI2-NEXT:    andnl %edi, %eax, %eax
+; X64-BMI2-NEXT:    retq
+  %notmask = shl i32 3, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %zmask = zext i32 %mask to i64
+  %masked = and i64 %zmask, %val
+  ret i64 %masked
+}

>From 2f2bbda120d18a9dccc0f30970f8f915ade3c0a0 Mon Sep 17 00:00:00 2001
From: Chris Kennelly <ckennelly at ckennelly.com>
Date: Sat, 3 Oct 2026 21:25:52 +0000
Subject: [PATCH 2/2] [DAGCombiner] Widen zero-extended low-bits masks to
 select and-not

(and X, (zext (not (shl -1, Y)))) is the same low-bits mask built in the
wide type: for shift amounts within the narrow width the values are
equal, and beyond it the narrow shift is poison. Build the mask in the
wide type when the target has an and-not instruction, so the and selects
as one: on AArch64, lsl + bic instead of lsl + mvn + and. On X86 the
same shape now selects the 64-bit bzhi or bextr directly instead of the
32-bit form followed by a zero-extension.

This is the canonical form of a 32-bit low-bits mask applied to a 64-bit
value, whether written as (1u << n) - 1 in source, as in brotli's
hashers and zlib's inflate, or produced from a 32-bit mask table by
AggressiveInstCombine's fold (394c8dd14933), as for zstd's BIT_mask on
AArch64.

Measured on Neoverse-V2 against main, interleaved rounds pinned to one
core: Fleetbench's compression family is 0.5% +/- 0.2% faster in
geomean (zstd compress +0.5%, zstd decompress +1.0%, snappy +0.1% to
+0.4%); zstd 1.5.7 on enwik8 compresses 0.6% faster at level 1 and 0.5%
at level 3, with level 9 and decompression unchanged within half a
percent; brotli and snappy are unchanged within noise (brotli's hashers
lose the mvn but it is not measurable; snappy has no such masks). On
Zen 3 the only difference in the generated code is bzhil becoming
bzhiq, and the measured deltas are within noise.

Speed change with the fold, mean +/- standard deviation of the per-round
ratio over 10 interleaved rounds pinned to one core:

                                      Neoverse-V2         Zen 3
  Fleetbench compression (geomean)   +0.49% +/- 0.17%   +0.22% +/- 1.69%
    ZSTD compress                    +0.46% +/- 0.21%   -0.84% +/- 1.68%
    ZSTD decompress                  +1.02% +/- 0.78%   -0.62% +/- 1.66%
    Snappy compress                  +0.36% +/- 0.28%   +1.68% +/- 4.62%
    Snappy decompress                +0.14% +/- 0.45%   +0.81% +/- 3.50%
  zstd 1.5.7, enwik8
    compress   L1                    +0.59% +/- 0.02%   -0.33% +/- 0.39%
    compress   L3                    +0.54% +/- 0.59%   -0.31% +/- 1.59%
    compress   L9                    -0.05% +/- 0.41%   -1.64% +/- 5.04%
    decompress L1                    +0.31% +/- 0.12%   +0.00% +/- 0.23%
    decompress L3                    -0.09% +/- 0.16%   +1.53% +/- 3.68%
    decompress L9                    +0.21% +/- 0.20%   -0.77% +/- 2.66%
  brotli 1.1.0, enwik8
    compress   q5                    -0.15% +/- 0.32%   +0.55% +/- 0.96%
    compress   q9                    +0.06% +/- 0.21%   +1.72% +/- 3.34%
    decompress q5                    +1.08% +/- 3.17%   +2.14% +/- 5.11%
    decompress q9                    +0.94% +/- 2.94%   +4.02% +/- 8.30%
  snappy 1.2.2, 77 benchmarks        -0.04% +/- 0.15%   -0.11% +/- 0.57%

Assisted-by: Claude Code
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 17 +++++++
 llvm/test/CodeGen/AArch64/extract-lowbits.ll  | 49 ++++++++++---------
 llvm/test/CodeGen/X86/andnot-patterns.ll      | 22 ++++-----
 llvm/test/CodeGen/X86/extract-lowbits.ll      | 44 ++++++++++-------
 4 files changed, 77 insertions(+), 55 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index c8a984a9dc56e..982be8ba4ec6c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -8237,6 +8237,23 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
     if (SDValue Folded = foldBitwiseOpWithNeg(N, DL, VT))
       return Folded;
 
+  // Fold (and X, (zext (not (shl -1, Y)))) -> (and X, (not (shl -1, Y)))
+  // A low-bits mask built in a narrower type and zero-extended is the same
+  // mask built in the wide type: for shift amounts within the narrow width
+  // the values are equal, and beyond it the narrow shift is poison. The wide
+  // form selects as a single and-not, and nothing needs zero-extending.
+  if (TLI.hasAndNot(SDValue(N, 0)) && VT.isScalarInteger() &&
+      TLI.isTypeLegal(VT) &&
+      (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) &&
+      sd_match(N, m_And(m_Value(X), m_OneUse(m_ZExt(m_OneUse(m_Not(m_OneUse(
+                                        m_Shl(m_AllOnes(), m_Value(Y))))))))))
+    return DAG.getNode(
+        ISD::AND, DL, VT, X,
+        DAG.getNOT(DL,
+                   DAG.getNode(ISD::SHL, DL, VT, DAG.getAllOnesConstant(DL, VT),
+                               DAG.getShiftAmountOperand(VT, Y)),
+                   VT));
+
   // Fold (and (srl X, C), 1) -> (srl X, BW-1) for signbit extraction
   // If we are shifting down an extended sign bit, see if we can simplify
   // this to shifting the MSB directly to expose further simplifications.
diff --git a/llvm/test/CodeGen/AArch64/extract-lowbits.ll b/llvm/test/CodeGen/AArch64/extract-lowbits.ll
index a199a5fe0191c..7179c6b06f3a2 100644
--- a/llvm/test/CodeGen/AArch64/extract-lowbits.ll
+++ b/llvm/test/CodeGen/AArch64/extract-lowbits.ll
@@ -777,10 +777,10 @@ define i64 @bzhi64_constant_mask8_load(ptr %val) nounwind {
 define i64 @bzhi64_32_e0(i64 %val, i32 %numlowbits) nounwind {
 ; CHECK-LABEL: bzhi64_32_e0:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    mvn w8, w8
-; CHECK-NEXT:    and x0, x8, x0
+; CHECK-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    lsl x8, x8, x1
+; CHECK-NEXT:    bic x0, x0, x8
 ; CHECK-NEXT:    ret
   %notmask = shl i32 -1, %numlowbits
   %mask = xor i32 %notmask, -1
@@ -792,10 +792,10 @@ define i64 @bzhi64_32_e0(i64 %val, i32 %numlowbits) nounwind {
 define i64 @bzhi64_32_e1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
 ; CHECK-LABEL: bzhi64_32_e1_indexzext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    mvn w8, w8
-; CHECK-NEXT:    and x0, x8, x0
+; CHECK-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    lsl x8, x8, x1
+; CHECK-NEXT:    bic x0, x0, x8
 ; CHECK-NEXT:    ret
   %conv = zext i8 %numlowbits to i32
   %notmask = shl i32 -1, %conv
@@ -808,11 +808,11 @@ define i64 @bzhi64_32_e1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
 define i64 @bzhi64_32_e2_load(ptr %w, i32 %numlowbits) nounwind {
 ; CHECK-LABEL: bzhi64_32_e2_load:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
+; CHECK-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    ldr x9, [x0]
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    mvn w8, w8
-; CHECK-NEXT:    and x0, x8, x9
+; CHECK-NEXT:    lsl x8, x8, x1
+; CHECK-NEXT:    bic x0, x9, x8
 ; CHECK-NEXT:    ret
   %val = load i64, ptr %w
   %notmask = shl i32 -1, %numlowbits
@@ -825,10 +825,10 @@ define i64 @bzhi64_32_e2_load(ptr %w, i32 %numlowbits) nounwind {
 define i64 @bzhi64_32_e3_commutative(i64 %val, i32 %numlowbits) nounwind {
 ; CHECK-LABEL: bzhi64_32_e3_commutative:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    mvn w8, w8
-; CHECK-NEXT:    and x0, x0, x8
+; CHECK-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    lsl x8, x8, x1
+; CHECK-NEXT:    bic x0, x0, x8
 ; CHECK-NEXT:    ret
   %notmask = shl i32 -1, %numlowbits
   %mask = xor i32 %notmask, -1
@@ -842,10 +842,11 @@ define i64 @bzhi64_32_e3_commutative(i64 %val, i32 %numlowbits) nounwind {
 define i64 @bzhi64_32_e4_shifted_val(i64 %val, i32 %numlowbits) nounwind {
 ; CHECK-LABEL: bzhi64_32_e4_shifted_val:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    mvn w8, w8
-; CHECK-NEXT:    and x0, x8, x0, lsr #40
+; CHECK-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    lsr x9, x0, #40
+; CHECK-NEXT:    lsl x8, x8, x1
+; CHECK-NEXT:    bic x0, x9, x8
 ; CHECK-NEXT:    ret
   %shifted = lshr i64 %val, 40
   %notmask = shl i32 -1, %numlowbits
@@ -858,10 +859,10 @@ define i64 @bzhi64_32_e4_shifted_val(i64 %val, i32 %numlowbits) nounwind {
 define i64 @bzhi64_16_e0(i64 %val, i16 %numlowbits) nounwind {
 ; CHECK-LABEL: bzhi64_16_e0:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-1 // =0xffffffff
-; CHECK-NEXT:    lsl w8, w8, w1
-; CHECK-NEXT:    bic w8, w0, w8
-; CHECK-NEXT:    and x0, x8, #0xffff
+; CHECK-NEXT:    mov x8, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    lsl x8, x8, x1
+; CHECK-NEXT:    bic x0, x0, x8
 ; CHECK-NEXT:    ret
   %notmask = shl i16 -1, %numlowbits
   %mask = xor i16 %notmask, -1
diff --git a/llvm/test/CodeGen/X86/andnot-patterns.ll b/llvm/test/CodeGen/X86/andnot-patterns.ll
index 710cce093798d..6d7f252b328b3 100644
--- a/llvm/test/CodeGen/X86/andnot-patterns.ll
+++ b/llvm/test/CodeGen/X86/andnot-patterns.ll
@@ -1236,8 +1236,9 @@ define i64 @andnot_zext_lowmask_i64_i32(i64 %a0, i32 %a1) nounwind {
 ;
 ; X64-BMI-LABEL: andnot_zext_lowmask_i64_i32:
 ; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI-NEXT:    shll $8, %esi
-; X64-BMI-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI-NEXT:    retq
   %notmask = shl i32 -1, %a1
   %mask = xor i32 %notmask, -1
@@ -1260,10 +1261,9 @@ define i64 @andnot_zext_lowmask_i64_i16(i64 %a0, i16 %a1) nounwind {
 ;
 ; X86-BMI-LABEL: andnot_zext_lowmask_i64_i16:
 ; X86-BMI:       # %bb.0:
-; X86-BMI-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI-NEXT:    shll $8, %ecx
-; X86-BMI-NEXT:    bextrl %ecx, %eax, %eax
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    shll $8, %eax
+; X86-BMI-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
 ; X86-BMI-NEXT:    xorl %edx, %edx
 ; X86-BMI-NEXT:    retl
 ;
@@ -1280,9 +1280,9 @@ define i64 @andnot_zext_lowmask_i64_i16(i64 %a0, i16 %a1) nounwind {
 ;
 ; X64-BMI-LABEL: andnot_zext_lowmask_i64_i16:
 ; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI-NEXT:    shll $8, %esi
-; X64-BMI-NEXT:    bextrl %esi, %edi, %eax
-; X64-BMI-NEXT:    movzwl %ax, %eax
+; X64-BMI-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI-NEXT:    retq
   %notmask = shl i16 -1, %a1
   %mask = xor i16 %notmask, -1
@@ -1304,10 +1304,9 @@ define i32 @andnot_zext_lowmask_i32_i16(i32 %a0, i16 %a1) nounwind {
 ;
 ; X86-BMI-LABEL: andnot_zext_lowmask_i32_i16:
 ; X86-BMI:       # %bb.0:
-; X86-BMI-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI-NEXT:    shll $8, %ecx
-; X86-BMI-NEXT:    bextrl %ecx, %eax, %eax
+; X86-BMI-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    shll $8, %eax
+; X86-BMI-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
 ; X86-BMI-NEXT:    retl
 ;
 ; X64-NOBMI-LABEL: andnot_zext_lowmask_i32_i16:
@@ -1325,7 +1324,6 @@ define i32 @andnot_zext_lowmask_i32_i16(i32 %a0, i16 %a1) nounwind {
 ; X64-BMI:       # %bb.0:
 ; X64-BMI-NEXT:    shll $8, %esi
 ; X64-BMI-NEXT:    bextrl %esi, %edi, %eax
-; X64-BMI-NEXT:    movzwl %ax, %eax
 ; X64-BMI-NEXT:    retq
   %notmask = shl i16 -1, %a1
   %mask = xor i16 %notmask, -1
diff --git a/llvm/test/CodeGen/X86/extract-lowbits.ll b/llvm/test/CodeGen/X86/extract-lowbits.ll
index 29f5ce8e19687..0e201372956ba 100644
--- a/llvm/test/CodeGen/X86/extract-lowbits.ll
+++ b/llvm/test/CodeGen/X86/extract-lowbits.ll
@@ -4659,13 +4659,15 @@ define i64 @bzhi64_32_e0(i64 %val, i32 %numlowbits) nounwind {
 ;
 ; X64-BMI1-LABEL: bzhi64_32_e0:
 ; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI1-NEXT:    shll $8, %esi
-; X64-BMI1-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI1-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: bzhi64_32_e0:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI2-NEXT:    bzhiq %rsi, %rdi, %rax
 ; X64-BMI2-NEXT:    retq
   %notmask = shl i32 -1, %numlowbits
   %mask = xor i32 %notmask, -1
@@ -4712,13 +4714,15 @@ define i64 @bzhi64_32_e1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
 ;
 ; X64-BMI1-LABEL: bzhi64_32_e1_indexzext:
 ; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI1-NEXT:    shll $8, %esi
-; X64-BMI1-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI1-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: bzhi64_32_e1_indexzext:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI2-NEXT:    bzhiq %rsi, %rdi, %rax
 ; X64-BMI2-NEXT:    retq
   %conv = zext i8 %numlowbits to i32
   %notmask = shl i32 -1, %conv
@@ -4769,13 +4773,15 @@ define i64 @bzhi64_32_e2_load(ptr %w, i32 %numlowbits) nounwind {
 ;
 ; X64-BMI1-LABEL: bzhi64_32_e2_load:
 ; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI1-NEXT:    shll $8, %esi
-; X64-BMI1-NEXT:    bextrl %esi, (%rdi), %eax
+; X64-BMI1-NEXT:    bextrq %rsi, (%rdi), %rax
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: bzhi64_32_e2_load:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    bzhil %esi, (%rdi), %eax
+; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI2-NEXT:    bzhiq %rsi, (%rdi), %rax
 ; X64-BMI2-NEXT:    retq
   %val = load i64, ptr %w
   %notmask = shl i32 -1, %numlowbits
@@ -4823,13 +4829,15 @@ define i64 @bzhi64_32_e3_commutative(i64 %val, i32 %numlowbits) nounwind {
 ;
 ; X64-BMI1-LABEL: bzhi64_32_e3_commutative:
 ; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI1-NEXT:    shll $8, %esi
-; X64-BMI1-NEXT:    bextrl %esi, %edi, %eax
+; X64-BMI1-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: bzhi64_32_e3_commutative:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
+; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI2-NEXT:    bzhiq %rsi, %rdi, %rax
 ; X64-BMI2-NEXT:    retq
   %notmask = shl i32 -1, %numlowbits
   %mask = xor i32 %notmask, -1
@@ -4917,18 +4925,16 @@ define i64 @bzhi64_16_e0(i64 %val, i16 %numlowbits) nounwind {
 ;
 ; X86-BMI1-LABEL: bzhi64_16_e0:
 ; X86-BMI1:       # %bb.0:
-; X86-BMI1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1-NEXT:    shll $8, %ecx
-; X86-BMI1-NEXT:    bextrl %ecx, %eax, %eax
+; X86-BMI1-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT:    shll $8, %eax
+; X86-BMI1-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
 ; X86-BMI1-NEXT:    xorl %edx, %edx
 ; X86-BMI1-NEXT:    retl
 ;
 ; X86-BMI2-LABEL: bzhi64_16_e0:
 ; X86-BMI2:       # %bb.0:
-; X86-BMI2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT:    bzhil %ecx, %eax, %eax
+; X86-BMI2-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT:    bzhil %eax, {{[0-9]+}}(%esp), %eax
 ; X86-BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI2-NEXT:    retl
 ;
@@ -4945,15 +4951,15 @@ define i64 @bzhi64_16_e0(i64 %val, i16 %numlowbits) nounwind {
 ;
 ; X64-BMI1-LABEL: bzhi64_16_e0:
 ; X64-BMI1:       # %bb.0:
+; X64-BMI1-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64-BMI1-NEXT:    shll $8, %esi
-; X64-BMI1-NEXT:    bextrl %esi, %edi, %eax
-; X64-BMI1-NEXT:    movzwl %ax, %eax
+; X64-BMI1-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1-NEXT:    retq
 ;
 ; X64-BMI2-LABEL: bzhi64_16_e0:
 ; X64-BMI2:       # %bb.0:
-; X64-BMI2-NEXT:    bzhil %esi, %edi, %eax
-; X64-BMI2-NEXT:    movzwl %ax, %eax
+; X64-BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI2-NEXT:    bzhiq %rsi, %rdi, %rax
 ; X64-BMI2-NEXT:    retq
   %notmask = shl i16 -1, %numlowbits
   %mask = xor i16 %notmask, -1



More information about the llvm-commits mailing list