[llvm] [CodeGenPrepare] Hoist cross-block zext to its source block (PR #223010)
Lydia Kim via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 11 11:51:14 PDT 2026
https://github.com/lydkim updated https://github.com/llvm/llvm-project/pull/223010
>From 4543adeea1678a707b6ddf16fcc33b2b4b26c906 Mon Sep 17 00:00:00 2001
From: Lydia Kim <lydkim at fb.com>
Date: Tue, 8 Sep 2026 08:31:09 -0700
Subject: [PATCH] [CodeGenPrepare] Hoist cross-block zext to its source block
When a narrow bit-clearing operation (e.g. `and i16 %x, 16383`) feeds a
`zext` in a different basic block, SelectionDAG emits a redundant `movzx`:
it processes one basic block at a time and cannot see that the upper bits
are already zero.
Add `hoistZExtToSourceBlock` to CodeGenPrepare, which moves such a `zext`
into its source instruction's block. Once both are in one block the pattern
reaches `DAGCombiner::visitZERO_EXTEND` and folds away, so the extension
disappears instead of becoming a `movzx`.
Hoisting is restricted to cases that provably fold:
- `computeKnownBits` must show cleared upper bits on the source. Without
them the zext is real work, and moving it to a possibly hotter block
would be a pessimization.
- The extension must not already be free (`isZExtFree`), and both the
source and destination types must be legal. Illegal types are promoted
by type legalization, which folds the extension implicitly.
- The source must not have another use in its own block that needs the
narrow value, which would leave the same number live in two registers.
This shows up in hashmap probe-mask code, where an `and` and a `movzwl`
separated by a branch collapse into a single 32-bit `and`.
The transform can be disabled with `-cgp-hoist-zext=false`.
---
llvm/lib/CodeGen/CodeGenPrepare.cpp | 105 ++++++-
llvm/test/CodeGen/X86/pr33828.ll | 6 +-
.../CodeGenPrepare/X86/cross-bb-zext-and.ll | 259 ++++++++++++++++++
3 files changed, 365 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/cross-bb-zext-and.ll
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 52ed1255ab9c4..7a77495f6a76e 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -89,6 +89,7 @@
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/KnownBits.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
@@ -136,6 +137,8 @@ STATISTIC(NumRetsDup, "Number of return instructions duplicated");
STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved");
STATISTIC(NumSelectsExpanded, "Number of selects turned into branches");
STATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed");
+STATISTIC(NumZExtsHoisted,
+ "Number of zext instructions hoisted to source block");
static cl::opt<bool> DisableBranchOpts(
"disable-cgp-branch-opts", cl::Hidden, cl::init(false),
@@ -278,6 +281,10 @@ static cl::opt<bool>
DisableDeletePHIs("disable-cgp-delete-phis", cl::Hidden, cl::init(false),
cl::desc("Disable elimination of dead PHI nodes."));
+static cl::opt<bool> EnableZExtHoisting(
+ "cgp-hoist-zext", cl::Hidden, cl::init(true),
+ cl::desc("Enable hoisting zext instructions to their source block"));
+
namespace {
enum ExtType {
@@ -471,6 +478,7 @@ class CodeGenPrepare {
bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
bool combineToUAddWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
bool unfoldPowerOf2Test(CmpInst *Cmp);
+ bool hoistZExtToSourceBlock(ZExtInst *ZExt);
void verifyBFIUpdates(Function &F);
bool _run(Function &F);
};
@@ -7212,6 +7220,95 @@ bool CodeGenPrepare::canFormExtLd(
return TLI->isExtLoad(LI, Inst, *DL);
}
+/// Hoist a cross-block zext to the block where its source is defined when
+/// doing so enables additional folding, working around SelectionDAG's
+/// single-block limitation.
+///
+/// IR before: IR after:
+/// bb.0: bb.0:
+/// %a = and i16 %x, 16383 %a = and i16 %x, 16383
+/// %c = icmp eq i16 %a, 0 %w = zext i16 %a to i32 ; hoisted
+/// br i1 %c, bb.1, bb.2 %c = icmp eq i16 %a, 0
+/// bb.1: br i1 %c, bb.1, bb.2
+/// %w = zext i16 %a to i32 bb.1:
+/// ... ... (uses %w directly)
+///
+/// x86 asm before (without hoist): x86 asm after (with hoist):
+/// bb.0: bb.0:
+/// andw $16383, %ax andl $16383, %ecx
+/// ... ...
+/// bb.1: bb.1:
+/// movzwl %ax, %ecx ... (movzwl eliminated)
+/// ... ...
+///
+/// SelectionDAG processes one block at a time. Without the hoist, it
+/// cannot see across the block boundary to fold zext(and(x, c)) into
+/// and(zext(x), c), so it emits a 16-bit and in bb.0 followed by a
+/// movzwl in bb.1. With the hoist, both are in bb.0 and SelectionDAG
+/// folds them into a single 32-bit andl, eliminating the movzwl.
+bool CodeGenPrepare::hoistZExtToSourceBlock(ZExtInst *ZExt) {
+ if (!EnableZExtHoisting)
+ return false;
+
+ // Source must be an instruction in a different block.
+ // Same-block cases are already handled by SelectionDAG.
+ Value *Src = ZExt->getOperand(0);
+ auto *SrcInst = dyn_cast<Instruction>(Src);
+ if (!SrcInst || SrcInst->getParent() == ZExt->getParent())
+ return false;
+
+ if (SrcInst->isTerminator())
+ return false;
+
+ Type *SrcTy = Src->getType();
+ if (!SrcTy->isIntegerTy())
+ return false;
+ unsigned SrcBitWidth = SrcTy->getIntegerBitWidth();
+ if (SrcBitWidth < 8 || SrcBitWidth >= 32)
+ return false;
+ if (TLI->isZExtFree(SrcTy, ZExt->getType()))
+ return false;
+ EVT SrcVT = TLI->getValueType(*DL, SrcTy);
+ EVT WideVT = TLI->getValueType(*DL, ZExt->getType());
+ if (!TLI->isTypeLegal(SrcVT) || !TLI->isTypeLegal(WideVT))
+ return false;
+ // Only hoist patterns SelectionDAG will actually fold. Known-zero upper
+ // bits are the precondition for that: once the zext and its source share
+ // a block, the pattern reaches DAGCombiner::visitZERO_EXTEND, whose
+ // isTruncateOf() check folds (zext (truncate x)) away when the truncated
+ // bits are already zero. Without cleared bits the zext is real work, and
+ // hoisting it to a potentially higher-frequency block is a pessimization.
+ KnownBits Known = computeKnownBits(Src, *DL);
+ if (Known.countMinLeadingZeros() == 0)
+ return false;
+ // Don't hoist when the source has another use in its own block that needs
+ // the narrow value. Such a use keeps the narrow value live alongside the
+ // widened one, so the hoist would just compute the same number in two
+ // registers. A comparison is exempt because SelectionDAG can widen its
+ // operands along with the zext, and a second zext of the same source folds
+ // the same way.
+ for (User *U : SrcInst->users()) {
+ auto *UI = dyn_cast<Instruction>(U);
+ if (!UI || UI == ZExt || UI->getParent() != SrcInst->getParent())
+ continue;
+ if (isa<ICmpInst>(UI) || isa<PHINode>(UI) || isa<ZExtInst>(UI))
+ continue;
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "CGP: Hoisting zext to source block: " << *ZExt << "\n");
+
+ // Place after the source, or after all PHIs if the source is a PHI.
+ if (isa<PHINode>(SrcInst))
+ ZExt->moveBefore(*SrcInst->getParent(),
+ SrcInst->getParent()->getFirstNonPHIIt());
+ else
+ ZExt->moveAfter(SrcInst);
+
+ ++NumZExtsHoisted;
+ return true;
+}
+
/// Move a zext or sext fed by a load into the same basic block as the load,
/// unless conditions are unfavorable. This allows SelectionDAG to fold the
/// extend into the load.
@@ -9009,7 +9106,13 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, ModifyDT &ModifiedDT) {
return true;
bool MadeChange = optimizeExt(I);
- return MadeChange | optimizeExtUses(I);
+ if (MadeChange | optimizeExtUses(I))
+ return true;
+
+ // If no other extension optimization applied, try hoisting a
+ // cross-block zext to its source block as a last resort.
+ if (auto *ZExt = dyn_cast<ZExtInst>(I))
+ return hoistZExtToSourceBlock(ZExt);
}
}
return AnyChange;
diff --git a/llvm/test/CodeGen/X86/pr33828.ll b/llvm/test/CodeGen/X86/pr33828.ll
index a96543ee6ef7a..239923e502ce3 100644
--- a/llvm/test/CodeGen/X86/pr33828.ll
+++ b/llvm/test/CodeGen/X86/pr33828.ll
@@ -7,8 +7,7 @@
define void @foo(i8 %a0) {
; X86-LABEL: foo:
; X86: # %bb.0: # %entry
-; X86-NEXT: movsbl var_580, %eax
-; X86-NEXT: testl $-536870913, %eax # imm = 0xDFFFFFFF
+; X86-NEXT: cmpb $0, var_580
; X86-NEXT: jne .LBB0_1
; X86-NEXT: # %bb.2: # %if.end13
; X86-NEXT: retl
@@ -16,8 +15,7 @@ define void @foo(i8 %a0) {
;
; X64-LABEL: foo:
; X64: # %bb.0: # %entry
-; X64-NEXT: movsbl var_580(%rip), %eax
-; X64-NEXT: testl $-536870913, %eax # imm = 0xDFFFFFFF
+; X64-NEXT: cmpb $0, var_580(%rip)
; X64-NEXT: jne .LBB0_1
; X64-NEXT: # %bb.2: # %if.end13
; X64-NEXT: retq
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/cross-bb-zext-and.ll b/llvm/test/Transforms/CodeGenPrepare/X86/cross-bb-zext-and.ll
new file mode 100644
index 0000000000000..52004614d65b0
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/cross-bb-zext-and.ll
@@ -0,0 +1,259 @@
+; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -S < %s | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+;; Test: basic cross-block zext hoisting.
+;; This is the F14 hashmap find() pattern. The zext is hoisted to the entry
+;; block so SelectionDAG can fold it with the and.
+define i32 @basic_hoist(i16 %x) {
+; CHECK-LABEL: @basic_hoist(
+; CHECK: entry:
+; CHECK-NEXT: [[AND:%.*]] = and i16 %x, 16383
+; CHECK-NEXT: [[WIDE:%.*]] = zext i16 [[AND]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[AND]], 0
+; CHECK-NEXT: br i1 [[CMP]]
+; CHECK: forward:
+; CHECK-NEXT: ret i32 [[WIDE]]
+;
+entry:
+ %and = and i16 %x, 16383
+ %cmp = icmp eq i16 %and, 0
+ br i1 %cmp, label %else, label %forward
+
+forward:
+ %wide = zext i16 %and to i32
+ br label %use
+
+use:
+ ret i32 %wide
+
+else:
+ ret i32 0
+}
+
+;; Negative: 'or' does not clear upper bits — no proof the hoist helps.
+define i32 @no_hoist_or_no_known_zeros(i16 %x) {
+; CHECK-LABEL: @no_hoist_or_no_known_zeros(
+; CHECK: forward:
+; CHECK-NEXT: [[WIDE:%.*]] = zext i16 [[OR:%.*]] to i32
+; CHECK-NEXT: br label %use
+;
+entry:
+ %or = or i16 %x, 255
+ br label %forward
+
+forward:
+ %wide = zext i16 %or to i32
+ br label %use
+
+use:
+ ret i32 %wide
+}
+
+;; Test: i8 to i32 hoisting.
+define i32 @hoist_i8_to_i32(i8 %x) {
+; CHECK-LABEL: @hoist_i8_to_i32(
+; CHECK: entry:
+; CHECK-NEXT: [[AND:%.*]] = and i8 %x, 63
+; CHECK-NEXT: [[WIDE:%.*]] = zext i8 [[AND]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT: br i1 [[CMP]]
+;
+entry:
+ %and = and i8 %x, 63
+ %cmp = icmp eq i8 %and, 0
+ br i1 %cmp, label %else, label %forward
+
+forward:
+ %wide = zext i8 %and to i32
+ br label %use
+
+use:
+ ret i32 %wide
+
+else:
+ ret i32 0
+}
+
+
+;; Test: i16 to i64 hoisting (isZExtFree(i16, i64) is false on x86-64).
+define i64 @hoist_i16_to_i64(i16 %x) {
+; CHECK-LABEL: @hoist_i16_to_i64(
+; CHECK: entry:
+; CHECK-NEXT: [[AND:%.*]] = and i16 %x, 16383
+; CHECK-NEXT: [[WIDE:%.*]] = zext i16 [[AND]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[AND]], 0
+; CHECK-NEXT: br i1 [[CMP]]
+;
+entry:
+ %and = and i16 %x, 16383
+ %cmp = icmp eq i16 %and, 0
+ br i1 %cmp, label %else, label %then
+
+then:
+ %wide = zext i16 %and to i64
+ ret i64 %wide
+
+else:
+ ret i64 0
+}
+
+;; Test: multiple zexts down both paths of a branch. Both should be hoisted
+;; to the source block. SelectionDAG will CSE them in the same DAG.
+define i32 @hoist_multi_zext_both_paths(i16 %x) {
+; CHECK-LABEL: @hoist_multi_zext_both_paths(
+; CHECK: entry:
+; CHECK-NEXT: [[AND:%.*]] = and i16 %x, 16383
+; CHECK-NEXT: [[WIDE2:%.*]] = zext i16 [[AND]] to i32
+; CHECK-NEXT: [[WIDE1:%.*]] = zext i16 [[AND]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[AND]], 0
+; CHECK-NEXT: br i1 [[CMP]]
+; CHECK: then:
+; CHECK-NEXT: br label %merge
+; CHECK: else:
+; CHECK-NEXT: br label %merge
+; CHECK: merge:
+; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[WIDE1]], %then ], [ [[WIDE2]], %else ]
+; CHECK-NEXT: ret i32 [[PHI]]
+;
+entry:
+ %and = and i16 %x, 16383
+ %cmp = icmp eq i16 %and, 0
+ br i1 %cmp, label %then, label %else
+
+then:
+ %wide1 = zext i16 %and to i32
+ br label %merge
+
+else:
+ %wide2 = zext i16 %and to i32
+ br label %merge
+
+merge:
+ %phi = phi i32 [%wide1, %then], [%wide2, %else]
+ ret i32 %phi
+}
+
+
+;; Test: zext block has other instructions (call) — zext still hoisted.
+declare void @use(i32)
+define i32 @hoist_with_call_in_block(i16 %x) {
+; CHECK-LABEL: @hoist_with_call_in_block(
+; CHECK: entry:
+; CHECK-NEXT: [[AND:%.*]] = and i16 %x, 255
+; CHECK-NEXT: [[WIDE:%.*]] = zext i16 [[AND]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[AND]], 0
+; CHECK-NEXT: br i1 [[CMP]]
+; CHECK: then:
+; CHECK-NEXT: call void @use(i32 [[WIDE]])
+; CHECK-NEXT: ret i32 [[WIDE]]
+;
+entry:
+ %and = and i16 %x, 255
+ %cmp = icmp eq i16 %and, 0
+ br i1 %cmp, label %else, label %then
+
+then:
+ %wide = zext i16 %and to i32
+ call void @use(i32 %wide)
+ ret i32 %wide
+
+else:
+ ret i32 0
+}
+
+
+;; ===========================================================================
+;; Negative tests: cases where hoisting should NOT occur.
+;; ===========================================================================
+
+;; Negative: same block — no hoisting needed.
+define i32 @same_block_no_hoist(i16 %x) {
+; CHECK-LABEL: @same_block_no_hoist(
+; CHECK: [[AND:%.*]] = and i16 %x, 255
+; CHECK-NEXT: [[ZEXT:%.*]] = zext i16 [[AND]] to i32
+; CHECK-NEXT: ret i32 [[ZEXT]]
+;
+entry:
+ %and = and i16 %x, 255
+ %wide = zext i16 %and to i32
+ ret i32 %wide
+}
+
+;; Test: zext block has other instructions (not a pure forwarding block).
+;; The zext should still be hoisted to the source block.
+define i32 @hoist_non_forwarding(i16 %x) {
+; CHECK-LABEL: @hoist_non_forwarding(
+; CHECK: entry:
+; CHECK-NEXT: [[AND:%.*]] = and i16 %x, 255
+; CHECK-NEXT: [[WIDE:%.*]] = zext i16 [[AND]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[AND]], 0
+; CHECK-NEXT: br i1 [[CMP]]
+; CHECK: then:
+; CHECK-NEXT: ret i32 [[WIDE]]
+;
+entry:
+ %and = and i16 %x, 255
+ %cmp = icmp eq i16 %and, 0
+ br i1 %cmp, label %else, label %then
+
+then:
+ %wide = zext i16 %and to i32
+ ret i32 %wide
+
+else:
+ ret i32 0
+}
+
+;; Negative: i32 to i64 — isZExtFree returns true on x86-64.
+define i64 @no_hoist_i32_to_i64(i32 %x) {
+; CHECK-LABEL: @no_hoist_i32_to_i64(
+; CHECK: forward:
+; CHECK-NEXT: [[WIDE:%.*]] = zext i32 [[AND:%.*]] to i64
+; CHECK-NEXT: br label %use
+;
+entry:
+ %and = and i32 %x, 255
+ %cmp = icmp eq i32 %and, 0
+ br i1 %cmp, label %else, label %forward
+
+forward:
+ %wide = zext i32 %and to i64
+ br label %use
+
+use:
+ ret i64 %wide
+
+else:
+ ret i64 0
+}
+
+;; Negative: the source also feeds arithmetic in its own block. Hoisting would
+;; keep the narrow value live for the shl alongside the widened one, computing
+;; the same number in two registers.
+define i32 @no_hoist_narrow_use_in_source_block(i16 %x, ptr %p) {
+; CHECK-LABEL: @no_hoist_narrow_use_in_source_block(
+; CHECK: entry:
+; CHECK-NEXT: [[AND:%.*]] = and i16 %x, 16383
+; CHECK-NEXT: [[SHL:%.*]] = shl i16 [[AND]], 2
+; CHECK-NEXT: store i16 [[SHL]], ptr %p
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[AND]], 0
+; CHECK-NEXT: br i1 [[CMP]]
+; CHECK: forward:
+; CHECK-NEXT: [[WIDE:%.*]] = zext i16 [[AND]] to i32
+; CHECK-NEXT: ret i32 [[WIDE]]
+;
+entry:
+ %and = and i16 %x, 16383
+ %shl = shl i16 %and, 2
+ store i16 %shl, ptr %p
+ %cmp = icmp eq i16 %and, 0
+ br i1 %cmp, label %else, label %forward
+
+forward:
+ %wide = zext i16 %and to i32
+ ret i32 %wide
+
+else:
+ ret i32 0
+}
More information about the llvm-commits
mailing list