[llvm] [SelectionDAG] Widen even 33-bit-magic udiv on free-zext targets (PR #207634)
MITSUNARI Shigeo via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 00:02:53 PDT 2026
https://github.com/herumi updated https://github.com/llvm/llvm-project/pull/207634
>From 7d5ba8978c96dfd2cd5416d156527ddda35f427c Mon Sep 17 00:00:00 2001
From: MITSUNARI Shigeo <herumi at nifty.com>
Date: Mon, 6 Jul 2026 13:33:10 +0900
Subject: [PATCH 1/3] [SelectionDAG] Add tests for even 33-bit-magic udiv
widening
Add test coverage for 32-bit unsigned division by an even constant whose
odd part yields a 33-bit magic number (the GM IsAdd case), on X86,
AArch64 and RISC-V. The CHECK lines capture the current code generation:
the even-divisor rewrite pre-shifts the dividend and divides the odd part
with a 32-bit magic.
A follow-up commit changes the lowering on free-zext targets and updates
these CHECK lines accordingly.
---
.../AArch64/udiv-const-optimization.ll | 34 ++++++++++++++++++
.../CodeGen/RISCV/udiv-const-optimization.ll | 16 +++++++++
.../CodeGen/X86/udiv-const-optimization.ll | 36 +++++++++++++++++++
3 files changed, 86 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/udiv-const-optimization.ll b/llvm/test/CodeGen/AArch64/udiv-const-optimization.ll
index eed677921e764..4014d4b177c94 100644
--- a/llvm/test/CodeGen/AArch64/udiv-const-optimization.ll
+++ b/llvm/test/CodeGen/AArch64/udiv-const-optimization.ll
@@ -59,3 +59,37 @@ define i32 @udiv_by_3(i32 %x) nounwind {
%div = udiv i32 %x, 3
ret i32 %div
}
+
+; Even divisors with 33-bit magic. On AArch64 these use umulh (i32->i64
+; zero-extension is free).
+define i32 @udiv_by_14(i32 %x) nounwind {
+; CHECK-LABEL: udiv_by_14:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsr w8, w0, #1
+; CHECK-NEXT: mov w9, #9363 // =0x2493
+; CHECK-NEXT: movk w9, #37449, lsl #16
+; CHECK-NEXT: umull x8, w8, w9
+; CHECK-NEXT: lsr x0, x8, #34
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT: ret
+ %div = udiv i32 %x, 14
+ ret i32 %div
+}
+
+; Vector udiv is unaffected by the free-zext scalar rewrite: <4 x i32> keeps
+; the even-divisor rewrite (this function is unchanged by the follow-up commit).
+define <4 x i32> @udiv_by_14_vec(<4 x i32> %x) nounwind {
+; CHECK-LABEL: udiv_by_14_vec:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov w8, #9363 // =0x2493
+; CHECK-NEXT: ushr v0.4s, v0.4s, #1
+; CHECK-NEXT: movk w8, #37449, lsl #16
+; CHECK-NEXT: dup v1.4s, w8
+; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: uzp2 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: ushr v0.4s, v0.4s, #2
+; CHECK-NEXT: ret
+ %div = udiv <4 x i32> %x, <i32 14, i32 14, i32 14, i32 14>
+ ret <4 x i32> %div
+}
diff --git a/llvm/test/CodeGen/RISCV/udiv-const-optimization.ll b/llvm/test/CodeGen/RISCV/udiv-const-optimization.ll
index f72664587302f..9d9ce34fcec49 100644
--- a/llvm/test/CodeGen/RISCV/udiv-const-optimization.ll
+++ b/llvm/test/CodeGen/RISCV/udiv-const-optimization.ll
@@ -61,3 +61,19 @@ define i32 @udiv_by_3(i32 %x) nounwind {
%div = udiv i32 %x, 3
ret i32 %div
}
+
+; Even divisor with 33-bit magic. On RISC-V the widen path is not used (i32->i64
+; zero-extension is not free); the even-divisor rewrite is kept instead.
+define i32 @udiv_by_14(i32 %x) nounwind {
+; CHECK-LABEL: udiv_by_14:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a1, 299593
+; CHECK-NEXT: slli a1, a1, 1
+; CHECK-NEXT: srliw a0, a0, 1
+; CHECK-NEXT: addi a1, a1, 1171
+; CHECK-NEXT: mul a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 34
+; CHECK-NEXT: ret
+ %div = udiv i32 %x, 14
+ ret i32 %div
+}
diff --git a/llvm/test/CodeGen/X86/udiv-const-optimization.ll b/llvm/test/CodeGen/X86/udiv-const-optimization.ll
index a4fa413bab038..a754d9c5db28b 100644
--- a/llvm/test/CodeGen/X86/udiv-const-optimization.ll
+++ b/llvm/test/CodeGen/X86/udiv-const-optimization.ll
@@ -139,3 +139,39 @@ define i32 @udiv_by_3(i32 %x) nounwind {
%div = udiv i32 %x, 3
ret i32 %div
}
+
+; Even divisors with 33-bit magic. On 64-bit targets these use the widened
+; high-multiply path (i32->i64 zero-extension is free here).
+define i32 @udiv_by_14(i32 %x) nounwind {
+; X64-LABEL: udiv_by_14:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: shrl %edi
+; X64-NEXT: movl $2454267027, %eax # imm = 0x92492493
+; X64-NEXT: imulq %rdi, %rax
+; X64-NEXT: shrq $34, %rax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-NEXT: retq
+;
+; X64-BMI2-LABEL: udiv_by_14:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-BMI2-NEXT: shrl %edi
+; X64-BMI2-NEXT: movl $2454267027, %eax # imm = 0x92492493
+; X64-BMI2-NEXT: imulq %rdi, %rax
+; X64-BMI2-NEXT: shrq $34, %rax
+; X64-BMI2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-BMI2-NEXT: retq
+;
+; X86-LABEL: udiv_by_14:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shrl %eax
+; X86-NEXT: movl ${{-?[0-9]+}}, %ecx # imm = 0x92492493
+; X86-NEXT: mull %ecx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shrl $2, %eax
+; X86-NEXT: retl
+ %div = udiv i32 %x, 14
+ ret i32 %div
+}
>From c50d999b44532b6dfda9a0ea2dad7dd1d3e95ac3 Mon Sep 17 00:00:00 2001
From: MITSUNARI Shigeo <herumi at nifty.com>
Date: Wed, 8 Jul 2026 14:53:50 +0900
Subject: [PATCH 2/3] [SelectionDAG] Pin i386 immediates in
udiv-const-optimization.ll
The X86 checks for udiv_by_{19,21,3,14} used a {{-?[0-9]+}} regex
placeholder for the i386 immediate. update_llc_test_checks.py rewrites
that placeholder to the concrete signed literal on every regeneration,
so a full regeneration of the file churns these unrelated lines.
Pin them to the literals llc emits so that regenerating the checks is a
no-op. This keeps a later check update from touching functions that the
change does not affect. NFC.
---
llvm/test/CodeGen/X86/udiv-const-optimization.ll | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/X86/udiv-const-optimization.ll b/llvm/test/CodeGen/X86/udiv-const-optimization.ll
index a754d9c5db28b..8f84cabd599b7 100644
--- a/llvm/test/CodeGen/X86/udiv-const-optimization.ll
+++ b/llvm/test/CodeGen/X86/udiv-const-optimization.ll
@@ -62,7 +62,7 @@ define i32 @udiv_by_19(i32 %x) nounwind {
; X86-LABEL: udiv_by_19:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl ${{-?[0-9]+}}, %edx # imm = 0xAF286BCB
+; X86-NEXT: movl $-1356305461, %edx # imm = 0xAF286BCB
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: mull %edx
; X86-NEXT: subl %edx, %ecx
@@ -96,7 +96,7 @@ define i32 @udiv_by_21(i32 %x) nounwind {
; X86-LABEL: udiv_by_21:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl ${{-?[0-9]+}}, %edx # imm = 0x86186187
+; X86-NEXT: movl $-2045222521, %edx # imm = 0x86186187
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: mull %edx
; X86-NEXT: subl %edx, %ecx
@@ -131,7 +131,7 @@ define i32 @udiv_by_3(i32 %x) nounwind {
;
; X86-LABEL: udiv_by_3:
; X86: # %bb.0:
-; X86-NEXT: movl ${{-?[0-9]+}}, %eax # imm = 0xAAAAAAAB
+; X86-NEXT: movl $-1431655765, %eax # imm = 0xAAAAAAAB
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: movl %edx, %eax
; X86-NEXT: shrl %eax
@@ -167,7 +167,7 @@ define i32 @udiv_by_14(i32 %x) nounwind {
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: shrl %eax
-; X86-NEXT: movl ${{-?[0-9]+}}, %ecx # imm = 0x92492493
+; X86-NEXT: movl $-1840700269, %ecx # imm = 0x92492493
; X86-NEXT: mull %ecx
; X86-NEXT: movl %edx, %eax
; X86-NEXT: shrl $2, %eax
>From cb4528e6966b5f688d3ff8e37592c1500183691b Mon Sep 17 00:00:00 2001
From: MITSUNARI Shigeo <herumi at nifty.com>
Date: Wed, 8 Jul 2026 15:59:32 +0900
Subject: [PATCH 3/3] [SelectionDAG] Widen even 33-bit-magic udiv on free-zext
targets
Follow-up to #181288, which lowers a 32-bit unsigned division by a
constant with a 33-bit magic number (IsAdd) to a widened 64-bit
high-multiply (MULHU/UMUL_LOHI) on 64-bit targets. That patch only
reached odd divisors: for an even divisor the even-divisor rewrite in
UnsignedDivisionByConstantInfo::get() fires first, pre-shifting the
dividend and dividing by the odd part with a 32-bit magic, so the
widened path is never used.
On targets that zero-extend i32 to i64 for free (x86-64 mul/mulx,
AArch64 umulh) the widened path is a single high-multiply and beats the
even-divisor rewrite, which needs an extra shift. BuildUDIV now
suppresses the even-divisor rewrite for those targets by keying on
isZExtFree(i32, i64), so even divisors use the widened path too. Where
the extension is not free (e.g. RISC-V) the even-divisor rewrite is
kept and code generation is unchanged.
The decision is made in the caller; UnsignedDivisionByConstantInfo::get()
is unchanged and its two permission flags stay independent.
---
.../lib/CodeGen/SelectionDAG/TargetLowering.cpp | 8 +++++++-
.../CodeGen/AArch64/udiv-const-optimization.ll | 10 +++++-----
llvm/test/CodeGen/X86/divide-by-constant.ll | 8 ++++----
.../test/CodeGen/X86/udiv-const-optimization.ll | 17 +++++++----------
4 files changed, 23 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index a4bcc8cb01126..f67bf7d08439d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6921,6 +6921,12 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
isOperationLegalOrCustom(ISD::UMUL_LOHI, WideSVT, IsAfterLegalization);
const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);
+ // For even divisors with a 33-bit magic number, the widened high-multiply
+ // path is only worthwhile over the even-divisor rewrite on targets that
+ // zero-extend i32 to i64 for free (e.g. x86-64 and AArch64). Elsewhere (e.g.
+ // RISC-V) keep the even-divisor rewrite, which avoids the explicit extension.
+ const bool AllowEvenToWiden = AllowWiden && isZExtFree(VT, WideSVT);
+
bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
bool UseWiden = false;
SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
@@ -6943,7 +6949,7 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
UnsignedDivisionByConstantInfo magics =
UnsignedDivisionByConstantInfo::get(
Divisor, std::min(KnownLeadingZeros, Divisor.countl_zero()),
- /*AllowEvenDivisorOptimization=*/true,
+ /*AllowEvenDivisorOptimization=*/!AllowEvenToWiden,
/*AllowWidenOptimization=*/AllowWiden);
if (magics.Widen) {
diff --git a/llvm/test/CodeGen/AArch64/udiv-const-optimization.ll b/llvm/test/CodeGen/AArch64/udiv-const-optimization.ll
index 4014d4b177c94..03ead29ba7f4f 100644
--- a/llvm/test/CodeGen/AArch64/udiv-const-optimization.ll
+++ b/llvm/test/CodeGen/AArch64/udiv-const-optimization.ll
@@ -65,11 +65,11 @@ define i32 @udiv_by_3(i32 %x) nounwind {
define i32 @udiv_by_14(i32 %x) nounwind {
; CHECK-LABEL: udiv_by_14:
; CHECK: // %bb.0:
-; CHECK-NEXT: lsr w8, w0, #1
-; CHECK-NEXT: mov w9, #9363 // =0x2493
-; CHECK-NEXT: movk w9, #37449, lsl #16
-; CHECK-NEXT: umull x8, w8, w9
-; CHECK-NEXT: lsr x0, x8, #34
+; CHECK-NEXT: mov x8, #1342177280 // =0x50000000
+; CHECK-NEXT: mov w9, w0
+; CHECK-NEXT: movk x8, #9362, lsl #32
+; CHECK-NEXT: movk x8, #4681, lsl #48
+; CHECK-NEXT: umulh x0, x9, x8
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%div = udiv i32 %x, 14
diff --git a/llvm/test/CodeGen/X86/divide-by-constant.ll b/llvm/test/CodeGen/X86/divide-by-constant.ll
index d9e19df39f6b6..6a692e3c98b73 100644
--- a/llvm/test/CodeGen/X86/divide-by-constant.ll
+++ b/llvm/test/CodeGen/X86/divide-by-constant.ll
@@ -148,10 +148,10 @@ define i32 @test7(i32 %x) nounwind {
;
; X64-LABEL: test7:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: shrl $2, %edi
-; X64-NEXT: imulq $613566757, %rdi, %rax # imm = 0x24924925
-; X64-NEXT: shrq $32, %rax
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: movabsq $658812288404291584, %rcx # imm = 0x924924928000000
+; X64-NEXT: mulq %rcx
+; X64-NEXT: movq %rdx, %rax
; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%div = udiv i32 %x, 28
diff --git a/llvm/test/CodeGen/X86/udiv-const-optimization.ll b/llvm/test/CodeGen/X86/udiv-const-optimization.ll
index 8f84cabd599b7..5dfad9f896540 100644
--- a/llvm/test/CodeGen/X86/udiv-const-optimization.ll
+++ b/llvm/test/CodeGen/X86/udiv-const-optimization.ll
@@ -145,21 +145,18 @@ define i32 @udiv_by_3(i32 %x) nounwind {
define i32 @udiv_by_14(i32 %x) nounwind {
; X64-LABEL: udiv_by_14:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: shrl %edi
-; X64-NEXT: movl $2454267027, %eax # imm = 0x92492493
-; X64-NEXT: imulq %rdi, %rax
-; X64-NEXT: shrq $34, %rax
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: movabsq $1317624576808583168, %rcx # imm = 0x1249249250000000
+; X64-NEXT: mulq %rcx
+; X64-NEXT: movq %rdx, %rax
; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
;
; X64-BMI2-LABEL: udiv_by_14:
; X64-BMI2: # %bb.0:
-; X64-BMI2-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-BMI2-NEXT: shrl %edi
-; X64-BMI2-NEXT: movl $2454267027, %eax # imm = 0x92492493
-; X64-BMI2-NEXT: imulq %rdi, %rax
-; X64-BMI2-NEXT: shrq $34, %rax
+; X64-BMI2-NEXT: movl %edi, %edx
+; X64-BMI2-NEXT: movabsq $1317624576808583168, %rax # imm = 0x1249249250000000
+; X64-BMI2-NEXT: mulxq %rax, %rax, %rax
; X64-BMI2-NEXT: # kill: def $eax killed $eax killed $rax
; X64-BMI2-NEXT: retq
;
More information about the llvm-commits
mailing list