[llvm] [X86] Widen non-power-of-two vector div/rem under strictfp (PR #217572)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 11:58:02 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Adam Scott (as4230)
<details>
<summary>Changes</summary>
Non power of two vectors bail out of the FP divide fold under strictfp and scalarize. The SAE forms are 512-bit only so operands widen into a zmm, and a non power of two lane count has no machine type to widen through.
This widens to the next power of two first but only while the result still fits one divide. Past that it splits into two chains and a scalar divide on the odd lane is cheaper so a shape like v9i32 is unchanged.
Fixes #<!-- -->217558
---
Full diff: https://github.com/llvm/llvm-project/pull/217572.diff
2 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+14-2)
- (modified) llvm/test/CodeGen/X86/vector-idiv-strictfp.ll (+81)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 922657ca9e17c..1ec725f12c46c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50789,9 +50789,21 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
Attribute::StrictFP);
if (IsStrict) {
// The SAE forms are 512-bit only. Inputs widen into a zmm below, which
- // requires 512-bit types to be legal and a power of 2 lane count.
- if (!Subtarget.useAVX512Regs() || !isPowerOf2_32(VT.getVectorNumElements()))
+ // requires 512-bit types to be legal.
+ if (!Subtarget.useAVX512Regs())
return SDValue();
+ // Widen a non-power-of-two lane count to get a machine type, but only
+ // while it still fits one divide. Two chains lose to a chain plus a scalar.
+ unsigned NumElts = VT.getVectorNumElements();
+ if (!isPowerOf2_32(NumElts)) {
+ if (NextPowerOf2(NumElts) * FPSclVT.getSizeInBits() > 512)
+ return SDValue();
+ SDValue WideDividend = DAG.WidenVector(Dividend, DL);
+ EVT WideVT = WideDividend.getValueType();
+ SDValue WideDivisor = DAG.WidenVector(Divisor, DL);
+ SDValue Wide = DAG.getNode(Opc, DL, WideVT, WideDividend, WideDivisor);
+ return DAG.getExtractSubvector(DL, VT, Wide, 0);
+ }
} else if (!IsSigned && VT.getScalarSizeInBits() == 32 &&
!Subtarget.hasAVX2()) {
// Unsigned i32 needs FP_TO_UINT(f64->u32) which is emulated and a loss
diff --git a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
index 7add2e5ac4d6c..31f7bac000dab 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-strictfp.ll
@@ -201,3 +201,84 @@ define void @test_divv_32i32_strictfp(<32 x i32> %a, <32 x i32> %b, ptr %p) noun
store <32 x i32> %res, ptr %p
ret void
}
+
+define void @test_divv_7i32_strictfp(<7 x i32> %a, <7 x i32> %b, ptr %p) nounwind strictfp {
+; CHECK-LABEL: test_divv_7i32_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vcvtudq2pd %ymm1, %zmm1
+; CHECK-NEXT: vcvtudq2pd %ymm0, %zmm0
+; CHECK-NEXT: vdivpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttpd2udq {sae}, %zmm0, %ymm0
+; CHECK-NEXT: vextractf128 $1, %ymm0, %xmm1
+; CHECK-NEXT: vextractps $2, %xmm1, 24(%rdi)
+; CHECK-NEXT: vmovlps %xmm1, 16(%rdi)
+; CHECK-NEXT: vmovaps %xmm0, (%rdi)
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %res = udiv <7 x i32> %a, %b
+ store <7 x i32> %res, ptr %p
+ ret void
+}
+
+define void @test_divv_7i16_strictfp(<7 x i16> %a, <7 x i16> %b, ptr %p) nounwind strictfp {
+; CHECK-LABEL: test_divv_7i16_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-NEXT: vcvtdq2ps %ymm1, %ymm1
+; CHECK-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0
+; CHECK-NEXT: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttps2udq {sae}, %zmm0, %zmm0
+; CHECK-NEXT: vpmovdw %zmm0, %ymm0
+; CHECK-NEXT: vpextrw $6, %xmm0, 12(%rdi)
+; CHECK-NEXT: vpextrd $2, %xmm0, 8(%rdi)
+; CHECK-NEXT: vmovq %xmm0, (%rdi)
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %res = udiv <7 x i16> %a, %b
+ store <7 x i16> %res, ptr %p
+ ret void
+}
+
+define void @test_divv_15i8_strictfp(<15 x i8> %a, <15 x i8> %b, ptr %p) nounwind strictfp {
+; CHECK-LABEL: test_divv_15i8_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; CHECK-NEXT: vcvtdq2ps %zmm1, %zmm1
+; CHECK-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttps2udq {sae}, %zmm0, %zmm0
+; CHECK-NEXT: vpmovdb %zmm0, %xmm0
+; CHECK-NEXT: vpextrb $14, %xmm0, 14(%rdi)
+; CHECK-NEXT: vpextrw $6, %xmm0, 12(%rdi)
+; CHECK-NEXT: vpextrd $2, %xmm0, 8(%rdi)
+; CHECK-NEXT: vmovq %xmm0, (%rdi)
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %res = udiv <15 x i8> %a, %b
+ store <15 x i8> %res, ptr %p
+ ret void
+}
+
+define void @test_divv_15i16_strictfp(<15 x i16> %a, <15 x i16> %b, ptr %p) nounwind strictfp {
+; CHECK-LABEL: test_divv_15i16_strictfp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; CHECK-NEXT: vcvtdq2ps %zmm1, %zmm1
+; CHECK-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0
+; CHECK-NEXT: vdivps {rn-sae}, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vcvttps2udq {sae}, %zmm0, %zmm0
+; CHECK-NEXT: vpmovdw %zmm0, %ymm0
+; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1
+; CHECK-NEXT: vpextrw $6, %xmm1, 28(%rdi)
+; CHECK-NEXT: vpextrd $2, %xmm1, 24(%rdi)
+; CHECK-NEXT: vmovq %xmm1, 16(%rdi)
+; CHECK-NEXT: vmovdqa %xmm0, (%rdi)
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %res = udiv <15 x i16> %a, %b
+ store <15 x i16> %res, ptr %p
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/217572
More information about the llvm-commits
mailing list