[llvm] [X86] Vectorize non-power-of-two integer division (PR #215076)
Patrick Ribbsaeter via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 03:43:15 PDT 2026
https://github.com/patrickswedish updated https://github.com/llvm/llvm-project/pull/215076
>From 9215cfff69f10398c42c89de81ca811773227095 Mon Sep 17 00:00:00 2001
From: Patrick Ribbsaeter <patrickswedish at gmail.com>
Date: Sun, 9 Aug 2026 10:18:13 +0200
Subject: [PATCH 1/4] [X86] Vectorize non-power-of-two integer division
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 22 +++++++++++++---
.../CodeGen/X86/vector-idiv-udiv-non-pow2.ll | 26 +++++++++++++++++++
2 files changed, 44 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c74d342fed1cb..8902a9560f3cc 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50643,10 +50643,24 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
bool IsSigned = Opc == ISD::SDIV || Opc == ISD::SREM;
// If the result is only read back as scalar extracts, scalarization computes
- // just the demanded lanes.
- if (all_of(N->users(), [](const SDNode *U) {
- return U->getOpcode() == ISD::EXTRACT_VECTOR_ELT;
- }))
+ // just the demanded lanes. Keep the vector operation when every lane is
+ // extracted, which occurs when non-power-of-two vectors are returned.
+ APInt ExtractedElts = APInt::getZero(VT.getVectorNumElements());
+ bool OnlyExtracts = true;
+ for (const SDNode *U : N->users()) {
+ if (U->getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
+ OnlyExtracts = false;
+ break;
+ }
+ auto *Idx = dyn_cast<ConstantSDNode>(U->getOperand(1));
+ if (!Idx)
+ return SDValue();
+ uint64_t IdxVal = Idx->getZExtValue();
+ if (IdxVal >= VT.getVectorNumElements())
+ return SDValue();
+ ExtractedElts.setBit(IdxVal);
+ }
+ if (OnlyExtracts && !ExtractedElts.isAllOnes())
return SDValue();
// Magic multiply lowers constant divisors cheaper than a divide.
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll
new file mode 100644
index 0000000000000..4b5d55acfcc54
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll
@@ -0,0 +1,26 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s
+
+define <7 x i32> @udiv_v7i32(<7 x i32> %x, <7 x i32> %y) {
+; CHECK-LABEL: udiv_v7i32:
+; CHECK-NOT: divl
+; CHECK: vcvtudq2pd
+; CHECK: vcvtudq2pd
+; CHECK: vdivpd
+; CHECK: vcvttpd2udq
+; CHECK-NOT: divl
+; CHECK: retq
+ %div = udiv <7 x i32> %x, %y
+ ret <7 x i32> %div
+}
+
+; Keep scalarizing when only one result lane is demanded.
+define i32 @udiv_v7i32_extract0(<7 x i32> %x, <7 x i32> %y) {
+; CHECK-LABEL: udiv_v7i32_extract0:
+; CHECK-NOT: vdivpd
+; CHECK: divl
+; CHECK-NOT: vdivpd
+; CHECK: retq
+ %div = udiv <7 x i32> %x, %y
+ %elt = extractelement <7 x i32> %div, i32 0
+ ret i32 %elt
+}
>From fd567029164477446ace2af89ec7a1094c76a2a3 Mon Sep 17 00:00:00 2001
From: Patrick Ribbsaeter <hello at glasheldereindhoven.nl>
Date: Sun, 9 Aug 2026 20:14:28 +0200
Subject: [PATCH 2/4] [X86] Address vector idiv review
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 6 +++---
llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll | 2 +-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 8902a9560f3cc..5f415a1437f9d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50655,10 +50655,10 @@ static SDValue combineIntDivRem(SDNode *N, SelectionDAG &DAG,
auto *Idx = dyn_cast<ConstantSDNode>(U->getOperand(1));
if (!Idx)
return SDValue();
- uint64_t IdxVal = Idx->getZExtValue();
- if (IdxVal >= VT.getVectorNumElements())
+ const APInt &IdxVal = Idx->getAPIntValue();
+ if (IdxVal.uge(VT.getVectorNumElements()))
return SDValue();
- ExtractedElts.setBit(IdxVal);
+ ExtractedElts.setBit(IdxVal.getZExtValue());
}
if (OnlyExtracts && !ExtractedElts.isAllOnes())
return SDValue();
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll
index 4b5d55acfcc54..5d2ffb9fad12a 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 | FileCheck %s
define <7 x i32> @udiv_v7i32(<7 x i32> %x, <7 x i32> %y) {
; CHECK-LABEL: udiv_v7i32:
>From b63680b00c99db6fc96284856b2ba8dfc4fae680 Mon Sep 17 00:00:00 2001
From: Patrick Ribbsaeter <patrickswedish at gmail.com>
Date: Mon, 10 Aug 2026 12:39:20 +0200
Subject: [PATCH 3/4] ci: apply LLVM review fixes
---
.github/workflows/apply-llvm-review-fixes.yml | 115 ++++++++++++++++++
1 file changed, 115 insertions(+)
create mode 100644 .github/workflows/apply-llvm-review-fixes.yml
diff --git a/.github/workflows/apply-llvm-review-fixes.yml b/.github/workflows/apply-llvm-review-fixes.yml
new file mode 100644
index 0000000000000..e3808de58ede6
--- /dev/null
+++ b/.github/workflows/apply-llvm-review-fixes.yml
@@ -0,0 +1,115 @@
+name: Apply LLVM review fixes
+
+on:
+ push:
+ branches:
+ - agent/x86-non-pow2-int-div
+
+permissions:
+ contents: write
+
+jobs:
+ apply-review-fixes:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 90
+ steps:
+ - name: Checkout branch
+ uses: actions/checkout at v4
+ with:
+ fetch-depth: 0
+
+ - name: Rebase onto current upstream main
+ run: |
+ git config user.name "Patrick Ribbsaeter"
+ git config user.email "233187023+patrickswedish at users.noreply.github.com"
+ git remote add upstream https://github.com/llvm/llvm-project.git || true
+ git fetch upstream main
+ git rebase upstream/main
+
+ - name: Apply review fixes
+ run: |
+ python3 - <<'PY'
+ from pathlib import Path
+
+ source_path = Path('llvm/lib/Target/X86/X86ISelLowering.cpp')
+ source = source_path.read_text()
+ old = ''' auto *Idx = dyn_cast<ConstantSDNode>(U->getOperand(1));
+ if (!Idx)
+ return SDValue();
+ const APInt &IdxVal = Idx->getAPIntValue();
+ if (IdxVal.uge(VT.getVectorNumElements()))
+ return SDValue();
+ ExtractedElts.setBit(IdxVal.getZExtValue());'''
+ # Normalize indentation in the literal above to the source's six-space body indentation.
+ old = old.replace(' ', ' ')
+ new = old.replace('return SDValue();', 'continue;')
+ if old not in source:
+ raise SystemExit('Expected integer-div extract block was not found')
+ source_path.write_text(source.replace(old, new, 1))
+
+ test_path = Path('llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll')
+ test = test_path.read_text()
+ if 'define <7 x i32> @test_divv_7i32' not in test:
+ anchor = 'define <8 x i32> @test_divv_8i32'
+ start = test.index(anchor)
+ marker = '\n;\n; urem by 7\n;\n'
+ insert_at = test.index(marker, start)
+ additions = '''
+
+define <7 x i32> @test_divv_7i32(<7 x i32> %a, <7 x i32> %b) nounwind {
+ %res = udiv <7 x i32> %a, %b
+ ret <7 x i32> %res
+}
+
+define i32 @test_divv_7i32_extract0(<7 x i32> %a, <7 x i32> %b) nounwind {
+ %res = udiv <7 x i32> %a, %b
+ %elt = extractelement <7 x i32> %res, i32 0
+ ret i32 %elt
+}
+'''
+ test = test[:insert_at] + additions + test[insert_at:]
+ test_path.write_text(test)
+
+ Path('llvm/test/CodeGen/X86/vector-idiv-udiv-non-pow2.ll').unlink(missing_ok=True)
+ PY
+
+ - name: Configure LLVM
+ run: |
+ cmake -S llvm -B build -G Ninja \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_TARGETS_TO_BUILD=X86 \
+ -DLLVM_ENABLE_ASSERTIONS=ON
+
+ - name: Build llc
+ run: cmake --build build --target llc -j2
+
+ - name: Regenerate focused FileCheck assertions
+ run: |
+ python3 llvm/utils/update_llc_test_checks.py \
+ --llc-binary build/bin/llc \
+ --function test_divv_7i32 \
+ llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+ python3 llvm/utils/update_llc_test_checks.py \
+ --llc-binary build/bin/llc \
+ --function test_divv_7i32_extract0 \
+ llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll
+
+ - name: Run focused X86 regression matrix
+ run: |
+ build/bin/llvm-lit -v \
+ llvm/test/CodeGen/X86/vector-idiv-udiv-256.ll \
+ llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll \
+ llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll \
+ llvm/test/CodeGen/X86/vector-idiv-v2i32.ll \
+ llvm/test/CodeGen/X86/vector-idiv.ll \
+ llvm/test/CodeGen/X86/scalar_widen_div.ll
+ git diff --check
+
+ - name: Squash to one clean commit and push
+ run: |
+ rm -f .github/workflows/apply-llvm-review-fixes.yml
+ git add -A
+ git reset --soft upstream/main
+ git add -A
+ git commit -m "[X86] Vectorize non-power-of-two integer division"
+ git push --force-with-lease origin HEAD:agent/x86-non-pow2-int-div
>From c49de057ca58b8e646604713e2d356a99ddbd882 Mon Sep 17 00:00:00 2001
From: Patrick Ribbsaeter <patrickswedish at gmail.com>
Date: Mon, 10 Aug 2026 12:43:03 +0200
Subject: [PATCH 4/4] ci: fix LLVM review workflow
---
.github/workflows/apply-llvm-review-fixes.yml | 22 +++++++++++--------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/apply-llvm-review-fixes.yml b/.github/workflows/apply-llvm-review-fixes.yml
index e3808de58ede6..5ec5847afb760 100644
--- a/.github/workflows/apply-llvm-review-fixes.yml
+++ b/.github/workflows/apply-llvm-review-fixes.yml
@@ -34,15 +34,19 @@ jobs:
source_path = Path('llvm/lib/Target/X86/X86ISelLowering.cpp')
source = source_path.read_text()
old = ''' auto *Idx = dyn_cast<ConstantSDNode>(U->getOperand(1));
- if (!Idx)
- return SDValue();
- const APInt &IdxVal = Idx->getAPIntValue();
- if (IdxVal.uge(VT.getVectorNumElements()))
- return SDValue();
- ExtractedElts.setBit(IdxVal.getZExtValue());'''
- # Normalize indentation in the literal above to the source's six-space body indentation.
- old = old.replace(' ', ' ')
- new = old.replace('return SDValue();', 'continue;')
+ if (!Idx)
+ return SDValue();
+ const APInt &IdxVal = Idx->getAPIntValue();
+ if (IdxVal.uge(VT.getVectorNumElements()))
+ return SDValue();
+ ExtractedElts.setBit(IdxVal.getZExtValue());'''
+ new = ''' auto *Idx = dyn_cast<ConstantSDNode>(U->getOperand(1));
+ if (!Idx)
+ continue;
+ const APInt &IdxVal = Idx->getAPIntValue();
+ if (IdxVal.uge(VT.getVectorNumElements()))
+ continue;
+ ExtractedElts.setBit(IdxVal.getZExtValue());'''
if old not in source:
raise SystemExit('Expected integer-div extract block was not found')
source_path.write_text(source.replace(old, new, 1))
More information about the llvm-commits
mailing list