[llvm] [SLP][AArch64]Support masked div/rem on non-pow-2 vectors (PR #210623)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 11:36:11 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/210623
For fixed-width integer div/rem on AArch64 SVE, a non-power-of-2 vector
cannot execute as a single whole-register operation. When profitable,
pad the vector to the next full register and use the masked div/rem
intrinsics.
Fixes #207880
>From cb86f451d15e0edde5e9eeba11129c6dd52e9960 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 19 Jul 2026 11:35:58 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../AArch64/AArch64TargetTransformInfo.cpp | 12 +++
.../Transforms/Vectorize/SLPVectorizer.cpp | 86 ++++++++++++++++++-
.../Vectorize/SLPVectorizer/SLPUtils.cpp | 15 ++++
.../Vectorize/SLPVectorizer/SLPUtils.h | 6 ++
.../Analysis/CostModel/AArch64/sve-div.ll | 4 +-
.../AArch64/masked-div-rem-non-pow2-revec.ll | 5 +-
.../AArch64/masked-div-rem-non-pow2.ll | 10 ++-
7 files changed, 130 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 9d448b4a8681a..df0e3ef8a6d6b 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -12,6 +12,7 @@
#include "AArch64SMEAttributes.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/bit.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/BasicTTIImpl.h"
@@ -4832,6 +4833,17 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
if (nullptr != Entry)
return Entry->Cost;
}
+ // A non-power-of-2 count can't divide as a single whole-register op
+ // (an inactive lane's leftover value could be a zero divisor and
+ // trap), so the legalizer emits one div per whole register plus one
+ // per set bit of the remainder (e.g. <7 x i32> emits 3 divs, not 2).
+ if (auto *FVTy = dyn_cast<FixedVectorType>(Ty);
+ FVTy && LT.second.isFixedLengthVector()) {
+ unsigned NumElts = FVTy->getNumElements();
+ unsigned RegElts = LT.second.getVectorNumElements();
+ if (RegElts > 0)
+ Cost = (NumElts / RegElts + popcount(NumElts % RegElts)) * 2;
+ }
// For 8/16-bit elements, the cost is higher because the type
// requires promotion and possibly splitting:
if (LT.second.getScalarType() == MVT::i8)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6dde1850d58d5..dd8aa4a877fa2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -421,6 +421,48 @@ getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
return (Sz / RegVF) * RegVF;
}
+/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the
+/// padded full-register vector type if padding is structurally possible, or
+/// nullptr if the vector already fills a register or the opcode is not
+/// div/rem. Does not check profitability; see getMaskedDivRemCost for that.
+static FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI,
+ unsigned Opcode, Type *ScalarTy,
+ unsigned NumElts) {
+ if (!Instruction::isIntDivRem(Opcode) || has_single_bit(NumElts))
+ return nullptr;
+ unsigned PaddedNumElts =
+ getFullVectorNumberOfElements(TTI, ScalarTy, NumElts);
+ if (PaddedNumElts == NumElts)
+ return nullptr;
+ return cast<FixedVectorType>(getWidenedType(ScalarTy, PaddedNumElts));
+}
+
+/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, checks if
+/// padding to a full register and using the masked div/rem intrinsic is
+/// cheaper than the direct vector op. Returns the cost of the masked
+/// alternative, or an invalid cost if it is not applicable or not cheaper.
+static InstructionCost getMaskedDivRemCost(const TargetTransformInfo &TTI,
+ unsigned Opcode, Type *ScalarTy,
+ unsigned NumElts,
+ TTI::TargetCostKind CostKind) {
+ FixedVectorType *PaddedVecTy =
+ getMaskedDivRemType(TTI, Opcode, ScalarTy, NumElts);
+ if (!PaddedVecTy)
+ return InstructionCost::getInvalid();
+ // One mask bit per element of the padded vector, not per padded lane.
+ auto *MaskTy =
+ FixedVectorType::get(IntegerType::getInt1Ty(ScalarTy->getContext()),
+ PaddedVecTy->getNumElements());
+ InstructionCost DirectCost = TTI.getArithmeticInstrCost(
+ Opcode, getWidenedType(ScalarTy, NumElts), CostKind);
+ IntrinsicCostAttributes ICA(getMaskedDivRemIntrinsic(Opcode), PaddedVecTy,
+ {PaddedVecTy, PaddedVecTy, MaskTy});
+ InstructionCost MaskedCost = TTI.getIntrinsicInstrCost(ICA, CostKind);
+ if (!MaskedCost.isValid() || MaskedCost >= DirectCost)
+ return InstructionCost::getInvalid();
+ return MaskedCost;
+}
+
/// Checks if the vector of instructions can be represented as a shuffle, like:
/// %x0 = extractelement <4 x i8> %x, i32 0
/// %x3 = extractelement <4 x i8> %x, i32 3
@@ -17546,6 +17588,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return CommonCost;
}
}
+ // Masked path ignores Op1Info/Op2Info like its codegen does; keep it
+ // out of the operand-aware cost comparison below.
+ if (!E->isAltShuffle()) {
+ if (InstructionCost MaskedCost = getMaskedDivRemCost(
+ *TTI, ShuffleOrOp, ScalarTy, VL.size(), CostKind);
+ MaskedCost.isValid())
+ return MaskedCost + CommonCost;
+ }
unsigned OpIdx = isa<UnaryOperator>(VL0) ? 0 : 1;
TTI::OperandValueInfo Op1Info = getOperandInfo(E->getOperand(0));
TTI::OperandValueInfo Op2Info = getOperandInfo(E->getOperand(OpIdx));
@@ -23737,9 +23787,39 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
RHS = Builder.CreateIntCast(RHS, VecTy, GetOperandSignedness(1));
}
- Value *V = Builder.CreateBinOp(
- static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS,
- RHS);
+ // getMaskedDivRemType alone is not enough here: it only checks that
+ // padding is structurally possible, not that it is cheaper, so this
+ // must re-check cost to match getEntryCost's decision.
+ Value *V = nullptr;
+ if (!E->isAltShuffle()) {
+ unsigned NumElts = E->Scalars.size();
+ if (getMaskedDivRemCost(*TTI, ShuffleOrOp, ScalarTy, NumElts,
+ TTI::TCK_RecipThroughput)
+ .isValid()) {
+ FixedVectorType *PaddedVecTy =
+ getMaskedDivRemType(*TTI, ShuffleOrOp, ScalarTy, NumElts);
+ // Scale the lane count up to elements for REVEC, where each lane
+ // is itself a vector.
+ unsigned NumActiveElts = NumElts * getNumElements(ScalarTy);
+ Value *WidenedLHS = createInsertVector(
+ Builder, PoisonValue::get(PaddedVecTy), LHS, 0);
+ Value *WidenedRHS = createInsertVector(
+ Builder, PoisonValue::get(PaddedVecTy), RHS, 0);
+ SmallVector<Constant *> MaskValues(
+ PaddedVecTy->getNumElements(),
+ ConstantInt::getFalse(Builder.getContext()));
+ std::fill_n(MaskValues.begin(), NumActiveElts,
+ ConstantInt::getTrue(Builder.getContext()));
+ Value *DivRemMask = ConstantVector::get(MaskValues);
+ Value *MaskedV = Builder.CreateIntrinsic(
+ getMaskedDivRemIntrinsic(ShuffleOrOp), {PaddedVecTy},
+ {WidenedLHS, WidenedRHS, DivRemMask});
+ V = createExtractVector(Builder, MaskedV, NumActiveElts, 0);
+ }
+ }
+ if (!V)
+ V = Builder.CreateBinOp(
+ static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, RHS);
V = PropagateIRFlags(V);
V = FinalShuffle(V, E);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 93239b7cd2725..048559624b6ed 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -611,4 +611,19 @@ SmallVector<Constant *> replicateMask(ArrayRef<Constant *> Val, unsigned VF) {
return NewVal;
}
+Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
+ switch (Opcode) {
+ case Instruction::UDiv:
+ return Intrinsic::masked_udiv;
+ case Instruction::SDiv:
+ return Intrinsic::masked_sdiv;
+ case Instruction::URem:
+ return Intrinsic::masked_urem;
+ case Instruction::SRem:
+ return Intrinsic::masked_srem;
+ default:
+ llvm_unreachable("Unexpected opcode");
+ }
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index 51181b99078c5..d46e7b272557b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -20,6 +20,7 @@
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/IR/Intrinsics.h"
#include <optional>
#include <string>
@@ -273,6 +274,11 @@ SmallBitVector getAltInstrMask(ArrayRef<Value *> VL, Type *ScalarTy,
/// Replicates the given \p Val \p VF times.
SmallVector<Constant *> replicateMask(ArrayRef<Constant *> Val, unsigned VF);
+/// \returns the masked division/remainder intrinsic corresponding to \p
+/// Opcode. Disabled lanes of these intrinsics are poison rather than UB,
+/// unlike the plain opcode.
+Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPUTILS_H
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-div.ll b/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
index 84a9ca0fab6d8..c6494ab85a7e9 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-div.ll
@@ -20,7 +20,7 @@ define void @sdiv() {
; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2i8 = sdiv <2 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i8 = sdiv <4 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V6i8 = sdiv <6 x i8> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V6i8 = sdiv <6 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i8 = sdiv <8 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, undef
@@ -103,7 +103,7 @@ define void @udiv() {
; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V2i8 = udiv <2 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i8 = udiv <4 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V6i8 = udiv <6 x i8> undef, undef
+; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V6i8 = udiv <6 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i8 = udiv <8 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, undef
; CHECK-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, undef
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/masked-div-rem-non-pow2-revec.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/masked-div-rem-non-pow2-revec.ll
index ee831842f825f..44b58bc4fe977 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/masked-div-rem-non-pow2-revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/masked-div-rem-non-pow2-revec.ll
@@ -7,7 +7,10 @@ define void @udiv_v3v2i32(ptr noalias %dst, ptr noalias %x, ptr noalias %y) vsca
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <6 x i32>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <6 x i32>, ptr [[Y]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = udiv <6 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <6 x i32> [[TMP0]], <6 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <6 x i32> [[TMP1]], <6 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> [[TMP5]], <8 x i32> [[TMP3]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false>)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: store <6 x i32> [[TMP2]], ptr [[DST]], align 8
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/masked-div-rem-non-pow2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/masked-div-rem-non-pow2.ll
index 054f0799906cb..d7f61902c6753 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/masked-div-rem-non-pow2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/masked-div-rem-non-pow2.ll
@@ -7,7 +7,10 @@ define void @udiv_v7i32(ptr noalias %dst, ptr noalias %x, ptr noalias %y) vscale
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP1:%.*]] = load <7 x i32>, ptr [[X]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <7 x i32>, ptr [[Y]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = udiv <7 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <7 x i32> [[TMP1]], <7 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <7 x i32> [[TMP2]], <7 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> [[TMP5]], <8 x i32> [[TMP3]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false>)
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: store <7 x i32> [[TMP6]], ptr [[DST]], align 4
; CHECK-NEXT: ret void
;
@@ -67,7 +70,10 @@ define void @sdiv_v7i32(ptr noalias %dst, ptr noalias %x, ptr noalias %y) vscale
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP1:%.*]] = load <7 x i32>, ptr [[X]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <7 x i32>, ptr [[Y]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = sdiv <7 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <7 x i32> [[TMP1]], <7 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <7 x i32> [[TMP2]], <7 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> [[TMP5]], <8 x i32> [[TMP3]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false>)
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <7 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: store <7 x i32> [[TMP6]], ptr [[DST]], align 4
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list