[llvm] [SLP]Support for bswap pattern for bytes-based disjoint or reductions (PR #178513)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 28 13:02:46 PST 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/178513
If the reduction forms reversed bitcast, we can represent it as
a bitcast + bswap, if the source elements are byte sized
>From c71b38a7b6a05c5c791ea4be81b9f590d1324143 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Wed, 28 Jan 2026 13:02:32 -0800
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 70 +++++++++++++++----
.../X86/disjoint-or-reductions.ll | 4 +-
2 files changed, 57 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 73fc0a96692e7..774401811e5a3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2103,7 +2103,9 @@ class slpvectorizer::BoUpSLP {
/// Returns the opcode of the root node, or 0, if the root node is gather.
bool isReducedBitcastRoot() const {
return VectorizableTree.front()->hasState() &&
- VectorizableTree.front()->CombinedOp == TreeEntry::ReducedBitcast &&
+ (VectorizableTree.front()->CombinedOp == TreeEntry::ReducedBitcast ||
+ VectorizableTree.front()->CombinedOp ==
+ TreeEntry::ReducedBitcastBSwap) &&
VectorizableTree.front()->State == TreeEntry::Vectorize;
}
@@ -3929,7 +3931,9 @@ class slpvectorizer::BoUpSLP {
/// Checks if the tree represents disjoint or reduction of shl(zext, (0, 8,
/// .., 56))-like pattern.
/// If the int shifts unique, also strided, but not ordered, sets \p Order.
- bool matchesShlZExt(const TreeEntry &TE, OrdersType &Order) const;
+ /// If the node can be represented as a bitcast + bswap, sets \p IsBSwap.
+ bool matchesShlZExt(const TreeEntry &TE, OrdersType &Order,
+ bool &IsBSwap) const;
class TreeEntry {
public:
@@ -4061,6 +4065,7 @@ class slpvectorizer::BoUpSLP {
MinMax = Instruction::OtherOpsEnd + 1,
FMulAdd,
ReducedBitcast,
+ ReducedBitcastBSwap,
};
CombinedOpcode CombinedOp = NotCombinedOp;
@@ -13347,9 +13352,11 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
return FMACost < FMulPlusFAddCost ? FMACost : InstructionCost::getInvalid();
}
-bool BoUpSLP::matchesShlZExt(const TreeEntry &TE, OrdersType &Order) const {
+bool BoUpSLP::matchesShlZExt(const TreeEntry &TE, OrdersType &Order,
+ bool &IsBSwap) const {
assert(TE.hasState() && TE.getOpcode() == Instruction::Shl &&
"Expected Shl node.");
+ IsBSwap = false;
if (TE.State != TreeEntry::Vectorize || !TE.ReorderIndices.empty() ||
!TE.ReuseShuffleIndices.empty() || MinBWs.contains(&TE) ||
any_of(TE.Scalars, [](Value *V) { return !V->hasOneUse(); }))
@@ -13444,6 +13451,19 @@ bool BoUpSLP::matchesShlZExt(const TreeEntry &TE, OrdersType &Order) const {
BitcastCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, SrcVecTy,
Mask, CostKind);
}
+ // Check if the combination can be modeled as a bitcast+byteswap operation.
+ if (!Order.empty() && isReverseOrder(Order) &&
+ DL->getTypeStoreSize(SrcScalarTy) == 1) {
+ IntrinsicCostAttributes CostAttrs(Intrinsic::bswap, ScalarTy, {ScalarTy});
+ InstructionCost BSwapCost =
+ TTI->getCastInstrCost(Instruction::BitCast, ScalarTy, SrcVecTy, CastCtx,
+ CostKind) +
+ TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
+ if (BSwapCost <= BitcastCost) {
+ BitcastCost = BSwapCost;
+ IsBSwap = true;
+ }
+ }
return BitcastCost < VecCost;
}
@@ -13862,11 +13882,15 @@ void BoUpSLP::transformNodes() {
}))
break;
OrdersType Order;
- if (!matchesShlZExt(E, Order))
+ bool IsBSwap;
+ if (!matchesShlZExt(E, Order, IsBSwap))
break;
// This node is a (reduced disjoint or) bitcast node.
- E.CombinedOp = TreeEntry::ReducedBitcast;
- E.ReorderIndices = std::move(Order);
+ TreeEntry::CombinedOpcode Code =
+ IsBSwap ? TreeEntry::ReducedBitcastBSwap : TreeEntry::ReducedBitcast;
+ E.CombinedOp = Code;
+ if (!IsBSwap)
+ E.ReorderIndices = std::move(Order);
TreeEntry *ZExtEntry = getOperandEntry(&E, 0);
assert(ZExtEntry->UserTreeIndex &&
ZExtEntry->State == TreeEntry::Vectorize &&
@@ -13874,13 +13898,13 @@ void BoUpSLP::transformNodes() {
"Expected ZExt node.");
// The ZExt node is part of the combined node.
ZExtEntry->State = TreeEntry::CombinedVectorize;
- ZExtEntry->CombinedOp = TreeEntry::ReducedBitcast;
+ ZExtEntry->CombinedOp = Code;
TreeEntry *ConstEntry = getOperandEntry(&E, 1);
assert(ConstEntry->UserTreeIndex && ConstEntry->isGather() &&
"Expected ZExt node.");
// The ConstNode node is part of the combined node.
ConstEntry->State = TreeEntry::CombinedVectorize;
- ConstEntry->CombinedOp = TreeEntry::ReducedBitcast;
+ ConstEntry->CombinedOp = Code;
break;
}
default:
@@ -14925,7 +14949,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
ScalarTy = IntegerType::get(F->getContext(), It->second.first);
if (VecTy)
ScalarTy = getWidenedType(ScalarTy, VecTy->getNumElements());
- } else if (E->Idx == 0 && E->CombinedOp == TreeEntry::ReducedBitcast) {
+ } else if (E->Idx == 0 && isReducedBitcastRoot()) {
const TreeEntry *ZExt = getOperandEntry(E, /*Idx=*/0);
ScalarTy = cast<CastInst>(ZExt->getMainOp())->getSrcTy();
}
@@ -15499,7 +15523,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
- case TreeEntry::ReducedBitcast: {
+ case TreeEntry::ReducedBitcast:
+ case TreeEntry::ReducedBitcastBSwap: {
auto GetScalarCost = [&, &TTI = *TTI](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
@@ -15521,6 +15546,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
auto *SrcVecTy = getWidenedType(SrcScalarTy, LhsTE->getVectorFactor());
InstructionCost BitcastCost = TTI.getCastInstrCost(
Instruction::BitCast, ScalarTy, SrcVecTy, CastCtx, CostKind);
+ if (ShuffleOrOp == TreeEntry::ReducedBitcastBSwap) {
+ auto *OrigScalarTy = E->getMainOp()->getType();
+ IntrinsicCostAttributes CostAttrs(Intrinsic::bswap, OrigScalarTy,
+ {OrigScalarTy});
+ InstructionCost IntrinsicCost =
+ TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
+ BitcastCost += IntrinsicCost;
+ }
return BitcastCost + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
@@ -16299,7 +16332,8 @@ InstructionCost BoUpSLP::getSpillCost() {
SmallPtrSet<const Instruction *, 8> LastInstructions;
SmallPtrSet<const TreeEntry *, 8> ScalarOrPseudoEntries;
for (const auto &TEPtr : VectorizableTree) {
- if (TEPtr->CombinedOp == TreeEntry::ReducedBitcast) {
+ if (TEPtr->CombinedOp == TreeEntry::ReducedBitcast ||
+ TEPtr->CombinedOp == TreeEntry::ReducedBitcastBSwap) {
ScalarOrPseudoEntries.insert(TEPtr.get());
continue;
}
@@ -20223,7 +20257,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
if (!E->isAltShuffle()) {
switch (E->CombinedOp) {
case TreeEntry::ReducedBitcast:
- ShuffleOrOp = TreeEntry::ReducedBitcast;
+ case TreeEntry::ReducedBitcastBSwap:
+ ShuffleOrOp = E->CombinedOp;
break;
default:
break;
@@ -21159,7 +21194,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return V;
}
- case TreeEntry::ReducedBitcast: {
+ case TreeEntry::ReducedBitcast:
+ case TreeEntry::ReducedBitcastBSwap: {
assert(UserIgnoreList && "Expected reduction operations only.");
setInsertPointAfterBundle(E);
TreeEntry *ZExt = getOperandEntry(E, /*Idx=*/0);
@@ -21176,6 +21212,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Op, IntegerType::get(
Op->getContext(),
DL->getTypeSizeInBits(ZExt->getMainOp()->getType())));
+ if (ShuffleOrOp == TreeEntry::ReducedBitcastBSwap)
+ V = Builder.CreateUnaryIntrinsic(Intrinsic::bswap, V);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
@@ -21206,7 +21244,8 @@ Value *BoUpSLP::vectorizeTree(
for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
if (TE->isGather() || DeletedNodes.contains(TE.get()) ||
(TE->State == TreeEntry::CombinedVectorize &&
- TE->CombinedOp == TreeEntry::ReducedBitcast))
+ (TE->CombinedOp == TreeEntry::ReducedBitcast ||
+ TE->CombinedOp == TreeEntry::ReducedBitcastBSwap)))
continue;
(void)getLastInstructionInBundle(TE.get());
}
@@ -21764,7 +21803,8 @@ Value *BoUpSLP::vectorizeTree(
TransformedToGatherNodes.contains(Entry))
continue;
- if (Entry->CombinedOp == TreeEntry::ReducedBitcast) {
+ if (Entry->CombinedOp == TreeEntry::ReducedBitcast ||
+ Entry->CombinedOp == TreeEntry::ReducedBitcastBSwap) {
// Skip constant node
if (!Entry->hasState()) {
assert(allConstant(Entry->Scalars) && "Expected constants only.");
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/disjoint-or-reductions.ll b/llvm/test/Transforms/SLPVectorizer/X86/disjoint-or-reductions.ll
index 383edfc286da6..80a1b6bd85163 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/disjoint-or-reductions.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/disjoint-or-reductions.ll
@@ -6,8 +6,8 @@ define i64 @bswap(ptr noalias %p, ptr noalias %p1) {
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[P:%.*]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr [[P1:%.*]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i8> [[TMP3]], <8 x i8> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP4]] to i64
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[TMP3]] to i64
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.bswap.i64(i64 [[TMP4]])
; CHECK-NEXT: ret i64 [[TMP6]]
;
%g1 = getelementptr i8, ptr %p, i32 1
More information about the llvm-commits
mailing list