[llvm] [SLP]Initial compatibility support for shl v, 1 and add v, v (PR #181168)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 12 07:53:15 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
Allows to make shl v, 1 to be compatible with add operations by
modeling it as add v,v
---
Patch is 40.84 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/181168.diff
6 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+201-26)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll (+10-10)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/shl-compatible-with-add.ll (+10-12)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/shl-to-add-transformation.ll (+19-30)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/shl-to-add-transformation4.ll (+15-14)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/shl-to-add-transformation5.ll (+31-40)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 8fb88a1fac0ef..58c5451010852 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1002,8 +1002,12 @@ class BinOpSameOpcodeHelper {
/// preventing us from determining which instruction it should convert to.
bool trySet(MaskType OpcodeInMaskForm, MaskType InterchangeableMask) {
if (Mask & InterchangeableMask) {
- SeenBefore |= OpcodeInMaskForm;
- Mask &= InterchangeableMask;
+ MaskType TempSeenBefore = SeenBefore | OpcodeInMaskForm;
+ MaskType TempMask = Mask & InterchangeableMask;
+ if (!(TempMask & TempSeenBefore))
+ return false;
+ Mask = TempMask;
+ SeenBefore = TempSeenBefore;
return true;
}
return false;
@@ -1083,6 +1087,8 @@ class BinOpSameOpcodeHelper {
APInt ToCIValue;
switch (FromOpcode) {
case Instruction::Shl:
+ if (ToOpcode == Instruction::Add && FromCIValue.isOne())
+ return {I->getOperand(0), I->getOperand(0)};
if (ToOpcode == Instruction::Mul) {
ToCIValue = APInt::getOneBitSet(FromCIValueBitWidth,
FromCIValue.getZExtValue());
@@ -1205,6 +1211,8 @@ class BinOpSameOpcodeHelper {
case Instruction::Shl:
if (CIValue.ult(CIValue.getBitWidth()))
InterchangeableMask = CIValue.isZero() ? CanBeAll : MulBIT | ShlBIT;
+ if (CIValue.isOne())
+ InterchangeableMask |= AddBIT;
break;
case Instruction::Mul:
if (CIValue.isOne()) {
@@ -1402,6 +1410,49 @@ class InstructionsState {
Converter.hasAltOp() || !Converter.hasCandidateOpcode(getOpcode());
}
+ /// Checks if the value \p V is a transformed instruction, compatible either
+ /// with main or alternate ops.
+ bool isExpandedBinOp(Value *V) const {
+ assert(valid() && "InstructionsState is invalid.");
+ if (isCopyableElement(V))
+ return false;
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ return false;
+ auto CheckForTransformedOpcode = [](const Instruction *Op, Instruction *I) {
+ switch (Op->getOpcode()) {
+ case Instruction::Add:
+ switch (I->getOpcode()) {
+ case Instruction::Shl:
+ assert(match(I, m_Shl(m_Value(), m_One())) &&
+ "Expected shl x, 1 only.");
+ return true;
+ default:
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+ return false;
+ };
+ Instruction *Op = getMatchingMainOpOrAltOp(I);
+ return CheckForTransformedOpcode(Op, I);
+ }
+
+ /// Checks if the operand at index \p Idx of instruction \p I is an expanded
+ /// operand.
+ bool isExpandedOperand(Instruction *I, unsigned Idx) const {
+ assert(isExpandedBinOp(I) && "Expected an expanded binop.");
+ switch (I->getOpcode()) {
+ case Instruction::Shl:
+ assert(match(I, m_Shl(m_Value(), m_One())) && "Expected shl x, 1 only.");
+ return Idx == 1;
+ default:
+ llvm_unreachable("Unexpected opcode for an expanded operand.");
+ }
+ }
+
/// Checks if the value is non-schedulable.
bool isNonSchedulable(Value *V) const {
assert(valid() && "InstructionsState is invalid.");
@@ -4211,6 +4262,26 @@ class slpvectorizer::BoUpSLP {
return CopyableElements.contains(V);
}
+ /// Checks if the value \p V is a transformed instruction, compatible either
+ /// with main or alternate ops.
+ bool isExpandedBinOp(Value *V) const {
+ assert(hasState() && "InstructionsState is invalid.");
+ if (isCopyableElement(V))
+ return false;
+ return S.isExpandedBinOp(V);
+ }
+
+ /// Checks if the operand at index \p Idx of instruction \p I is an expanded
+ /// operand.
+ bool isExpandedOperand(Instruction *I, unsigned Idx) const {
+ assert(hasState() && "InstructionsState is invalid.");
+ if (isCopyableElement(I))
+ return false;
+ if (!isExpandedBinOp(I))
+ return false;
+ return S.isExpandedOperand(I, Idx);
+ }
+
/// Returns true if any scalar in the list is a copyable element.
bool hasCopyableElements() const { return !CopyableElements.empty(); }
@@ -4285,8 +4356,11 @@ class slpvectorizer::BoUpSLP {
dbgs().indent(2) << *V << "\n";
}
dbgs() << "Scalars: \n";
- for (Value *V : Scalars)
- dbgs().indent(2) << *V << "\n";
+ for (Value *V : Scalars) {
+ dbgs().indent(2) << *V
+ << ((S && S.isExpandedBinOp(V)) ? " [[Expanded]]\n"
+ : "\n");
+ }
dbgs() << "State: ";
if (S && hasCopyableElements())
dbgs() << "[[Copyable]] ";
@@ -5697,8 +5771,18 @@ class slpvectorizer::BoUpSLP {
for (const Use &U : In->operands()) {
if (auto *I = dyn_cast<Instruction>(U.get())) {
auto Res = OperandsUses.try_emplace(I, 0);
- ++Res.first->getSecond();
- ++TotalOpCount;
+ unsigned Inc = 1;
+ // Count all expanded operands in the binops.
+ for (ScheduleBundle *Bundle : Bundles) {
+ if (const TreeEntry *TE = Bundle->getTreeEntry()) {
+ if (TE->isExpandedBinOp(In))
+ ++Inc;
+ } else if (S.isExpandedBinOp(In)) {
+ ++Inc;
+ }
+ }
+ Res.first->getSecond() += Inc;
+ TotalOpCount += Inc;
}
}
}
@@ -5707,7 +5791,7 @@ class slpvectorizer::BoUpSLP {
auto DecrUnschedForInst =
[&](Instruction *I, TreeEntry *UserTE, unsigned OpIdx,
SmallDenseSet<std::pair<const ScheduleEntity *, unsigned>>
- &Checked) {
+ &Checked, bool IsExpandedOperand = false) {
if (!ScheduleCopyableDataMap.empty()) {
const EdgeInfo EI = {UserTE, OpIdx};
if (ScheduleCopyableData *CD =
@@ -5722,7 +5806,8 @@ class slpvectorizer::BoUpSLP {
assert(It != OperandsUses.end() && "Operand not found");
if (It->second > 0) {
if (ScheduleData *OpSD = getScheduleData(I)) {
- if (!Checked.insert(std::make_pair(OpSD, OpIdx)).second)
+ if (!IsExpandedOperand &&
+ !Checked.insert(std::make_pair(OpSD, OpIdx)).second)
return;
--It->getSecond();
assert(TotalOpCount > 0 && "No more operands to decrement");
@@ -5798,7 +5883,9 @@ class slpvectorizer::BoUpSLP {
Bundle->getTreeEntry()->getOperand(OpIdx)[Lane])) {
LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): "
<< *I << "\n");
- DecrUnschedForInst(I, Bundle->getTreeEntry(), OpIdx, Checked);
+ DecrUnschedForInst(
+ I, Bundle->getTreeEntry(), OpIdx, Checked,
+ Bundle->getTreeEntry()->isExpandedOperand(In, OpIdx));
}
// If parent node is schedulable, it will be handled correctly.
if (Bundle->getTreeEntry()->isCopyableElement(In))
@@ -5986,6 +6073,7 @@ class slpvectorizer::BoUpSLP {
/// bundles which depend on the original bundle.
void calculateDependencies(ScheduleBundle &Bundle, bool InsertInReadyList,
BoUpSLP *SLP,
+ const SmallPtrSetImpl<Value *> &ExpandedOps,
ArrayRef<ScheduleData *> ControlDeps = {});
/// Sets all instruction in the scheduling region to un-scheduled.
@@ -22487,6 +22575,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
(!EI.UserTE->hasCopyableElements() ||
!EI.UserTE->isCopyableElement(SD->getInst())))
SD->clearDirectDependencies();
+ const bool IsExpandedBinOp = S.isExpandedBinOp(SD->getInst());
for (const Use &U : SD->getInst()->operands()) {
unsigned &NumOps =
UserOpToNumOps
@@ -22502,6 +22591,15 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
if (RegionHasStackSave ||
!isGuaranteedToTransferExecutionToSuccessor(OpSD->getInst()))
ControlDependentMembers.push_back(OpSD);
+ continue;
+ }
+ }
+ if (IsExpandedBinOp) {
+ if (ScheduleData *OpSD = getScheduleData(U.get());
+ OpSD && OpSD->hasValidDependencies()) {
+ OpSD->clearDirectDependencies();
+ ControlDependentMembers.push_back(OpSD);
+ continue;
}
}
}
@@ -22528,20 +22626,24 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
});
ReSchedule = true;
}
+ SmallPtrSet<Value *, 4> ExpandedOps;
+ for (Value *V : VL) {
+ if (S.isExpandedBinOp(V))
+ ExpandedOps.insert(V);
+ }
// Check if the bundle data has deps for copyable elements already. In
// this case need to reset deps and recalculate it.
if (Bundle && !Bundle.getBundle().empty()) {
- if (S.areInstructionsWithCopyableElements() ||
- !ScheduleCopyableDataMap.empty())
+ if (!ScheduleCopyableDataMap.empty() || !ExpandedOps.empty())
CheckIfNeedToClearDeps(Bundle);
LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << Bundle << " in block "
<< BB->getName() << "\n");
calculateDependencies(Bundle, /*InsertInReadyList=*/!ReSchedule, SLP,
- ControlDependentMembers);
+ ExpandedOps, ControlDependentMembers);
} else if (!ControlDependentMembers.empty()) {
ScheduleBundle Invalid = ScheduleBundle::invalid();
calculateDependencies(Invalid, /*InsertInReadyList=*/!ReSchedule, SLP,
- ControlDependentMembers);
+ ExpandedOps, ControlDependentMembers);
}
if (ReSchedule) {
@@ -22606,7 +22708,8 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
ReadyInsts.remove(B);
}
- if (!S.isCopyableElement(V) && !BundleMember->isScheduled())
+ if (!S.isCopyableElement(V) && !S.isExpandedBinOp(V) &&
+ !BundleMember->isScheduled())
continue;
// A bundle member was scheduled as single instruction before and now
// needs to be scheduled as part of the bundle. We just get rid of the
@@ -22642,6 +22745,15 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
if (S.isNonSchedulable(V))
continue;
auto *I = cast<Instruction>(V);
+ if (S.isExpandedBinOp(I)) {
+ for (Value *Op : I->operands()) {
+ if (ScheduleData *OpSD = getScheduleData(Op);
+ OpSD && OpSD->hasValidDependencies()) {
+ OpSD->clearDirectDependencies();
+ ControlDependentMembers.push_back(OpSD);
+ }
+ }
+ }
if (S.isCopyableElement(I)) {
// Remove the copyable data from the scheduling region and restore
// previous mappings.
@@ -22688,6 +22800,17 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
if (RegionHasStackSave ||
!isGuaranteedToTransferExecutionToSuccessor(OpSD->getInst()))
ControlDependentMembers.push_back(OpSD);
+ if (any_of(VL, [&](Value *V) { return S.isExpandedBinOp(V); })) {
+ // Clear scheduling data for all operands, if this node is operand
+ // of the expanded instruction.
+ for (Value *Op : I->operands()) {
+ if (ScheduleData *OpSD = getScheduleData(Op);
+ OpSD && OpSD->hasValidDependencies()) {
+ OpSD->clearDirectDependencies();
+ ControlDependentMembers.push_back(OpSD);
+ }
+ }
+ }
}
continue;
}
@@ -22695,8 +22818,9 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
}
if (!ControlDependentMembers.empty()) {
ScheduleBundle Invalid = ScheduleBundle::invalid();
+ SmallPtrSet<Value *, 4> ExpandedOps;
calculateDependencies(Invalid, /*InsertInReadyList=*/false, SLP,
- ControlDependentMembers);
+ ExpandedOps, ControlDependentMembers);
}
return std::nullopt;
}
@@ -22835,6 +22959,7 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
void BoUpSLP::BlockScheduling::calculateDependencies(
ScheduleBundle &Bundle, bool InsertInReadyList, BoUpSLP *SLP,
+ const SmallPtrSetImpl<Value *> &ExpandedOps,
ArrayRef<ScheduleData *> ControlDeps) {
SmallVector<ScheduleEntity *> WorkList;
auto ProcessNode = [&](ScheduleEntity *SE) {
@@ -22873,9 +22998,13 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
}
} else if (Visited.insert(In).second) {
if (ScheduleData *UseSD = getScheduleData(In)) {
- CD->incDependencies();
+ unsigned Inc = 1;
+ // Increment twice, since the operand was expanded in binop.
+ if (EI.UserTE && EI.UserTE->isExpandedBinOp(In))
+ Inc = 2;
+ for_each(seq(Inc), [&](unsigned) { CD->incDependencies(); });
if (!UseSD->isScheduled())
- CD->incrementUnscheduledDeps(1);
+ CD->incrementUnscheduledDeps(Inc);
if (!UseSD->hasValidDependencies() ||
(InsertInReadyList && UseSD->isReady()))
WorkList.push_back(UseSD);
@@ -22923,9 +23052,17 @@ void BoUpSLP::BlockScheduling::calculateDependencies(
if (areAllOperandsReplacedByCopyableData(
cast<Instruction>(U), BundleMember->getInst(), *SLP, NumOps))
continue;
- BundleMember->incDependencies();
+ unsigned Inc = 1;
+ // Increment twice, since the operand was expanded in binop.
+ for (const TreeEntry *UserTE : SLP->getTreeEntries(U)) {
+ if (UserTE->isExpandedBinOp(U))
+ ++Inc;
+ }
+ if (ExpandedOps.contains(U))
+ ++Inc;
+ for_each(seq(Inc), [&](unsigned) { BundleMember->incDependencies(); });
if (!UseSD->isScheduled())
- BundleMember->incrementUnscheduledDeps(1);
+ BundleMember->incrementUnscheduledDeps(Inc);
if (!UseSD->hasValidDependencies() ||
(InsertInReadyList && UseSD->isReady()))
WorkList.push_back(UseSD);
@@ -23197,15 +23334,21 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
if (!Bundles.empty()) {
for (ScheduleBundle *Bundle : Bundles) {
Bundle->setSchedulingPriority(Idx++);
- if (!Bundle->hasValidDependencies())
- BS->calculateDependencies(*Bundle, /*InsertInReadyList=*/false, this);
+ if (!Bundle->hasValidDependencies()) {
+ SmallPtrSet<Value *, 4> ExpandedOps;
+ BS->calculateDependencies(*Bundle, /*InsertInReadyList=*/false, this,
+ ExpandedOps);
+ }
}
SmallVector<ScheduleCopyableData *> SDs = BS->getScheduleCopyableData(I);
for (ScheduleCopyableData *SD : reverse(SDs)) {
ScheduleBundle &Bundle = SD->getBundle();
Bundle.setSchedulingPriority(Idx++);
- if (!Bundle.hasValidDependencies())
- BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this);
+ if (!Bundle.hasValidDependencies()) {
+ SmallPtrSet<Value *, 4> ExpandedOps;
+ BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this,
+ ExpandedOps);
+ }
}
continue;
}
@@ -23231,14 +23374,19 @@ void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
// recalculate them.
ScheduleBundle Bundle;
Bundle.add(SD);
- BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this);
+ SmallPtrSet<Value *, 4> ExpandedOps;
+ BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this,
+ ExpandedOps);
}
}
for (ScheduleCopyableData *SD : reverse(CopyableData)) {
ScheduleBundle &Bundle = SD->getBundle();
Bundle.setSchedulingPriority(Idx++);
- if (!Bundle.hasValidDependencies())
- BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this);
+ if (!Bundle.hasValidDependencies()) {
+ SmallPtrSet<Value *, 4> ExpandedOps;
+ BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this,
+ ExpandedOps);
+ }
}
}
BS->initialFillReadyList(ReadyInsts);
@@ -25422,6 +25570,33 @@ class HorizontalReduction {
ReducedVals.erase(std::next(ReducedVals.begin(), SelectIdx));
}
}
+ // Check if shl %x, 1 can be merged with adds.
+ auto ShlIt = UsedReductionOpIds.find(Instruction::Shl);
+ auto AddIt = UsedReductionOpIds.find(Instruction::Add);
+ if (ShlIt != UsedReductionOpIds.end() &&
+ AddIt != UsedReductionOpIds.end()) {
+ unsigned ShlIdx = ShlIt->second;
+ unsigned AddIdx = AddIt->second;
+ if (ReducedVals[ShlIdx].size() < ReductionLimit) {
+ SmallVector<Value *> Shls;
+ SmallVector<Value *> Remaining;
+ for (Value *V : ReducedVals[ShlIdx]) {
+ if (match(V, m_Shl(m_Value(), m_One())))
+ Shls.push_back(V);
+ else
+ Remaining.push_back(V);
+ }
+ // Have compatible shls? Merge them to adds, if so.
+ if (!Shls.empty()) {
+ Shls.append(ReducedVals[AddIdx]);
+ ReducedVals[AddIdx].swap(Shls);
+ if (Remaining.empty())
+ ReducedVals.erase(std::next(ReducedVals.begin(), ShlIdx));
+ else
+ ReducedVals[ShlIdx].swap(Remaining);
+ }
+ }
+ }
}
public:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll b/llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll
index f75c8deddd9bb..38dddb064b751 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/entries-different-vf.ll
@@ -5,16 +5,16 @@ define i1 @test(i64 %v) {
; CHECK-LABEL: define i1 @test
; CHECK-SAME: (i64 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[V]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[V]], 3
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[V]], 7
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 0, i64 0, i64 0, i64 0>, i64 [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i64> <i64 undef, i64 undef, i64 0, i64 0>, i64 [[TMP2]], i32 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> [[TMP5]], i64 [[TMP0]], i32 1
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 0, i32 3, i32 1>
-; CHECK-NEXT: [[TMP8:%.*]] = or <8 x i64> [[TMP4]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = sub <8 x i64> [[TMP4]], [[TMP7]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[V]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> <i64 poison, i64 poison, i64 0, i64 0>, i64 [[V]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> <i64 7, i64 undef, i64 0, i64 0>, i64 [[V]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/181168
More information about the llvm-commits
mailing list