[llvm] [SLP] Normalize copyable operand order via majority voting (PR #191631)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 09:56:28 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/191631
>From b8336e29150190eac20761885da1ddd5f6bb4324 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 11 Apr 2026 07:20:20 -0700
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 63 +++++++++++++++++++
.../X86/bottom-to-top-reorder.ll | 2 +-
.../SLPVectorizer/X86/copyable_reorder.ll | 27 +++-----
.../reused-last-instruction-in-split-node.ll | 2 +-
4 files changed, 74 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index eedddf81d6877..88eb8b1d2dd49 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11932,12 +11932,75 @@ class InstructionsCompatibilityAnalysis {
if (S.areInstructionsWithCopyableElements()) {
MainOp = S.getMainOp();
MainOpcode = S.getOpcode();
+ const bool IsCommutative =
+ isCommutative(MainOp) && MainOp->getNumOperands() == 2;
Operands.assign(MainOp->getNumOperands(),
BoUpSLP::ValueList(VL.size(), nullptr));
+ // Build operands and simultaneously count (ID0, ID1) pair
+ // frequencies for commutative operand normalization. Pairs and
+ // their inverses are tracked under a canonical key so that
+ // (Load, Add) and (Add, Load) contribute to the same bucket.
+ struct PairInfo {
+ unsigned FwdCount = 0;
+ unsigned RevCount = 0;
+ unsigned MinID = 0;
+ unsigned MaxID = 0;
+ };
+ SmallMapVector<size_t, PairInfo, 8> PairCounts;
+ unsigned MajID0 = 0, MajID1 = 0;
for (auto [Idx, V] : enumerate(VL)) {
SmallVector<Value *> OperandsForValue = getOperands(S, V);
for (auto [OperandIdx, Operand] : enumerate(OperandsForValue))
Operands[OperandIdx][Idx] = Operand;
+ if (!IsCommutative || S.isCopyableElement(V) || isa<PoisonValue>(V))
+ continue;
+ unsigned ID0 = OperandsForValue[0]->getValueID();
+ unsigned ID1 = OperandsForValue[1]->getValueID();
+ if (ID0 == ID1)
+ continue;
+ unsigned MinID = std::min(ID0, ID1);
+ unsigned MaxID = std::max(ID0, ID1);
+ size_t Key = (static_cast<size_t>(MinID) << 16) | MaxID;
+ auto [It, Inserted] = PairCounts.try_emplace(Key);
+ PairInfo &Info = It->second;
+ if (Inserted) {
+ Info.MinID = MinID;
+ Info.MaxID = MaxID;
+ }
+ if (ID0 < ID1)
+ ++Info.FwdCount;
+ else
+ ++Info.RevCount;
+ }
+ // Find the most frequent (ID0, ID1) pair across non-copyable
+ // lanes. Select the orientation (original or inverse) that has
+ // more votes as the majority pattern.
+ unsigned BestCount = 0;
+ for (auto &[Key, Info] : PairCounts) {
+ unsigned Total = Info.FwdCount + Info.RevCount;
+ if (Total > BestCount) {
+ BestCount = Total;
+ if (Info.FwdCount >= Info.RevCount) {
+ MajID0 = Info.MinID;
+ MajID1 = Info.MaxID;
+ } else {
+ MajID0 = Info.MaxID;
+ MajID1 = Info.MinID;
+ }
+ }
+ }
+ // For commutative ops, swap lanes whose operand types are the
+ // exact inverse of the majority pattern, making the non-copyable
+ // lanes consistent.
+ if (BestCount > 0) {
+ for (auto [Idx, V] : enumerate(VL)) {
+ if (S.isCopyableElement(V) || isa<PoisonValue>(V))
+ continue;
+ unsigned ID0 = Operands[0][Idx]->getValueID();
+ unsigned ID1 = Operands[1][Idx]->getValueID();
+ if (ID0 == MajID1 && ID1 == MajID0)
+ std::swap(Operands[0][Idx], Operands[1][Idx]);
+ }
}
} else {
buildOriginalOperands(S, VL, Operands);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
index 54e5405e21c49..f303b79778c05 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
@@ -10,7 +10,7 @@ define void @test(ptr %0, ptr %1, ptr %2) {
; CHECK-NEXT: [[TMP11:%.*]] = sub <4 x i32> <i32 0, i32 0, i32 undef, i32 0>, [[TMP8]]
; CHECK-NEXT: [[TMP12:%.*]] = sub <4 x i32> [[TMP11]], [[TMP10]]
; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP6]]
-; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> <i32 0, i32 0, i32 1, i32 0>, [[TMP13]]
+; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[TMP13]], <i32 0, i32 0, i32 1, i32 0>
; CHECK-NEXT: [[TMP17:%.*]] = sub <4 x i32> [[TMP16]], zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = sub <4 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
index 44588dfd8ffe4..b432627e6726c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
@@ -136,24 +136,15 @@ entry:
define void @test_add_udiv_reorder_add(ptr %arr1, ptr %arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
; CHECK-LABEL: @test_add_udiv_reorder_add(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 2
-; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr i32, ptr [[ARR1]], i32 3
-; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[GEP1_2]], align 4
-; CHECK-NEXT: [[V3:%.*]] = load i32, ptr [[GEP1_3]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[A0:%.*]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[A1:%.*]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1146, i32 146>
-; CHECK-NEXT: [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
-; CHECK-NEXT: [[Y3:%.*]] = add nsw i32 [[A3:%.*]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[ARR1]], align 4
-; CHECK-NEXT: [[RES2:%.*]] = udiv i32 [[V2]], [[Y2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[RES2]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[V3]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[Y3]], i32 3
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP9]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[ARR1:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[A2:%.*]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], <i32 1, i32 1, i32 42, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[A0:%.*]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[A1:%.*]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[V3:%.*]], i32 3
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 1146, i32 146, i32 0, i32 0>, [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = udiv <4 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP7]], [[TMP8]]
; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[ARR2:%.*]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
index 830c7755fc53a..49164e59621fb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
@@ -8,7 +8,7 @@ define float @test() {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <12 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float undef, float 0.000000e+00, float 0.000000e+00, float undef>, float 0.000000e+00, i32 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <12 x float> [[TMP1]], float 0.000000e+00, i32 8
; CHECK-NEXT: [[TMP3:%.*]] = fmul <12 x float> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd <12 x float> <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float poison>, [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <12 x float> [[TMP3]], <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float poison>
; CHECK-NEXT: [[TMP5:%.*]] = fsub <8 x float> zeroinitializer, <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison>
; CHECK-NEXT: [[TMP6:%.*]] = fadd <12 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison>, [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <12 x float> [[TMP6]], <12 x float> poison, <20 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
>From 814fcfa3d62f4cae05b01d2b3412ae299c4a3d7a Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 11 Apr 2026 09:35:22 -0700
Subject: [PATCH 2/2] Simplifications, fixes
Created using spr 1.3.7
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 88eb8b1d2dd49..f2c9832a6fb07 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11946,7 +11946,7 @@ class InstructionsCompatibilityAnalysis {
unsigned MinID = 0;
unsigned MaxID = 0;
};
- SmallMapVector<size_t, PairInfo, 8> PairCounts;
+ SmallMapVector<std::pair<unsigned, unsigned>, PairInfo, 8> PairCounts;
unsigned MajID0 = 0, MajID1 = 0;
for (auto [Idx, V] : enumerate(VL)) {
SmallVector<Value *> OperandsForValue = getOperands(S, V);
@@ -11960,8 +11960,8 @@ class InstructionsCompatibilityAnalysis {
continue;
unsigned MinID = std::min(ID0, ID1);
unsigned MaxID = std::max(ID0, ID1);
- size_t Key = (static_cast<size_t>(MinID) << 16) | MaxID;
- auto [It, Inserted] = PairCounts.try_emplace(Key);
+ auto [It, Inserted] =
+ PairCounts.try_emplace(std::make_pair(MinID, MaxID));
PairInfo &Info = It->second;
if (Inserted) {
Info.MinID = MinID;
@@ -11976,7 +11976,8 @@ class InstructionsCompatibilityAnalysis {
// lanes. Select the orientation (original or inverse) that has
// more votes as the majority pattern.
unsigned BestCount = 0;
- for (auto &[Key, Info] : PairCounts) {
+ for (const auto &P : PairCounts) {
+ const PairInfo &Info = P.second;
unsigned Total = Info.FwdCount + Info.RevCount;
if (Total > BestCount) {
BestCount = Total;
More information about the llvm-commits
mailing list