[llvm] [SLP] Normalize copyable operand order via majority voting (PR #191631)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Apr 11 07:20:31 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/191631
When building operands for entries with copyable elements, non-copyable
lanes of commutative ops may have inconsistent operand order (e.g. some
lanes have load,add while others have add,load). This prevents
VLOperands::reorder() from grouping consecutive loads on one side,
degrading downstream vectorization.
Add majority-voting normalization during buildOperands: track the
(ValueID, ValueID) pair frequency across non-copyable lanes and swap
any lane whose operand types are the exact inverse of the most common
pattern. This makes operand order consistent, enabling better load
grouping.
This is part 1 of #189181.
>From b8336e29150190eac20761885da1ddd5f6bb4324 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 11 Apr 2026 07:20:20 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 63 +++++++++++++++++++
.../X86/bottom-to-top-reorder.ll | 2 +-
.../SLPVectorizer/X86/copyable_reorder.ll | 27 +++-----
.../reused-last-instruction-in-split-node.ll | 2 +-
4 files changed, 74 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index eedddf81d6877..88eb8b1d2dd49 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11932,12 +11932,75 @@ class InstructionsCompatibilityAnalysis {
if (S.areInstructionsWithCopyableElements()) {
MainOp = S.getMainOp();
MainOpcode = S.getOpcode();
+ const bool IsCommutative =
+ isCommutative(MainOp) && MainOp->getNumOperands() == 2;
Operands.assign(MainOp->getNumOperands(),
BoUpSLP::ValueList(VL.size(), nullptr));
+ // Build operands and simultaneously count (ID0, ID1) pair
+ // frequencies for commutative operand normalization. Pairs and
+ // their inverses are tracked under a canonical key so that
+ // (Load, Add) and (Add, Load) contribute to the same bucket.
+ struct PairInfo {
+ unsigned FwdCount = 0;
+ unsigned RevCount = 0;
+ unsigned MinID = 0;
+ unsigned MaxID = 0;
+ };
+ SmallMapVector<size_t, PairInfo, 8> PairCounts;
+ unsigned MajID0 = 0, MajID1 = 0;
for (auto [Idx, V] : enumerate(VL)) {
SmallVector<Value *> OperandsForValue = getOperands(S, V);
for (auto [OperandIdx, Operand] : enumerate(OperandsForValue))
Operands[OperandIdx][Idx] = Operand;
+ if (!IsCommutative || S.isCopyableElement(V) || isa<PoisonValue>(V))
+ continue;
+ unsigned ID0 = OperandsForValue[0]->getValueID();
+ unsigned ID1 = OperandsForValue[1]->getValueID();
+ if (ID0 == ID1)
+ continue;
+ unsigned MinID = std::min(ID0, ID1);
+ unsigned MaxID = std::max(ID0, ID1);
+ size_t Key = (static_cast<size_t>(MinID) << 16) | MaxID;
+ auto [It, Inserted] = PairCounts.try_emplace(Key);
+ PairInfo &Info = It->second;
+ if (Inserted) {
+ Info.MinID = MinID;
+ Info.MaxID = MaxID;
+ }
+ if (ID0 < ID1)
+ ++Info.FwdCount;
+ else
+ ++Info.RevCount;
+ }
+ // Find the most frequent (ID0, ID1) pair across non-copyable
+ // lanes. Select the orientation (original or inverse) that has
+ // more votes as the majority pattern.
+ unsigned BestCount = 0;
+ for (auto &[Key, Info] : PairCounts) {
+ unsigned Total = Info.FwdCount + Info.RevCount;
+ if (Total > BestCount) {
+ BestCount = Total;
+ if (Info.FwdCount >= Info.RevCount) {
+ MajID0 = Info.MinID;
+ MajID1 = Info.MaxID;
+ } else {
+ MajID0 = Info.MaxID;
+ MajID1 = Info.MinID;
+ }
+ }
+ }
+ // For commutative ops, swap lanes whose operand types are the
+ // exact inverse of the majority pattern, making the non-copyable
+ // lanes consistent.
+ if (BestCount > 0) {
+ for (auto [Idx, V] : enumerate(VL)) {
+ if (S.isCopyableElement(V) || isa<PoisonValue>(V))
+ continue;
+ unsigned ID0 = Operands[0][Idx]->getValueID();
+ unsigned ID1 = Operands[1][Idx]->getValueID();
+ if (ID0 == MajID1 && ID1 == MajID0)
+ std::swap(Operands[0][Idx], Operands[1][Idx]);
+ }
}
} else {
buildOriginalOperands(S, VL, Operands);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
index 54e5405e21c49..f303b79778c05 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
@@ -10,7 +10,7 @@ define void @test(ptr %0, ptr %1, ptr %2) {
; CHECK-NEXT: [[TMP11:%.*]] = sub <4 x i32> <i32 0, i32 0, i32 undef, i32 0>, [[TMP8]]
; CHECK-NEXT: [[TMP12:%.*]] = sub <4 x i32> [[TMP11]], [[TMP10]]
; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP6]]
-; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> <i32 0, i32 0, i32 1, i32 0>, [[TMP13]]
+; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[TMP13]], <i32 0, i32 0, i32 1, i32 0>
; CHECK-NEXT: [[TMP17:%.*]] = sub <4 x i32> [[TMP16]], zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = sub <4 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
index 44588dfd8ffe4..b432627e6726c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
@@ -136,24 +136,15 @@ entry:
define void @test_add_udiv_reorder_add(ptr %arr1, ptr %arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
; CHECK-LABEL: @test_add_udiv_reorder_add(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 2
-; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr i32, ptr [[ARR1]], i32 3
-; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[GEP1_2]], align 4
-; CHECK-NEXT: [[V3:%.*]] = load i32, ptr [[GEP1_3]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[A0:%.*]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[A1:%.*]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1146, i32 146>
-; CHECK-NEXT: [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
-; CHECK-NEXT: [[Y3:%.*]] = add nsw i32 [[A3:%.*]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[ARR1]], align 4
-; CHECK-NEXT: [[RES2:%.*]] = udiv i32 [[V2]], [[Y2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[RES2]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[V3]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[Y3]], i32 3
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP9]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[ARR1:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[A2:%.*]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], <i32 1, i32 1, i32 42, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[A0:%.*]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[A1:%.*]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[V3:%.*]], i32 3
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 1146, i32 146, i32 0, i32 0>, [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = udiv <4 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP7]], [[TMP8]]
; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[ARR2:%.*]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
index 830c7755fc53a..49164e59621fb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
@@ -8,7 +8,7 @@ define float @test() {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <12 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float undef, float 0.000000e+00, float 0.000000e+00, float undef>, float 0.000000e+00, i32 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <12 x float> [[TMP1]], float 0.000000e+00, i32 8
; CHECK-NEXT: [[TMP3:%.*]] = fmul <12 x float> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd <12 x float> <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float poison>, [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <12 x float> [[TMP3]], <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float poison>
; CHECK-NEXT: [[TMP5:%.*]] = fsub <8 x float> zeroinitializer, <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison>
; CHECK-NEXT: [[TMP6:%.*]] = fadd <12 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison>, [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <12 x float> [[TMP6]], <12 x float> poison, <20 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
More information about the llvm-commits
mailing list