[llvm] e90f463 - [SLP] Normalize copyable operand order via majority voting
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 09:56:49 PDT 2026
Author: Alexey Bataev
Date: 2026-04-16T12:56:44-04:00
New Revision: e90f463db5c2d28ae0d65e2ceccfaa03d4ea7c53
URL: https://github.com/llvm/llvm-project/commit/e90f463db5c2d28ae0d65e2ceccfaa03d4ea7c53
DIFF: https://github.com/llvm/llvm-project/commit/e90f463db5c2d28ae0d65e2ceccfaa03d4ea7c53.diff
LOG: [SLP] Normalize copyable operand order via majority voting
When building operands for entries with copyable elements, non-copyable
lanes of commutative ops may have inconsistent operand order (e.g. some
lanes have load,add while others have add,load). This prevents
VLOperands::reorder() from grouping consecutive loads on one side,
degrading downstream vectorization.
Add majority-voting normalization during buildOperands: track the
(ValueID, ValueID) pair frequency across non-copyable lanes and swap
any lane whose operand types are the exact inverse of the most common
pattern. This makes operand order consistent, enabling better load
grouping.
This is part 1 of #189181.
Reviewers: RKSimon, hiraditya
Pull Request: https://github.com/llvm/llvm-project/pull/191631
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
llvm/test/Transforms/SLPVectorizer/X86/operand-reorder-with-copyables.ll
llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 385e1a795b057..fd981afcaf581 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -12029,12 +12029,70 @@ class InstructionsCompatibilityAnalysis {
if (S.areInstructionsWithCopyableElements()) {
MainOp = S.getMainOp();
MainOpcode = S.getOpcode();
+ const bool IsCommutative =
+ isCommutative(MainOp) && MainOp->getNumOperands() == 2;
Operands.assign(MainOp->getNumOperands(),
BoUpSLP::ValueList(VL.size(), nullptr));
+ // Build operands and simultaneously count (ID0, ID1) pair
+ // frequencies for commutative operand normalization. Pairs and
+ // their inverses are tracked under a canonical key so that
+ // (Load, Add) and (Add, Load) contribute to the same bucket.
+ struct PairInfo {
+ unsigned FwdCount = 0;
+ unsigned RevCount = 0;
+ };
+ SmallMapVector<std::pair<unsigned, unsigned>, PairInfo, 8> PairCounts;
+ unsigned MajID0 = 0, MajID1 = 0;
for (auto [Idx, V] : enumerate(VL)) {
SmallVector<Value *> OperandsForValue = getOperands(S, V);
for (auto [OperandIdx, Operand] : enumerate(OperandsForValue))
Operands[OperandIdx][Idx] = Operand;
+ if (!IsCommutative || S.isCopyableElement(V) || isa<PoisonValue>(V))
+ continue;
+ unsigned ID0 = OperandsForValue[0]->getValueID();
+ unsigned ID1 = OperandsForValue[1]->getValueID();
+ if (ID0 == ID1)
+ continue;
+ unsigned MinID = std::min(ID0, ID1);
+ unsigned MaxID = std::max(ID0, ID1);
+ auto [It, Inserted] =
+ PairCounts.try_emplace(std::make_pair(MinID, MaxID));
+ PairInfo &Info = It->second;
+ if (ID0 < ID1)
+ ++Info.FwdCount;
+ else
+ ++Info.RevCount;
+ }
+ // Find the most frequent (ID0, ID1) pair across non-copyable
+ // lanes. Select the orientation (original or inverse) that has
+ // more votes as the majority pattern.
+ unsigned BestCount = 0;
+ for (const auto &P : PairCounts) {
+ const PairInfo &Info = P.second;
+ unsigned Total = Info.FwdCount + Info.RevCount;
+ if (Total > BestCount) {
+ BestCount = Total;
+ if (Info.FwdCount >= Info.RevCount) {
+ MajID0 = P.first.first;
+ MajID1 = P.first.second;
+ } else {
+ MajID0 = P.first.second;
+ MajID1 = P.first.first;
+ }
+ }
+ }
+ // For commutative ops, swap lanes whose operand types are the
+ // exact inverse of the majority pattern, making the non-copyable
+ // lanes consistent.
+ if (BestCount > 0) {
+ for (auto [Idx, V] : enumerate(VL)) {
+ if (S.isCopyableElement(V) || isa<PoisonValue>(V))
+ continue;
+ unsigned ID0 = Operands[0][Idx]->getValueID();
+ unsigned ID1 = Operands[1][Idx]->getValueID();
+ if (ID0 == MajID1 && ID1 == MajID0)
+ std::swap(Operands[0][Idx], Operands[1][Idx]);
+ }
}
} else {
buildOriginalOperands(S, VL, Operands);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
index 54e5405e21c49..f303b79778c05 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bottom-to-top-reorder.ll
@@ -10,7 +10,7 @@ define void @test(ptr %0, ptr %1, ptr %2) {
; CHECK-NEXT: [[TMP11:%.*]] = sub <4 x i32> <i32 0, i32 0, i32 undef, i32 0>, [[TMP8]]
; CHECK-NEXT: [[TMP12:%.*]] = sub <4 x i32> [[TMP11]], [[TMP10]]
; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP6]]
-; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> <i32 0, i32 0, i32 1, i32 0>, [[TMP13]]
+; CHECK-NEXT: [[TMP16:%.*]] = add <4 x i32> [[TMP13]], <i32 0, i32 0, i32 1, i32 0>
; CHECK-NEXT: [[TMP17:%.*]] = sub <4 x i32> [[TMP16]], zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = sub <4 x i32> [[TMP17]], zeroinitializer
; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
index 44588dfd8ffe4..b432627e6726c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/copyable_reorder.ll
@@ -136,24 +136,15 @@ entry:
define void @test_add_udiv_reorder_add(ptr %arr1, ptr %arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
; CHECK-LABEL: @test_add_udiv_reorder_add(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 2
-; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr i32, ptr [[ARR1]], i32 3
-; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[GEP1_2]], align 4
-; CHECK-NEXT: [[V3:%.*]] = load i32, ptr [[GEP1_3]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[A0:%.*]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[A1:%.*]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1146, i32 146>
-; CHECK-NEXT: [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
-; CHECK-NEXT: [[Y3:%.*]] = add nsw i32 [[A3:%.*]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[ARR1]], align 4
-; CHECK-NEXT: [[RES2:%.*]] = udiv i32 [[V2]], [[Y2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[RES2]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[V3]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[Y3]], i32 3
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP9]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[ARR1:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[A2:%.*]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], <i32 1, i32 1, i32 42, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[A0:%.*]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[A1:%.*]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[V3:%.*]], i32 3
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <4 x i32> <i32 1146, i32 146, i32 0, i32 0>, [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = udiv <4 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = add nsw <4 x i32> [[TMP7]], [[TMP8]]
; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[ARR2:%.*]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operand-reorder-with-copyables.ll b/llvm/test/Transforms/SLPVectorizer/X86/operand-reorder-with-copyables.ll
index 278f2c3735da2..22bca2072636a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operand-reorder-with-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operand-reorder-with-copyables.ll
@@ -14,19 +14,21 @@ define void @test(ptr %p0, ptr %p1, ptr %dst, i32 %a0, i32 %a1, i32 %a2, i32 %a3
; CHECK-NEXT: [[Y2:%.*]] = load i32, ptr [[R1]], align 4
; CHECK-NEXT: [[Y3:%.*]] = load i32, ptr [[R2]], align 4
; CHECK-NEXT: [[C2:%.*]] = sub i32 [[X2]], [[Y2]]
-; CHECK-NEXT: [[C3:%.*]] = sub i32 [[X3]], [[Y3]]
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[P0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P1]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = sub <2 x i32> [[TMP0]], [[TMP1]]
; CHECK-NEXT: [[RES2:%.*]] = udiv i32 [[C2]], [[A2]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[A0]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[A1]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[C3]], i32 3
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[A3]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[RES2]], i32 2
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[RES2]], i32 2
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[X3]], i32 3
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> [[TMP8]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP5]], [[TMP9]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[Y3]], i32 3
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP13]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[TMP15:%.*]] = sub <4 x i32> [[TMP9]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 poison>, i32 [[A0]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[A1]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[A3]], i32 3
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP15]], [[TMP12]]
; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[DST]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
index 830c7755fc53a..49164e59621fb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-last-instruction-in-split-node.ll
@@ -8,7 +8,7 @@ define float @test() {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <12 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float undef, float 0.000000e+00, float 0.000000e+00, float undef>, float 0.000000e+00, i32 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <12 x float> [[TMP1]], float 0.000000e+00, i32 8
; CHECK-NEXT: [[TMP3:%.*]] = fmul <12 x float> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = fadd <12 x float> <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float poison>, [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <12 x float> [[TMP3]], <float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float 0.000000e+00, float -0.000000e+00, float 0.000000e+00, float poison>
; CHECK-NEXT: [[TMP5:%.*]] = fsub <8 x float> zeroinitializer, <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison>
; CHECK-NEXT: [[TMP6:%.*]] = fadd <12 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison>, [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <12 x float> [[TMP6]], <12 x float> poison, <20 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
More information about the llvm-commits
mailing list