[llvm] [SLP] Prefer immediate opcode matches on score ties (PR #225869)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 10:51:41 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Kevin Choi (choikwa)
<details>
<summary>Changes</summary>
Keep the cumulative lookahead score as the primary ordering criterion, then prefer immediate same-opcode matches over alternate opcodes before considering external-use bonuses. This avoids selecting crossed commutative operands that force otherwise vectorizable child bundles to be gathered or scalarized.
Assisted-by: OpenAI Codex
Fixes https://github.com/llvm/llvm-project/issues/222423
---
Full diff: https://github.com/llvm/llvm-project/pull/225869.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+14-2)
- (added) llvm/test/Transforms/SLPVectorizer/AMDGPU/lookahead-root-opcode.ll (+61)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 451607e6f805c..7059fb15b3133 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1290,11 +1290,14 @@ class slpvectorizer::BoUpSLP {
/// Luís F. W. Góes
int getScoreAtLevelRec(Value *LHS, Value *RHS, Instruction *U1,
Instruction *U2, int CurrLevel,
- ArrayRef<Value *> MainAltOps) const {
+ ArrayRef<Value *> MainAltOps,
+ int *RootScore = nullptr) const {
// Get the shallow score of V1 and V2.
int ShallowScoreAtThisLevel =
getShallowScore(LHS, RHS, U1, U2, MainAltOps);
+ if (RootScore)
+ *RootScore = ShallowScoreAtThisLevel;
// If reached MaxLevel,
// or if V1 and V2 are not instructions,
@@ -1540,9 +1543,14 @@ class slpvectorizer::BoUpSLP {
LookAheadMaxDepth);
// Keep track of the instruction stack as we recurse into the operands
// during the look-ahead score exploration.
+ int OpcodeScore;
int Score =
LookAhead.getScoreAtLevelRec(LHS, RHS, /*U1=*/nullptr, /*U2=*/nullptr,
- /*CurrLevel=*/1, MainAltOps);
+ /*CurrLevel=*/1, MainAltOps,
+ &OpcodeScore);
+ if (OpcodeScore != LookAheadHeuristics::ScoreSameOpcode &&
+ OpcodeScore != LookAheadHeuristics::ScoreAltOpcodes)
+ OpcodeScore = 0;
if (Score) {
int SplatScore =
getSplatScore(Lane, OpIdx, Idx, UsedLanes) * ScoreScaleFactor;
@@ -1559,7 +1567,11 @@ class slpvectorizer::BoUpSLP {
const int SF = (LHS == RHS && isConstant(LHS))
? ScoreConstantScaleFactor
: ScoreScaleFactor;
+ // Scale the cumulative score and use immediate compatibility to
+ // break ties before considering whether all users are vectorized.
Score *= SF;
+ Score += OpcodeScore;
+ Score *= SF; // Scale Opcode as well
Score += getExternalUseScore(Lane, OpIdx, Idx);
IsUsed = true;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/lookahead-root-opcode.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/lookahead-root-opcode.ll
new file mode 100644
index 0000000000000..0937b70ba700f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/lookahead-root-opcode.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=slp-vectorizer -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck %s
+
+define <2 x float> @commutative_chain(float %x, <8 x float> %v) {
+; CHECK-LABEL: define <2 x float> @commutative_chain(
+; CHECK-SAME: float [[X:%.*]], <8 x float> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[X]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[V]], <8 x float> poison, <2 x i32> <i32 poison, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> <float 0.000000e+00, float poison>, <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x float> [[TMP4]], splat (float 5.000000e-01)
+; CHECK-NEXT: [[TMP6:%.*]] = fmul <2 x float> [[TMP5]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = fmul <2 x float> [[TMP6]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = fadd <2 x float> [[TMP7]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x float> [[TMP5]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> [[TMP9]], <2 x float> splat (float -1.000000e+00))
+; CHECK-NEXT: [[TMP11:%.*]] = call <2 x float> @llvm.minnum.v2f32(<2 x float> [[TMP10]], <2 x float> splat (float 1.000000e+00))
+; CHECK-NEXT: [[TMP12:%.*]] = fadd <2 x float> [[TMP11]], splat (float 1.000000e+00)
+; CHECK-NEXT: [[TMP13:%.*]] = fmul <2 x float> [[TMP12]], splat (float 5.000000e-01)
+; CHECK-NEXT: [[TMP14:%.*]] = fmul <2 x float> [[TMP13]], splat (float f0x3A000000)
+; CHECK-NEXT: [[TMP15:%.*]] = fmul <2 x float> [[TMP4]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = select <2 x i1> zeroinitializer, <2 x float> [[TMP15]], <2 x float> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[TMP16]]
+;
+entry:
+ %elt = extractelement <8 x float> %v, i64 3
+ %a0 = fmul float %x, %elt
+ %b0 = fmul float %x, 0.000000e+00
+ %b1 = fmul float %b0, 5.000000e-01
+ %b2 = fmul float %b1, %b1
+ %a1 = fmul float %a0, 5.000000e-01
+ %a2 = fmul float %a1, %a1
+ %a3 = fmul float %a2, 0.000000e+00
+ %b3 = fmul float %b2, 0.000000e+00
+ %b4 = fadd float %b3, 1.000000e+00
+ %b5 = fmul float %b1, %b4
+ %a4 = fadd float %a3, 1.000000e+00
+ %a5 = fmul float %a1, %a4
+ %a6 = call float @llvm.maxnum.f32(float %a5, float -1.000000e+00)
+ %b6 = call float @llvm.maxnum.f32(float %b5, float -1.000000e+00)
+ %b7 = call float @llvm.minnum.f32(float %b6, float 1.000000e+00)
+ %b8 = fadd float %b7, 1.000000e+00
+ %a7 = call float @llvm.minnum.f32(float %a6, float 1.000000e+00)
+ %a8 = fadd float %a7, 1.000000e+00
+ %a9 = fmul float %a8, 5.000000e-01
+ %a10 = fmul float %a9, f0x3A000000
+ %b9 = fmul float %b8, 5.000000e-01
+ %b10 = fmul float %b9, f0x3A000000
+ %b11 = fmul float %b0, %b10
+ %b12 = select i1 false, float %b11, float 0.000000e+00
+ %r0 = insertelement <2 x float> poison, float %b12, i64 0
+ %a11 = fmul float %a0, %a10
+ %a12 = select i1 false, float %a11, float 0.000000e+00
+ %r1 = insertelement <2 x float> %r0, float %a12, i64 1
+ ret <2 x float> %r1
+}
+
+declare float @llvm.maxnum.f32(float, float)
+declare float @llvm.minnum.f32(float, float)
``````````
</details>
https://github.com/llvm/llvm-project/pull/225869
More information about the llvm-commits
mailing list