[llvm] [SLP][X86][NFC] Add pre-commit test for width-3 shared-weight reduction (PR #222243)
Milin Bhade via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 22:50:02 PDT 2026
https://github.com/mbhade-amd created https://github.com/llvm/llvm-project/pull/222243
Pre-commit test for three independent reductions over contiguous bytes sharing a single broadcast weight. It captures current codegen: even with -slp-vectorize-non-power-of-2=true the width-3 tree is discarded to scalar because the non-power-of-2 <3 x i8> load is over-priced (the folded vpinsr*(mem) is double-counted). A follow-up X86 cost fix flips the NPOT run to the width-3 vector form.
>From 078fb87914c84e3073c28129b29c216ee85c70c8 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Wed, 9 Sep 2026 10:54:49 +0530
Subject: [PATCH] [SLP][X86][NFC] Add pre-commit test for width-3 shared-weight
reduction
Pre-commit test for three independent reductions over contiguous bytes
sharing a single broadcast weight. It captures current codegen: even with
-slp-vectorize-non-power-of-2=true the width-3 tree is discarded to scalar
because the non-power-of-2 <3 x i8> load is over-priced (the folded
vpinsr*(mem) is double-counted). A follow-up X86 cost fix flips the NPOT
run to the width-3 vector form.
Co-authored-by: Cursor <cursoragent at cursor.com>
---
.../X86/nonpow2-shared-weight-reduction.ll | 163 ++++++++++++++++++
1 file changed, 163 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/nonpow2-shared-weight-reduction.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/nonpow2-shared-weight-reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/nonpow2-shared-weight-reduction.ll
new file mode 100644
index 0000000000000..b9c7ebe1ddc64
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/nonpow2-shared-weight-reduction.ll
@@ -0,0 +1,163 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 \
+; RUN: -slp-vectorize-non-power-of-2=true -S < %s | FileCheck %s --check-prefixes=NPOT
+; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 \
+; RUN: -slp-vectorize-non-power-of-2=false -S < %s | FileCheck %s --check-prefixes=SCALAR
+
+; Three independent reductions over contiguous RGB bytes sharing a
+; single broadcast weight. SLP forms a width-3 tree, but at baseline the
+; non-power-of-2 <3 x i8> load is over-priced (the folded vpinsrb(mem) is
+; double-counted), so the tree cost is 0 and the tree is discarded to scalar.
+; Once the load is priced correctly the tree cost goes negative and the width-3
+; form is kept (vpmovzxbd + vpbroadcastd + vpmulld + vpaddd).
+
+define void @ship3(ptr noundef readonly %weights, ptr noundef readonly %src, i64 noundef %bpp, ptr noundef writeonly %dst, i32 noundef %n) {
+; NPOT-LABEL: @ship3(
+; NPOT-NEXT: [[N_EXT:%.*]] = sext i32 [[N:%.*]] to i64
+; NPOT-NEXT: [[POS:%.*]] = icmp sgt i32 [[N]], 0
+; NPOT-NEXT: br i1 [[POS]], label [[LOOP:%.*]], label [[EXIT:%.*]]
+; NPOT: loop:
+; NPOT-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; NPOT-NEXT: [[ACCB:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[ADDB:%.*]], [[LOOP]] ]
+; NPOT-NEXT: [[ACCG:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[ADDG:%.*]], [[LOOP]] ]
+; NPOT-NEXT: [[ACCR:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[ADDR:%.*]], [[LOOP]] ]
+; NPOT-NEXT: [[WGEP:%.*]] = getelementptr inbounds i32, ptr [[WEIGHTS:%.*]], i64 [[IV]]
+; NPOT-NEXT: [[W:%.*]] = load i32, ptr [[WGEP]], align 4
+; NPOT-NEXT: [[OFF:%.*]] = mul nsw i64 [[IV]], [[BPP:%.*]]
+; NPOT-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[OFF]]
+; NPOT-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P0]], i64 1
+; NPOT-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P0]], i64 2
+; NPOT-NEXT: [[B:%.*]] = load i8, ptr [[P0]], align 1
+; NPOT-NEXT: [[G:%.*]] = load i8, ptr [[P1]], align 1
+; NPOT-NEXT: [[R:%.*]] = load i8, ptr [[P2]], align 1
+; NPOT-NEXT: [[ZB:%.*]] = zext i8 [[B]] to i32
+; NPOT-NEXT: [[ZG:%.*]] = zext i8 [[G]] to i32
+; NPOT-NEXT: [[ZR:%.*]] = zext i8 [[R]] to i32
+; NPOT-NEXT: [[MB:%.*]] = mul nsw i32 [[W]], [[ZB]]
+; NPOT-NEXT: [[MG:%.*]] = mul nsw i32 [[W]], [[ZG]]
+; NPOT-NEXT: [[MR:%.*]] = mul nsw i32 [[W]], [[ZR]]
+; NPOT-NEXT: [[ADDB]] = add nsw i32 [[MB]], [[ACCB]]
+; NPOT-NEXT: [[ADDG]] = add nsw i32 [[MG]], [[ACCG]]
+; NPOT-NEXT: [[ADDR]] = add nsw i32 [[MR]], [[ACCR]]
+; NPOT-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NPOT-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_EXT]]
+; NPOT-NEXT: br i1 [[DONE]], label [[EXIT_LOOP:%.*]], label [[LOOP]]
+; NPOT: exit.loop:
+; NPOT-NEXT: [[SB:%.*]] = lshr i32 [[ADDB]], 16
+; NPOT-NEXT: [[SG:%.*]] = lshr i32 [[ADDG]], 16
+; NPOT-NEXT: [[SR:%.*]] = lshr i32 [[ADDR]], 16
+; NPOT-NEXT: [[TB:%.*]] = trunc i32 [[SB]] to i8
+; NPOT-NEXT: [[TG:%.*]] = trunc i32 [[SG]] to i8
+; NPOT-NEXT: [[TR:%.*]] = trunc i32 [[SR]] to i8
+; NPOT-NEXT: br label [[EXIT]]
+; NPOT: exit:
+; NPOT-NEXT: [[TMP11:%.*]] = phi i8 [ 0, [[TMP0]] ], [ [[TB]], [[EXIT_LOOP]] ]
+; NPOT-NEXT: [[TMP12:%.*]] = phi i8 [ 0, [[TMP0]] ], [ [[TG]], [[EXIT_LOOP]] ]
+; NPOT-NEXT: [[TMP13:%.*]] = phi i8 [ 0, [[TMP0]] ], [ [[TR]], [[EXIT_LOOP]] ]
+; NPOT-NEXT: [[D1:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 1
+; NPOT-NEXT: [[D2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 2
+; NPOT-NEXT: store i8 [[TMP11]], ptr [[DST]], align 1
+; NPOT-NEXT: store i8 [[TMP12]], ptr [[D1]], align 1
+; NPOT-NEXT: store i8 [[TMP13]], ptr [[D2]], align 1
+; NPOT-NEXT: ret void
+;
+; SCALAR-LABEL: @ship3(
+; SCALAR-NEXT: [[N_EXT:%.*]] = sext i32 [[N:%.*]] to i64
+; SCALAR-NEXT: [[POS:%.*]] = icmp sgt i32 [[N]], 0
+; SCALAR-NEXT: br i1 [[POS]], label [[LOOP:%.*]], label [[EXIT:%.*]]
+; SCALAR: loop:
+; SCALAR-NEXT: [[IV:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; SCALAR-NEXT: [[ACCB:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[ADDB:%.*]], [[LOOP]] ]
+; SCALAR-NEXT: [[ACCG:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[ADDG:%.*]], [[LOOP]] ]
+; SCALAR-NEXT: [[ACCR:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[ADDR:%.*]], [[LOOP]] ]
+; SCALAR-NEXT: [[WGEP:%.*]] = getelementptr inbounds i32, ptr [[WEIGHTS:%.*]], i64 [[IV]]
+; SCALAR-NEXT: [[W:%.*]] = load i32, ptr [[WGEP]], align 4
+; SCALAR-NEXT: [[OFF:%.*]] = mul nsw i64 [[IV]], [[BPP:%.*]]
+; SCALAR-NEXT: [[P0:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[OFF]]
+; SCALAR-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[P0]], i64 1
+; SCALAR-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[P0]], i64 2
+; SCALAR-NEXT: [[B:%.*]] = load i8, ptr [[P0]], align 1
+; SCALAR-NEXT: [[G:%.*]] = load i8, ptr [[P1]], align 1
+; SCALAR-NEXT: [[R:%.*]] = load i8, ptr [[P2]], align 1
+; SCALAR-NEXT: [[ZB:%.*]] = zext i8 [[B]] to i32
+; SCALAR-NEXT: [[ZG:%.*]] = zext i8 [[G]] to i32
+; SCALAR-NEXT: [[ZR:%.*]] = zext i8 [[R]] to i32
+; SCALAR-NEXT: [[MB:%.*]] = mul nsw i32 [[W]], [[ZB]]
+; SCALAR-NEXT: [[MG:%.*]] = mul nsw i32 [[W]], [[ZG]]
+; SCALAR-NEXT: [[MR:%.*]] = mul nsw i32 [[W]], [[ZR]]
+; SCALAR-NEXT: [[ADDB]] = add nsw i32 [[MB]], [[ACCB]]
+; SCALAR-NEXT: [[ADDG]] = add nsw i32 [[MG]], [[ACCG]]
+; SCALAR-NEXT: [[ADDR]] = add nsw i32 [[MR]], [[ACCR]]
+; SCALAR-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; SCALAR-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_EXT]]
+; SCALAR-NEXT: br i1 [[DONE]], label [[EXIT_LOOP:%.*]], label [[LOOP]]
+; SCALAR: exit.loop:
+; SCALAR-NEXT: [[SB:%.*]] = lshr i32 [[ADDB]], 16
+; SCALAR-NEXT: [[SG:%.*]] = lshr i32 [[ADDG]], 16
+; SCALAR-NEXT: [[SR:%.*]] = lshr i32 [[ADDR]], 16
+; SCALAR-NEXT: [[TB:%.*]] = trunc i32 [[SB]] to i8
+; SCALAR-NEXT: [[TG:%.*]] = trunc i32 [[SG]] to i8
+; SCALAR-NEXT: [[TR:%.*]] = trunc i32 [[SR]] to i8
+; SCALAR-NEXT: br label [[EXIT]]
+; SCALAR: exit:
+; SCALAR-NEXT: [[OB:%.*]] = phi i8 [ 0, [[TMP0]] ], [ [[TB]], [[EXIT_LOOP]] ]
+; SCALAR-NEXT: [[OG:%.*]] = phi i8 [ 0, [[TMP0]] ], [ [[TG]], [[EXIT_LOOP]] ]
+; SCALAR-NEXT: [[OR:%.*]] = phi i8 [ 0, [[TMP0]] ], [ [[TR]], [[EXIT_LOOP]] ]
+; SCALAR-NEXT: [[D1:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 1
+; SCALAR-NEXT: [[D2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 2
+; SCALAR-NEXT: store i8 [[OB]], ptr [[DST]], align 1
+; SCALAR-NEXT: store i8 [[OG]], ptr [[D1]], align 1
+; SCALAR-NEXT: store i8 [[OR]], ptr [[D2]], align 1
+; SCALAR-NEXT: ret void
+;
+ %n.ext = sext i32 %n to i64
+ %pos = icmp sgt i32 %n, 0
+ br i1 %pos, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %0 ], [ %iv.next, %loop ]
+ %accb = phi i32 [ 0, %0 ], [ %addb, %loop ]
+ %accg = phi i32 [ 0, %0 ], [ %addg, %loop ]
+ %accr = phi i32 [ 0, %0 ], [ %addr, %loop ]
+ %wgep = getelementptr inbounds i32, ptr %weights, i64 %iv
+ %w = load i32, ptr %wgep, align 4
+ %off = mul nsw i64 %iv, %bpp
+ %p0 = getelementptr inbounds i8, ptr %src, i64 %off
+ %p1 = getelementptr inbounds i8, ptr %p0, i64 1
+ %p2 = getelementptr inbounds i8, ptr %p0, i64 2
+ %b = load i8, ptr %p0, align 1
+ %g = load i8, ptr %p1, align 1
+ %r = load i8, ptr %p2, align 1
+ %zb = zext i8 %b to i32
+ %zg = zext i8 %g to i32
+ %zr = zext i8 %r to i32
+ %mb = mul nsw i32 %w, %zb
+ %mg = mul nsw i32 %w, %zg
+ %mr = mul nsw i32 %w, %zr
+ %addb = add nsw i32 %mb, %accb
+ %addg = add nsw i32 %mg, %accg
+ %addr = add nsw i32 %mr, %accr
+ %iv.next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n.ext
+ br i1 %done, label %exit.loop, label %loop
+
+exit.loop:
+ %sb = lshr i32 %addb, 16
+ %sg = lshr i32 %addg, 16
+ %sr = lshr i32 %addr, 16
+ %tb = trunc i32 %sb to i8
+ %tg = trunc i32 %sg to i8
+ %tr = trunc i32 %sr to i8
+ br label %exit
+
+exit:
+ %ob = phi i8 [ 0, %0 ], [ %tb, %exit.loop ]
+ %og = phi i8 [ 0, %0 ], [ %tg, %exit.loop ]
+ %or = phi i8 [ 0, %0 ], [ %tr, %exit.loop ]
+ %d1 = getelementptr inbounds i8, ptr %dst, i64 1
+ %d2 = getelementptr inbounds i8, ptr %dst, i64 2
+ store i8 %ob, ptr %dst, align 1
+ store i8 %og, ptr %d1, align 1
+ store i8 %or, ptr %d2, align 1
+ ret void
+}
More information about the llvm-commits
mailing list