[llvm] [SLP] Merge redundant input operands of binary-operator nodes (PR #208670)
Balakrishna Bandlapalli via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 00:59:26 PDT 2026
https://github.com/amd-bbandlap updated https://github.com/llvm/llvm-project/pull/208670
>From 656babca4b733bfb0d6a56d9ca151badd86ac102 Mon Sep 17 00:00:00 2001
From: Balakrishna Bandlapalli <bbandlap at gmail.com>
Date: Thu, 16 Jul 2026 13:27:02 +0530
Subject: [PATCH 1/2] [SLP][X86] Add pre-commit tests for merging
binary-operator input operands
Add baseline SLP vectorizer tests covering the merge-input-operands
transform, including cases that exercise IR flag propagation (fast-math
flags, integer nsw/nuw wrap flags, and the exact flag on shifts) onto
the merged operand node. These checks reflect current (pre-transform)
codegen; a follow-up patch enables the transform and updates the checks
to show the delta.
Co-authored-by: Cursor <cursoragent at cursor.com>
---
.../SLPVectorizer/X86/merge-input-operands.ll | 462 ++++++++++++++++++
1 file changed, 462 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/merge-input-operands.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/merge-input-operands.ll b/llvm/test/Transforms/SLPVectorizer/X86/merge-input-operands.ll
new file mode 100644
index 0000000000000..16786b42e426e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/merge-input-operands.ll
@@ -0,0 +1,462 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-unknown-linux -mcpu=znver4 -passes=slp-vectorizer < %s | FileCheck %s
+
+; The MergeInputOps transform targets a binary-operator bundle whose two operand
+; bundles each reuse a small set of scalars. Here a 4-wide fmul has
+; left operands = [s0, s1, s0, s1]
+; right operands = [s2, s3, s3, s2]
+; and the four sums s0..s3 are all fadd. With merging enabled the union
+; [s0,s1,s2,s3] becomes one <4 x float> add feeding two reuse shuffles; with it
+; disabled the sums are built as two separate <2 x float> adds.
+define void @merge_products_of_sums(ptr %A, ptr %Out) {
+; CHECK-LABEL: @merge_products_of_sums(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
+; CHECK-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
+; CHECK-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x float> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = fmul <4 x float> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %a0 = load float, ptr %A
+ %p1 = getelementptr float, ptr %A, i64 1
+ %a1 = load float, ptr %p1
+ %p2 = getelementptr float, ptr %A, i64 2
+ %a2 = load float, ptr %p2
+ %p3 = getelementptr float, ptr %A, i64 3
+ %a3 = load float, ptr %p3
+ %p4 = getelementptr float, ptr %A, i64 4
+ %a4 = load float, ptr %p4
+ %p5 = getelementptr float, ptr %A, i64 5
+ %a5 = load float, ptr %p5
+ %p6 = getelementptr float, ptr %A, i64 6
+ %a6 = load float, ptr %p6
+ %p7 = getelementptr float, ptr %A, i64 7
+ %a7 = load float, ptr %p7
+
+ %s0 = fadd float %a0, %a4
+ %s1 = fadd float %a1, %a5
+ %s2 = fadd float %a2, %a6
+ %s3 = fadd float %a3, %a7
+
+ %m0 = fmul float %s0, %s2
+ %m1 = fmul float %s1, %s3
+ %m2 = fmul float %s0, %s3
+ %m3 = fmul float %s1, %s2
+
+ store float %m0, ptr %Out
+ %q1 = getelementptr float, ptr %Out, i64 1
+ store float %m1, ptr %q1
+ %q2 = getelementptr float, ptr %Out, i64 2
+ store float %m2, ptr %q2
+ %q3 = getelementptr float, ptr %Out, i64 3
+ store float %m3, ptr %q3
+ ret void
+}
+
+; Negative: the two operand sides have different opcodes (add vs mul), so the
+; merged bundle would be an alt-shuffle node. Merging must be rejected and the
+; output must be identical with the flag on or off.
+define void @no_merge_mixed_opcode(ptr %A, ptr %Out) {
+; CHECK-LABEL: @no_merge_mixed_opcode(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[V:%.*]] = load <16 x i32>, ptr [[A:%.*]], align 64
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <16 x i32> [[V]], <16 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[V]], <16 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i32> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: [[TMP6:%.*]] = sub <8 x i32> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: store <8 x i32> [[TMP6]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <16 x i32>, ptr %A
+ %a0 = extractelement <16 x i32> %v, i64 0
+ %a1 = extractelement <16 x i32> %v, i64 1
+ %a2 = extractelement <16 x i32> %v, i64 2
+ %a3 = extractelement <16 x i32> %v, i64 3
+ %a4 = extractelement <16 x i32> %v, i64 4
+ %a5 = extractelement <16 x i32> %v, i64 5
+ %a6 = extractelement <16 x i32> %v, i64 6
+ %a7 = extractelement <16 x i32> %v, i64 7
+ %l0 = add i32 %a0, %a1
+ %l1 = add i32 %a2, %a3
+ %l2 = add i32 %a4, %a5
+ %l3 = add i32 %a6, %a7
+ %r0 = mul i32 %a0, %a1
+ %r1 = mul i32 %a2, %a3
+ %r2 = mul i32 %a4, %a5
+ %r3 = mul i32 %a6, %a7
+ %m0 = sub i32 %l0, %r0
+ %m1 = sub i32 %l1, %r1
+ %m2 = sub i32 %l2, %r2
+ %m3 = sub i32 %l3, %r3
+ %m4 = sub i32 %l0, %r1
+ %m5 = sub i32 %l1, %r2
+ %m6 = sub i32 %l2, %r3
+ %m7 = sub i32 %l3, %r0
+ store i32 %m0, ptr %Out
+ %o1 = getelementptr i32, ptr %Out, i64 1
+ store i32 %m1, ptr %o1
+ %o2 = getelementptr i32, ptr %Out, i64 2
+ store i32 %m2, ptr %o2
+ %o3 = getelementptr i32, ptr %Out, i64 3
+ store i32 %m3, ptr %o3
+ %o4 = getelementptr i32, ptr %Out, i64 4
+ store i32 %m4, ptr %o4
+ %o5 = getelementptr i32, ptr %Out, i64 5
+ store i32 %m5, ptr %o5
+ %o6 = getelementptr i32, ptr %Out, i64 6
+ store i32 %m6, ptr %o6
+ %o7 = getelementptr i32, ptr %Out, i64 7
+ store i32 %m7, ptr %o7
+ ret void
+}
+
+; Negative: a value (s0) is used on both operand sides, so the merged bundle
+; would contain a duplicate lane. Merging must be rejected.
+define void @no_merge_overlapping_operands(ptr %A, ptr %Out) {
+; CHECK-LABEL: @no_merge_overlapping_operands(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P1:%.*]] = getelementptr float, ptr [[A:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[P1]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <6 x float> @llvm.masked.load.v6f32.p0(ptr align 4 [[A]], <6 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true>, <6 x float> poison)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <6 x float> [[TMP1]], <6 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 5>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x float> [[TMP0]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <6 x float> [[TMP1]], <6 x float> poison, <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <6 x float> [[TMP8]], <6 x float> [[TMP1]], <2 x i32> <i32 0, i32 11>
+; CHECK-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[TMP6]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP12:%.*]] = fmul <4 x float> [[TMP5]], [[TMP11]]
+; CHECK-NEXT: store <4 x float> [[TMP12]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %a0 = load float, ptr %A
+ %p1 = getelementptr float, ptr %A, i64 1
+ %a1 = load float, ptr %p1
+ %p2 = getelementptr float, ptr %A, i64 2
+ %a2 = load float, ptr %p2
+ %p3 = getelementptr float, ptr %A, i64 3
+ %a3 = load float, ptr %p3
+ %p4 = getelementptr float, ptr %A, i64 4
+ %a4 = load float, ptr %p4
+ %p5 = getelementptr float, ptr %A, i64 5
+ %a5 = load float, ptr %p5
+ %s0 = fadd float %a0, %a1
+ %s1 = fadd float %a2, %a3
+ %s2 = fadd float %a4, %a5
+ %m0 = fmul float %s0, %s0
+ %m1 = fmul float %s1, %s2
+ %m2 = fmul float %s0, %s2
+ %m3 = fmul float %s1, %s0
+ store float %m0, ptr %Out
+ %q1 = getelementptr float, ptr %Out, i64 1
+ store float %m1, ptr %q1
+ %q2 = getelementptr float, ptr %Out, i64 2
+ store float %m2, ptr %q2
+ %q3 = getelementptr float, ptr %Out, i64 3
+ store float %m3, ptr %q3
+ ret void
+}
+
+; Flag propagation: every merged scalar (the four fadd sums) carries the same
+; fast-math flags, so the single merged fadd must keep the full "fast" flag set.
+define void @merge_fmf_uniform(ptr %A, ptr %Out) {
+; CHECK-LABEL: @merge_fmf_uniform(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
+; CHECK-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
+; CHECK-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x float> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <2 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %a0 = load float, ptr %A
+ %p1 = getelementptr float, ptr %A, i64 1
+ %a1 = load float, ptr %p1
+ %p2 = getelementptr float, ptr %A, i64 2
+ %a2 = load float, ptr %p2
+ %p3 = getelementptr float, ptr %A, i64 3
+ %a3 = load float, ptr %p3
+ %p4 = getelementptr float, ptr %A, i64 4
+ %a4 = load float, ptr %p4
+ %p5 = getelementptr float, ptr %A, i64 5
+ %a5 = load float, ptr %p5
+ %p6 = getelementptr float, ptr %A, i64 6
+ %a6 = load float, ptr %p6
+ %p7 = getelementptr float, ptr %A, i64 7
+ %a7 = load float, ptr %p7
+
+ %s0 = fadd fast float %a0, %a4
+ %s1 = fadd fast float %a1, %a5
+ %s2 = fadd fast float %a2, %a6
+ %s3 = fadd fast float %a3, %a7
+
+ %m0 = fmul fast float %s0, %s2
+ %m1 = fmul fast float %s1, %s3
+ %m2 = fmul fast float %s0, %s3
+ %m3 = fmul fast float %s1, %s2
+
+ store float %m0, ptr %Out
+ %q1 = getelementptr float, ptr %Out, i64 1
+ store float %m1, ptr %q1
+ %q2 = getelementptr float, ptr %Out, i64 2
+ store float %m2, ptr %q2
+ %q3 = getelementptr float, ptr %Out, i64 3
+ store float %m3, ptr %q3
+ ret void
+}
+
+; Flag propagation: the merged scalars carry different fast-math flags. Two sums
+; are "fast" and two are only "nnan ninf", so the single merged fadd must keep
+; only the common subset ("nnan ninf"), not the full "fast" set.
+define void @merge_fmf_mixed(ptr %A, ptr %Out) {
+; CHECK-LABEL: @merge_fmf_mixed(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
+; CHECK-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
+; CHECK-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x float> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = fadd nnan ninf <2 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %a0 = load float, ptr %A
+ %p1 = getelementptr float, ptr %A, i64 1
+ %a1 = load float, ptr %p1
+ %p2 = getelementptr float, ptr %A, i64 2
+ %a2 = load float, ptr %p2
+ %p3 = getelementptr float, ptr %A, i64 3
+ %a3 = load float, ptr %p3
+ %p4 = getelementptr float, ptr %A, i64 4
+ %a4 = load float, ptr %p4
+ %p5 = getelementptr float, ptr %A, i64 5
+ %a5 = load float, ptr %p5
+ %p6 = getelementptr float, ptr %A, i64 6
+ %a6 = load float, ptr %p6
+ %p7 = getelementptr float, ptr %A, i64 7
+ %a7 = load float, ptr %p7
+
+ %s0 = fadd fast float %a0, %a4
+ %s1 = fadd fast float %a1, %a5
+ %s2 = fadd nnan ninf float %a2, %a6
+ %s3 = fadd nnan ninf float %a3, %a7
+
+ %m0 = fmul fast float %s0, %s2
+ %m1 = fmul fast float %s1, %s3
+ %m2 = fmul fast float %s0, %s3
+ %m3 = fmul fast float %s1, %s2
+
+ store float %m0, ptr %Out
+ %q1 = getelementptr float, ptr %Out, i64 1
+ store float %m1, ptr %q1
+ %q2 = getelementptr float, ptr %Out, i64 2
+ store float %m2, ptr %q2
+ %q3 = getelementptr float, ptr %Out, i64 3
+ store float %m3, ptr %q3
+ ret void
+}
+
+; Flag propagation: integer wrap flags. Every merged scalar (the four adds)
+; carries "nsw nuw", so the single merged add must keep both flags.
+define void @merge_int_wrap_uniform(ptr %A, ptr %Out) {
+; CHECK-LABEL: @merge_int_wrap_uniform(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
+; CHECK-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
+; CHECK-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw <2 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %a0 = load i32, ptr %A
+ %p1 = getelementptr i32, ptr %A, i64 1
+ %a1 = load i32, ptr %p1
+ %p2 = getelementptr i32, ptr %A, i64 2
+ %a2 = load i32, ptr %p2
+ %p3 = getelementptr i32, ptr %A, i64 3
+ %a3 = load i32, ptr %p3
+ %p4 = getelementptr i32, ptr %A, i64 4
+ %a4 = load i32, ptr %p4
+ %p5 = getelementptr i32, ptr %A, i64 5
+ %a5 = load i32, ptr %p5
+ %p6 = getelementptr i32, ptr %A, i64 6
+ %a6 = load i32, ptr %p6
+ %p7 = getelementptr i32, ptr %A, i64 7
+ %a7 = load i32, ptr %p7
+
+ %s0 = add nsw nuw i32 %a0, %a4
+ %s1 = add nsw nuw i32 %a1, %a5
+ %s2 = add nsw nuw i32 %a2, %a6
+ %s3 = add nsw nuw i32 %a3, %a7
+
+ %m0 = mul i32 %s0, %s2
+ %m1 = mul i32 %s1, %s3
+ %m2 = mul i32 %s0, %s3
+ %m3 = mul i32 %s1, %s2
+
+ store i32 %m0, ptr %Out
+ %q1 = getelementptr i32, ptr %Out, i64 1
+ store i32 %m1, ptr %q1
+ %q2 = getelementptr i32, ptr %Out, i64 2
+ store i32 %m2, ptr %q2
+ %q3 = getelementptr i32, ptr %Out, i64 3
+ store i32 %m3, ptr %q3
+ ret void
+}
+
+; Flag propagation: mixed integer wrap flags. Two adds carry "nsw nuw" and two
+; carry only "nsw", so the single merged add must keep only "nsw".
+define void @merge_int_wrap_mixed(ptr %A, ptr %Out) {
+; CHECK-LABEL: @merge_int_wrap_mixed(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
+; CHECK-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
+; CHECK-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw <2 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %a0 = load i32, ptr %A
+ %p1 = getelementptr i32, ptr %A, i64 1
+ %a1 = load i32, ptr %p1
+ %p2 = getelementptr i32, ptr %A, i64 2
+ %a2 = load i32, ptr %p2
+ %p3 = getelementptr i32, ptr %A, i64 3
+ %a3 = load i32, ptr %p3
+ %p4 = getelementptr i32, ptr %A, i64 4
+ %a4 = load i32, ptr %p4
+ %p5 = getelementptr i32, ptr %A, i64 5
+ %a5 = load i32, ptr %p5
+ %p6 = getelementptr i32, ptr %A, i64 6
+ %a6 = load i32, ptr %p6
+ %p7 = getelementptr i32, ptr %A, i64 7
+ %a7 = load i32, ptr %p7
+
+ %s0 = add nsw nuw i32 %a0, %a4
+ %s1 = add nsw nuw i32 %a1, %a5
+ %s2 = add nsw i32 %a2, %a6
+ %s3 = add nsw i32 %a3, %a7
+
+ %m0 = mul i32 %s0, %s2
+ %m1 = mul i32 %s1, %s3
+ %m2 = mul i32 %s0, %s3
+ %m3 = mul i32 %s1, %s2
+
+ store i32 %m0, ptr %Out
+ %q1 = getelementptr i32, ptr %Out, i64 1
+ store i32 %m1, ptr %q1
+ %q2 = getelementptr i32, ptr %Out, i64 2
+ store i32 %m2, ptr %q2
+ %q3 = getelementptr i32, ptr %Out, i64 3
+ store i32 %m3, ptr %q3
+ ret void
+}
+
+; Flag propagation: the "exact" flag on shifts. Every merged scalar (the four
+; lshr ops) is "exact", so the single merged lshr must keep "exact".
+define void @merge_exact_shift(ptr %A, ptr %Out) {
+; CHECK-LABEL: @merge_exact_shift(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
+; CHECK-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
+; CHECK-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = lshr exact <2 x i32> [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = lshr exact <2 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %a0 = load i32, ptr %A
+ %p1 = getelementptr i32, ptr %A, i64 1
+ %a1 = load i32, ptr %p1
+ %p2 = getelementptr i32, ptr %A, i64 2
+ %a2 = load i32, ptr %p2
+ %p3 = getelementptr i32, ptr %A, i64 3
+ %a3 = load i32, ptr %p3
+ %p4 = getelementptr i32, ptr %A, i64 4
+ %a4 = load i32, ptr %p4
+ %p5 = getelementptr i32, ptr %A, i64 5
+ %a5 = load i32, ptr %p5
+ %p6 = getelementptr i32, ptr %A, i64 6
+ %a6 = load i32, ptr %p6
+ %p7 = getelementptr i32, ptr %A, i64 7
+ %a7 = load i32, ptr %p7
+
+ %s0 = lshr exact i32 %a0, %a4
+ %s1 = lshr exact i32 %a1, %a5
+ %s2 = lshr exact i32 %a2, %a6
+ %s3 = lshr exact i32 %a3, %a7
+
+ %m0 = mul i32 %s0, %s2
+ %m1 = mul i32 %s1, %s3
+ %m2 = mul i32 %s0, %s3
+ %m3 = mul i32 %s1, %s2
+
+ store i32 %m0, ptr %Out
+ %q1 = getelementptr i32, ptr %Out, i64 1
+ store i32 %m1, ptr %q1
+ %q2 = getelementptr i32, ptr %Out, i64 2
+ store i32 %m2, ptr %q2
+ %q3 = getelementptr i32, ptr %Out, i64 3
+ store i32 %m3, ptr %q3
+ ret void
+}
>From 6b6678b01906214ee2c5db766ee811faeae4abdc Mon Sep 17 00:00:00 2001
From: Balakrishna Bandlapalli <bbandlap at gmail.com>
Date: Fri, 10 Jul 2026 14:46:39 +0530
Subject: [PATCH 2/2] [SLP] Merge redundant input operands of binary-operator
nodes
Add a new SLP transformation, controlled by -slp-merge-inputops, that
reduces redundant vector computation for a binary-operator bundle whose
two operand bundles each reuse a small set of scalars. Both operands are
built from a single merged operand node, with reuse-shuffle masks
selecting the per-operand lanes.
Co-authored-by: Cursor <cursoragent at cursor.com>
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 338 +++++++++++++++++-
.../SLPVectorizer/X86/merge-input-operands.ll | 267 +++++++++-----
2 files changed, 501 insertions(+), 104 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 48fb4beba6935..9af9830b7d5c2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -151,6 +151,11 @@ static cl::opt<bool> SLPInstCountCheck(
cl::desc("Reject vectorization if vector instruction count exceeds "
"scalar instruction count"));
+static cl::opt<bool> SLPMergeInputOps(
+ "slp-merge-inputops", cl::init(true), cl::Hidden,
+ cl::desc("Attempt to vectorize by merging input operands of a binary "
+ "operator node into a single wider node"));
+
static cl::opt<int>
MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
cl::desc("Attempt to vectorize for this register size in bits"));
@@ -2670,6 +2675,15 @@ class slpvectorizer::BoUpSLP {
};
friend struct DenseMapInfo<EdgeInfo>;
+ /// When building a merged operand node (see handleMergedOperands()), the
+ /// reuse shuffle mask must be recomputed against the merged input list so
+ /// that the lanes line up with the user node. Helper used by
+ /// tryToFindDuplicates().
+ void updateReuseShuffleForMergedOps(
+ const EdgeInfo &UserTreeIdx, SmallVectorImpl<int> &ReuseShuffleIndices,
+ SmallDenseMap<Value *, unsigned, 16> &UniquePositions,
+ ArrayRef<Value *> UniqueValues, bool MergedOp) const;
+
/// A helper class used for scoring candidates for two consecutive lanes.
class LookAheadHeuristics {
const TargetLibraryInfo &TLI;
@@ -4111,7 +4125,37 @@ class slpvectorizer::BoUpSLP {
/// This is the recursive part of buildTree.
void buildTreeRec(ArrayRef<Value *> Roots, unsigned Depth, const EdgeInfo &EI,
- unsigned InterleaveFactor = 0);
+ unsigned InterleaveFactor = 0, bool MergedOp = false);
+
+ /// \returns the tree entry built for the \p UserEdge.EdgeIdx'th operand of
+ /// \p UserEdge.UserTE, or nullptr if no such entry exists.
+ TreeEntry *getTreeEntry(const EdgeInfo &UserEdge) {
+ for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree)
+ if (TE->UserTreeIndex.UserTE == UserEdge.UserTE &&
+ TE->UserTreeIndex.EdgeIdx == UserEdge.EdgeIdx)
+ return TE.get();
+ return nullptr;
+ }
+
+ /// \returns true if the operand list \p InVL of a binary operator bundle can
+ /// be vectorized on its own. On success, the (deduplicated) scalars are
+ /// appended to \p MergedVL.
+ bool isInputVectorizable(ArrayRef<Value *> ParentVL, ArrayRef<Value *> InVL,
+ SmallVectorImpl<Value *> &MergedVL);
+
+ /// \returns true if both input operands (\p InLeft and \p InRight) of a
+ /// binary operator bundle \p InVL can be merged into a single wider operand
+ /// node. On success, \p MergedVL holds the merged scalars.
+ bool canMergeInputOperands(ArrayRef<Value *> InVL, ArrayRef<Value *> InLeft,
+ ArrayRef<Value *> InRight,
+ SmallVectorImpl<Value *> &MergedVL);
+
+ /// Tries to merge the input operands \p Left and \p Right of the binary
+ /// operator node \p TE (built from \p VL) into a single wider operand node.
+ /// \returns true on success, in which case the operand nodes have been built.
+ bool handleMergedOperands(TreeEntry *TE, ArrayRef<Value *> VL,
+ ArrayRef<Value *> Left, ArrayRef<Value *> Right,
+ unsigned Depth);
/// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
/// be vectorized to use the original vector (or aggregate "bitcast" to a
@@ -4343,6 +4387,8 @@ class slpvectorizer::BoUpSLP {
}
return false;
}
+ if (isMergedVL(VL))
+ return true;
return IsSame(Scalars, ReuseShuffleIndices);
}
@@ -4449,6 +4495,38 @@ class slpvectorizer::BoUpSLP {
/// other entry kinds.
SmallVector<unsigned, 1> StructEVIndices;
+ /// Set to true when the input operands of this (binary operator) node were
+ /// merged into a single wider operand node. See handleMergedOperands().
+ bool OperandsMerged = false;
+
+ /// Set to true for an operand node that was produced by merging the input
+ /// operands of its user node (see handleMergedOperands()).
+ bool MergedOp = false;
+
+ /// The merged list of scalars used to build the merged operand nodes of
+ /// this node. Only valid when \a OperandsMerged is true.
+ ValueList MergedVL;
+
+ /// Records the merged list of scalars used to build the merged operand
+ /// nodes of this node.
+ void setOperandsMerged(ArrayRef<Value *> VL) {
+ OperandsMerged = true;
+ MergedVL.assign(VL.begin(), VL.end());
+ }
+
+ /// Marks this operand node as one that was built by merging the input
+ /// operands of its user node.
+ void setMergedOp() { MergedOp = true; }
+
+ /// \returns true if this is a merged operand node and all values in \p VL
+ /// are part of its scalars.
+ bool isMergedVL(ArrayRef<Value *> VL) const {
+ if (!MergedOp)
+ return false;
+ SmallPtrSet<Value *, 4> Values(Scalars.begin(), Scalars.end());
+ return all_of(VL, [&](Value *V) { return Values.contains(V); });
+ }
+
private:
/// The operands of each instruction in each lane Operands[op_index][lane].
/// Note: This helps avoid the replication of the code that performs the
@@ -4468,6 +4546,7 @@ class slpvectorizer::BoUpSLP {
/// True if the node does not require scheduling.
bool DoesNotNeedToSchedule = false;
+ public:
/// Set this bundle's \p OpIdx'th operand to \p OpVL.
void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
if (Operands.size() < OpIdx + 1)
@@ -4614,7 +4693,7 @@ class slpvectorizer::BoUpSLP {
if (!ReorderIndices.empty())
FoundLane = ReorderIndices[FoundLane];
assert(FoundLane < Scalars.size() && "Couldn't find extract lane");
- if (ReuseShuffleIndices.empty())
+ if (ReuseShuffleIndices.empty() || MergedOp)
break;
if (auto *RIt = find(ReuseShuffleIndices, FoundLane);
RIt != ReuseShuffleIndices.end()) {
@@ -8451,6 +8530,14 @@ static bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
std::optional<BoUpSLP::OrdersType>
BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
bool IgnoreReorder) {
+ // An entry whose input operands were merged (handleMergedOperands), and the
+ // merged operand entries themselves, share a single operand subtree and
+ // carry ReuseShuffleIndices computed against the build-time (operand-driven)
+ // lane order. Reordering such an entry permutes its Scalars/reuse mask but
+ // not the shared operands' production order, desyncing them and producing a
+ // wrong shuffle. Keep merged entries in their build order.
+ if (TE.OperandsMerged || TE.MergedOp)
+ return std::nullopt;
// No need to reorder if need to shuffle reuses, still need to shuffle the
// node.
if (!TE.ReuseShuffleIndices.empty()) {
@@ -9878,6 +9965,10 @@ void BoUpSLP::buildExternalUses(
})) {
LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
<< ".\n");
+ // A merged operand node shares its scalars with its user node; the
+ // shared scalar is consumed by the merged vector, not extracted.
+ if (UseEntries[0]->MergedOp)
+ continue;
assert(none_of(UseEntries,
[](TreeEntry *UseEntry) {
return UseEntry->isGather();
@@ -11682,6 +11773,18 @@ getMainAltOpsNoStateVL(ArrayRef<Value *> VL) {
return std::make_pair(MainOp, AltOp);
}
+/// For a merged operand node (see handleMergedOperands()) the vectorized value
+/// is normally a shufflevector wrapping the real source operand; this returns
+/// the source so it can be fed into the shuffle builder. When the operand node
+/// had an empty reuse mask (or was constant-folded) the value may not be a
+/// shufflevector, so it is only unwrapped when it actually is one.
+static Value *getShuffleInput(bool MergedOp, Value *Vec) {
+ if (MergedOp && Vec)
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(Vec))
+ Vec = SVI->getOperand(0);
+ return Vec;
+}
+
/// Checks that every instruction appears once in the list and if not, packs
/// them, building \p ReuseShuffleIndices mask and mutating \p VL. The list of
/// unique scalars is extended by poison values to the whole register size.
@@ -11694,7 +11797,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
const TargetLibraryInfo &TLI,
const InstructionsState &S,
const BoUpSLP::EdgeInfo &UserTreeIdx,
- const BoUpSLP &R, bool BuildGatherOnly = true) {
+ const BoUpSLP &R, bool BuildGatherOnly = true,
+ bool MergedOp = false) {
// TODO: Reordering of struct types is not supported.
if (isa<StructType>(getValueType(VL.front()))) {
LLVM_DEBUG(dbgs() << "SLP: struct type in bundle.\n");
@@ -11740,6 +11844,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
unsigned NumUniqueScalarValues = UniqueValues.size();
if (NumUniqueScalarValues == VL.size()) {
ReuseShuffleIndices.clear();
+ R.updateReuseShuffleForMergedOps(UserTreeIdx, ReuseShuffleIndices,
+ UniquePositions, UniqueValues, MergedOp);
return true;
}
@@ -11909,8 +12015,13 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
} else {
// Better to use uniques + reshuffle.
LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
- VL = std::move(UniqueValues);
+ // For a merged operand node keep the full merged list of scalars; the
+ // reuse shuffle mask is recomputed against the user node below.
+ if (!MergedOp)
+ VL = std::move(UniqueValues);
}
+ R.updateReuseShuffleForMergedOps(UserTreeIdx, ReuseShuffleIndices,
+ UniquePositions, UniqueValues, MergedOp);
return true;
}
@@ -13145,9 +13256,162 @@ BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
return ScalarsVectorizationLegality(S, /*IsLegal=*/true);
}
+void BoUpSLP::updateReuseShuffleForMergedOps(
+ const EdgeInfo &UserTreeIdx, SmallVectorImpl<int> &ReuseShuffleIndices,
+ SmallDenseMap<Value *, unsigned, 16> &UniquePositions,
+ ArrayRef<Value *> UniqueValues, bool MergedOp) const {
+ if (!MergedOp)
+ return;
+ // The operand node is built from the merged scalar list, but the reuse
+ // shuffle mask must map the user node's operand lanes onto the (unique)
+ // merged scalars so that codegen reads the correct lane.
+ ArrayRef<Value *> InVL = UserTreeIdx.UserTE->getOperand(UserTreeIdx.EdgeIdx);
+ ReuseShuffleIndices.clear();
+ for (Value *V : InVL) {
+ auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
+ ReuseShuffleIndices.emplace_back(Res.first->second);
+ assert(!Res.second && "Merged operand value not found in unique values.");
+ }
+}
+
+bool BoUpSLP::isInputVectorizable(ArrayRef<Value *> ParentVL,
+ ArrayRef<Value *> InVL,
+ SmallVectorImpl<Value *> &MergedVL) {
+ SmallVector<Value *> VL(InVL.begin(), InVL.end());
+ assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
+
+ SmallVector<int> ReuseShuffleIndices;
+ SmallVector<Value *> UniqueValues;
+ auto TryToFindDuplicates = [&]() {
+ SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
+ for (Value *V : VL) {
+ if (isConstant(V)) {
+ ReuseShuffleIndices.emplace_back(
+ isa<UndefValue>(V) ? PoisonMaskElem : UniqueValues.size());
+ UniqueValues.emplace_back(V);
+ continue;
+ }
+ auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
+ ReuseShuffleIndices.emplace_back(Res.first->second);
+ if (Res.second)
+ UniqueValues.emplace_back(V);
+ }
+ size_t NumUniqueScalarValues = UniqueValues.size();
+ if (NumUniqueScalarValues == VL.size()) {
+ ReuseShuffleIndices.clear();
+ } else {
+ if (NumUniqueScalarValues <= 1 ||
+ (UniquePositions.size() == 1 &&
+ all_of(UniqueValues,
+ [](Value *V) {
+ return isa<UndefValue>(V) || !isConstant(V);
+ })) ||
+ !has_single_bit(NumUniqueScalarValues))
+ return false;
+ VL = UniqueValues;
+ }
+ return true;
+ };
+ InstructionsState S = getSameOpcode(VL, *TLI);
+ if (!S)
+ return false;
+ if (!isa<BinaryOperator>(S.getMainOp()))
+ return false;
+ // Check that every instruction appears once in this bundle.
+ if (!TryToFindDuplicates())
+ return false;
+
+ // Bail out if the unique operations in the input operand vector are not
+ // (at least) half of the parent VL - otherwise merging is not profitable.
+ if (PowerOf2Ceil(ParentVL.size()) / 2 < PowerOf2Ceil(VL.size()))
+ return false;
+ // The merged operands must be vectorizable on their own.
+ OrdersType CurrentOrder;
+ SmallVector<Value *> PointerOps;
+ StridedPtrInfo SPtrInfo;
+ SmallVector<int> ExpandShuffleMask;
+ TreeEntry::EntryState State = getScalarsVectorizationState(
+ S, VL, /*IsScatterVectorizeUserTE=*/false, CurrentOrder, PointerOps,
+ SPtrInfo, ExpandShuffleMask);
+ if (State != TreeEntry::Vectorize)
+ return false;
+
+ MergedVL.append(VL.begin(), VL.end());
+ return true;
+}
+
+bool BoUpSLP::canMergeInputOperands(ArrayRef<Value *> InVL,
+ ArrayRef<Value *> InLeft,
+ ArrayRef<Value *> InRight,
+ SmallVectorImpl<Value *> &MergedVL) {
+ // We expect the minimum size of the merged VL to be at least 4; smaller
+ // sizes are better vectorized with splats if not merged.
+ if (PowerOf2Ceil(InVL.size()) < 4)
+ return false;
+ if (!isInputVectorizable(InVL, InLeft, MergedVL) ||
+ !isInputVectorizable(InVL, InRight, MergedVL))
+ return false;
+ if (MergedVL.size() != InLeft.size() || MergedVL.size() != InRight.size())
+ return false;
+ // Both operand nodes are built from the single MergedVL bundle and are
+ // distinguished only by their reuse-shuffle masks selecting lanes out of one
+ // merged vector. For codegen to reconstruct each operand correctly:
+ // - every merged scalar must occupy a distinct lane (otherwise the two
+ // operand nodes can no longer be told apart by their reuse masks and they
+ // collapse to the same vector), and
+ // - the merged bundle must vectorize as a single uniform node. A
+ // mixed-opcode bundle would become an alt-shuffle node whose vectorized
+ // value is a shuffle of two different source vectors; the merged-operand
+ // codegen only unwraps a single source operand (getShuffleInput) and
+ // would silently drop the other half.
+ SmallPtrSet<Value *, 8> SeenVals;
+ for (Value *V : MergedVL)
+ if (!SeenVals.insert(V).second)
+ return false;
+ InstructionsState MergedS = getSameOpcode(MergedVL, *TLI);
+ if (!MergedS || MergedS.isAltShuffle())
+ return false;
+ OrdersType MergedOrder;
+ SmallVector<Value *> MergedPointerOps;
+ StridedPtrInfo MergedSPtrInfo;
+ SmallVector<int> MergedExpandShuffleMask;
+ return getScalarsVectorizationState(MergedS, MergedVL,
+ /*IsScatterVectorizeUserTE=*/false,
+ MergedOrder, MergedPointerOps,
+ MergedSPtrInfo,
+ MergedExpandShuffleMask) ==
+ TreeEntry::Vectorize;
+}
+
+bool BoUpSLP::handleMergedOperands(TreeEntry *TE, ArrayRef<Value *> VL,
+ ArrayRef<Value *> Left,
+ ArrayRef<Value *> Right, unsigned Depth) {
+ if (!SLPMergeInputOps)
+ return false;
+ SmallVector<Value *> MergedVL;
+ if (!canMergeInputOperands(VL, Left, Right, MergedVL))
+ return false;
+ LLVM_DEBUG(dbgs() << "SLP: Merging input operands in " << F->getName()
+ << "\n");
+
+ TE->setOperand(0, Left);
+ TE->setOperand(1, Right);
+ buildTreeRec(MergedVL, Depth + 1, {TE, 0}, /*InterleaveFactor=*/0,
+ /*MergedOp=*/true);
+ buildTreeRec(MergedVL, Depth + 1, {TE, 1}, /*InterleaveFactor=*/0,
+ /*MergedOp=*/true);
+ TE->setOperandsMerged(MergedVL);
+ TreeEntry *Op1 = getTreeEntry({TE, 0});
+ TreeEntry *Op2 = getTreeEntry({TE, 1});
+ assert(Op1 && Op2 && "Expected operand entries for merged operands.");
+ Op1->setMergedOp();
+ Op2->setMergedOp();
+ return true;
+}
+
void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
const EdgeInfo &UserTreeIdx,
- unsigned InterleaveFactor) {
+ unsigned InterleaveFactor, bool MergedOp) {
assert((allConstant(VLRef) || allSameType(VLRef)) && "Invalid types!");
SmallVector<int> ReuseShuffleIndices;
@@ -13215,7 +13479,8 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
}
if (Legality.tryToFindDuplicates())
(void)tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S,
- UserTreeIdx, *this);
+ UserTreeIdx, *this, /*BuildGatherOnly=*/true,
+ MergedOp);
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
@@ -13227,7 +13492,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Check that every instruction appears once in this bundle.
if (!tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx,
- *this, /*BuildGatherOnly=*/false)) {
+ *this, /*BuildGatherOnly=*/false, MergedOp)) {
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
@@ -13678,6 +13943,11 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
"(SelectInst/UnaryOperator/BinaryOperator/FreezeInst).\n";
TE->dump());
+ // Try to merge both input operands of a binary operator node into a
+ // single wider operand node when profitable.
+ if (isa<BinaryOperator>(VL0) && Operands.size() == 2 &&
+ handleMergedOperands(TE, VL, Operands[0], Operands[1], Depth))
+ return;
if (isa<BinaryOperator>(VL0) && isCommutative(VL0)) {
VLOperands Ops(VL, Operands, S, *this);
Ops.reorder();
@@ -16846,6 +17116,20 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(const TreeEntry *E,
unsigned Idx) const {
TreeEntry *Op = OperandsToTreeEntry.at({E, Idx});
+ // When E's input operands were merged (handleMergedOperands), the operand
+ // entry was built from E's merged VL and may then be reordered/deduplicated
+ // by the reorder machinery, so its scalar order no longer matches
+ // E->getOperand(Idx) (or even the stored MergedVL). getOperandEntry only
+ // needs to return the entry recorded for {E,Idx}; the lane permutation is
+ // tracked by ReorderIndices for codegen. Relax the order-sensitive sanity
+ // check to a value-set membership test for the merged case.
+ if (E->OperandsMerged) {
+ SmallPtrSet<Value *, 8> MergedVals(E->MergedVL.begin(), E->MergedVL.end());
+ assert(all_of(Op->Scalars,
+ [&](Value *V) { return MergedVals.contains(V); }) &&
+ "Operands mismatch!");
+ return Op;
+ }
assert(Op->isSame(E->getOperand(Idx)) && "Operands mismatch!");
return Op;
}
@@ -23011,6 +23295,27 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
// node. Cost is 0.
LLVM_DEBUG(dbgs() << "SLP: perfect diamond match for gather bundle "
<< shortBundleName(E->Scalars, E->Idx) << ".\n");
+ // If the matched front entry's vectorized value is still a postponed
+ // gather placeholder (a load from a poison pointer), a merged-operand
+ // entry that diamond-matches it must also be postponed - otherwise it
+ // would consume the placeholder. Emit an equivalent placeholder and
+ // postpone E. Codegen path only (ResTy == Value *).
+ if constexpr (std::is_same_v<ResTy, Value *>) {
+ const TreeEntry *MergedFrontTE = Entries.front().front();
+ if (auto *LI =
+ dyn_cast_or_null<LoadInst>(MergedFrontTE->VectorizedValue)) {
+ if (isa<PoisonValue>(LI->getPointerOperand()) &&
+ PostponedGathers.contains(MergedFrontTE) && E->MergedOp) {
+ PostponedGathers.insert(E);
+ auto *ResVecTy = getWidenedType(ScalarTy, E->getVectorFactor());
+ return Builder.CreateAlignedLoad(
+ ResVecTy,
+ PoisonValue::get(
+ PointerType::getUnqual(ScalarTy->getContext())),
+ MaybeAlign());
+ }
+ }
+ }
// Restore the mask for previous partially matched values.
Mask.resize(E->Scalars.size());
const TreeEntry *FrontTE = Entries.front().front();
@@ -23043,6 +23348,15 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
std::iota(Mask.begin(), Mask.end(), 0);
ShuffleBuilder.add(*FrontTE, Mask);
Res = ShuffleBuilder.finalize({}, {}, {});
+ } else if (E->MergedOp) {
+ // For merged-operand entries the vectorized value is a
+ // shufflevector; feed its source operand into the builder.
+ Value *Vec = FrontTE->VectorizedValue;
+ if (!Vec)
+ Vec = Constant::getNullValue(
+ getWidenedType(ScalarTy, FrontTE->getVectorFactor()));
+ ShuffleBuilder.add(getShuffleInput(E->MergedOp, Vec), Mask);
+ Res = ShuffleBuilder.finalize(E->getCommonMask(), {}, {});
} else {
ShuffleBuilder.add(*FrontTE, Mask);
Res = ShuffleBuilder.finalize(E->getCommonMask(), {}, {});
@@ -25002,6 +25316,16 @@ Value *BoUpSLP::vectorizeTree(
Value *Vec = E->VectorizedValue;
assert(Vec && "Can't find vectorizable value");
+ // For a merged-operand entry the vectorized value is a shufflevector
+ // wrapping the real source operand; extract from the source operand.
+ if (E->MergedOp) {
+ assert((isa<ShuffleVectorInst>(Vec) || E->ReuseShuffleIndices.empty() ||
+ isa<Constant>(Vec)) &&
+ "Unexpected vectorized value for merged operand.");
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(Vec))
+ Vec = SVI->getOperand(0);
+ }
+
Value *Lane = Builder.getInt32(ExternalUse.Lane);
auto ExtractAndExtendIfNeeded = [&](Value *Vec) {
if (isa<InsertValueInst>(Scalar))
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/merge-input-operands.ll b/llvm/test/Transforms/SLPVectorizer/X86/merge-input-operands.ll
index 16786b42e426e..661ee80898841 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/merge-input-operands.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/merge-input-operands.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=x86_64-unknown-linux -mcpu=znver4 -passes=slp-vectorizer < %s | FileCheck %s
+; RUN: opt -S -mtriple=x86_64-unknown-linux -mcpu=znver4 -passes=slp-vectorizer -slp-merge-inputops=true < %s | FileCheck %s --check-prefixes=CHECK,MERGE
+; RUN: opt -S -mtriple=x86_64-unknown-linux -mcpu=znver4 -passes=slp-vectorizer -slp-merge-inputops=false < %s | FileCheck %s --check-prefixes=CHECK,NOMERGE
; The MergeInputOps transform targets a binary-operator bundle whose two operand
; bundles each reuse a small set of scalars. Here a 4-wide fmul has
@@ -9,22 +10,34 @@
; [s0,s1,s2,s3] becomes one <4 x float> add feeding two reuse shuffles; with it
; disabled the sums are built as two separate <2 x float> adds.
define void @merge_products_of_sums(ptr %A, ptr %Out) {
-; CHECK-LABEL: @merge_products_of_sums(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
-; CHECK-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
-; CHECK-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x float> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x float> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = fmul <4 x float> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
-; CHECK-NEXT: ret void
+; MERGE-LABEL: @merge_products_of_sums(
+; MERGE-NEXT: entry:
+; MERGE-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A:%.*]], i64 4
+; MERGE-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 4
+; MERGE-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[P4]], align 4
+; MERGE-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[TMP0]], [[TMP1]]
+; MERGE-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; MERGE-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 3, i32 2>
+; MERGE-NEXT: [[TMP5:%.*]] = fmul <4 x float> [[TMP3]], [[TMP4]]
+; MERGE-NEXT: store <4 x float> [[TMP5]], ptr [[OUT:%.*]], align 4
+; MERGE-NEXT: ret void
+;
+; NOMERGE-LABEL: @merge_products_of_sums(
+; NOMERGE-NEXT: entry:
+; NOMERGE-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
+; NOMERGE-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
+; NOMERGE-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
+; NOMERGE-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
+; NOMERGE-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
+; NOMERGE-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
+; NOMERGE-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
+; NOMERGE-NEXT: [[TMP4:%.*]] = fadd <2 x float> [[TMP0]], [[TMP2]]
+; NOMERGE-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; NOMERGE-NEXT: [[TMP6:%.*]] = fadd <2 x float> [[TMP1]], [[TMP3]]
+; NOMERGE-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; NOMERGE-NEXT: [[TMP8:%.*]] = fmul <4 x float> [[TMP5]], [[TMP7]]
+; NOMERGE-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
+; NOMERGE-NEXT: ret void
;
entry:
%a0 = load float, ptr %A
@@ -178,22 +191,34 @@ entry:
; Flag propagation: every merged scalar (the four fadd sums) carries the same
; fast-math flags, so the single merged fadd must keep the full "fast" flag set.
define void @merge_fmf_uniform(ptr %A, ptr %Out) {
-; CHECK-LABEL: @merge_fmf_uniform(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
-; CHECK-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
-; CHECK-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x float> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <2 x float> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
-; CHECK-NEXT: ret void
+; MERGE-LABEL: @merge_fmf_uniform(
+; MERGE-NEXT: entry:
+; MERGE-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A:%.*]], i64 4
+; MERGE-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 4
+; MERGE-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[P4]], align 4
+; MERGE-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[TMP0]], [[TMP1]]
+; MERGE-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; MERGE-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 3, i32 2>
+; MERGE-NEXT: [[TMP5:%.*]] = fmul fast <4 x float> [[TMP3]], [[TMP4]]
+; MERGE-NEXT: store <4 x float> [[TMP5]], ptr [[OUT:%.*]], align 4
+; MERGE-NEXT: ret void
+;
+; NOMERGE-LABEL: @merge_fmf_uniform(
+; NOMERGE-NEXT: entry:
+; NOMERGE-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
+; NOMERGE-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
+; NOMERGE-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
+; NOMERGE-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
+; NOMERGE-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
+; NOMERGE-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
+; NOMERGE-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
+; NOMERGE-NEXT: [[TMP4:%.*]] = fadd fast <2 x float> [[TMP0]], [[TMP2]]
+; NOMERGE-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; NOMERGE-NEXT: [[TMP6:%.*]] = fadd fast <2 x float> [[TMP1]], [[TMP3]]
+; NOMERGE-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; NOMERGE-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP7]]
+; NOMERGE-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
+; NOMERGE-NEXT: ret void
;
entry:
%a0 = load float, ptr %A
@@ -236,22 +261,34 @@ entry:
; are "fast" and two are only "nnan ninf", so the single merged fadd must keep
; only the common subset ("nnan ninf"), not the full "fast" set.
define void @merge_fmf_mixed(ptr %A, ptr %Out) {
-; CHECK-LABEL: @merge_fmf_mixed(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
-; CHECK-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
-; CHECK-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x float> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP6:%.*]] = fadd nnan ninf <2 x float> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
-; CHECK-NEXT: ret void
+; MERGE-LABEL: @merge_fmf_mixed(
+; MERGE-NEXT: entry:
+; MERGE-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A:%.*]], i64 4
+; MERGE-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[A]], align 4
+; MERGE-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[P4]], align 4
+; MERGE-NEXT: [[TMP2:%.*]] = fadd nnan ninf <4 x float> [[TMP0]], [[TMP1]]
+; MERGE-NEXT: [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; MERGE-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 3, i32 2>
+; MERGE-NEXT: [[TMP5:%.*]] = fmul fast <4 x float> [[TMP3]], [[TMP4]]
+; MERGE-NEXT: store <4 x float> [[TMP5]], ptr [[OUT:%.*]], align 4
+; MERGE-NEXT: ret void
+;
+; NOMERGE-LABEL: @merge_fmf_mixed(
+; NOMERGE-NEXT: entry:
+; NOMERGE-NEXT: [[P2:%.*]] = getelementptr float, ptr [[A:%.*]], i64 2
+; NOMERGE-NEXT: [[P4:%.*]] = getelementptr float, ptr [[A]], i64 4
+; NOMERGE-NEXT: [[P6:%.*]] = getelementptr float, ptr [[A]], i64 6
+; NOMERGE-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4
+; NOMERGE-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[P2]], align 4
+; NOMERGE-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[P4]], align 4
+; NOMERGE-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[P6]], align 4
+; NOMERGE-NEXT: [[TMP4:%.*]] = fadd fast <2 x float> [[TMP0]], [[TMP2]]
+; NOMERGE-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; NOMERGE-NEXT: [[TMP6:%.*]] = fadd nnan ninf <2 x float> [[TMP1]], [[TMP3]]
+; NOMERGE-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; NOMERGE-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP5]], [[TMP7]]
+; NOMERGE-NEXT: store <4 x float> [[TMP8]], ptr [[OUT:%.*]], align 4
+; NOMERGE-NEXT: ret void
;
entry:
%a0 = load float, ptr %A
@@ -293,22 +330,34 @@ entry:
; Flag propagation: integer wrap flags. Every merged scalar (the four adds)
; carries "nsw nuw", so the single merged add must keep both flags.
define void @merge_int_wrap_uniform(ptr %A, ptr %Out) {
-; CHECK-LABEL: @merge_int_wrap_uniform(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
-; CHECK-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
-; CHECK-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw <2 x i32> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
-; CHECK-NEXT: ret void
+; MERGE-LABEL: @merge_int_wrap_uniform(
+; MERGE-NEXT: entry:
+; MERGE-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 4
+; MERGE-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; MERGE-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[P4]], align 4
+; MERGE-NEXT: [[TMP2:%.*]] = add nuw nsw <4 x i32> [[TMP0]], [[TMP1]]
+; MERGE-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; MERGE-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 3, i32 2>
+; MERGE-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[TMP3]], [[TMP4]]
+; MERGE-NEXT: store <4 x i32> [[TMP5]], ptr [[OUT:%.*]], align 4
+; MERGE-NEXT: ret void
+;
+; NOMERGE-LABEL: @merge_int_wrap_uniform(
+; NOMERGE-NEXT: entry:
+; NOMERGE-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
+; NOMERGE-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
+; NOMERGE-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
+; NOMERGE-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; NOMERGE-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
+; NOMERGE-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
+; NOMERGE-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
+; NOMERGE-NEXT: [[TMP4:%.*]] = add nuw nsw <2 x i32> [[TMP0]], [[TMP2]]
+; NOMERGE-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; NOMERGE-NEXT: [[TMP6:%.*]] = add nuw nsw <2 x i32> [[TMP1]], [[TMP3]]
+; NOMERGE-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; NOMERGE-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
+; NOMERGE-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
+; NOMERGE-NEXT: ret void
;
entry:
%a0 = load i32, ptr %A
@@ -350,22 +399,34 @@ entry:
; Flag propagation: mixed integer wrap flags. Two adds carry "nsw nuw" and two
; carry only "nsw", so the single merged add must keep only "nsw".
define void @merge_int_wrap_mixed(ptr %A, ptr %Out) {
-; CHECK-LABEL: @merge_int_wrap_mixed(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
-; CHECK-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
-; CHECK-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw <2 x i32> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP6:%.*]] = add nsw <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
-; CHECK-NEXT: ret void
+; MERGE-LABEL: @merge_int_wrap_mixed(
+; MERGE-NEXT: entry:
+; MERGE-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 4
+; MERGE-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; MERGE-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[P4]], align 4
+; MERGE-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP0]], [[TMP1]]
+; MERGE-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; MERGE-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 3, i32 2>
+; MERGE-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[TMP3]], [[TMP4]]
+; MERGE-NEXT: store <4 x i32> [[TMP5]], ptr [[OUT:%.*]], align 4
+; MERGE-NEXT: ret void
+;
+; NOMERGE-LABEL: @merge_int_wrap_mixed(
+; NOMERGE-NEXT: entry:
+; NOMERGE-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
+; NOMERGE-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
+; NOMERGE-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
+; NOMERGE-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; NOMERGE-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
+; NOMERGE-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
+; NOMERGE-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
+; NOMERGE-NEXT: [[TMP4:%.*]] = add nuw nsw <2 x i32> [[TMP0]], [[TMP2]]
+; NOMERGE-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; NOMERGE-NEXT: [[TMP6:%.*]] = add nsw <2 x i32> [[TMP1]], [[TMP3]]
+; NOMERGE-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; NOMERGE-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
+; NOMERGE-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
+; NOMERGE-NEXT: ret void
;
entry:
%a0 = load i32, ptr %A
@@ -407,22 +468,34 @@ entry:
; Flag propagation: the "exact" flag on shifts. Every merged scalar (the four
; lshr ops) is "exact", so the single merged lshr must keep "exact".
define void @merge_exact_shift(ptr %A, ptr %Out) {
-; CHECK-LABEL: @merge_exact_shift(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
-; CHECK-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
-; CHECK-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = lshr exact <2 x i32> [[TMP0]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP6:%.*]] = lshr exact <2 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
-; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
-; CHECK-NEXT: ret void
+; MERGE-LABEL: @merge_exact_shift(
+; MERGE-NEXT: entry:
+; MERGE-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 4
+; MERGE-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[A]], align 4
+; MERGE-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[P4]], align 4
+; MERGE-NEXT: [[TMP2:%.*]] = lshr exact <4 x i32> [[TMP0]], [[TMP1]]
+; MERGE-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; MERGE-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 3, i32 2>
+; MERGE-NEXT: [[TMP5:%.*]] = mul <4 x i32> [[TMP3]], [[TMP4]]
+; MERGE-NEXT: store <4 x i32> [[TMP5]], ptr [[OUT:%.*]], align 4
+; MERGE-NEXT: ret void
+;
+; NOMERGE-LABEL: @merge_exact_shift(
+; NOMERGE-NEXT: entry:
+; NOMERGE-NEXT: [[P2:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 2
+; NOMERGE-NEXT: [[P4:%.*]] = getelementptr i32, ptr [[A]], i64 4
+; NOMERGE-NEXT: [[P6:%.*]] = getelementptr i32, ptr [[A]], i64 6
+; NOMERGE-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[A]], align 4
+; NOMERGE-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P2]], align 4
+; NOMERGE-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[P4]], align 4
+; NOMERGE-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[P6]], align 4
+; NOMERGE-NEXT: [[TMP4:%.*]] = lshr exact <2 x i32> [[TMP0]], [[TMP2]]
+; NOMERGE-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; NOMERGE-NEXT: [[TMP6:%.*]] = lshr exact <2 x i32> [[TMP1]], [[TMP3]]
+; NOMERGE-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; NOMERGE-NEXT: [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
+; NOMERGE-NEXT: store <4 x i32> [[TMP8]], ptr [[OUT:%.*]], align 4
+; NOMERGE-NEXT: ret void
;
entry:
%a0 = load i32, ptr %A
More information about the llvm-commits
mailing list