[llvm] [AMDGPU] Limit the fmul fusion discount to a matching context type (PR #229845)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 11 13:37:36 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/229845
>From ebfca51713dd3cd8fccfa14d64edfb3a3a55e86a Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 7 Oct 2026 11:47:05 +0200
Subject: [PATCH 1/2] [AMDGPU] Limit the fmul fusion discount to a matching
context type
The fmul is free when its context instruction feeds a fusable fadd or
fsub. A vector fmul priced with a scalar lane as the context inherits
that fusion only when it is emitted lane by lane. On a packed type the
scalar user does not show that the vector fmul feeds a vector fadd, and
products extracted into a scalar fadd chain keep the packed fmul while
the fma is lost.
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 7 +-
.../AMDGPU/alt-fmul-fadd-cost.ll | 18 ++--
.../SLPVectorizer/AMDGPU/complex-mul-fma.ll | 2 +-
.../AMDGPU/fma-operand-contract-selection.ll | 42 +++++++--
.../AMDGPU/fmul-extract-fadd-chain.ll | 88 +++++++++++++++++++
.../ordered-reduction-coalesced-loads.ll | 80 +++++++++++++----
.../AMDGPU/ordered-reduction-fma-fusion.ll | 75 ++++++++--------
7 files changed, 239 insertions(+), 73 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-extract-fadd-chain.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 13c62bb0781c5..4952766f1c80b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -635,8 +635,11 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost(
case ISD::FMUL:
// Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for
// fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
- // fused operation.
- if (CtxI && CtxI->hasOneUse()) {
+ // fused operation. A vector fmul priced with a scalar context fuses with
+ // the scalar fadd|fsub only when it is emitted lane by lane.
+ if (CtxI && CtxI->hasOneUse() &&
+ (CtxI->getType() == Ty ||
+ getMaximumVF(Ty->getScalarSizeInBits(), Opcode) == 1)) {
const auto *FAddSub = dyn_cast<BinaryOperator>(*CtxI->user_begin());
if (FAddSub &&
(FAddSub->getOpcode() == Instruction::FAdd ||
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/alt-fmul-fadd-cost.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/alt-fmul-fadd-cost.ll
index 0968c8f98c310..3edd0b6a35ba5 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/alt-fmul-fadd-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/alt-fmul-fadd-cost.ll
@@ -10,17 +10,13 @@
define <2 x float> @alt_fmul_fadd(float %a0, float %a1, float %b0, float %b1, float %c0, float %c1) {
; CHECK-LABEL: define <2 x float> @alt_fmul_fadd(
; CHECK-SAME: float [[A0:%.*]], float [[A1:%.*]], float [[B0:%.*]], float [[B1:%.*]], float [[C0:%.*]], float [[C1:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[A0]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> [[TMP1]], float [[A1]], i64 1
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[B0]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[B1]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = fmul contract <2 x float> [[TMP2]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = fadd contract <2 x float> [[TMP2]], [[TMP4]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> [[TMP6]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x float> poison, float [[C0]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x float> [[TMP8]], float [[C1]], i64 1
-; CHECK-NEXT: [[TMP10:%.*]] = fsub contract <2 x float> [[TMP7]], [[TMP9]]
-; CHECK-NEXT: ret <2 x float> [[TMP10]]
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[A0]], [[B0]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[A1]], [[B1]]
+; CHECK-NEXT: [[SUB0:%.*]] = fsub contract float [[MUL]], [[C0]]
+; CHECK-NEXT: [[SUB1:%.*]] = fsub contract float [[ADD]], [[C1]]
+; CHECK-NEXT: [[INS0:%.*]] = insertelement <2 x float> poison, float [[SUB0]], i64 0
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <2 x float> [[INS0]], float [[SUB1]], i64 1
+; CHECK-NEXT: ret <2 x float> [[INS1]]
;
%mul = fmul contract float %a0, %b0
%add = fadd contract float %a1, %b1
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/complex-mul-fma.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/complex-mul-fma.ll
index 3d3b2c151ecf0..37de983451669 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/complex-mul-fma.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/complex-mul-fma.ll
@@ -1,7 +1,7 @@
; RUN: opt -passes=slp-vectorizer -mtriple=amdgpu9.42-amd-amdhsa -pass-remarks=slp-vectorizer -disable-output < %s 2>&1 | FileCheck %s
; The second fmul of each fsub/fadd does not fuse, so packing those pays off.
-; CHECK: Stores SLP vectorized with cost -2
+; CHECK: Stores SLP vectorized with cost -1
define void @cmul_store(ptr addrspace(1) %out, float %xr, float %xi, float %wr, float %wi) {
%xr.wr = fmul contract float %xr, %wr
%xi.wi = fmul contract float %xi, %wi
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
index 9b37c518981e3..5d86fb9fb8825 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll
@@ -241,12 +241,42 @@ define void @mixed_contract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b,
; THR15-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0
; THR15-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0
; THR15-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0
-; THR15-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4
-; THR15-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4
-; THR15-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4
-; THR15-NEXT: [[TMP3:%.*]] = fmul <4 x float> [[TMP0]], [[TMP1]]
-; THR15-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]]
-; THR15-NEXT: store <4 x float> [[TMP4]], ptr [[DP0]], align 4
+; THR15-NEXT: [[A0:%.*]] = load float, ptr [[AP0]], align 4
+; THR15-NEXT: [[B0:%.*]] = load float, ptr [[BP0]], align 4
+; THR15-NEXT: [[C0:%.*]] = load float, ptr [[CP0]], align 4
+; THR15-NEXT: [[M0:%.*]] = fmul contract float [[A0]], [[B0]]
+; THR15-NEXT: [[R0:%.*]] = fadd contract float [[M0]], [[C0]]
+; THR15-NEXT: store float [[R0]], ptr [[DP0]], align 4
+; THR15-NEXT: [[AP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
+; THR15-NEXT: [[BP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 1
+; THR15-NEXT: [[CP1:%.*]] = getelementptr inbounds float, ptr [[C]], i64 1
+; THR15-NEXT: [[DP1:%.*]] = getelementptr inbounds float, ptr [[D]], i64 1
+; THR15-NEXT: [[A1:%.*]] = load float, ptr [[AP1]], align 4
+; THR15-NEXT: [[B1:%.*]] = load float, ptr [[BP1]], align 4
+; THR15-NEXT: [[C1:%.*]] = load float, ptr [[CP1]], align 4
+; THR15-NEXT: [[M1:%.*]] = fmul float [[A1]], [[B1]]
+; THR15-NEXT: [[R1:%.*]] = fadd contract float [[M1]], [[C1]]
+; THR15-NEXT: store float [[R1]], ptr [[DP1]], align 4
+; THR15-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
+; THR15-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2
+; THR15-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2
+; THR15-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2
+; THR15-NEXT: [[A2:%.*]] = load float, ptr [[AP2]], align 4
+; THR15-NEXT: [[B2:%.*]] = load float, ptr [[BP2]], align 4
+; THR15-NEXT: [[C2:%.*]] = load float, ptr [[CP2]], align 4
+; THR15-NEXT: [[M2:%.*]] = fmul float [[A2]], [[B2]]
+; THR15-NEXT: [[R2:%.*]] = fadd contract float [[M2]], [[C2]]
+; THR15-NEXT: store float [[R2]], ptr [[DP2]], align 4
+; THR15-NEXT: [[AP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; THR15-NEXT: [[BP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 3
+; THR15-NEXT: [[CP3:%.*]] = getelementptr inbounds float, ptr [[C]], i64 3
+; THR15-NEXT: [[DP3:%.*]] = getelementptr inbounds float, ptr [[D]], i64 3
+; THR15-NEXT: [[A3:%.*]] = load float, ptr [[AP3]], align 4
+; THR15-NEXT: [[B3:%.*]] = load float, ptr [[BP3]], align 4
+; THR15-NEXT: [[C3:%.*]] = load float, ptr [[CP3]], align 4
+; THR15-NEXT: [[M3:%.*]] = fmul float [[A3]], [[B3]]
+; THR15-NEXT: [[R3:%.*]] = fadd contract float [[M3]], [[C3]]
+; THR15-NEXT: store float [[R3]], ptr [[DP3]], align 4
; THR15-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-extract-fadd-chain.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-extract-fadd-chain.ll
new file mode 100644
index 0000000000000..2cc40f25d7339
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-extract-fadd-chain.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck %s --check-prefix=GFX1250
+; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck %s --check-prefix=GFX1030
+
+; The products feed a scalar contract fadd chain one lane at a time. A packed
+; vector fmul cannot fuse into those fadds, so it costs real instructions and
+; the scalar products, which do fuse, stay. Without packed f32 instructions
+; the vector fmul is emitted lane by lane and every lane still fuses, so the
+; loads and converts remain worth packing.
+
+define float @dot4_i16_ordered(ptr addrspace(1) %a, ptr addrspace(1) %b, float %acc) {
+; GFX1250-LABEL: define float @dot4_i16_ordered(
+; GFX1250-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX1250-NEXT: [[ENTRY:.*:]]
+; GFX1250-NEXT: [[A1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 2
+; GFX1250-NEXT: [[A2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 4
+; GFX1250-NEXT: [[A3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 6
+; GFX1250-NEXT: [[B1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 4
+; GFX1250-NEXT: [[B2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 8
+; GFX1250-NEXT: [[B3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 12
+; GFX1250-NEXT: [[I0:%.*]] = load i16, ptr addrspace(1) [[A]], align 2
+; GFX1250-NEXT: [[I1:%.*]] = load i16, ptr addrspace(1) [[A1]], align 2
+; GFX1250-NEXT: [[I2:%.*]] = load i16, ptr addrspace(1) [[A2]], align 2
+; GFX1250-NEXT: [[I3:%.*]] = load i16, ptr addrspace(1) [[A3]], align 2
+; GFX1250-NEXT: [[F0:%.*]] = sitofp i16 [[I0]] to float
+; GFX1250-NEXT: [[F1:%.*]] = sitofp i16 [[I1]] to float
+; GFX1250-NEXT: [[F2:%.*]] = sitofp i16 [[I2]] to float
+; GFX1250-NEXT: [[F3:%.*]] = sitofp i16 [[I3]] to float
+; GFX1250-NEXT: [[W0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
+; GFX1250-NEXT: [[W1:%.*]] = load float, ptr addrspace(1) [[B1]], align 4
+; GFX1250-NEXT: [[W2:%.*]] = load float, ptr addrspace(1) [[B2]], align 4
+; GFX1250-NEXT: [[W3:%.*]] = load float, ptr addrspace(1) [[B3]], align 4
+; GFX1250-NEXT: [[M0:%.*]] = fmul contract float [[F0]], [[W0]]
+; GFX1250-NEXT: [[M1:%.*]] = fmul contract float [[F1]], [[W1]]
+; GFX1250-NEXT: [[M2:%.*]] = fmul contract float [[F2]], [[W2]]
+; GFX1250-NEXT: [[M3:%.*]] = fmul contract float [[F3]], [[W3]]
+; GFX1250-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
+; GFX1250-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
+; GFX1250-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
+; GFX1250-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
+; GFX1250-NEXT: ret float [[S3]]
+;
+; GFX1030-LABEL: define float @dot4_i16_ordered(
+; GFX1030-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX1030-NEXT: [[ENTRY:.*:]]
+; GFX1030-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
+; GFX1030-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
+; GFX1030-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
+; GFX1030-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; GFX1030-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
+; GFX1030-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
+; GFX1030-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
+; GFX1030-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
+; GFX1030-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
+; GFX1030-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
+; GFX1030-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; GFX1030-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
+; GFX1030-NEXT: ret float [[S3]]
+;
+entry:
+ %a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
+ %a2 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 4
+ %a3 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 6
+ %b1 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 4
+ %b2 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 8
+ %b3 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 12
+ %i0 = load i16, ptr addrspace(1) %a, align 2
+ %i1 = load i16, ptr addrspace(1) %a1, align 2
+ %i2 = load i16, ptr addrspace(1) %a2, align 2
+ %i3 = load i16, ptr addrspace(1) %a3, align 2
+ %f0 = sitofp i16 %i0 to float
+ %f1 = sitofp i16 %i1 to float
+ %f2 = sitofp i16 %i2 to float
+ %f3 = sitofp i16 %i3 to float
+ %w0 = load float, ptr addrspace(1) %b, align 4
+ %w1 = load float, ptr addrspace(1) %b1, align 4
+ %w2 = load float, ptr addrspace(1) %b2, align 4
+ %w3 = load float, ptr addrspace(1) %b3, align 4
+ %m0 = fmul contract float %f0, %w0
+ %m1 = fmul contract float %f1, %w1
+ %m2 = fmul contract float %f2, %w2
+ %m3 = fmul contract float %f3, %w3
+ %s0 = fadd contract float %acc, %m0
+ %s1 = fadd contract float %s0, %m1
+ %s2 = fadd contract float %s1, %m2
+ %s3 = fadd contract float %s2, %m3
+ ret float %s3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
index 7a790a37acba4..be90212aae68a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
@@ -167,22 +167,70 @@ declare i64 @_Z13get_global_idj(i32 noundef)
; coalesced loads and stay scalar too.
define float @dot_i16_window(ptr addrspace(1) %a, ptr addrspace(1) %b, float %acc) {
-; CHECK-LABEL: define float @dot_i16_window(
-; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
-; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[M0:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
-; CHECK-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
-; CHECK-NEXT: [[M1:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
-; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
-; CHECK-NEXT: [[M2:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
-; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
-; CHECK-NEXT: [[M3:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
-; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
-; CHECK-NEXT: ret float [[S3]]
+; GFX9-LABEL: define float @dot_i16_window(
+; GFX9-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
+; GFX9-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
+; GFX9-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
+; GFX9-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; GFX9-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
+; GFX9-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
+; GFX9-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
+; GFX9-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
+; GFX9-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
+; GFX9-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
+; GFX9-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; GFX9-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
+; GFX9-NEXT: ret float [[S3]]
+;
+; GFX1030-LABEL: define float @dot_i16_window(
+; GFX1030-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; GFX1030-NEXT: [[ENTRY:.*:]]
+; GFX1030-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
+; GFX1030-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
+; GFX1030-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
+; GFX1030-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; GFX1030-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
+; GFX1030-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
+; GFX1030-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
+; GFX1030-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
+; GFX1030-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
+; GFX1030-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
+; GFX1030-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; GFX1030-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
+; GFX1030-NEXT: ret float [[S3]]
+;
+; GFX1250-LABEL: define float @dot_i16_window(
+; GFX1250-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; GFX1250-NEXT: [[ENTRY:.*:]]
+; GFX1250-NEXT: [[A1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 2
+; GFX1250-NEXT: [[A2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 4
+; GFX1250-NEXT: [[A3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 6
+; GFX1250-NEXT: [[B1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 4
+; GFX1250-NEXT: [[B2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 8
+; GFX1250-NEXT: [[B3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 12
+; GFX1250-NEXT: [[I0:%.*]] = load i16, ptr addrspace(1) [[A]], align 2
+; GFX1250-NEXT: [[I1:%.*]] = load i16, ptr addrspace(1) [[A1]], align 2
+; GFX1250-NEXT: [[I2:%.*]] = load i16, ptr addrspace(1) [[A2]], align 2
+; GFX1250-NEXT: [[I3:%.*]] = load i16, ptr addrspace(1) [[A3]], align 2
+; GFX1250-NEXT: [[F0:%.*]] = sitofp i16 [[I0]] to float
+; GFX1250-NEXT: [[F1:%.*]] = sitofp i16 [[I1]] to float
+; GFX1250-NEXT: [[F2:%.*]] = sitofp i16 [[I2]] to float
+; GFX1250-NEXT: [[F3:%.*]] = sitofp i16 [[I3]] to float
+; GFX1250-NEXT: [[W0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
+; GFX1250-NEXT: [[W1:%.*]] = load float, ptr addrspace(1) [[B1]], align 4
+; GFX1250-NEXT: [[W2:%.*]] = load float, ptr addrspace(1) [[B2]], align 4
+; GFX1250-NEXT: [[W3:%.*]] = load float, ptr addrspace(1) [[B3]], align 4
+; GFX1250-NEXT: [[M0:%.*]] = fmul contract float [[F0]], [[W0]]
+; GFX1250-NEXT: [[M1:%.*]] = fmul contract float [[F1]], [[W1]]
+; GFX1250-NEXT: [[M2:%.*]] = fmul contract float [[F2]], [[W2]]
+; GFX1250-NEXT: [[M3:%.*]] = fmul contract float [[F3]], [[W3]]
+; GFX1250-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
+; GFX1250-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
+; GFX1250-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
+; GFX1250-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
+; GFX1250-NEXT: ret float [[S3]]
;
entry:
%a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
index 6a7a93a44f8c9..a25409530b4bb 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-fma-fusion.ll
@@ -8,44 +8,44 @@
define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-LABEL: @conv_contract(
-; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT:%.*]], i64 0
-; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK:%.*]], i64 0
-; CHECK-NEXT: [[IP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 4
+; CHECK-NEXT: [[IP4:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT:%.*]], i64 0
; CHECK-NEXT: [[IV4:%.*]] = load float, ptr addrspace(1) [[IP4]], align 4
-; CHECK-NEXT: [[IP5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 8
+; CHECK-NEXT: [[MP1:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK:%.*]], i64 0
+; CHECK-NEXT: [[MV0:%.*]] = load float, ptr addrspace(4) [[MP1]], align 4
+; CHECK-NEXT: [[PROD0:%.*]] = fmul contract float [[IV4]], [[MV0]]
+; CHECK-NEXT: [[IP0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 1
+; CHECK-NEXT: [[MP0:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 1
+; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 8
+; CHECK-NEXT: [[IP5:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 12
; CHECK-NEXT: [[IV5:%.*]] = load float, ptr addrspace(1) [[IP5]], align 4
-; CHECK-NEXT: [[IP6:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 9
-; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 11
-; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16
-; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 18
-; CHECK-NEXT: [[IP14:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 20
+; CHECK-NEXT: [[IP14:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 16
; CHECK-NEXT: [[IV14:%.*]] = load float, ptr addrspace(1) [[IP14]], align 4
-; CHECK-NEXT: [[IP15:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 24
-; CHECK-NEXT: [[IV15:%.*]] = load float, ptr addrspace(1) [[IP15]], align 4
+; CHECK-NEXT: [[IP8:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 17
+; CHECK-NEXT: [[IP10:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 19
+; CHECK-NEXT: [[IP12:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 24
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[IP0]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(1) [[IP6]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[IP6]], align 4
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr addrspace(1) [[IP8]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr addrspace(1) [[IP10]], align 4
; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr addrspace(1) [[IP12]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = load <16 x float>, ptr addrspace(4) [[MP0]], align 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x float> poison, float [[IV4]], i64 4
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x float> [[TMP7]], float [[IV5]], i64 5
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <16 x float> [[TMP8]], float [[IV14]], i64 14
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <16 x float> [[TMP22]], float [[IV15]], i64 15
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x float> poison, float [[IV5]], i64 8
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <16 x float> [[TMP7]], float [[IV14]], i64 9
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x float> [[TMP23]], <16 x float> [[TMP11]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x float> [[TMP24]], <16 x float> [[TMP25]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x float> [[TMP16]], <16 x float> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x float> [[TMP18]], <16 x float> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP20]], [[TMP6]]
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP20]], <16 x float> [[TMP22]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; CHECK-NEXT: [[TMP21:%.*]] = fmul contract <16 x float> [[TMP24]], [[TMP6]]
; CHECK-NEXT: [[ACC14:%.*]] = extractelement <16 x float> [[TMP21]], i64 0
+; CHECK-NEXT: [[ACC1:%.*]] = fadd contract float [[PROD0]], [[ACC14]]
; CHECK-NEXT: [[PROD15:%.*]] = extractelement <16 x float> [[TMP21]], i64 1
-; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC14]], [[PROD15]]
+; CHECK-NEXT: [[ACC15:%.*]] = fadd contract float [[ACC1]], [[PROD15]]
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP21]], i64 2
; CHECK-NEXT: [[ACC16:%.*]] = fadd contract float [[ACC15]], [[TMP9]]
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP21]], i64 3
@@ -74,15 +74,22 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[ACC25:%.*]] = fadd contract float [[ACC13]], [[TMP36]]
; CHECK-NEXT: [[TMP37:%.*]] = extractelement <16 x float> [[TMP21]], i64 15
; CHECK-NEXT: [[ACC26:%.*]] = fadd contract float [[ACC25]], [[TMP37]]
-; CHECK-NEXT: [[IP16:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 25
-; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 16
+; CHECK-NEXT: [[IP17:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 26
+; CHECK-NEXT: [[MP16:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 17
+; CHECK-NEXT: [[IP19:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 28
+; CHECK-NEXT: [[IV19:%.*]] = load float, ptr addrspace(1) [[IP19]], align 4
; CHECK-NEXT: [[IP20:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 32
-; CHECK-NEXT: [[TMP38:%.*]] = load <4 x float>, ptr addrspace(1) [[IP16]], align 4
-; CHECK-NEXT: [[TMP39:%.*]] = load <4 x float>, ptr addrspace(1) [[IP20]], align 4
+; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP20]], align 4
+; CHECK-NEXT: [[IP21:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 33
+; CHECK-NEXT: [[TMP38:%.*]] = load <2 x float>, ptr addrspace(1) [[IP17]], align 4
+; CHECK-NEXT: [[TMP39:%.*]] = load <4 x float>, ptr addrspace(1) [[IP21]], align 4
; CHECK-NEXT: [[TMP40:%.*]] = load <8 x float>, ptr addrspace(4) [[MP16]], align 4
-; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP41:%.*]] = insertelement <8 x float> poison, float [[IV19]], i64 2
+; CHECK-NEXT: [[TMP53:%.*]] = insertelement <8 x float> [[TMP41]], float [[IV20]], i64 3
; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <4 x float> [[TMP39]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <4 x float> [[TMP38]], <4 x float> [[TMP39]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <8 x float> [[TMP53]], <8 x float> [[TMP42]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <2 x float> [[TMP38]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <8 x float> [[TMP54]], <8 x float> [[TMP55]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP44:%.*]] = fmul contract <8 x float> [[TMP43]], [[TMP40]]
; CHECK-NEXT: [[TMP45:%.*]] = extractelement <8 x float> [[TMP44]], i64 0
; CHECK-NEXT: [[ACC27:%.*]] = fadd contract float [[ACC26]], [[TMP45]]
@@ -99,13 +106,7 @@ define float @conv_contract(ptr addrspace(1) %input, ptr addrspace(4) %mask) {
; CHECK-NEXT: [[TMP51:%.*]] = extractelement <8 x float> [[TMP44]], i64 6
; CHECK-NEXT: [[ACC22:%.*]] = fadd contract float [[ACC21]], [[TMP51]]
; CHECK-NEXT: [[TMP52:%.*]] = extractelement <8 x float> [[TMP44]], i64 7
-; CHECK-NEXT: [[ACC23:%.*]] = fadd contract float [[ACC22]], [[TMP52]]
-; CHECK-NEXT: [[IP24:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[INPUT]], i64 36
-; CHECK-NEXT: [[IV20:%.*]] = load float, ptr addrspace(1) [[IP24]], align 4
-; CHECK-NEXT: [[MP20:%.*]] = getelementptr inbounds float, ptr addrspace(4) [[MASK]], i64 24
-; CHECK-NEXT: [[MV20:%.*]] = load float, ptr addrspace(4) [[MP20]], align 4
-; CHECK-NEXT: [[PROD20:%.*]] = fmul contract float [[IV20]], [[MV20]]
-; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC23]], [[PROD20]]
+; CHECK-NEXT: [[ACC24:%.*]] = fadd contract float [[ACC22]], [[TMP52]]
; CHECK-NEXT: ret float [[ACC24]]
;
%ip0 = getelementptr inbounds float, ptr addrspace(1) %input, i64 0
>From 48397d76650afd48a9bb83ff297e728732b24e1a Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Wed, 7 Oct 2026 20:11:45 +0200
Subject: [PATCH 2/2] update tests
---
.../AMDGPU/fmul-extract-fadd-chain.ll | 88 -------------------
.../ordered-reduction-coalesced-loads.ll | 80 ++++-------------
2 files changed, 16 insertions(+), 152 deletions(-)
delete mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-extract-fadd-chain.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-extract-fadd-chain.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-extract-fadd-chain.ll
deleted file mode 100644
index 2cc40f25d7339..0000000000000
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fmul-extract-fadd-chain.ll
+++ /dev/null
@@ -1,88 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck %s --check-prefix=GFX1250
-; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck %s --check-prefix=GFX1030
-
-; The products feed a scalar contract fadd chain one lane at a time. A packed
-; vector fmul cannot fuse into those fadds, so it costs real instructions and
-; the scalar products, which do fuse, stay. Without packed f32 instructions
-; the vector fmul is emitted lane by lane and every lane still fuses, so the
-; loads and converts remain worth packing.
-
-define float @dot4_i16_ordered(ptr addrspace(1) %a, ptr addrspace(1) %b, float %acc) {
-; GFX1250-LABEL: define float @dot4_i16_ordered(
-; GFX1250-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) #[[ATTR0:[0-9]+]] {
-; GFX1250-NEXT: [[ENTRY:.*:]]
-; GFX1250-NEXT: [[A1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 2
-; GFX1250-NEXT: [[A2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 4
-; GFX1250-NEXT: [[A3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 6
-; GFX1250-NEXT: [[B1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 4
-; GFX1250-NEXT: [[B2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 8
-; GFX1250-NEXT: [[B3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 12
-; GFX1250-NEXT: [[I0:%.*]] = load i16, ptr addrspace(1) [[A]], align 2
-; GFX1250-NEXT: [[I1:%.*]] = load i16, ptr addrspace(1) [[A1]], align 2
-; GFX1250-NEXT: [[I2:%.*]] = load i16, ptr addrspace(1) [[A2]], align 2
-; GFX1250-NEXT: [[I3:%.*]] = load i16, ptr addrspace(1) [[A3]], align 2
-; GFX1250-NEXT: [[F0:%.*]] = sitofp i16 [[I0]] to float
-; GFX1250-NEXT: [[F1:%.*]] = sitofp i16 [[I1]] to float
-; GFX1250-NEXT: [[F2:%.*]] = sitofp i16 [[I2]] to float
-; GFX1250-NEXT: [[F3:%.*]] = sitofp i16 [[I3]] to float
-; GFX1250-NEXT: [[W0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
-; GFX1250-NEXT: [[W1:%.*]] = load float, ptr addrspace(1) [[B1]], align 4
-; GFX1250-NEXT: [[W2:%.*]] = load float, ptr addrspace(1) [[B2]], align 4
-; GFX1250-NEXT: [[W3:%.*]] = load float, ptr addrspace(1) [[B3]], align 4
-; GFX1250-NEXT: [[M0:%.*]] = fmul contract float [[F0]], [[W0]]
-; GFX1250-NEXT: [[M1:%.*]] = fmul contract float [[F1]], [[W1]]
-; GFX1250-NEXT: [[M2:%.*]] = fmul contract float [[F2]], [[W2]]
-; GFX1250-NEXT: [[M3:%.*]] = fmul contract float [[F3]], [[W3]]
-; GFX1250-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
-; GFX1250-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
-; GFX1250-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
-; GFX1250-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
-; GFX1250-NEXT: ret float [[S3]]
-;
-; GFX1030-LABEL: define float @dot4_i16_ordered(
-; GFX1030-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) #[[ATTR0:[0-9]+]] {
-; GFX1030-NEXT: [[ENTRY:.*:]]
-; GFX1030-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
-; GFX1030-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
-; GFX1030-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
-; GFX1030-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; GFX1030-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
-; GFX1030-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
-; GFX1030-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
-; GFX1030-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
-; GFX1030-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
-; GFX1030-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
-; GFX1030-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
-; GFX1030-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
-; GFX1030-NEXT: ret float [[S3]]
-;
-entry:
- %a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
- %a2 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 4
- %a3 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 6
- %b1 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 4
- %b2 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 8
- %b3 = getelementptr inbounds nuw i8, ptr addrspace(1) %b, i64 12
- %i0 = load i16, ptr addrspace(1) %a, align 2
- %i1 = load i16, ptr addrspace(1) %a1, align 2
- %i2 = load i16, ptr addrspace(1) %a2, align 2
- %i3 = load i16, ptr addrspace(1) %a3, align 2
- %f0 = sitofp i16 %i0 to float
- %f1 = sitofp i16 %i1 to float
- %f2 = sitofp i16 %i2 to float
- %f3 = sitofp i16 %i3 to float
- %w0 = load float, ptr addrspace(1) %b, align 4
- %w1 = load float, ptr addrspace(1) %b1, align 4
- %w2 = load float, ptr addrspace(1) %b2, align 4
- %w3 = load float, ptr addrspace(1) %b3, align 4
- %m0 = fmul contract float %f0, %w0
- %m1 = fmul contract float %f1, %w1
- %m2 = fmul contract float %f2, %w2
- %m3 = fmul contract float %f3, %w3
- %s0 = fadd contract float %acc, %m0
- %s1 = fadd contract float %s0, %m1
- %s2 = fadd contract float %s1, %m2
- %s3 = fadd contract float %s2, %m3
- ret float %s3
-}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
index be90212aae68a..60cf5cbef69fd 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/ordered-reduction-coalesced-loads.ll
@@ -167,70 +167,22 @@ declare i64 @_Z13get_global_idj(i32 noundef)
; coalesced loads and stay scalar too.
define float @dot_i16_window(ptr addrspace(1) %a, ptr addrspace(1) %b, float %acc) {
-; GFX9-LABEL: define float @dot_i16_window(
-; GFX9-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
-; GFX9-NEXT: [[ENTRY:.*:]]
-; GFX9-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
-; GFX9-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
-; GFX9-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
-; GFX9-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; GFX9-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
-; GFX9-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
-; GFX9-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
-; GFX9-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
-; GFX9-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
-; GFX9-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
-; GFX9-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
-; GFX9-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
-; GFX9-NEXT: ret float [[S3]]
-;
-; GFX1030-LABEL: define float @dot_i16_window(
-; GFX1030-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
-; GFX1030-NEXT: [[ENTRY:.*:]]
-; GFX1030-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
-; GFX1030-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
-; GFX1030-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
-; GFX1030-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
-; GFX1030-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
-; GFX1030-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
-; GFX1030-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
-; GFX1030-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
-; GFX1030-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
-; GFX1030-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
-; GFX1030-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
-; GFX1030-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
-; GFX1030-NEXT: ret float [[S3]]
-;
-; GFX1250-LABEL: define float @dot_i16_window(
-; GFX1250-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
-; GFX1250-NEXT: [[ENTRY:.*:]]
-; GFX1250-NEXT: [[A1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 2
-; GFX1250-NEXT: [[A2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 4
-; GFX1250-NEXT: [[A3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 6
-; GFX1250-NEXT: [[B1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 4
-; GFX1250-NEXT: [[B2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 8
-; GFX1250-NEXT: [[B3:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[B]], i64 12
-; GFX1250-NEXT: [[I0:%.*]] = load i16, ptr addrspace(1) [[A]], align 2
-; GFX1250-NEXT: [[I1:%.*]] = load i16, ptr addrspace(1) [[A1]], align 2
-; GFX1250-NEXT: [[I2:%.*]] = load i16, ptr addrspace(1) [[A2]], align 2
-; GFX1250-NEXT: [[I3:%.*]] = load i16, ptr addrspace(1) [[A3]], align 2
-; GFX1250-NEXT: [[F0:%.*]] = sitofp i16 [[I0]] to float
-; GFX1250-NEXT: [[F1:%.*]] = sitofp i16 [[I1]] to float
-; GFX1250-NEXT: [[F2:%.*]] = sitofp i16 [[I2]] to float
-; GFX1250-NEXT: [[F3:%.*]] = sitofp i16 [[I3]] to float
-; GFX1250-NEXT: [[W0:%.*]] = load float, ptr addrspace(1) [[B]], align 4
-; GFX1250-NEXT: [[W1:%.*]] = load float, ptr addrspace(1) [[B1]], align 4
-; GFX1250-NEXT: [[W2:%.*]] = load float, ptr addrspace(1) [[B2]], align 4
-; GFX1250-NEXT: [[W3:%.*]] = load float, ptr addrspace(1) [[B3]], align 4
-; GFX1250-NEXT: [[M0:%.*]] = fmul contract float [[F0]], [[W0]]
-; GFX1250-NEXT: [[M1:%.*]] = fmul contract float [[F1]], [[W1]]
-; GFX1250-NEXT: [[M2:%.*]] = fmul contract float [[F2]], [[W2]]
-; GFX1250-NEXT: [[M3:%.*]] = fmul contract float [[F3]], [[W3]]
-; GFX1250-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[M0]]
-; GFX1250-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[M1]]
-; GFX1250-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[M2]]
-; GFX1250-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[M3]]
-; GFX1250-NEXT: ret float [[S3]]
+; CHECK-LABEL: define float @dot_i16_window(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], float [[ACC:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[A]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i16> [[TMP0]] to <4 x float>
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i64 0
+; CHECK-NEXT: [[S0:%.*]] = fadd contract float [[ACC]], [[TMP4]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i64 1
+; CHECK-NEXT: [[S1:%.*]] = fadd contract float [[S0]], [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i64 2
+; CHECK-NEXT: [[S2:%.*]] = fadd contract float [[S1]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; CHECK-NEXT: [[S3:%.*]] = fadd contract float [[S2]], [[TMP7]]
+; CHECK-NEXT: ret float [[S3]]
;
entry:
%a1 = getelementptr inbounds nuw i8, ptr addrspace(1) %a, i64 2
More information about the llvm-commits
mailing list