[llvm] [AArch64][TTI] Fix extract cost for scalar fmul users (PR #212739)
Anutosh Bhat via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 20:58:37 PDT 2026
https://github.com/anutosh491 updated https://github.com/llvm/llvm-project/pull/212739
>From 4b847b9db0d056b97371fbdbf9252e11ac22fd2d Mon Sep 17 00:00:00 2001
From: anutosh491 <andersonbhat491 at gmail.com>
Date: Wed, 29 Jul 2026 16:34:16 +0530
Subject: [PATCH] [AArch64][TTI] Fix extract cost for scalar fmul users
---
.../AArch64/AArch64TargetTransformInfo.cpp | 26 +++---
.../AArch64/extract-fmul-cost.ll | 29 +++----
.../SLPVectorizer/reordering-single-phi.ll | 86 ++++++++++++-------
3 files changed, 81 insertions(+), 60 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 207080ba7c971..946045185820c 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4433,8 +4433,10 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
if (Scalar) {
DenseMap<User *, unsigned> UserToExtractIdx;
- for (auto *U : Scalar->users()) {
- if (!IsUserFMulScalarTy(U))
+ for (auto &[S, U, L] : ScalarUserAndIdx) {
+ if (S != Scalar)
+ continue;
+ if (!U || !IsUserFMulScalarTy(U))
return false;
// Recording entry for the user is important. Index value is not
// important.
@@ -4442,19 +4444,19 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
}
if (UserToExtractIdx.empty())
return false;
- for (auto &[S, U, L] : ScalarUserAndIdx) {
- for (auto *U : S->users()) {
- if (UserToExtractIdx.contains(U)) {
- auto *FMul = cast<BinaryOperator>(U);
- auto *Op0 = FMul->getOperand(0);
- auto *Op1 = FMul->getOperand(1);
- if ((Op0 == S && Op1 == S) || Op0 != S || Op1 != S) {
- UserToExtractIdx[U] = L;
- break;
- }
+
+ for (auto &[U, L] : UserToExtractIdx) {
+ auto *FMul = cast<BinaryOperator>(U);
+ Value *OtherOp = FMul->getOperand(0) == Scalar ? FMul->getOperand(1)
+ : FMul->getOperand(0);
+ for (auto &[S, ScalarUser, Lane] : ScalarUserAndIdx) {
+ if (S == OtherOp && ScalarUser == U) {
+ L = Lane;
+ break;
}
}
}
+
for (auto &[U, L] : UserToExtractIdx) {
if (!IsExtractLaneEquivalentToZero(Index, Val->getScalarSizeInBits()) &&
!IsExtractLaneEquivalentToZero(L, Val->getScalarSizeInBits()))
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/extract-fmul-cost.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/extract-fmul-cost.ll
index fb578ec6d265b..a2d1b1f78c9d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/extract-fmul-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/extract-fmul-cost.ll
@@ -2,36 +2,27 @@
; RUN: opt -passes=slp-vectorizer -slp-threshold=3 \
; RUN: -mtriple=aarch64-unknown-linux -S < %s | FileCheck %s
-; TODO: The first fadd pair should also be vectorized because the scalar fmul
-; can use the required vector lanes without separate extract instructions.
-
define double @lane1_times_lane2(ptr %a, ptr %b, ptr %out) {
; CHECK-LABEL: define double @lane1_times_lane2(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[OUT:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[A0P:%.*]] = getelementptr double, ptr [[A]], i64 0
-; CHECK-NEXT: [[A1P:%.*]] = getelementptr double, ptr [[A]], i64 1
; CHECK-NEXT: [[A2P:%.*]] = getelementptr double, ptr [[A]], i64 2
; CHECK-NEXT: [[B0P:%.*]] = getelementptr double, ptr [[B]], i64 0
-; CHECK-NEXT: [[B1P:%.*]] = getelementptr double, ptr [[B]], i64 1
; CHECK-NEXT: [[B2P:%.*]] = getelementptr double, ptr [[B]], i64 2
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A0P]], align 8
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[A1P]], align 8
-; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[B0P]], align 8
-; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[B1P]], align 8
-; CHECK-NEXT: [[X0:%.*]] = fadd double [[A0]], [[B0]]
-; CHECK-NEXT: [[X1:%.*]] = fadd double [[A1]], [[B1]]
; CHECK-NEXT: [[O0:%.*]] = getelementptr double, ptr [[OUT]], i64 0
-; CHECK-NEXT: [[O1:%.*]] = getelementptr double, ptr [[OUT]], i64 1
; CHECK-NEXT: [[O2:%.*]] = getelementptr double, ptr [[OUT]], i64 2
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[A2P]], align 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[B2P]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[A0P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[B0P]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: store double [[X0]], ptr [[O0]], align 8
-; CHECK-NEXT: store double [[X1]], ptr [[O1]], align 8
-; CHECK-NEXT: store <2 x double> [[TMP2]], ptr [[O2]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
-; CHECK-NEXT: [[MUL:%.*]] = fmul double [[X1]], [[TMP3]]
+; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[A2P]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[B2P]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: store <2 x double> [[TMP2]], ptr [[O0]], align 8
+; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[O2]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
+; CHECK-NEXT: [[MUL:%.*]] = fmul double [[TMP6]], [[TMP7]]
; CHECK-NEXT: ret double [[MUL]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/reordering-single-phi.ll b/llvm/test/Transforms/SLPVectorizer/reordering-single-phi.ll
index b06032bb4929d..6a71b23ebf5ff 100644
--- a/llvm/test/Transforms/SLPVectorizer/reordering-single-phi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reordering-single-phi.ll
@@ -1,37 +1,65 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s | FileCheck %s --check-prefix=X86 %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux < %s | FileCheck %s --check-prefix=AARCH64 %}
@a = external global [32000 x float], align 64
define void @test() {
-; CHECK-LABEL: define void @test() {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP16:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP1]]
-; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
-; CHECK-NEXT: [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
-; CHECK-NEXT: [[TMP14:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP14]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP7]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP15:%.*]] = fmul fast <4 x float> [[TMP11]], [[TMP14]]
-; CHECK-NEXT: store <4 x float> [[TMP15]], ptr [[ARRAYIDX6]], align 4
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; CHECK-NEXT: [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
-; CHECK-NEXT: [[TMP16]] = load float, ptr [[ARRAYIDX41]], align 4
-; CHECK-NEXT: [[MUL45:%.*]] = fmul fast float [[TMP16]], [[TMP6]]
-; CHECK-NEXT: store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
-; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i64 [[INDVARS_IV]], 31990
-; CHECK-NEXT: br i1 [[CMP2]], label %[[FOR_BODY]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
+; X86-LABEL: define void @test() {
+; X86-NEXT: [[ENTRY:.*]]:
+; X86-NEXT: br label %[[FOR_BODY:.*]]
+; X86: [[FOR_BODY]]:
+; X86-NEXT: [[TMP0:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP9:%.*]], %[[FOR_BODY]] ]
+; X86-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; X86-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; X86-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP1]]
+; X86-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
+; X86-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; X86-NEXT: [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP2]]
+; X86-NEXT: [[TMP3:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
+; X86-NEXT: [[TMP4:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
+; X86-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
+; X86-NEXT: [[TMP6:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; X86-NEXT: [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; X86-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP7]], [[TMP4]]
+; X86-NEXT: store <4 x float> [[TMP8]], ptr [[ARRAYIDX6]], align 4
+; X86-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
+; X86-NEXT: [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
+; X86-NEXT: [[TMP9]] = load float, ptr [[ARRAYIDX41]], align 4
+; X86-NEXT: [[MUL45:%.*]] = fmul fast float [[TMP9]], [[TMP3]]
+; X86-NEXT: store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
+; X86-NEXT: [[CMP2:%.*]] = icmp ult i64 [[INDVARS_IV]], 31990
+; X86-NEXT: br i1 [[CMP2]], label %[[FOR_BODY]], label %[[EXIT:.*]]
+; X86: [[EXIT]]:
+; X86-NEXT: ret void
+;
+; AARCH64-LABEL: define void @test() {
+; AARCH64-NEXT: [[ENTRY:.*]]:
+; AARCH64-NEXT: br label %[[FOR_BODY:.*]]
+; AARCH64: [[FOR_BODY]]:
+; AARCH64-NEXT: [[TMP0:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP8:%.*]], %[[FOR_BODY]] ]
+; AARCH64-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; AARCH64-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AARCH64-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP1]]
+; AARCH64-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
+; AARCH64-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
+; AARCH64-NEXT: [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP2]]
+; AARCH64-NEXT: [[TMP3:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
+; AARCH64-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
+; AARCH64-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
+; AARCH64-NEXT: [[TMP6:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP5]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; AARCH64-NEXT: [[TMP7:%.*]] = fmul fast <4 x float> [[TMP6]], [[TMP3]]
+; AARCH64-NEXT: store <4 x float> [[TMP7]], ptr [[ARRAYIDX6]], align 4
+; AARCH64-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
+; AARCH64-NEXT: [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
+; AARCH64-NEXT: [[TMP8]] = load float, ptr [[ARRAYIDX41]], align 4
+; AARCH64-NEXT: [[TMP9:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; AARCH64-NEXT: [[MUL45:%.*]] = fmul fast float [[TMP8]], [[TMP9]]
+; AARCH64-NEXT: store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
+; AARCH64-NEXT: [[CMP2:%.*]] = icmp ult i64 [[INDVARS_IV]], 31990
+; AARCH64-NEXT: br i1 [[CMP2]], label %[[FOR_BODY]], label %[[EXIT:.*]]
+; AARCH64: [[EXIT]]:
+; AARCH64-NEXT: ret void
;
entry:
br label %for.body
More information about the llvm-commits
mailing list