[llvm] [CostModel][X86] Add more realistic v8i64/v16i32 + v8f64/v16f32 add reduction costs (PR #206124)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 09:24:26 PDT 2026
https://github.com/RKSimon created https://github.com/llvm/llvm-project/pull/206124
Fixes failure to fold to v16i32 reduction on ax512 targets
We still need to determine better CostKind values - but that can wait until #194621 is complete
>From 0629952eefb009b3df7404291d5820a9a22e8772 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Fri, 26 Jun 2026 17:22:35 +0100
Subject: [PATCH] [CostModel][X86] Add more realistic v8i64/v16i32 +
v8f64/v16f32 add reduction costs
Fixes failure to fodl to v16i32 reduction on ax512 targets
We still need to determine better CostKind values - but that can wait until #194621 is complete
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 20 +++++++++
.../test/Analysis/CostModel/X86/reduce-add.ll | 16 +++----
.../Analysis/CostModel/X86/reduce-fadd.ll | 4 +-
.../X86/horizontal-reduce-add.ll | 45 +++++--------------
.../X86/redux-feed-buildvector.ll | 23 ++++------
.../X86/reduction-of-truncations.ll | 21 ++++-----
6 files changed, 59 insertions(+), 70 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index e139be77e4a44..8838fd7e71f02 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -5671,6 +5671,18 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
{ ISD::ADD, MVT::v32i8, 4 },
};
+ static const CostTblEntry AVX512FCostTbl[] = {
+ { ISD::FADD, MVT::v8f64, 4 },
+ { ISD::FADD, MVT::v16f32, 5 },
+ { ISD::ADD, MVT::v8i64, 4 },
+ { ISD::ADD, MVT::v16i32, 6 },
+ };
+
+ static const CostTblEntry AVX512BWCostTbl[] = {
+ { ISD::ADD, MVT::v32i16, 7 },
+ { ISD::ADD, MVT::v64i8, 4 },
+ };
+
int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
@@ -5684,6 +5696,14 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
return Entry->Cost;
+ if (ST->hasBWI())
+ if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
+ return Entry->Cost;
+
+ if (ST->hasAVX512())
+ if (const auto *Entry = CostTableLookup(AVX512FCostTbl, ISD, MTy))
+ return Entry->Cost;
+
if (ST->hasAVX())
if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
return Entry->Cost;
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-add.ll b/llvm/test/Analysis/CostModel/X86/reduce-add.ll
index 9a717e7dbef73..b458efef9e54d 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-add.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-add.ll
@@ -39,7 +39,7 @@ define i32 @reduce_i64(i32 %arg) {
; AVX512F-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of 3 for: %V4 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:8 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
+; AVX512F-NEXT: Cost Model: Found costs of 4 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
@@ -47,7 +47,7 @@ define i32 @reduce_i64(i32 %arg) {
; AVX512BW-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 3 for: %V4 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
+; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:8 for: %V16 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
@@ -55,7 +55,7 @@ define i32 @reduce_i64(i32 %arg) {
; AVX512DQ-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 3 for: %V4 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:8 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
@@ -104,7 +104,7 @@ define i32 @reduce_i32(i32 %arg) {
; AVX512F-NEXT: Cost Model: Found costs of 2 for: %V2 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of 3 for: %V4 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of 5 for: %V8 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:10 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
+; AVX512F-NEXT: Cost Model: Found costs of 6 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:11 for: %V32 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
@@ -112,7 +112,7 @@ define i32 @reduce_i32(i32 %arg) {
; AVX512BW-NEXT: Cost Model: Found costs of 2 for: %V2 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 3 for: %V4 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 5 for: %V8 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
+; AVX512BW-NEXT: Cost Model: Found costs of 6 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:10 for: %V32 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
@@ -120,7 +120,7 @@ define i32 @reduce_i32(i32 %arg) {
; AVX512DQ-NEXT: Cost Model: Found costs of 2 for: %V2 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 3 for: %V4 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 5 for: %V8 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:10 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
+; AVX512DQ-NEXT: Cost Model: Found costs of 6 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:11 for: %V32 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
@@ -182,7 +182,7 @@ define i32 @reduce_i16(i32 %arg) {
; AVX512BW-NEXT: Cost Model: Found costs of 3 for: %V4 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %V8 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 5 for: %V16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> undef)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:15 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> undef)
+; AVX512BW-NEXT: Cost Model: Found costs of 7 for: %V32 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:16 SizeLat:12 for: %V64 = call i16 @llvm.vector.reduce.add.v64i16(<64 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
@@ -260,7 +260,7 @@ define i32 @reduce_i8(i32 %arg) {
; AVX512BW-NEXT: Cost Model: Found costs of 2 for: %V8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 3 for: %V16 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %V32 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> undef)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:13 CodeSize:13 Lat:17 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> undef)
+; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %V64 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:18 SizeLat:14 for: %V128 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
diff --git a/llvm/test/Analysis/CostModel/X86/reduce-fadd.ll b/llvm/test/Analysis/CostModel/X86/reduce-fadd.ll
index 122903d1e7a1c..3befb6741cd20 100644
--- a/llvm/test/Analysis/CostModel/X86/reduce-fadd.ll
+++ b/llvm/test/Analysis/CostModel/X86/reduce-fadd.ll
@@ -202,7 +202,7 @@ define void @reduce_f64_fast(double %arg) {
; AVX512-NEXT: Cost Model: Found costs of 0 for: %V1 = call fast double @llvm.vector.reduce.fadd.v1f64(double %arg, <1 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of 2 for: %V2 = call fast double @llvm.vector.reduce.fadd.v2f64(double %arg, <2 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of 3 for: %V4 = call fast double @llvm.vector.reduce.fadd.v4f64(double %arg, <4 x double> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:17 SizeLat:6 for: %V8 = call fast double @llvm.vector.reduce.fadd.v8f64(double %arg, <8 x double> undef)
+; AVX512-NEXT: Cost Model: Found costs of 4 for: %V8 = call fast double @llvm.vector.reduce.fadd.v8f64(double %arg, <8 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:21 SizeLat:7 for: %V16 = call fast double @llvm.vector.reduce.fadd.v16f64(double %arg, <16 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
@@ -274,7 +274,7 @@ define void @reduce_f32_fast(float %arg) {
; AVX512-NEXT: Cost Model: Found costs of 2 for: %V2 = call fast float @llvm.vector.reduce.fadd.v2f32(float %arg, <2 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of 3 for: %V4 = call fast float @llvm.vector.reduce.fadd.v4f32(float %arg, <4 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of 4 for: %V8 = call fast float @llvm.vector.reduce.fadd.v8f32(float %arg, <8 x float> undef)
-; AVX512-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:24 SizeLat:8 for: %V16 = call fast float @llvm.vector.reduce.fadd.v16f32(float %arg, <16 x float> undef)
+; AVX512-NEXT: Cost Model: Found costs of 5 for: %V16 = call fast float @llvm.vector.reduce.fadd.v16f32(float %arg, <16 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:28 SizeLat:9 for: %V32 = call fast float @llvm.vector.reduce.fadd.v32f32(float %arg, <32 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/horizontal-reduce-add.ll b/llvm/test/Transforms/PhaseOrdering/X86/horizontal-reduce-add.ll
index 893c12c965ff0..cf456f86b4ac5 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/horizontal-reduce-add.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/horizontal-reduce-add.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64 | FileCheck %s
+; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v2 | FileCheck %s
+; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v3 | FileCheck %s
+; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v4 | FileCheck %s
+; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s
+; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s
+; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
+; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s
; PR37890 - subvector reduction followed by shuffle reduction
@@ -83,31 +83,10 @@ define i16 @PR37890_v16i16(<16 x i16> %a) {
}
define i32 @PR37890_v16i32(<16 x i32> %a) {
-; SSE-LABEL: define i32 @PR37890_v16i32(
-; SSE-SAME: <16 x i32> [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; SSE-NEXT: [[E:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[A]])
-; SSE-NEXT: ret i32 [[E]]
-;
-; AVX2-LABEL: define i32 @PR37890_v16i32(
-; AVX2-SAME: <16 x i32> [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; AVX2-NEXT: [[E:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[A]])
-; AVX2-NEXT: ret i32 [[E]]
-;
-; AVX512-LABEL: define i32 @PR37890_v16i32(
-; AVX512-SAME: <16 x i32> [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; AVX512-NEXT: [[HI0:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX512-NEXT: [[LO0:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT: [[SUM0:%.*]] = add <8 x i32> [[LO0]], [[HI0]]
-; AVX512-NEXT: [[HI1:%.*]] = shufflevector <8 x i32> [[SUM0]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT: [[LO1:%.*]] = shufflevector <8 x i32> [[SUM0]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; AVX512-NEXT: [[SUM1:%.*]] = add <4 x i32> [[LO1]], [[HI1]]
-; AVX512-NEXT: [[HI2:%.*]] = shufflevector <4 x i32> [[SUM1]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
-; AVX512-NEXT: [[LO2:%.*]] = shufflevector <4 x i32> [[SUM1]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; AVX512-NEXT: [[SUM2:%.*]] = add <2 x i32> [[LO2]], [[HI2]]
-; AVX512-NEXT: [[HI3:%.*]] = shufflevector <2 x i32> [[SUM2]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
-; AVX512-NEXT: [[SUM3:%.*]] = add <2 x i32> [[HI3]], [[SUM2]]
-; AVX512-NEXT: [[E:%.*]] = extractelement <2 x i32> [[SUM3]], i64 0
-; AVX512-NEXT: ret i32 [[E]]
+; CHECK-LABEL: define i32 @PR37890_v16i32(
+; CHECK-SAME: <16 x i32> [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CHECK-NEXT: [[E:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[A]])
+; CHECK-NEXT: ret i32 [[E]]
;
%hi0 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%lo0 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
index 885e28dec2df2..cc7638431e03b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
@@ -10,22 +10,17 @@ declare void @llvm.masked.scatter.v2f64.v2p0(<2 x double>, <2 x ptr>, i32 immarg
define void @test(ptr nocapture readonly %arg, ptr nocapture readonly %arg1, ptr nocapture %arg2) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP1_0:%.*]] = getelementptr inbounds double, ptr [[ARG:%.*]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x ptr> poison, ptr [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x ptr> [[TMP0]], <8 x ptr> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP1]], <8 x i64> <i64 1, i64 3, i64 5, i64 7, i64 9, i64 11, i64 13, i64 15>
; CHECK-NEXT: [[GEP2_0:%.*]] = getelementptr inbounds double, ptr [[ARG1:%.*]], i64 16
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[GEP2_0]], align 8
-; CHECK-NEXT: [[GEP2_4:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 20
-; CHECK-NEXT: [[TMP1:%.*]] = call <15 x double> @llvm.masked.load.v15f64.p0(ptr align 8 [[GEP1_0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <15 x double> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <15 x double> [[TMP1]], <15 x double> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[TMP3:%.*]] = load <8 x double>, ptr [[ARG1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP5]], <8 x i1> splat (i1 true), <8 x double> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = load <8 x double>, ptr [[GEP2_0]], align 8
; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <8 x double> [[TMP3]], [[TMP2]]
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x double> [[TMP0]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP4]])
-; CHECK-NEXT: [[TMP8:%.*]] = load <4 x double>, ptr [[GEP2_4]], align 8
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <15 x double> [[TMP1]], <15 x double> poison, <4 x i32> <i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[TMP10:%.*]] = fmul fast <4 x double> [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[RDX_OP:%.*]] = fadd fast <4 x double> [[TMP6]], [[TMP10]]
-; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[RDX_OP]])
+; CHECK-NEXT: [[TMP6:%.*]] = load <8 x double>, ptr [[ARG1]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <8 x double> [[TMP6]], [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP8]])
+; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP4]])
; CHECK-NEXT: [[I142:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
; CHECK-NEXT: [[I143:%.*]] = insertelement <2 x double> [[I142]], double [[TMP11]], i64 1
; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds double, ptr [[ARG2:%.*]], <2 x i64> <i64 0, i64 16>
diff --git a/llvm/test/Transforms/VectorCombine/X86/reduction-of-truncations.ll b/llvm/test/Transforms/VectorCombine/X86/reduction-of-truncations.ll
index f09ac41793b51..4b70254167506 100644
--- a/llvm/test/Transforms/VectorCombine/X86/reduction-of-truncations.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/reduction-of-truncations.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,X64
-; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,X64
-; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,X64
-; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s
+; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s
+; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
+; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s
;
; Fold reduce(trunc(X)) -> trunc(reduce(X)) if more cost efficient
@@ -10,15 +10,10 @@
; Cheap AVX512 v8i64 -> v8i32 truncation
define i32 @reduce_add_trunc_v8i64_i32(<8 x i64> %a0) {
-; X64-LABEL: @reduce_add_trunc_v8i64_i32(
-; X64-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[A0:%.*]])
-; X64-NEXT: [[RED:%.*]] = trunc i64 [[TMP1]] to i32
-; X64-NEXT: ret i32 [[RED]]
-;
-; AVX512-LABEL: @reduce_add_trunc_v8i64_i32(
-; AVX512-NEXT: [[TR:%.*]] = trunc <8 x i64> [[A0:%.*]] to <8 x i32>
-; AVX512-NEXT: [[RED:%.*]] = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TR]])
-; AVX512-NEXT: ret i32 [[RED]]
+; CHECK-LABEL: @reduce_add_trunc_v8i64_i32(
+; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[A0:%.*]])
+; CHECK-NEXT: [[RED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT: ret i32 [[RED]]
;
%tr = trunc <8 x i64> %a0 to <8 x i32>
%red = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %tr)
More information about the llvm-commits
mailing list