[llvm] [InstCombine] Fold shuffles through all trivially vectorizable intrinsics (PR #141979)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Thu May 29 09:49:18 PDT 2025
https://github.com/lukel97 created https://github.com/llvm/llvm-project/pull/141979
This addresses a TODO in foldShuffledIntrinsicOperands to use isTriviallyVectorizable instead of a hardcoded list of intrinsics, which in turn allows more intriniscs to be scalarized by VectorCombine.
>From what I can tell every intrinsic here should be speculatable so an assertion was added.
Because this enables intrinsics like abs which have a scalar operand, we need to also check isVectorIntrinsicWithScalarOpAtArg.
>From 46d23a731a12b19cc1043fd30b2a9155796ba67c Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 29 May 2025 17:44:06 +0100
Subject: [PATCH 1/2] Precommit tests
---
llvm/test/Transforms/InstCombine/abs-1.ll | 11 +++++++++++
llvm/test/Transforms/InstCombine/fma.ll | 15 +++++++++++++++
llvm/test/Transforms/InstCombine/sqrt.ll | 11 +++++++++++
3 files changed, 37 insertions(+)
diff --git a/llvm/test/Transforms/InstCombine/abs-1.ll b/llvm/test/Transforms/InstCombine/abs-1.ll
index 7037647d116ba..8841b25fb249d 100644
--- a/llvm/test/Transforms/InstCombine/abs-1.ll
+++ b/llvm/test/Transforms/InstCombine/abs-1.ll
@@ -978,3 +978,14 @@ define i32 @abs_diff_signed_slt_no_nsw_swap(i32 %a, i32 %b) {
%cond = select i1 %cmp, i32 %sub_ba, i32 %sub_ab
ret i32 %cond
}
+
+define <2 x i32> @abs_unary_shuffle_ops(<2 x i32> %x) {
+; CHECK-LABEL: @abs_unary_shuffle_ops(
+; CHECK-NEXT: [[R1:%.*]] = shufflevector <2 x i32> [[R:%.*]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[R2:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[R1]], i1 false)
+; CHECK-NEXT: ret <2 x i32> [[R2]]
+;
+ %a = shufflevector <2 x i32> %x, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+ %r = call <2 x i32> @llvm.abs(<2 x i32> %a, i1 false)
+ ret <2 x i32> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/fma.ll b/llvm/test/Transforms/InstCombine/fma.ll
index f0d4f776a5d90..800f9beb974d9 100644
--- a/llvm/test/Transforms/InstCombine/fma.ll
+++ b/llvm/test/Transforms/InstCombine/fma.ll
@@ -972,6 +972,21 @@ define <2 x half> @fma_negone_vec_partial_undef(<2 x half> %x, <2 x half> %y) {
ret <2 x half> %sub
}
+define <2 x float> @fmuladd_unary_shuffle_ops(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @fmuladd_unary_shuffle_ops(
+; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[B:%.*]] = shufflevector <2 x float> [[Y:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[C:%.*]] = shufflevector <2 x float> [[Z:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[R:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[A]], <2 x float> [[B]], <2 x float> [[C]])
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %a = shufflevector <2 x float> %x, <2 x float> poison, <2 x i32> <i32 1, i32 0>
+ %b = shufflevector <2 x float> %y, <2 x float> poison, <2 x i32> <i32 1, i32 0>
+ %c = shufflevector <2 x float> %z, <2 x float> poison, <2 x i32> <i32 1, i32 0>
+ %r = call <2 x float> @llvm.fmuladd(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+ ret <2 x float> %r
+}
+
; negative tests
define half @fma_non_negone(half %x, half %y) {
diff --git a/llvm/test/Transforms/InstCombine/sqrt.ll b/llvm/test/Transforms/InstCombine/sqrt.ll
index 0f4db3b3a65ae..ccc8fc693d58d 100644
--- a/llvm/test/Transforms/InstCombine/sqrt.ll
+++ b/llvm/test/Transforms/InstCombine/sqrt.ll
@@ -201,6 +201,17 @@ define <2 x float> @sqrt_exp_vec(<2 x float> %x) {
ret <2 x float> %res
}
+define <2 x float> @sqrt_unary_shuffle_ops(<2 x float> %x) {
+; CHECK-LABEL: @sqrt_unary_shuffle_ops(
+; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[R:%.*]] = call <2 x float> @llvm.sqrt.v2f32(<2 x float> [[A]])
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %a = shufflevector <2 x float> %x, <2 x float> poison, <2 x i32> <i32 1, i32 0>
+ %r = call <2 x float> @llvm.sqrt(<2 x float> %a)
+ ret <2 x float> %r
+}
+
declare i32 @foo(double)
declare double @sqrt(double) readnone
declare float @sqrtf(float)
>From 4c47e2b8737f8570e9fd34b0dc910aa612cf6318 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 29 May 2025 17:45:34 +0100
Subject: [PATCH 2/2] [InstCombine] Fold shuffles through all trivially
vectorizable intrinsics
This addresses a TODO in foldShuffledIntrinsicOperands to use isTriviallyVectorizable instead of a hardcoded list of intrinsics, which in turn allows more intriniscs to be scalarized by VectorCombine.
>From what I can tell every intrinsic here should be speculatable so an assertion was added.
Because this enables intrinsics like abs which have a scalar operand, we need to also check isVectorIntrinsicWithScalarOpAtArg.
---
.../InstCombine/InstCombineCalls.cpp | 41 ++++++++++---------
llvm/test/Transforms/InstCombine/abs-1.ll | 6 +--
llvm/test/Transforms/InstCombine/fma.ll | 8 ++--
llvm/test/Transforms/InstCombine/sqrt.ll | 6 +--
4 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index e101edf4a6208..5eb466f6c6df1 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1401,26 +1401,25 @@ static Instruction *factorizeMinMaxTree(IntrinsicInst *II) {
/// try to shuffle after the intrinsic.
Instruction *
InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {
- // TODO: This should be extended to handle other intrinsics like fshl, ctpop,
- // etc. Use llvm::isTriviallyVectorizable() and related to determine
- // which intrinsics are safe to shuffle?
- switch (II->getIntrinsicID()) {
- case Intrinsic::smax:
- case Intrinsic::smin:
- case Intrinsic::umax:
- case Intrinsic::umin:
- case Intrinsic::fma:
- case Intrinsic::fshl:
- case Intrinsic::fshr:
- break;
- default:
+ if (!isTriviallyVectorizable(II->getIntrinsicID()))
+ return nullptr;
+
+ assert(isSafeToSpeculativelyExecute(II) &&
+ "Trivially vectorizable but not safe to speculatively execute?");
+
+ // fabs is canonicalized to fabs (shuffle ...) in foldShuffleOfUnaryOps, so
+ // avoid undoing it.
+ if (match(II, m_FAbs(m_Value())))
return nullptr;
- }
Value *X;
Constant *C;
ArrayRef<int> Mask;
- auto *NonConstArg = find_if_not(II->args(), IsaPred<Constant>);
+ auto *NonConstArg = find_if_not(II->args(), [&II](Use &Arg) {
+ return isa<Constant>(Arg.get()) ||
+ isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
+ Arg.getOperandNo(), nullptr);
+ });
if (!NonConstArg ||
!match(NonConstArg, m_Shuffle(m_Value(X), m_Poison(), m_Mask(Mask))))
return nullptr;
@@ -1432,11 +1431,15 @@ InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {
// See if all arguments are shuffled with the same mask.
SmallVector<Value *, 4> NewArgs;
Type *SrcTy = X->getType();
- for (Value *Arg : II->args()) {
- if (match(Arg, m_Shuffle(m_Value(X), m_Poison(), m_SpecificMask(Mask))) &&
- X->getType() == SrcTy)
+ for (Use &Arg : II->args()) {
+ if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
+ Arg.getOperandNo(), nullptr))
+ NewArgs.push_back(Arg);
+ else if (match(&Arg,
+ m_Shuffle(m_Value(X), m_Poison(), m_SpecificMask(Mask))) &&
+ X->getType() == SrcTy)
NewArgs.push_back(X);
- else if (match(Arg, m_ImmConstant(C))) {
+ else if (match(&Arg, m_ImmConstant(C))) {
// If it's a constant, try find the constant that would be shuffled to C.
if (Constant *ShuffledC =
unshuffleConstant(Mask, C, cast<VectorType>(SrcTy)))
diff --git a/llvm/test/Transforms/InstCombine/abs-1.ll b/llvm/test/Transforms/InstCombine/abs-1.ll
index 8841b25fb249d..fd67fc3421498 100644
--- a/llvm/test/Transforms/InstCombine/abs-1.ll
+++ b/llvm/test/Transforms/InstCombine/abs-1.ll
@@ -981,9 +981,9 @@ define i32 @abs_diff_signed_slt_no_nsw_swap(i32 %a, i32 %b) {
define <2 x i32> @abs_unary_shuffle_ops(<2 x i32> %x) {
; CHECK-LABEL: @abs_unary_shuffle_ops(
-; CHECK-NEXT: [[R1:%.*]] = shufflevector <2 x i32> [[R:%.*]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[R2:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[R1]], i1 false)
-; CHECK-NEXT: ret <2 x i32> [[R2]]
+; CHECK-NEXT: [[R2:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[R1:%.*]], i1 false)
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[R2]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: ret <2 x i32> [[R]]
;
%a = shufflevector <2 x i32> %x, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
%r = call <2 x i32> @llvm.abs(<2 x i32> %a, i1 false)
diff --git a/llvm/test/Transforms/InstCombine/fma.ll b/llvm/test/Transforms/InstCombine/fma.ll
index 800f9beb974d9..e3d3e722bcc23 100644
--- a/llvm/test/Transforms/InstCombine/fma.ll
+++ b/llvm/test/Transforms/InstCombine/fma.ll
@@ -974,11 +974,9 @@ define <2 x half> @fma_negone_vec_partial_undef(<2 x half> %x, <2 x half> %y) {
define <2 x float> @fmuladd_unary_shuffle_ops(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
; CHECK-LABEL: @fmuladd_unary_shuffle_ops(
-; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[B:%.*]] = shufflevector <2 x float> [[Y:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[C:%.*]] = shufflevector <2 x float> [[Z:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[R:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[A]], <2 x float> [[B]], <2 x float> [[C]])
-; CHECK-NEXT: ret <2 x float> [[R]]
+; CHECK-NEXT: [[R:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[A:%.*]], <2 x float> [[B:%.*]], <2 x float> [[C:%.*]])
+; CHECK-NEXT: [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: ret <2 x float> [[R1]]
;
%a = shufflevector <2 x float> %x, <2 x float> poison, <2 x i32> <i32 1, i32 0>
%b = shufflevector <2 x float> %y, <2 x float> poison, <2 x i32> <i32 1, i32 0>
diff --git a/llvm/test/Transforms/InstCombine/sqrt.ll b/llvm/test/Transforms/InstCombine/sqrt.ll
index ccc8fc693d58d..2fda5bc37d023 100644
--- a/llvm/test/Transforms/InstCombine/sqrt.ll
+++ b/llvm/test/Transforms/InstCombine/sqrt.ll
@@ -203,9 +203,9 @@ define <2 x float> @sqrt_exp_vec(<2 x float> %x) {
define <2 x float> @sqrt_unary_shuffle_ops(<2 x float> %x) {
; CHECK-LABEL: @sqrt_unary_shuffle_ops(
-; CHECK-NEXT: [[A:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[R:%.*]] = call <2 x float> @llvm.sqrt.v2f32(<2 x float> [[A]])
-; CHECK-NEXT: ret <2 x float> [[R]]
+; CHECK-NEXT: [[R:%.*]] = call <2 x float> @llvm.sqrt.v2f32(<2 x float> [[A:%.*]])
+; CHECK-NEXT: [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: ret <2 x float> [[R1]]
;
%a = shufflevector <2 x float> %x, <2 x float> poison, <2 x i32> <i32 1, i32 0>
%r = call <2 x float> @llvm.sqrt(<2 x float> %a)
More information about the llvm-commits
mailing list