[llvm] [InstCombine] Fold binary op of reductions. (PR #121567)
Mikhail Gudim via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 8 14:20:23 PST 2025
https://github.com/mgudim updated https://github.com/llvm/llvm-project/pull/121567
>From d6833553d5d02d25c781d44e95e0f0ee717d77fc Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at ventanamicro.com>
Date: Thu, 2 Jan 2025 08:26:22 -0800
Subject: [PATCH] [InstCombine] Fold binary op of reductions.
Replace binary of of two reductions with one reduction of the binary op
applied to vectors. For example:
```
%v0_red = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %v0)
%v1_red = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %v1)
%res = add i32 %v0_red, %v1_red
```
gets transformed to:
```
%1 = add <16 x i32> %v0, %v1
%res = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
```
---
.../InstCombine/InstCombineAddSub.cpp | 18 ++----
.../InstCombine/InstCombineAndOrXor.cpp | 9 +++
.../InstCombine/InstCombineInternal.h | 1 +
.../InstCombine/InstCombineMulDivRem.cpp | 3 +
.../InstCombine/InstructionCombining.cpp | 57 +++++++++++++++++++
5 files changed, 75 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 73876d00e73a7c..cf61ebd4a662bb 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1528,6 +1528,9 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
if (Instruction *X = foldVectorBinop(I))
return X;
+ if (Instruction *X = foldBinopOfReductions(I))
+ return replaceInstUsesWith(I, X);
+
if (Instruction *Phi = foldBinopWithPhiOperands(I))
return Phi;
@@ -2378,19 +2381,8 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
}
}
- auto m_AddRdx = [](Value *&Vec) {
- return m_OneUse(m_Intrinsic<Intrinsic::vector_reduce_add>(m_Value(Vec)));
- };
- Value *V0, *V1;
- if (match(Op0, m_AddRdx(V0)) && match(Op1, m_AddRdx(V1)) &&
- V0->getType() == V1->getType()) {
- // Difference of sums is sum of differences:
- // add_rdx(V0) - add_rdx(V1) --> add_rdx(V0 - V1)
- Value *Sub = Builder.CreateSub(V0, V1);
- Value *Rdx = Builder.CreateIntrinsic(Intrinsic::vector_reduce_add,
- {Sub->getType()}, {Sub});
- return replaceInstUsesWith(I, Rdx);
- }
+ if (Instruction *X = foldBinopOfReductions(I))
+ return replaceInstUsesWith(I, X);
if (Constant *C = dyn_cast<Constant>(Op0)) {
Value *X;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 184c75a1dd860e..4d0cbb7165a3c1 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -2399,6 +2399,9 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
if (Instruction *X = foldVectorBinop(I))
return X;
+ if (Instruction *X = foldBinopOfReductions(I))
+ return replaceInstUsesWith(I, X);
+
if (Instruction *Phi = foldBinopWithPhiOperands(I))
return Phi;
@@ -3585,6 +3588,9 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
if (Instruction *X = foldVectorBinop(I))
return X;
+ if (Instruction *X = foldBinopOfReductions(I))
+ return replaceInstUsesWith(I, X);
+
if (Instruction *Phi = foldBinopWithPhiOperands(I))
return Phi;
@@ -4696,6 +4702,9 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
if (Instruction *X = foldVectorBinop(I))
return X;
+ if (Instruction *X = foldBinopOfReductions(I))
+ return replaceInstUsesWith(I, X);
+
if (Instruction *Phi = foldBinopWithPhiOperands(I))
return Phi;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index b31ae374540bbd..c4f892d695ac0a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -597,6 +597,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
/// Canonicalize the position of binops relative to shufflevector.
Instruction *foldVectorBinop(BinaryOperator &Inst);
+ Instruction *foldBinopOfReductions(BinaryOperator &Inst);
Instruction *foldVectorSelect(SelectInst &Sel);
Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 0c34cf01bdf1a9..842697a83038b5 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -201,6 +201,9 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) {
if (Instruction *X = foldVectorBinop(I))
return X;
+ if (Instruction *X = foldBinopOfReductions(I))
+ return replaceInstUsesWith(I, X);
+
if (Instruction *Phi = foldBinopWithPhiOperands(I))
return Phi;
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 553435c937a70a..b91246a2c1c35d 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -2313,6 +2313,63 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
return nullptr;
}
+static Intrinsic::ID getReductionForBinop(Instruction::BinaryOps Opc) {
+ switch (Opc) {
+ default:
+ break;
+ case Instruction::Add:
+ return Intrinsic::vector_reduce_add;
+ case Instruction::Mul:
+ return Intrinsic::vector_reduce_mul;
+ case Instruction::And:
+ return Intrinsic::vector_reduce_and;
+ case Instruction::Or:
+ return Intrinsic::vector_reduce_or;
+ case Instruction::Xor:
+ return Intrinsic::vector_reduce_xor;
+ }
+ return Intrinsic::not_intrinsic;
+}
+
+Instruction *InstCombinerImpl::foldBinopOfReductions(BinaryOperator &Inst) {
+ Instruction::BinaryOps BinOpOpc = Inst.getOpcode();
+ Intrinsic::ID ReductionIID = getReductionForBinop(BinOpOpc);
+ if (BinOpOpc == Instruction::Sub)
+ ReductionIID = Intrinsic::vector_reduce_add;
+ if (ReductionIID == Intrinsic::not_intrinsic)
+ return nullptr;
+
+ auto checkIntrinsicAndGetItsArgument = [](Value *V,
+ Intrinsic::ID IID) -> Value * {
+ IntrinsicInst *II = dyn_cast<IntrinsicInst>(V);
+ if (!II)
+ return nullptr;
+ if ((II->getIntrinsicID() == IID) && II->hasOneUse())
+ return II->getArgOperand(0);
+ return nullptr;
+ };
+
+ Value *V0 = checkIntrinsicAndGetItsArgument(Inst.getOperand(0), ReductionIID);
+ if (!V0)
+ return nullptr;
+ Value *V1 = checkIntrinsicAndGetItsArgument(Inst.getOperand(1), ReductionIID);
+ if (!V1)
+ return nullptr;
+
+ Type *VTy = V0->getType();
+ if (V1->getType() != VTy)
+ return nullptr;
+
+ Value *VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
+
+ if (PossiblyDisjointInst *PDInst = dyn_cast<PossiblyDisjointInst>(&Inst))
+ if (auto *PDVectorBO = dyn_cast<PossiblyDisjointInst>(VectorBO))
+ PDVectorBO->setIsDisjoint(PDInst->isDisjoint());
+
+ Instruction *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
+ return Rdx;
+}
+
/// Try to narrow the width of a binop if at least 1 operand is an extend of
/// of a value. This requires a potentially expensive known bits check to make
/// sure the narrow op does not overflow.
More information about the llvm-commits
mailing list