[llvm] [LV] Create in-loop sub reductions (PR #147026)
Sam Tebbs via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 17 01:58:33 PDT 2025
https://github.com/SamTebbs33 updated https://github.com/llvm/llvm-project/pull/147026
>From 39f3dab6bb392a79e1888fc1c9aa82d9259c5576 Mon Sep 17 00:00:00 2001
From: Samuel Tebbs <samuel.tebbs at arm.com>
Date: Tue, 1 Jul 2025 13:38:04 +0100
Subject: [PATCH 1/3] [LV] Create in-loop sub reductions
This PR allows the loop vectorizer to handle sub reductions by forming a
normal add reduction with a negated input.
---
llvm/lib/Analysis/IVDescriptors.cpp | 4 ++++
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 8 ++++++++
llvm/test/Transforms/LoopVectorize/reduction-inloop.ll | 9 +++++----
3 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index b275b1064cef2..2b7d06e0c682e 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1263,6 +1263,10 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
if (isFMulAddIntrinsic(Cur))
return true;
+ // Recognize a sub reduction. It gets canonicalized to add(sub (0, ...)).
+ if (Cur->getOpcode() == Instruction::Sub && getOpcode() == Instruction::Add)
+ return true;
+
return Cur->getOpcode() == getOpcode();
};
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 907839711a39c..1cfbcf1336620 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9144,6 +9144,14 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
CurrentLinkI->getFastMathFlags());
LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
VecOp = FMulRecipe;
+ } else if (PhiR->isInLoop() && Kind == RecurKind::Add &&
+ CurrentLinkI->getOpcode() == Instruction::Sub) {
+ Type *PhiTy = PhiR->getUnderlyingValue()->getType();
+ auto *Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
+ VPWidenRecipe *Sub = new VPWidenRecipe(
+ *CurrentLinkI, {Zero, CurrentLink->getOperand(1)});
+ LinkVPBB->insert(Sub, CurrentLink->getIterator());
+ VecOp = Sub;
} else {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
if (isa<VPWidenRecipe>(CurrentLink)) {
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
index e762c9ff81322..4c6e0e487d0dd 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
@@ -627,7 +627,7 @@ for.end: ; preds = %for.body, %entry
ret float %result.0.lcssa
}
-; Sub we can create a reduction, but not inloop
+; Sub we can create a reduction inloop
define i32 @reduction_sub_lhs(ptr noalias nocapture %A) {
; CHECK-LABEL: @reduction_sub_lhs(
; CHECK-NEXT: entry:
@@ -636,15 +636,16 @@ define i32 @reduction_sub_lhs(ptr noalias nocapture %A) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1]] = sub <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <4 x i32> zeroinitializer, [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
; CHECK-NEXT: br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
>From eb2b4687d5a631e47cfb9d68d0c99b5c731dbb91 Mon Sep 17 00:00:00 2001
From: Samuel Tebbs <samuel.tebbs at arm.com>
Date: Wed, 16 Jul 2025 10:08:16 +0100
Subject: [PATCH 2/3] Use empty flags for negation
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 ++-
llvm/test/Transforms/LoopVectorize/reduction-inloop.ll | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1cfbcf1336620..d26479a41a4e8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9149,7 +9149,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
Type *PhiTy = PhiR->getUnderlyingValue()->getType();
auto *Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
VPWidenRecipe *Sub = new VPWidenRecipe(
- *CurrentLinkI, {Zero, CurrentLink->getOperand(1)});
+ Instruction::Sub, {Zero, CurrentLink->getOperand(1)}, {},
+ CurrentLinkI->getDebugLoc());
LinkVPBB->insert(Sub, CurrentLink->getIterator());
VecOp = Sub;
} else {
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
index 4c6e0e487d0dd..8d8cc474c6f41 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
@@ -639,7 +639,7 @@ define i32 @reduction_sub_lhs(ptr noalias nocapture %A) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <4 x i32> zeroinitializer, [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP1]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
>From 60780a4a1bb5e5949a3993cff6b1e2e23988f261 Mon Sep 17 00:00:00 2001
From: Samuel Tebbs <samuel.tebbs at arm.com>
Date: Wed, 16 Jul 2025 13:33:02 +0100
Subject: [PATCH 3/3] Improve test comment
---
llvm/test/Transforms/LoopVectorize/reduction-inloop.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
index 8d8cc474c6f41..a2645cd7f3398 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
@@ -627,7 +627,7 @@ for.end: ; preds = %for.body, %entry
ret float %result.0.lcssa
}
-; Sub we can create a reduction inloop
+; We can create an in-loop reduction for sub-reductions by negating the input.
define i32 @reduction_sub_lhs(ptr noalias nocapture %A) {
; CHECK-LABEL: @reduction_sub_lhs(
; CHECK-NEXT: entry:
More information about the llvm-commits
mailing list