[llvm] [LV] Add support for absolute difference partial reductions (PR #188043)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 15 03:38:38 PDT 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/188043
>From 2aabda527dfb8acb3dce8fedc21c8e7eb7157e1c Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 23 Mar 2026 13:18:39 +0000
Subject: [PATCH 1/6] [LV] Add support for absolute difference partial
reductions
This adds support for partial reductions where the extended operand is
a signed or unsigned absolute difference.
We match the absolute difference as `abs(sub(ext(X), ext(Y)))`, where
type(X) == type(Y) and both extends are the same kind (sext/zext). This
is then handled the same as an operand without a binop (`ext(...)`), as
we will transform the operand to `ext(absolute-difference(A, B))` when
we rewrite the reduction chain to partial reductions.
The `absolute-difference` is represented with two new VPInstructions
SignedAbsoluteDifference and UnsignedAbsoluteDifference, which are
expanded during plan execution.
Note: A few AArch64 CodeGen tests have been added to demonstrate the
absolute difference expansions generate the desired instructions.
This is an alternative to #162296.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +
.../Transforms/Vectorize/VPlanAnalysis.cpp | 3 +
.../Transforms/Vectorize/VPlanPatternMatch.h | 9 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 21 ++
.../Transforms/Vectorize/VPlanTransforms.cpp | 46 ++++
.../AArch64/narrow-absolute-difference.ll | 50 ++++
.../AArch64/partial-reduce-usabs.ll | 214 ++++++++++++++++++
.../VPlan/AArch64/partial-reduce-usabs.ll | 143 ++++++++++++
8 files changed, 487 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/narrow-absolute-difference.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index cc93bb924998b..9767e13e19356 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1309,6 +1309,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
LastActiveLane,
// Returns a reversed vector for the operand.
Reverse,
+ // Calculates the [signed|unsigned] absolute difference between two vector
+ // operands.
+ SignedAbsoluteDifference,
+ UnsignedAbsoluteDifference,
// The opcodes below are used for VPInstructionWithType.
//
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 2f6377614a9bf..71b778dcdcba6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -126,6 +126,9 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
// Assume that the maximum possible number of elements in a vector fits
// within the index type for the default address space.
return DL.getIndexType(Ctx, 0);
+ case VPInstruction::SignedAbsoluteDifference:
+ case VPInstruction::UnsignedAbsoluteDifference:
+ return inferScalarType(R->getOperand(0));
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
assert(inferScalarType(R->getOperand(0))->isIntegerTy(1) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 2ce012c5a9f10..e694a6c432a02 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1063,10 +1063,13 @@ m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
}
+template <Intrinsic::ID IntrID, typename... T>
+inline auto m_WidenIntrinsic(const T &...Ops) {
+ return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
+}
+
inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
-/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
-/// and bind the source element type and operands.
struct GetElementPtr_match {
Type *&SourceElementType;
ArrayRef<VPValue *> &Operands;
@@ -1120,7 +1123,7 @@ template <typename SubPattern_t> struct OneUse_match {
OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
- template <typename OpTy> bool match(OpTy *V) {
+ template <typename OpTy> bool match(OpTy *V) const {
return V->hasOneUse() && SubPattern.match(V);
}
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 2c2f3915c41d4..bd589385b677e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -481,6 +481,8 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::WidePtrAdd:
case VPInstruction::WideIVStep:
case VPInstruction::CalculateTripCountMinusVF:
+ case VPInstruction::SignedAbsoluteDifference:
+ case VPInstruction::UnsignedAbsoluteDifference:
return 2;
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
@@ -911,6 +913,17 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
+ case VPInstruction::SignedAbsoluteDifference:
+ case VPInstruction::UnsignedAbsoluteDifference: {
+ bool IsSigned = getOpcode() == VPInstruction::SignedAbsoluteDifference;
+ Value *X = Builder.CreateFreeze(State.get(getOperand(0)));
+ Value *Y = Builder.CreateFreeze(State.get(getOperand(1)));
+ Value *Max = Builder.CreateBinaryIntrinsic(
+ IsSigned ? Intrinsic::smax : Intrinsic::umax, X, Y);
+ Value *Min = Builder.CreateBinaryIntrinsic(
+ IsSigned ? Intrinsic::smin : Intrinsic::umin, X, Y);
+ return Builder.CreateSub(Max, Min);
+ }
default:
llvm_unreachable("Unsupported opcode for instruction");
}
@@ -1357,6 +1370,8 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::Reverse:
case VPInstruction::VScale:
case VPInstruction::Unpack:
+ case VPInstruction::SignedAbsoluteDifference:
+ case VPInstruction::UnsignedAbsoluteDifference:
return false;
default:
return true;
@@ -1537,6 +1552,12 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::LastActiveLane:
O << "last-active-lane";
break;
+ case VPInstruction::SignedAbsoluteDifference:
+ O << "signed-absolute-difference";
+ break;
+ case VPInstruction::UnsignedAbsoluteDifference:
+ O << "unsigned-absolute-difference";
+ break;
case VPInstruction::ReductionStartVector:
O << "reduction-start-vector";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 83ab470e1516f..0bdc97dc227ad 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6030,6 +6030,24 @@ optimizeExtendsForPartialReduction(VPSingleDefRecipe *BinOp,
return BinOp;
}
+ // reduce.add(abs(sub(ext(A), ext(B))))
+ // -> reduce.add(ext(absolute-difference(A, B)))
+ VPValue *X, *Y;
+ if (match(BinOp,
+ m_Intrinsic<Intrinsic::abs>(m_Sub(m_ZExtOrSExt(m_VPValue(X)),
+ m_ZExtOrSExt(m_VPValue(Y)))))) {
+ auto *Ext = cast<VPWidenCastRecipe>(
+ BinOp->getOperand(0)->getDefiningRecipe()->getOperand(0));
+ bool IsSigned = Ext->getOpcode() == Instruction::SExt;
+ VPBuilder Builder(BinOp);
+ VPValue *AbsDiff = Builder.createNaryOp(
+ IsSigned ? VPInstruction::SignedAbsoluteDifference
+ : VPInstruction::UnsignedAbsoluteDifference,
+ {X, Y});
+ return Builder.createWidenCast(Instruction::CastOps::ZExt, AbsDiff,
+ TypeInfo.inferScalarType(BinOp));
+ }
+
// reduce.add(ext(mul(ext(A), ext(B))))
// -> reduce.add(mul(wider_ext(A), wider_ext(B)))
// TODO: Support this optimization for float types.
@@ -6208,6 +6226,7 @@ static ExtendKind getPartialReductionExtendKind(VPWidenCastRecipe *Cast) {
/// - UpdateR(PrevValue, neg(BinOp(ext(...), Constant)))
/// - UpdateR(PrevValue, ext(mul(ext(...), ext(...))))
/// - UpdateR(PrevValue, ext(mul(ext(...), Constant)))
+/// - UpdateR(PrevValue, abs(sub(ext(...), ext(...)))
///
/// Note: The second operand of UpdateR corresponds to \p Op in the examples.
static std::optional<ExtendedReductionOperand>
@@ -6216,6 +6235,33 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op,
assert(is_contained(UpdateR->operands(), Op) &&
"Op should be operand of UpdateR");
+ // Try matching an absolute difference operand of the form:
+ // `abs(sub(ext(A), ext(B)))`. We can rewrite these to
+ // `ext(absolute-difference(A, B))`. This allows us to perform the absolute
+ // difference on a wider type and get the extend for "free" from the partial
+ // reduction.
+ VPValue *X, *Y;
+ if (Op->hasOneUse() &&
+ match(Op, m_WidenIntrinsic<Intrinsic::abs>(
+ m_OneUse(m_Sub(m_WidenAnyExtend(m_VPValue(X)),
+ m_WidenAnyExtend(m_VPValue(Y))))))) {
+ auto *Abs = cast<VPWidenIntrinsicRecipe>(Op);
+ auto *Sub = cast<VPWidenRecipe>(Abs->getOperand(0));
+ auto *LHSExt = cast<VPWidenCastRecipe>(Sub->getOperand(0));
+ auto *RHSExt = cast<VPWidenCastRecipe>(Sub->getOperand(1));
+ Type *LHSInputType = TypeInfo.inferScalarType(X);
+ Type *RHSInputType = TypeInfo.inferScalarType(Y);
+ if (LHSInputType != RHSInputType ||
+ LHSExt->getOpcode() != RHSExt->getOpcode())
+ return std::nullopt;
+ // Note: This is essentially the same as matching ext(...) as we will
+ // rewrite this operand to ext(absolute-difference(A, B)).
+ return ExtendedReductionOperand{
+ Sub,
+ /*ExtendA=*/{LHSInputType, getPartialReductionExtendKind(LHSExt)},
+ /*ExtendB=*/{}};
+ }
+
std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
if (match(Op, m_WidenAnyExtend(m_VPValue()))) {
auto *CastRecipe = cast<VPWidenCastRecipe>(Op);
diff --git a/llvm/test/CodeGen/AArch64/narrow-absolute-difference.ll b/llvm/test/CodeGen/AArch64/narrow-absolute-difference.ll
new file mode 100644
index 0000000000000..2086188c3690e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/narrow-absolute-difference.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mattr=+sve < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define <vscale x 16 x i8> @sabs_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sabs_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: sabd z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+ %smax = tail call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %smin = tail call <vscale x 16 x i8> @llvm.smin.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %sabs = sub <vscale x 16 x i8> %smax, %smin
+ ret <vscale x 16 x i8> %sabs
+}
+
+define <vscale x 16 x i8> @uabs_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uabs_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: uabd z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+ %umax = tail call <vscale x 16 x i8> @llvm.umax.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %umin = tail call <vscale x 16 x i8> @llvm.umin.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %uabs = sub <vscale x 16 x i8> %umax, %umin
+ ret <vscale x 16 x i8> %uabs
+}
+
+define <16 x i8> @sabs_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: sabs_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sabd v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %smax = tail call <16 x i8> @llvm.smax.v16i8(<16 x i8> %a, <16 x i8> %b)
+ %smin = tail call <16 x i8> @llvm.smin.v16i8(<16 x i8> %a, <16 x i8> %b)
+ %sabs = sub <16 x i8> %smax, %smin
+ ret <16 x i8> %sabs
+}
+
+define <16 x i8> @uabs_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: uabs_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uabd v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %umax = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %a, <16 x i8> %b)
+ %umin = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %a, <16 x i8> %b)
+ %uabs = sub <16 x i8> %umax, %umin
+ ret <16 x i8> %uabs
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
new file mode 100644
index 0000000000000..9c01a2646c297
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
@@ -0,0 +1,214 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -mattr=+neon,+dotprod -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s
+
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @unsigned_absolute_difference(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @unsigned_absolute_difference(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[FOR_BODY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[EXIT1]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[PARTIAL_REDUCE:%.*]], %[[EXIT1]] ]
+; CHECK-NEXT: [[X_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[X_PTR1]], align 1
+; CHECK-NEXT: [[Y_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[Y_PTR1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <16 x i8> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <16 x i8> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <16 x i8> @llvm.umax.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i8> @llvm.umin.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub <16 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[TMP6]] to <16 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP7]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV1]], 16
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 8000
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT2:.*]], label %[[EXIT1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT2]]:
+; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = zext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.diff, %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @signed_absolute_difference(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @signed_absolute_difference(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[FOR_BODY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[EXIT1]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[PARTIAL_REDUCE:%.*]], %[[EXIT1]] ]
+; CHECK-NEXT: [[X_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[X_PTR1]], align 1
+; CHECK-NEXT: [[Y_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[Y_PTR1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <16 x i8> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <16 x i8> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <16 x i8> @llvm.smax.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i8> @llvm.smin.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub <16 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[TMP6]] to <16 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP7]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV1]], 16
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 8000
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT2:.*]], label %[[EXIT1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT2]]:
+; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = sext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = sext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.diff, %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+; Negative test: Mismatched sign and zero extend.
+define i32 @mismatched_extends(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @mismatched_extends(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP8]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = sext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.diff, %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+; Negative test: Mismatched source types (i16 and i8).
+define i32 @mismatched_src_types(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @mismatched_src_types(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = sext <16 x i16> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP8]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i16, ptr %x, i64 %iv
+ %x.val = load i16, ptr %x.ptr, align 2
+ %ext.x = sext i16 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.diff, %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
new file mode 100644
index 0000000000000..34ca6b0d77ade
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --version 6
+; RUN: opt -passes=loop-vectorize -mattr=+neon,+dotprod -force-vector-interleave=1 -vplan-print-after="optimize$" \
+; RUN: -force-vector-width=16 -disable-output 2>&1 < %s | FileCheck %s
+
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @unsigned_absolute_difference(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: VPlan for loop in 'unsigned_absolute_difference'
+; CHECK: VPlan 'Initial VPlan for VF={16},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<8000> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.0> = phi vp<[[VP3]]>, vp<[[VP9:%[0-9]+]]> (VF scaled by 1/4)
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%x.ptr> = getelementptr inbounds nuw ir<%x>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw ir<%x.ptr>
+; CHECK-NEXT: WIDEN ir<%x.val> = load vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%y.ptr> = getelementptr inbounds nuw ir<%y>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%y.ptr>
+; CHECK-NEXT: WIDEN ir<%y.val> = load vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = unsigned-absolute-difference ir<%x.val>, ir<%y.val>
+; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = ir<%sum.0> + partial.reduce.add (vp<[[VP8]]> zext to i32)
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<8000>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %sum.1.lcssa = phi i32 [ %sum.1, %for.body ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = zext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.diff, %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @signed_absolute_difference(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: VPlan for loop in 'signed_absolute_difference'
+; CHECK: VPlan 'Initial VPlan for VF={16},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<8000> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.0> = phi vp<[[VP3]]>, vp<[[VP9:%[0-9]+]]> (VF scaled by 1/4)
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%x.ptr> = getelementptr inbounds nuw ir<%x>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw ir<%x.ptr>
+; CHECK-NEXT: WIDEN ir<%x.val> = load vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%y.ptr> = getelementptr inbounds nuw ir<%y>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%y.ptr>
+; CHECK-NEXT: WIDEN ir<%y.val> = load vp<[[VP7]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = signed-absolute-difference ir<%x.val>, ir<%y.val>
+; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = ir<%sum.0> + partial.reduce.add (vp<[[VP8]]> zext to i32)
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<8000>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %sum.1.lcssa = phi i32 [ %sum.1, %for.body ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = sext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = sext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.diff, %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
>From 1ac38cdcf0be16a4376c695e1f584bebae40c828 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 7 Apr 2026 15:47:52 +0000
Subject: [PATCH 2/6] Expand abs diff inline
---
.../Vectorize/LoopVectorizationPlanner.h | 7 +++--
llvm/lib/Transforms/Vectorize/VPlan.h | 4 ---
.../Transforms/Vectorize/VPlanAnalysis.cpp | 3 --
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 21 --------------
.../Transforms/Vectorize/VPlanTransforms.cpp | 15 +++++++---
.../VPlan/AArch64/partial-reduce-usabs.ll | 28 ++++++++++++-------
6 files changed, 34 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index b0cf7823ce9f8..3c3bb99798089 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -146,8 +146,11 @@ class VPBuilder {
InsertPt = IP->getIterator();
}
- /// Insert \p R at the current insertion point.
- void insert(VPRecipeBase *R) { BB->insert(R, InsertPt); }
+ /// Insert \p R at the current insertion point. Returns \p R unchanged.
+ template <typename T> [[maybe_unused]] T *insert(T *R) {
+ BB->insert(R, InsertPt);
+ return R;
+ }
/// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as
/// its underlying Instruction.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 9767e13e19356..cc93bb924998b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1309,10 +1309,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
LastActiveLane,
// Returns a reversed vector for the operand.
Reverse,
- // Calculates the [signed|unsigned] absolute difference between two vector
- // operands.
- SignedAbsoluteDifference,
- UnsignedAbsoluteDifference,
// The opcodes below are used for VPInstructionWithType.
//
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 71b778dcdcba6..2f6377614a9bf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -126,9 +126,6 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
// Assume that the maximum possible number of elements in a vector fits
// within the index type for the default address space.
return DL.getIndexType(Ctx, 0);
- case VPInstruction::SignedAbsoluteDifference:
- case VPInstruction::UnsignedAbsoluteDifference:
- return inferScalarType(R->getOperand(0));
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
assert(inferScalarType(R->getOperand(0))->isIntegerTy(1) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bd589385b677e..2c2f3915c41d4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -481,8 +481,6 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::WidePtrAdd:
case VPInstruction::WideIVStep:
case VPInstruction::CalculateTripCountMinusVF:
- case VPInstruction::SignedAbsoluteDifference:
- case VPInstruction::UnsignedAbsoluteDifference:
return 2;
case Instruction::Select:
case VPInstruction::ActiveLaneMask:
@@ -913,17 +911,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Result;
}
- case VPInstruction::SignedAbsoluteDifference:
- case VPInstruction::UnsignedAbsoluteDifference: {
- bool IsSigned = getOpcode() == VPInstruction::SignedAbsoluteDifference;
- Value *X = Builder.CreateFreeze(State.get(getOperand(0)));
- Value *Y = Builder.CreateFreeze(State.get(getOperand(1)));
- Value *Max = Builder.CreateBinaryIntrinsic(
- IsSigned ? Intrinsic::smax : Intrinsic::umax, X, Y);
- Value *Min = Builder.CreateBinaryIntrinsic(
- IsSigned ? Intrinsic::smin : Intrinsic::umin, X, Y);
- return Builder.CreateSub(Max, Min);
- }
default:
llvm_unreachable("Unsupported opcode for instruction");
}
@@ -1370,8 +1357,6 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::Reverse:
case VPInstruction::VScale:
case VPInstruction::Unpack:
- case VPInstruction::SignedAbsoluteDifference:
- case VPInstruction::UnsignedAbsoluteDifference:
return false;
default:
return true;
@@ -1552,12 +1537,6 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::LastActiveLane:
O << "last-active-lane";
break;
- case VPInstruction::SignedAbsoluteDifference:
- O << "signed-absolute-difference";
- break;
- case VPInstruction::UnsignedAbsoluteDifference:
- O << "unsigned-absolute-difference";
- break;
case VPInstruction::ReductionStartVector:
O << "reduction-start-vector";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 0bdc97dc227ad..41e41dda46e78 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6040,10 +6040,17 @@ optimizeExtendsForPartialReduction(VPSingleDefRecipe *BinOp,
BinOp->getOperand(0)->getDefiningRecipe()->getOperand(0));
bool IsSigned = Ext->getOpcode() == Instruction::SExt;
VPBuilder Builder(BinOp);
- VPValue *AbsDiff = Builder.createNaryOp(
- IsSigned ? VPInstruction::SignedAbsoluteDifference
- : VPInstruction::UnsignedAbsoluteDifference,
- {X, Y});
+ Type *SrcTy = TypeInfo.inferScalarType(X);
+ auto *FreezeX = Builder.insert(new VPWidenRecipe(Instruction::Freeze, {X}));
+ auto *FreezeY = Builder.insert(new VPWidenRecipe(Instruction::Freeze, {Y}));
+ auto *Max = Builder.insert(
+ new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smax : Intrinsic::umax,
+ {FreezeX, FreezeY}, SrcTy));
+ auto *Min = Builder.insert(
+ new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smin : Intrinsic::umin,
+ {FreezeX, FreezeY}, SrcTy));
+ auto *AbsDiff =
+ Builder.insert(new VPWidenRecipe(Instruction::Sub, {Max, Min}));
return Builder.createWidenCast(Instruction::CastOps::ZExt, AbsDiff,
TypeInfo.inferScalarType(BinOp));
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
index 34ca6b0d77ade..62790bd6859a3 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
@@ -22,7 +22,7 @@ define i32 @unsigned_absolute_difference(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.0> = phi vp<[[VP3]]>, vp<[[VP9:%[0-9]+]]> (VF scaled by 1/4)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.0> = phi vp<[[VP3]]>, vp<[[VP13:%[0-9]+]]> (VF scaled by 1/4)
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%x.ptr> = getelementptr inbounds nuw ir<%x>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw ir<%x.ptr>
@@ -30,8 +30,12 @@ define i32 @unsigned_absolute_difference(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: CLONE ir<%y.ptr> = getelementptr inbounds nuw ir<%y>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%y.ptr>
; CHECK-NEXT: WIDEN ir<%y.val> = load vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = unsigned-absolute-difference ir<%x.val>, ir<%y.val>
-; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = ir<%sum.0> + partial.reduce.add (vp<[[VP8]]> zext to i32)
+; CHECK-NEXT: WIDEN vp<[[VP8:%[0-9]+]]> = freeze ir<%x.val>
+; CHECK-NEXT: WIDEN vp<[[VP9:%[0-9]+]]> = freeze ir<%y.val>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP10:%[0-9]+]]> = call llvm.umax(vp<[[VP8]]>, vp<[[VP9]]>)
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP11:%[0-9]+]]> = call llvm.umin(vp<[[VP8]]>, vp<[[VP9]]>)
+; CHECK-NEXT: WIDEN vp<[[VP12:%[0-9]+]]> = sub vp<[[VP10]]>, vp<[[VP11]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP13]]> = ir<%sum.0> + partial.reduce.add (vp<[[VP12]]> zext to i32)
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -39,13 +43,13 @@ define i32 @unsigned_absolute_difference(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<8000>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %sum.1.lcssa = phi i32 [ %sum.1, %for.body ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: IR %sum.1.lcssa = phi i32 [ %sum.1, %for.body ] (extra operand: vp<[[VP15]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -91,7 +95,7 @@ define i32 @signed_absolute_difference(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.0> = phi vp<[[VP3]]>, vp<[[VP9:%[0-9]+]]> (VF scaled by 1/4)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.0> = phi vp<[[VP3]]>, vp<[[VP13:%[0-9]+]]> (VF scaled by 1/4)
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: CLONE ir<%x.ptr> = getelementptr inbounds nuw ir<%x>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw ir<%x.ptr>
@@ -99,8 +103,12 @@ define i32 @signed_absolute_difference(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: CLONE ir<%y.ptr> = getelementptr inbounds nuw ir<%y>, vp<[[VP5]]>
; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%y.ptr>
; CHECK-NEXT: WIDEN ir<%y.val> = load vp<[[VP7]]>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = signed-absolute-difference ir<%x.val>, ir<%y.val>
-; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = ir<%sum.0> + partial.reduce.add (vp<[[VP8]]> zext to i32)
+; CHECK-NEXT: WIDEN vp<[[VP8:%[0-9]+]]> = freeze ir<%x.val>
+; CHECK-NEXT: WIDEN vp<[[VP9:%[0-9]+]]> = freeze ir<%y.val>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP10:%[0-9]+]]> = call llvm.smax(vp<[[VP8]]>, vp<[[VP9]]>)
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP11:%[0-9]+]]> = call llvm.smin(vp<[[VP8]]>, vp<[[VP9]]>)
+; CHECK-NEXT: WIDEN vp<[[VP12:%[0-9]+]]> = sub vp<[[VP10]]>, vp<[[VP11]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP13]]> = ir<%sum.0> + partial.reduce.add (vp<[[VP12]]> zext to i32)
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -108,13 +116,13 @@ define i32 @signed_absolute_difference(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<8000>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %sum.1.lcssa = phi i32 [ %sum.1, %for.body ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: IR %sum.1.lcssa = phi i32 [ %sum.1, %for.body ] (extra operand: vp<[[VP15]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
>From ff8548480ef03daebf8cbc7f451b6194a4967a42 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 9 Apr 2026 12:28:22 +0000
Subject: [PATCH 3/6] Add comment back
---
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index e694a6c432a02..b4bb29c43aeb2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1070,6 +1070,7 @@ inline auto m_WidenIntrinsic(const T &...Ops) {
inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
+/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
struct GetElementPtr_match {
Type *&SourceElementType;
ArrayRef<VPValue *> &Operands;
>From 52c4cb8914069b9064d6e83b311b8a3d2a8aa301 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 9 Apr 2026 12:32:11 +0000
Subject: [PATCH 4/6] Add rest of comment
---
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index b4bb29c43aeb2..939dcccfb58fc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1071,6 +1071,7 @@ inline auto m_WidenIntrinsic(const T &...Ops) {
inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
+/// and bind the source element type and operands.
struct GetElementPtr_match {
Type *&SourceElementType;
ArrayRef<VPValue *> &Operands;
>From 7faf6502e7172f456a3fa696d1af15cf023b482e Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 14 Apr 2026 16:16:13 +0000
Subject: [PATCH 5/6] Fixups
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 41e41dda46e78..2604759a7f77b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6036,8 +6036,11 @@ optimizeExtendsForPartialReduction(VPSingleDefRecipe *BinOp,
if (match(BinOp,
m_Intrinsic<Intrinsic::abs>(m_Sub(m_ZExtOrSExt(m_VPValue(X)),
m_ZExtOrSExt(m_VPValue(Y)))))) {
- auto *Ext = cast<VPWidenCastRecipe>(
- BinOp->getOperand(0)->getDefiningRecipe()->getOperand(0));
+ auto *Sub = BinOp->getOperand(0)->getDefiningRecipe();
+ auto *Ext = cast<VPWidenCastRecipe>(Sub->getOperand(0));
+ assert(Ext->getOpcode() ==
+ cast<VPWidenCastRecipe>(Sub->getOperand(1))->getOpcode() &&
+ "Expected both the LHS and RHS extends to be the same");
bool IsSigned = Ext->getOpcode() == Instruction::SExt;
VPBuilder Builder(BinOp);
Type *SrcTy = TypeInfo.inferScalarType(X);
@@ -6242,8 +6245,8 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op,
assert(is_contained(UpdateR->operands(), Op) &&
"Op should be operand of UpdateR");
- // Try matching an absolute difference operand of the form:
- // `abs(sub(ext(A), ext(B)))`. We can rewrite these to
+ // Try matching an absolute difference operand of the form
+ // `abs(sub(ext(A), ext(B)))`. This will be later transformed into
// `ext(absolute-difference(A, B))`. This allows us to perform the absolute
// difference on a wider type and get the extend for "free" from the partial
// reduction.
>From d74d7815980045075d0e077c92b53b6d9c44ab01 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 15 Apr 2026 10:36:58 +0000
Subject: [PATCH 6/6] Update tests
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 4 +-
.../AArch64/narrow-absolute-difference.ll | 50 -----
.../AArch64/partial-reduce-usabs.ll | 186 +++++++++++++++++-
3 files changed, 180 insertions(+), 60 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/narrow-absolute-difference.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2604759a7f77b..65be9fd5c5fae 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6034,8 +6034,8 @@ optimizeExtendsForPartialReduction(VPSingleDefRecipe *BinOp,
// -> reduce.add(ext(absolute-difference(A, B)))
VPValue *X, *Y;
if (match(BinOp,
- m_Intrinsic<Intrinsic::abs>(m_Sub(m_ZExtOrSExt(m_VPValue(X)),
- m_ZExtOrSExt(m_VPValue(Y)))))) {
+ m_WidenIntrinsic<Intrinsic::abs>(m_Sub(
+ m_ZExtOrSExt(m_VPValue(X)), m_ZExtOrSExt(m_VPValue(Y)))))) {
auto *Sub = BinOp->getOperand(0)->getDefiningRecipe();
auto *Ext = cast<VPWidenCastRecipe>(Sub->getOperand(0));
assert(Ext->getOpcode() ==
diff --git a/llvm/test/CodeGen/AArch64/narrow-absolute-difference.ll b/llvm/test/CodeGen/AArch64/narrow-absolute-difference.ll
deleted file mode 100644
index 2086188c3690e..0000000000000
--- a/llvm/test/CodeGen/AArch64/narrow-absolute-difference.ll
+++ /dev/null
@@ -1,50 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mattr=+sve < %s | FileCheck %s
-
-target triple = "aarch64-unknown-linux-gnu"
-
-define <vscale x 16 x i8> @sabs_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: sabs_nxv16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: sabd z0.b, p0/m, z0.b, z1.b
-; CHECK-NEXT: ret
- %smax = tail call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
- %smin = tail call <vscale x 16 x i8> @llvm.smin.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
- %sabs = sub <vscale x 16 x i8> %smax, %smin
- ret <vscale x 16 x i8> %sabs
-}
-
-define <vscale x 16 x i8> @uabs_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: uabs_nxv16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: uabd z0.b, p0/m, z0.b, z1.b
-; CHECK-NEXT: ret
- %umax = tail call <vscale x 16 x i8> @llvm.umax.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
- %umin = tail call <vscale x 16 x i8> @llvm.umin.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
- %uabs = sub <vscale x 16 x i8> %umax, %umin
- ret <vscale x 16 x i8> %uabs
-}
-
-define <16 x i8> @sabs_v16i8(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: sabs_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sabd v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: ret
- %smax = tail call <16 x i8> @llvm.smax.v16i8(<16 x i8> %a, <16 x i8> %b)
- %smin = tail call <16 x i8> @llvm.smin.v16i8(<16 x i8> %a, <16 x i8> %b)
- %sabs = sub <16 x i8> %smax, %smin
- ret <16 x i8> %sabs
-}
-
-define <16 x i8> @uabs_v16i8(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: uabs_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uabd v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: ret
- %umax = tail call <16 x i8> @llvm.umax.v16i8(<16 x i8> %a, <16 x i8> %b)
- %umin = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %a, <16 x i8> %b)
- %uabs = sub <16 x i8> %umax, %umin
- ret <16 x i8> %uabs
-}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
index 9c01a2646c297..65289f28bf7d6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
@@ -66,10 +66,10 @@ define i32 @signed_absolute_difference(ptr noalias %x, ptr noalias %y) {
; CHECK: [[EXIT1]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[EXIT1]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[PARTIAL_REDUCE:%.*]], %[[EXIT1]] ]
-; CHECK-NEXT: [[X_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[X_PTR1]], align 1
-; CHECK-NEXT: [[Y_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[IV1]]
-; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[Y_PTR1]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = freeze <16 x i8> [[WIDE_LOAD]]
; CHECK-NEXT: [[TMP3:%.*]] = freeze <16 x i8> [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = call <16 x i8> @llvm.smax.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
@@ -81,10 +81,10 @@ define i32 @signed_absolute_difference(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 8000
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT2:.*]], label %[[EXIT1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT2]]:
-; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-NEXT: ret i32 [[TMP9]]
;
entry:
br label %for.body
@@ -109,6 +109,60 @@ exit:
ret i32 %sum.1
}
+; This case is a single level extend (i32 -> i64). This could lower to a sabal[bt] pair.
+define i64 @signed_absolute_difference_i64_to_i64(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i64 @signed_absolute_difference_i64_to_i64(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i32> [[TMP6]] to <4 x i64>
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v4i64(<2 x i64> [[VEC_PHI]], <4 x i64> [[TMP7]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 [[TMP9]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i64 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i32, ptr %x, i64 %iv
+ %x.val = load i32, ptr %x.ptr, align 1
+ %ext.x = sext i32 %x.val to i64
+ %y.ptr = getelementptr inbounds nuw i32, ptr %y, i64 %iv
+ %y.val = load i32, ptr %y.ptr, align 1
+ %ext.y = sext i32 %y.val to i64
+ %sub = sub nsw i64 %ext.x, %ext.y
+ %abs.diff = tail call i64 @llvm.abs.i64(i64 %sub, i1 true)
+ %sum.1 = add nuw nsw i64 %abs.diff, %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i64 %sum.1
+}
+
; Negative test: Mismatched sign and zero extend.
define i32 @mismatched_extends(ptr noalias %x, ptr noalias %y) {
; CHECK-LABEL: define i32 @mismatched_extends(
@@ -131,7 +185,7 @@ define i32 @mismatched_extends(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -183,7 +237,7 @@ define i32 @mismatched_src_types(ptr noalias %x, ptr noalias %y) {
; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
; CHECK-NEXT: br label %[[EXIT:.*]]
@@ -212,3 +266,119 @@ for.body:
exit:
ret i32 %sum.1
}
+
+; Negative test: %sub has more than one user.
+define i32 @additional_user_of_sub(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @additional_user_of_sub(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP7]] = add <16 x i32> [[TMP4]], [[VEC_PHI1]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP7]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RET:%.*]] = xor i32 [[TMP10]], [[TMP9]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %redux.0 = phi i32 [ 0, %entry ], [ %redux.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = zext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.diff, %sum.0
+ %redux.1 = add nuw nsw i32 %sub, %redux.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ %ret = xor i32 %redux.1, %sum.1
+ ret i32 %ret
+}
+
+; Negative test: Additional use of %abs.diff (prevents extends folding away).
+; TODO: This could be profitable if the other user is a partial reduction.
+define i32 @uabs_additional_user_of_abs(ptr noalias %x, ptr noalias %y, ptr noalias %z) {
+; CHECK-LABEL: define i32 @uabs_additional_user_of_abs(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], ptr noalias [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[Z]], i64 [[INDEX]]
+; CHECK-NEXT: store <16 x i32> [[TMP5]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP9]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = zext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.diff, %sum.0
+ %z.ptr = getelementptr inbounds nuw i32, ptr %z, i64 %iv
+ store i32 %abs.diff, ptr %z.ptr, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
More information about the llvm-commits
mailing list