[llvm] f943685 - [LV] Add support for absolute difference partial reductions (#188043)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 01:24:52 PDT 2026
Author: Benjamin Maxwell
Date: 2026-04-16T09:24:47+01:00
New Revision: f943685879121b02d568c0ae7e0c95a89a2323e3
URL: https://github.com/llvm/llvm-project/commit/f943685879121b02d568c0ae7e0c95a89a2323e3
DIFF: https://github.com/llvm/llvm-project/commit/f943685879121b02d568c0ae7e0c95a89a2323e3.diff
LOG: [LV] Add support for absolute difference partial reductions (#188043)
This adds support for partial reductions where the extended operand is a
signed or unsigned absolute difference.
We match the absolute difference as `abs(sub(ext(X), ext(Y)))`, where
`type(X) == type(Y)` and both extends are the same kind (sext/zext).
This is then handled the same as an operand without a binop
(`ext(...)`), as we will transform the operand to
`ext(absolute-difference(A, B))` when we rewrite the reduction chain to
partial reductions.
This is an alternative to #162296.
Added:
llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index f2e820b28e9d7..8a9082c463e1e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -146,8 +146,11 @@ class VPBuilder {
InsertPt = IP->getIterator();
}
- /// Insert \p R at the current insertion point.
- void insert(VPRecipeBase *R) { BB->insert(R, InsertPt); }
+ /// Insert \p R at the current insertion point. Returns \p R unchanged.
+ template <typename T> [[maybe_unused]] T *insert(T *R) {
+ BB->insert(R, InsertPt);
+ return R;
+ }
/// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as
/// its underlying Instruction.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 4113188cfd846..ed5e5d0698993 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1007,6 +1007,11 @@ m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
}
+template <Intrinsic::ID IntrID, typename... T>
+inline auto m_WidenIntrinsic(const T &...Ops) {
+ return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
+}
+
inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
@@ -1064,7 +1069,7 @@ template <typename SubPattern_t> struct OneUse_match {
OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
- template <typename OpTy> bool match(OpTy *V) {
+ template <typename OpTy> bool match(OpTy *V) const {
return V->hasOneUse() && SubPattern.match(V);
}
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 52f299dd5fdca..d1ff014907dc9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6042,6 +6042,34 @@ optimizeExtendsForPartialReduction(VPSingleDefRecipe *BinOp,
return BinOp;
}
+ // reduce.add(abs(sub(ext(A), ext(B))))
+ // -> reduce.add(ext(absolute-
diff erence(A, B)))
+ VPValue *X, *Y;
+ if (match(BinOp,
+ m_WidenIntrinsic<Intrinsic::abs>(m_Sub(
+ m_ZExtOrSExt(m_VPValue(X)), m_ZExtOrSExt(m_VPValue(Y)))))) {
+ auto *Sub = BinOp->getOperand(0)->getDefiningRecipe();
+ auto *Ext = cast<VPWidenCastRecipe>(Sub->getOperand(0));
+ assert(Ext->getOpcode() ==
+ cast<VPWidenCastRecipe>(Sub->getOperand(1))->getOpcode() &&
+ "Expected both the LHS and RHS extends to be the same");
+ bool IsSigned = Ext->getOpcode() == Instruction::SExt;
+ VPBuilder Builder(BinOp);
+ Type *SrcTy = TypeInfo.inferScalarType(X);
+ auto *FreezeX = Builder.insert(new VPWidenRecipe(Instruction::Freeze, {X}));
+ auto *FreezeY = Builder.insert(new VPWidenRecipe(Instruction::Freeze, {Y}));
+ auto *Max = Builder.insert(
+ new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smax : Intrinsic::umax,
+ {FreezeX, FreezeY}, SrcTy));
+ auto *Min = Builder.insert(
+ new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smin : Intrinsic::umin,
+ {FreezeX, FreezeY}, SrcTy));
+ auto *AbsDiff =
+ Builder.insert(new VPWidenRecipe(Instruction::Sub, {Max, Min}));
+ return Builder.createWidenCast(Instruction::CastOps::ZExt, AbsDiff,
+ TypeInfo.inferScalarType(BinOp));
+ }
+
// reduce.add(ext(mul(ext(A), ext(B))))
// -> reduce.add(mul(wider_ext(A), wider_ext(B)))
// TODO: Support this optimization for float types.
@@ -6220,6 +6248,7 @@ static ExtendKind getPartialReductionExtendKind(VPWidenCastRecipe *Cast) {
/// - UpdateR(PrevValue, neg(BinOp(ext(...), Constant)))
/// - UpdateR(PrevValue, ext(mul(ext(...), ext(...))))
/// - UpdateR(PrevValue, ext(mul(ext(...), Constant)))
+/// - UpdateR(PrevValue, abs(sub(ext(...), ext(...)))
///
/// Note: The second operand of UpdateR corresponds to \p Op in the examples.
static std::optional<ExtendedReductionOperand>
@@ -6228,6 +6257,33 @@ matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op,
assert(is_contained(UpdateR->operands(), Op) &&
"Op should be operand of UpdateR");
+ // Try matching an absolute
diff erence operand of the form
+ // `abs(sub(ext(A), ext(B)))`. This will be later transformed into
+ // `ext(absolute-
diff erence(A, B))`. This allows us to perform the absolute
+ //
diff erence on a wider type and get the extend for "free" from the partial
+ // reduction.
+ VPValue *X, *Y;
+ if (Op->hasOneUse() &&
+ match(Op, m_WidenIntrinsic<Intrinsic::abs>(
+ m_OneUse(m_Sub(m_WidenAnyExtend(m_VPValue(X)),
+ m_WidenAnyExtend(m_VPValue(Y))))))) {
+ auto *Abs = cast<VPWidenIntrinsicRecipe>(Op);
+ auto *Sub = cast<VPWidenRecipe>(Abs->getOperand(0));
+ auto *LHSExt = cast<VPWidenCastRecipe>(Sub->getOperand(0));
+ auto *RHSExt = cast<VPWidenCastRecipe>(Sub->getOperand(1));
+ Type *LHSInputType = TypeInfo.inferScalarType(X);
+ Type *RHSInputType = TypeInfo.inferScalarType(Y);
+ if (LHSInputType != RHSInputType ||
+ LHSExt->getOpcode() != RHSExt->getOpcode())
+ return std::nullopt;
+ // Note: This is essentially the same as matching ext(...) as we will
+ // rewrite this operand to ext(absolute-
diff erence(A, B)).
+ return ExtendedReductionOperand{
+ Sub,
+ /*ExtendA=*/{LHSInputType, getPartialReductionExtendKind(LHSExt)},
+ /*ExtendB=*/{}};
+ }
+
std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
if (match(Op, m_WidenAnyExtend(m_VPValue()))) {
auto *CastRecipe = cast<VPWidenCastRecipe>(Op);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
new file mode 100644
index 0000000000000..65289f28bf7d6
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-usabs.ll
@@ -0,0 +1,384 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -mattr=+neon,+dotprod -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s
+
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @unsigned_absolute_
diff erence(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @unsigned_absolute_
diff erence(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[FOR_BODY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[EXIT1]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[PARTIAL_REDUCE:%.*]], %[[EXIT1]] ]
+; CHECK-NEXT: [[X_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[X_PTR1]], align 1
+; CHECK-NEXT: [[Y_PTR1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[Y_PTR1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <16 x i8> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <16 x i8> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <16 x i8> @llvm.umax.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i8> @llvm.umin.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub <16 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[TMP6]] to <16 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP7]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV1]], 16
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 8000
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT2:.*]], label %[[EXIT1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT2]]:
+; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = zext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.
diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.
diff , %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @signed_absolute_
diff erence(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @signed_absolute_
diff erence(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[FOR_BODY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[IV_NEXT:%.*]], %[[EXIT1]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[PARTIAL_REDUCE:%.*]], %[[EXIT1]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <16 x i8> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <16 x i8> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <16 x i8> @llvm.smax.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i8> @llvm.smin.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub <16 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext <16 x i8> [[TMP6]] to <16 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP7]])
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV1]], 16
+; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 8000
+; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT2:.*]], label %[[EXIT1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT2]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP9]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = sext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = sext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.
diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.
diff , %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+; This case is a single level extend (i32 -> i64). This could lower to a sabal[bt] pair.
+define i64 @signed_absolute_
diff erence_i64_to_i64(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i64 @signed_absolute_
diff erence_i64_to_i64(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i32, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = sub <4 x i32> [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i32> [[TMP6]] to <4 x i64>
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v4i64(<2 x i64> [[VEC_PHI]], <4 x i64> [[TMP7]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> [[PARTIAL_REDUCE]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 [[TMP9]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i64 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i32, ptr %x, i64 %iv
+ %x.val = load i32, ptr %x.ptr, align 1
+ %ext.x = sext i32 %x.val to i64
+ %y.ptr = getelementptr inbounds nuw i32, ptr %y, i64 %iv
+ %y.val = load i32, ptr %y.ptr, align 1
+ %ext.y = sext i32 %y.val to i64
+ %sub = sub nsw i64 %ext.x, %ext.y
+ %abs.
diff = tail call i64 @llvm.abs.i64(i64 %sub, i1 true)
+ %sum.1 = add nuw nsw i64 %abs.
diff , %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i64 %sum.1
+}
+
+; Negative test: Mismatched sign and zero extend.
+define i32 @mismatched_extends(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @mismatched_extends(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP8]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = sext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.
diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.
diff , %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+; Negative test: Mismatched source types (i16 and i8).
+define i32 @mismatched_src_types(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @mismatched_src_types(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i16>, ptr [[TMP0]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = sext <16 x i16> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP8]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i16, ptr %x, i64 %iv
+ %x.val = load i16, ptr %x.ptr, align 2
+ %ext.x = sext i16 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.
diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.
diff , %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+; Negative test: %sub has more than one user.
+define i32 @additional_user_of_sub(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: define i32 @additional_user_of_sub(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP7]] = add <16 x i32> [[TMP4]], [[VEC_PHI1]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP7]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RET:%.*]] = xor i32 [[TMP10]], [[TMP9]]
+; CHECK-NEXT: ret i32 [[RET]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %redux.0 = phi i32 [ 0, %entry ], [ %redux.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = zext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.
diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.
diff , %sum.0
+ %redux.1 = add nuw nsw i32 %sub, %redux.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ %ret = xor i32 %redux.1, %sum.1
+ ret i32 %ret
+}
+
+; Negative test: Additional use of %abs.
diff (prevents extends folding away).
+; TODO: This could be profitable if the other user is a partial reduction.
+define i32 @uabs_additional_user_of_abs(ptr noalias %x, ptr noalias %y, ptr noalias %z) {
+; CHECK-LABEL: define i32 @uabs_additional_user_of_abs(
+; CHECK-SAME: ptr noalias [[X:%.*]], ptr noalias [[Y:%.*]], ptr noalias [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <16 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.abs.v16i32(<16 x i32> [[TMP4]], i1 true)
+; CHECK-NEXT: [[TMP6]] = add <16 x i32> [[TMP5]], [[VEC_PHI]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i32, ptr [[Z]], i64 [[INDEX]]
+; CHECK-NEXT: store <16 x i32> [[TMP5]], ptr [[TMP7]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8000
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP6]])
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[TMP9]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = zext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.
diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.
diff , %sum.0
+ %z.ptr = getelementptr inbounds nuw i32, ptr %z, i64 %iv
+ store i32 %abs.
diff , ptr %z.ptr, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
new file mode 100644
index 0000000000000..62790bd6859a3
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/partial-reduce-usabs.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --version 6
+; RUN: opt -passes=loop-vectorize -mattr=+neon,+dotprod -force-vector-interleave=1 -vplan-print-after="optimize$" \
+; RUN: -force-vector-width=16 -disable-output 2>&1 < %s | FileCheck %s
+
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @unsigned_absolute_
diff erence(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: VPlan for loop in 'unsigned_absolute_
diff erence'
+; CHECK: VPlan 'Initial VPlan for VF={16},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<8000> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.0> = phi vp<[[VP3]]>, vp<[[VP13:%[0-9]+]]> (VF scaled by 1/4)
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%x.ptr> = getelementptr inbounds nuw ir<%x>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw ir<%x.ptr>
+; CHECK-NEXT: WIDEN ir<%x.val> = load vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%y.ptr> = getelementptr inbounds nuw ir<%y>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%y.ptr>
+; CHECK-NEXT: WIDEN ir<%y.val> = load vp<[[VP7]]>
+; CHECK-NEXT: WIDEN vp<[[VP8:%[0-9]+]]> = freeze ir<%x.val>
+; CHECK-NEXT: WIDEN vp<[[VP9:%[0-9]+]]> = freeze ir<%y.val>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP10:%[0-9]+]]> = call llvm.umax(vp<[[VP8]]>, vp<[[VP9]]>)
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP11:%[0-9]+]]> = call llvm.umin(vp<[[VP8]]>, vp<[[VP9]]>)
+; CHECK-NEXT: WIDEN vp<[[VP12:%[0-9]+]]> = sub vp<[[VP10]]>, vp<[[VP11]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP13]]> = ir<%sum.0> + partial.reduce.add (vp<[[VP12]]> zext to i32)
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP13]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<8000>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %sum.1.lcssa = phi i32 [ %sum.1, %for.body ] (extra operand: vp<[[VP15]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = zext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = zext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.
diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.
diff , %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @signed_absolute_
diff erence(ptr noalias %x, ptr noalias %y) {
+; CHECK-LABEL: VPlan for loop in 'signed_absolute_
diff erence'
+; CHECK: VPlan 'Initial VPlan for VF={16},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<8000> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.0> = phi vp<[[VP3]]>, vp<[[VP13:%[0-9]+]]> (VF scaled by 1/4)
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%x.ptr> = getelementptr inbounds nuw ir<%x>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds nuw ir<%x.ptr>
+; CHECK-NEXT: WIDEN ir<%x.val> = load vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%y.ptr> = getelementptr inbounds nuw ir<%y>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%y.ptr>
+; CHECK-NEXT: WIDEN ir<%y.val> = load vp<[[VP7]]>
+; CHECK-NEXT: WIDEN vp<[[VP8:%[0-9]+]]> = freeze ir<%x.val>
+; CHECK-NEXT: WIDEN vp<[[VP9:%[0-9]+]]> = freeze ir<%y.val>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP10:%[0-9]+]]> = call llvm.smax(vp<[[VP8]]>, vp<[[VP9]]>)
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP11:%[0-9]+]]> = call llvm.smin(vp<[[VP8]]>, vp<[[VP9]]>)
+; CHECK-NEXT: WIDEN vp<[[VP12:%[0-9]+]]> = sub vp<[[VP10]]>, vp<[[VP11]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP13]]> = ir<%sum.0> + partial.reduce.add (vp<[[VP12]]> zext to i32)
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = compute-reduction-result (add) vp<[[VP13]]>
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<8000>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %sum.1.lcssa = phi i32 [ %sum.1, %for.body ] (extra operand: vp<[[VP15]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %sum.0 = phi i32 [ 0, %entry ], [ %sum.1, %for.body ]
+ %x.ptr = getelementptr inbounds nuw i8, ptr %x, i64 %iv
+ %x.val = load i8, ptr %x.ptr, align 1
+ %ext.x = sext i8 %x.val to i32
+ %y.ptr = getelementptr inbounds nuw i8, ptr %y, i64 %iv
+ %y.val = load i8, ptr %y.ptr, align 1
+ %ext.y = sext i8 %y.val to i32
+ %sub = sub nsw i32 %ext.x, %ext.y
+ %abs.
diff = tail call i32 @llvm.abs.i32(i32 %sub, i1 true)
+ %sum.1 = add nuw nsw i32 %abs.
diff , %sum.0
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 8000
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
More information about the llvm-commits
mailing list