[llvm] [LV][ValueTracking] Narrow type-promoted reductions using known bits (PR #222183)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 15:47:44 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Konstantin Bogdanov (thevar1able)
<details>
<summary>Changes</summary>
Fixes #<!-- -->222142.
Since 4eb56cf497aa ([InstSimplify] Fold value-preserving zext/sext of trunc, #<!-- -->204089), a byte-wise OR reduction with a `uint8_t` accumulator is vectorized with `<8 x i32>` lanes and a zext per load instead of `<32 x i8>` lanes on AVX2.
The frontend promotes the accumulator to i32, so the loop-carried value is `trunc nuw i32 -> i8` on the backedge and `zext i8 -> i32` on the next use. InstCombine pushes the zext through the phi, and the new `zext (trunc nuw X) -> X` fold then removes the round trip entirely, leaving an i32 phi with no cast and no mask in the loop. Before the fold, the same step produced `and %phi, 255`, which RecurrenceDescriptor recognises as a type-promoted reduction and narrows.
Two changes:
* ValueTracking: for or/xor recurrences, bits that are zero in both the start value and the step stay zero; for or, bits that are one in the start stay one. For and recurrences, bits that are zero in the start stay zero and bits that are one in both stay one. Previously only trailing zeros were derived for these recurrences.
* IVDescriptors: when a reduction phi is not fed through an `and` mask, still run `computeRecurrenceType` for add/mul/or/and/xor and adopt the narrower type if the exit value is known to fit in it. These operations are exact modulo 2^N, so truncating the lanes and extending the reduced result is value-preserving. Values that fit in fewer than 8 bits are left alone, so boolean accumulators don't drive up the VF of the surrounding loop.
With this, the reproducer from the issue is vectorized as `<32 x i8>` again and the AVX2 loop body is a single `vpor` per 32 bytes.
One existing test changes because its reduction now narrows: the `add nuw` reduction in reduction-with-invariant-store.ll narrows to i32 via its known sign bits.
---
Patch is 28.88 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/222183.diff
6 Files Affected:
- (modified) llvm/lib/Analysis/IVDescriptors.cpp (+12)
- (modified) llvm/lib/Analysis/ValueTracking.cpp (+10)
- (modified) llvm/test/Transforms/InstCombine/recurrence.ll (+198)
- (added) llvm/test/Transforms/LoopVectorize/reduction-narrow-known-bits.ll (+272)
- (modified) llvm/test/Transforms/LoopVectorize/reduction-with-invariant-store.ll (+6-3)
- (added) llvm/test/Transforms/PhaseOrdering/X86/or-reduction-i8-promoted-acc.ll (+68)
``````````diff
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 6efbc9177c3c6..990bfdca4f4d9 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -805,6 +805,18 @@ bool RecurrenceDescriptor::AddReductionVar(
computeRecurrenceType(ExitInstruction, DB, AC, DT);
if (ComputedType != RecurrenceType)
return false;
+ } else if (RecurrenceType->isIntegerTy() && DB && AC && DT &&
+ (Kind == RecurKind::Add || Kind == RecurKind::Mul ||
+ Kind == RecurKind::Or || Kind == RecurKind::And ||
+ Kind == RecurKind::Xor)) {
+ auto [ComputedType, ComputedIsSigned] =
+ computeRecurrenceType(ExitInstruction, DB, AC, DT);
+ unsigned ComputedBits = ComputedType->getScalarSizeInBits();
+ if (ComputedBits >= 8 &&
+ ComputedBits < RecurrenceType->getScalarSizeInBits()) {
+ RecurrenceType = ComputedType;
+ IsSigned = ComputedIsSigned;
+ }
}
// Collect cast instructions and the minimum width used by the recurrence.
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 950b125028b99..0b7d9af071679 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1882,6 +1882,7 @@ static void computeKnownBitsFromOperator(const Operator *I,
case Instruction::Sub:
case Instruction::And:
case Instruction::Or:
+ case Instruction::Xor:
case Instruction::Mul: {
// Change the context instruction to the "edge" that flows into the
// phi. This is important because that is where the value is actually
@@ -1906,6 +1907,15 @@ static void computeKnownBitsFromOperator(const Operator *I,
Known.Zero.setLowBits(std::min(Known2.countMinTrailingZeros(),
Known3.countMinTrailingZeros()));
+ if (Opcode == Instruction::Or || Opcode == Instruction::Xor) {
+ Known.Zero |= Known2.Zero & Known3.Zero;
+ if (Opcode == Instruction::Or)
+ Known.One |= Known2.One;
+ } else if (Opcode == Instruction::And) {
+ Known.Zero |= Known2.Zero;
+ Known.One |= Known2.One & Known3.One;
+ }
+
auto *OverflowOp = dyn_cast<OverflowingBinaryOperator>(BO);
if (!OverflowOp || !Q.IIQ.hasNoSignedWrap(OverflowOp))
break;
diff --git a/llvm/test/Transforms/InstCombine/recurrence.ll b/llvm/test/Transforms/InstCombine/recurrence.ll
index 6207009b531d5..3da3dc5e6f5a4 100644
--- a/llvm/test/Transforms/InstCombine/recurrence.ll
+++ b/llvm/test/Transforms/InstCombine/recurrence.ll
@@ -163,3 +163,201 @@ loop: ; preds = %loop, %entry
}
declare void @use(i64)
+
+; The high bits of an or recurrence are known zero if they are zero in both
+; the start value and the step.
+define i32 @test_or_known_zero_high_bits(ptr %p, i64 %n) {
+; CHECK-LABEL: @test_or_known_zero_high_bits(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ACC_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[X_EXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT: [[ACC_NEXT]] = or i32 [[ACC]], [[X_EXT]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK: exit:
+; CHECK-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %gep = getelementptr inbounds i8, ptr %p, i64 %iv
+ %x = load i8, ptr %gep
+ %x.ext = zext i8 %x to i32
+ %acc.next = or i32 %acc, %x.ext
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %masked = and i32 %acc.next, 255
+ ret i32 %masked
+}
+
+; Negative test: the start value has unknown high bits.
+define i32 @test_or_unknown_start(ptr %p, i64 %n, i32 %start) {
+; CHECK-LABEL: @test_or_unknown_start(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ [[START:%.*]], [[ENTRY]] ], [ [[ACC_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[X_EXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT: [[ACC_NEXT]] = or i32 [[ACC]], [[X_EXT]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK: exit:
+; CHECK-NEXT: [[MASKED:%.*]] = and i32 [[ACC_NEXT]], 255
+; CHECK-NEXT: ret i32 [[MASKED]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ %start, %entry ], [ %acc.next, %loop ]
+ %gep = getelementptr inbounds i8, ptr %p, i64 %iv
+ %x = load i8, ptr %gep
+ %x.ext = zext i8 %x to i32
+ %acc.next = or i32 %acc, %x.ext
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %masked = and i32 %acc.next, 255
+ ret i32 %masked
+}
+
+; Bits that are one in the start value stay one in an or recurrence, but bits
+; that are only one in the step do not apply to the first iteration.
+define i64 @test_or_known_one_start(i64 %a) {
+; CHECK-LABEL: @test_or_known_one_start(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: tail call void @use(i64 1)
+; CHECK-NEXT: br label [[LOOP]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop ]
+ %iv.next = or i64 %iv, %a
+ %bit0 = and i64 %iv, 1
+ tail call void @use(i64 %bit0)
+ br label %loop
+}
+
+; The high bits of an and recurrence are known zero if they are zero in the
+; start value.
+define i32 @test_and_known_zero_start(ptr %p, i64 %n) {
+; CHECK-LABEL: @test_and_known_zero_start(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ 255, [[ENTRY]] ], [ [[ACC_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds [4 x i8], ptr [[P:%.*]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[ACC_NEXT]] = and i32 [[ACC]], [[X]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK: exit:
+; CHECK-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 255, %entry ], [ %acc.next, %loop ]
+ %gep = getelementptr inbounds i32, ptr %p, i64 %iv
+ %x = load i32, ptr %gep
+ %acc.next = and i32 %acc, %x
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %masked = and i32 %acc.next, 255
+ ret i32 %masked
+}
+
+; Negative test: an and recurrence does not inherit the zero bits of the step,
+; because the step does not apply to the first iteration.
+define i32 @test_and_step_zero_bits(i32 %start) {
+; CHECK-LABEL: @test_and_step_zero_bits(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: tail call void @use(i64 0)
+; CHECK-NEXT: br i1 false, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: [[MASKED:%.*]] = and i32 [[START:%.*]], 4
+; CHECK-NEXT: ret i32 [[MASKED]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ]
+ %iv.next = and i32 %iv, 4
+ %masked = and i32 %iv, 4
+ tail call void @use(i64 0)
+ br i1 false, label %loop, label %exit
+
+exit:
+ ret i32 %masked
+}
+
+; The high bits of an xor recurrence are known zero if they are zero in both
+; the start value and the step.
+define i32 @test_xor_known_zero_high_bits(ptr %p, i64 %n) {
+; CHECK-LABEL: @test_xor_known_zero_high_bits(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ACC_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[P:%.*]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[X_EXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT: [[ACC_NEXT]] = xor i32 [[ACC]], [[X_EXT]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK: exit:
+; CHECK-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %gep = getelementptr inbounds i8, ptr %p, i64 %iv
+ %x = load i8, ptr %gep
+ %x.ext = zext i8 %x to i32
+ %acc.next = xor i32 %acc, %x.ext
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ %masked = and i32 %acc.next, 255
+ ret i32 %masked
+}
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-narrow-known-bits.ll b/llvm/test/Transforms/LoopVectorize/reduction-narrow-known-bits.ll
new file mode 100644
index 0000000000000..c9972c607301c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reduction-narrow-known-bits.ll
@@ -0,0 +1,272 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+; Reductions whose accumulator was promoted to a wider type without leaving an
+; 'and' mask behind (e.g. after InstCombine folded the zext/trunc round trip
+; into the phi) should still be narrowed to the width the exit value is known
+; to fit in.
+
+; The or of zext'd i8 values fits in i8: expect an i8 recurrence type, i.e. a
+; trunc/zext pair around the reduction op and an i8 reduction.
+define i32 @or_zext_i8(ptr %p, i64 %n) {
+; CHECK-LABEL: define i32 @or_zext_i8(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = or <4 x i32> [[VEC_PHI]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8>
+; CHECK-NEXT: [[TMP4]] = zext <4 x i8> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> [[TMP3]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[X_EXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT: [[ACC_NEXT]] = or i32 [[ACC]], [[X_EXT]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[ACC_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[ACC_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %gep = getelementptr inbounds i8, ptr %p, i64 %iv
+ %x = load i8, ptr %gep, align 1
+ %x.ext = zext i8 %x to i32
+ %acc.next = or i32 %acc, %x.ext
+ %iv.next = add nuw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; Same for xor.
+define i32 @xor_zext_i8(ptr %p, i64 %n) {
+; CHECK-LABEL: define i32 @xor_zext_i8(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i32> [[VEC_PHI]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8>
+; CHECK-NEXT: [[TMP4]] = zext <4 x i8> [[TMP5]] to <4 x i32>
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP6:%.*]] = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[X_EXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT: [[ACC_NEXT]] = xor i32 [[ACC]], [[X_EXT]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[ACC_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i32 [[ACC_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %gep = getelementptr inbounds i8, ptr %p, i64 %iv
+ %x = load i8, ptr %gep, align 1
+ %x.ext = zext i8 %x to i32
+ %acc.next = xor i32 %acc, %x.ext
+ %iv.next = add nuw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; The start value has unknown high bits, so the reduction must stay i32.
+define i32 @or_zext_i8_unknown_start(ptr %p, i64 %n, i32 %start) {
+; CHECK-LABEL: define i32 @or_zext_i8_unknown_start(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[START]], i64 0
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP6]], %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
+; CHECK-NEXT: [[TMP3]] = or <4 x i32> [[VEC_PHI]], [[TMP2]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP3]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[X_EXT:%.*]] = zext i8 [[X]] to i32
+; CHECK-NEXT: [[ACC_NEXT]] = or i32 [[ACC]], [[X_EXT]]
+; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[ACC_NEXT]]...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/222183
More information about the llvm-commits
mailing list