[llvm-branch-commits] [llvm] [LoopUtils] Check the first outer iteration for co-moving pointer groups (PR #243836)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 18 04:53:01 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-loopvectorize
@llvm/pr-subscribers-llvm-vectorizers
Author: Valeriy Savchenko (SavchenkoValeriy)
<details>
<summary>Changes</summary>
When runtime checks are hoisted out of the outer loop, the range of each
pointer group is widened to cover all iterations of the outer loop. If both
groups of a check advance by the same step in the outer loop, e.g. two rows
of the same image:
```cpp
for (y = 0; y < H; y += 2)
for (x = 0; x < n; x++) {
img[y*S + 3*x] = ...;
img[(y+1)*S + 3*x] = ...;
}
```
the widened ranges overlap as soon as the outer loop runs twice, so the
check always fails and the vector loop never runs.
If all bounds of both groups are affine AddRecs of the outer loop with the
same step, the relative position of the two ranges is the same in every
outer iteration, so they overlap in some iteration iff they overlap in the
first one.
This change uses the bounds of the first iteration: the check is exact,
invariant in the outer loop, and needs neither the outer trip count nor
a stride check.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
---
Patch is 55.71 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/243836.diff
6 Files Affected:
- (modified) llvm/lib/Transforms/Utils/LoopUtils.cpp (+22-5)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll (+63-15)
- (modified) llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll (+2-3)
- (added) llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll (+626)
- (modified) llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll (+18-14)
- (modified) llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll (+2-3)
``````````diff
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index cefd05ce4413e..de7db30bc8184 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -2200,8 +2200,8 @@ getGroupBounds(const RuntimeCheckingPtrGroup *CG, const Loop *TheLoop,
// which can be significant for low trip counts. The disadvantage is that
// there is a chance we may now never enter the vectorized inner loop,
// whereas using a restricted range check could have allowed us to enter at
- // least once. This is why the behaviour is not currently the default and is
- // controlled by the parameter 'HoistRuntimeChecks'.
+ // least once. This is why the behaviour is controlled by the parameter
+ // 'HoistRuntimeChecks'.
const Loop *OuterLoop = TheLoop->getParentLoop();
if (HoistRuntimeChecks && OuterLoop &&
haveSameStepIn({Low, High}, OuterLoop, SE)) {
@@ -2235,16 +2235,33 @@ RuntimeCheckBounds
llvm::getRuntimeCheckBounds(const RuntimePointerCheck &Check,
const Loop *TheLoop, ScalarEvolution &SE,
bool HoistRuntimeChecks) {
+ // If both groups advance by the same step in the parent loop, their relative
+ // position is the same in every iteration of it, so they overlap in some
+ // iteration iff they overlap in the first one. Checking the first iteration
+ // is exact and invariant in the parent loop, and needs neither widening nor
+ // a stride check.
+ const Loop *ParentLoop = TheLoop->getParentLoop();
+ const auto &[A, B] = Check;
+ bool UseFirstIteration =
+ HoistRuntimeChecks && ParentLoop &&
+ haveSameStepIn({A->Low, A->High, B->Low, B->High}, ParentLoop, SE);
auto GetBounds = [&](const RuntimeCheckingPtrGroup *CG) {
LLVM_DEBUG(dbgs() << "LAA: Adding RT check for range:\n");
- RuntimeCheckGroupBounds Bounds =
- getGroupBounds(CG, TheLoop, SE, HoistRuntimeChecks);
+ RuntimeCheckGroupBounds Bounds;
+ if (UseFirstIteration) {
+ LLVM_DEBUG(dbgs() << "LAA: Using RT check range of the first outer loop "
+ "iteration in order to permit hoisting\n");
+ Bounds = {cast<SCEVAddRecExpr>(CG->Low)->getStart(),
+ cast<SCEVAddRecExpr>(CG->High)->getStart(), nullptr};
+ } else {
+ Bounds = getGroupBounds(CG, TheLoop, SE, HoistRuntimeChecks);
+ }
LLVM_DEBUG(dbgs() << "Start: " << *Bounds.Low << " End: " << *Bounds.High
<< "\n");
return Bounds;
};
// Braced initialization evaluates the groups in order.
- return {GetBounds(Check.first), GetBounds(Check.second)};
+ return {GetBounds(A), GetBounds(B)};
}
/// Expand code for the lower and upper bound \p Bounds of the pointer group
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
index 565e59948e182..488f8fe0c4b90 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
@@ -32,8 +32,46 @@ inner.exit:
define void @outer_no_tc(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_no_tc'
; CHECK: Calculating cost of runtime checks:
-; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 2
-; CHECK: Total cost of runtime checks: 2
+; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 3
+; CHECK: Total cost of runtime checks: 3
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
+entry:
+ br label %outer.loop
+
+outer.loop:
+ %outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
+ %mul.us = mul nsw i64 %outer.iv, %n
+ %mul.b = shl nsw i64 %mul.us, 1
+ br label %inner.loop
+
+inner.loop:
+ %inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
+ %add.us = add nuw nsw i64 %inner.iv, %mul.us
+ %add.b = add nuw nsw i64 %inner.iv, %mul.b
+ %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
+ %0 = load i8, ptr %arrayidx.us, align 1
+ %arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
+ %1 = load i8, ptr %arrayidx7.us, align 1
+ %add9.us = add i8 %1, %0
+ store i8 %add9.us, ptr %arrayidx7.us, align 1
+ %inner.iv.next = add nuw nsw i64 %inner.iv, 1
+ %exitcond.not = icmp eq i64 %inner.iv.next, %n
+ br i1 %exitcond.not, label %inner.exit, label %inner.loop
+
+inner.exit:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %exitcond27.not = icmp eq i64 %outer.iv.next, %m
+ br i1 %exitcond27.not, label %outer.exit, label %outer.loop
+
+outer.exit:
+ ret void
+}
+
+define void @outer_no_tc_same_step(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
+; CHECK-LABEL: LV: Checking a loop in 'outer_no_tc_same_step'
+; CHECK: Calculating cost of runtime checks:
+; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 3 to 1
+; CHECK: Total cost of runtime checks: 1
; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -69,8 +107,8 @@ outer.exit:
define void @outer_known_tc3(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc3'
; CHECK: Calculating cost of runtime checks:
-; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
-; CHECK: Total cost of runtime checks: 1
+; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 2
+; CHECK: Total cost of runtime checks: 2
; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -78,12 +116,14 @@ entry:
outer.loop:
%outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
%mul.us = mul nsw i64 %outer.iv, %n
+ %mul.b = shl nsw i64 %mul.us, 1
br label %inner.loop
inner.loop:
%inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
%add.us = add nuw nsw i64 %inner.iv, %mul.us
- %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.us
+ %add.b = add nuw nsw i64 %inner.iv, %mul.b
+ %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
%0 = load i8, ptr %arrayidx.us, align 1
%arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
%1 = load i8, ptr %arrayidx7.us, align 1
@@ -106,7 +146,7 @@ outer.exit:
define void @outer_known_tc64(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc64'
; CHECK: Calculating cost of runtime checks:
-; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
+; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 1
; CHECK: Total cost of runtime checks: 1
; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
@@ -115,12 +155,14 @@ entry:
outer.loop:
%outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
%mul.us = mul nsw i64 %outer.iv, %n
+ %mul.b = shl nsw i64 %mul.us, 1
br label %inner.loop
inner.loop:
%inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
%add.us = add nuw nsw i64 %inner.iv, %mul.us
- %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.us
+ %add.b = add nuw nsw i64 %inner.iv, %mul.b
+ %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
%0 = load i8, ptr %arrayidx.us, align 1
%arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
%1 = load i8, ptr %arrayidx7.us, align 1
@@ -143,8 +185,8 @@ outer.exit:
define void @outer_pgo_3(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_pgo_3'
; CHECK: Calculating cost of runtime checks:
-; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
-; CHECK: Total cost of runtime checks: 1
+; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 2
+; CHECK: Total cost of runtime checks: 2
; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
br label %outer.loop
@@ -152,12 +194,14 @@ entry:
outer.loop:
%outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
%mul.us = mul nsw i64 %outer.iv, %n
+ %mul.b = shl nsw i64 %mul.us, 1
br label %inner.loop
inner.loop:
%inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
%add.us = add nuw nsw i64 %inner.iv, %mul.us
- %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.us
+ %add.b = add nuw nsw i64 %inner.iv, %mul.b
+ %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
%0 = load i8, ptr %arrayidx.us, align 1
%arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
%1 = load i8, ptr %arrayidx7.us, align 1
@@ -180,7 +224,7 @@ outer.exit:
define void @outer_pgo_minus1(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, i64 noundef %m, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_pgo_minus1'
; CHECK: Calculating cost of runtime checks:
-; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
+; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 1
; CHECK: Total cost of runtime checks: 1
; CHECK: LV: Minimum required TC for runtime checks to be profitable:16
entry:
@@ -189,12 +233,14 @@ entry:
outer.loop:
%outer.iv = phi i64 [ %outer.iv.next, %inner.exit ], [ 0, %entry ]
%mul.us = mul nsw i64 %outer.iv, %n
+ %mul.b = shl nsw i64 %mul.us, 1
br label %inner.loop
inner.loop:
%inner.iv = phi i64 [ 0, %outer.loop ], [ %inner.iv.next, %inner.loop ]
%add.us = add nuw nsw i64 %inner.iv, %mul.us
- %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.us
+ %add.b = add nuw nsw i64 %inner.iv, %mul.b
+ %arrayidx.us = getelementptr inbounds i8, ptr %b, i64 %add.b
%0 = load i8, ptr %arrayidx.us, align 1
%arrayidx7.us = getelementptr inbounds i8, ptr %a, i64 %add.us
%1 = load i8, ptr %arrayidx7.us, align 1
@@ -217,8 +263,8 @@ outer.exit:
define void @outer_known_tc3_full_range_checks(ptr nocapture noundef %dst, ptr nocapture noundef readonly %src, i64 noundef %n) {
; CHECK-LABEL: LV: Checking a loop in 'outer_known_tc3_full_range_checks'
; CHECK: Calculating cost of runtime checks:
-; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 5 to 1
-; CHECK: Total cost of runtime checks: 1
+; CHECK: We expect runtime memory checks to be hoisted out of the outer loop. Cost reduced from 7 to 2
+; CHECK: Total cost of runtime checks: 2
; CHECK: LV: Minimum required TC for runtime checks to be profitable:4
entry:
br label %outer.loop
@@ -226,12 +272,14 @@ entry:
outer.loop:
%outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %inner.exit ]
%0 = mul nsw i64 %outer.iv, %n
+ %mul.src = shl nsw i64 %0, 1
br label %inner.loop
inner.loop:
%iv.inner = phi i64 [ 0, %outer.loop ], [ %iv.inner.next, %inner.loop ]
%1 = add nuw nsw i64 %iv.inner, %0
- %arrayidx.us = getelementptr inbounds i32, ptr %src, i64 %1
+ %add.src = add nuw nsw i64 %iv.inner, %mul.src
+ %arrayidx.us = getelementptr inbounds i32, ptr %src, i64 %add.src
%2 = load i32, ptr %arrayidx.us, align 4
%arrayidx8.us = getelementptr inbounds i32, ptr %dst, i64 %1
%3 = load i32, ptr %arrayidx8.us, align 4
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll
index 5ca0e8c289af1..dd0c7df895837 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference-rmw-hoist.ll
@@ -78,9 +78,8 @@ define void @rmw_invariant_distance(ptr %dst, ptr %src, i64 %rows) {
; HOIST-LABEL: define void @rmw_invariant_distance(
; HOIST-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 [[ROWS:%.*]]) {
; HOIST-NEXT: [[ENTRY:.*]]:
-; HOIST-NEXT: [[TMP0:%.*]] = shl i64 [[ROWS]], 6
-; HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP0]]
-; HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
+; HOIST-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 64
+; HOIST-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 64
; HOIST-NEXT: br label %[[OUTER_HEADER:.*]]
; HOIST: [[OUTER_HEADER]]:
; HOIST-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll
new file mode 100644
index 0000000000000..686126042e43c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist-same-outer-step.ll
@@ -0,0 +1,626 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
+
+define void @two_rows_same_buffer(ptr %img, ptr %src, i64 %h, i64 %s, i64 %n) {
+; CHECK-LABEL: define void @two_rows_same_buffer(
+; CHECK-SAME: ptr [[IMG:%.*]], ptr [[SRC:%.*]], i64 [[H:%.*]], i64 [[S:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[H]], i64 2)
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[UMAX]], -1
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -2
+; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[S]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[N]], 3
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], -2
+; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[S]], 1
+; CHECK-NEXT: [[SCEVGEP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[N]]
+; CHECK-NEXT: [[TMP8:%.*]] = lshr i64 [[TMP1]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[S]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP9]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], -2
+; CHECK-NEXT: [[SCEVGEP14:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[S]], [[TMP5]]
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP13]], -2
+; CHECK-NEXT: [[SCEVGEP15:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP14]]
+; CHECK-NEXT: [[SCEVGEP16:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[S]]
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP5]], -2
+; CHECK-NEXT: [[SCEVGEP17:%.*]] = getelementptr i8, ptr [[IMG]], i64 [[TMP15]]
+; CHECK-NEXT: br label %[[OUTER:.*]]
+; CHECK: [[OUTER]]:
+; CHECK-NEXT: [[Y:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[Y_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT: [[ROW0:%.*]] = mul i64 [[Y]], [[S]]
+; CHECK-NEXT: [[Y1:%.*]] = add i64 [[Y]], 1
+; CHECK-NEXT: [[ROW1:%.*]] = mul i64 [[Y1]], [[S]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK: [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[IMG]], [[SCEVGEP15]]
+; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP16]], [[SCEVGEP17]]
+; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT: [[BOUND018:%.*]] = icmp ult ptr [[IMG]], [[SCEVGEP13]]
+; CHECK-NEXT: [[BOUND119:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP14]]
+; CHECK-NEXT: [[FOUND_CONFLICT20:%.*]] = and i1 [[BOUND018]], [[BOUND119]]
+; CHECK-NEXT: [[STRIDE_CHECK:%.*]] = icmp slt i64 [[TMP0]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[FOUND_CONFLICT20]], [[STRIDE_CHECK]]
+; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[TMP16]]
+; CHECK-NEXT: [[BOUND021:%.*]] = icmp ult ptr [[SCEVGEP16]], [[SCEVGEP13]]
+; CHECK-NEXT: [[BOUND122:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT: [[FOUND_CONFLICT23:%.*]] = and i1 [[BOUND021]], [[BOUND122]]
+; CHECK-NEXT: [[CONFLICT_RDX24:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT23]]
+; CHECK-NEXT: br i1 [[CONFLICT_RDX24]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP17:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP17]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[ROW0]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT25:%.*]] = insertelement <4 x i64> poison, i64 [[ROW1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT26:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT25]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP18:%.*]] = mul <4 x i64> [[VEC_IND]], splat (i64 3)
+; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP20:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], [[TMP18]]
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP20]], i64 0
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP21]]
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[TMP20]], i64 1
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP23]]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i64> [[TMP20]], i64 2
+; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP25]]
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP20]], i64 3
+; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP27]]
+; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: store i8 [[TMP29]], ptr [[TMP22]], align 1, !alias.scope [[META3:![0-9]+]], !noalias [[META5:![0-9]+]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: store i8 [[TMP30]], ptr [[TMP24]], align 1, !alias.scope [[META3]], !noalias [[META5]]
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 2
+; CHECK-NEXT: store i8 [[TMP31]], ptr [[TMP26]], align 1, !alias.scope [[META3]], !noalias [[META5]]
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 3
+; CHECK-NEXT: store i8 [[TMP32]], ptr [[TMP28]], align 1, !alias.scope [[META3]], !noalias [[META5]]
+; CHECK-NEXT: [[TMP33:%.*]] = add <4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT: [[TMP34:%.*]] = add <4 x i64> [[BROADCAST_SPLAT26]], [[TMP18]]
+; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i64> [[TMP34]], i64 0
+; CHECK-NEXT: [[TMP36:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i64> [[TMP34]], i64 1
+; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP37]]
+; CHECK-NEXT: [[TMP39:%.*]] = extractelement <4 x i64> [[TMP34]], i64 2
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[IMG]], i64 [[TMP39]]
+; CHECK-NEXT: [[TMP41:%.*]] = extractelemen...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/243836
More information about the llvm-branch-commits
mailing list