[llvm] [VPlan] Distribute execution frequencies with BlockFrequencyInfo's code. (PR #226841)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 27 14:55:19 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Florian Hahn (fhahn)
<details>
<summary>Changes</summary>
Instead of hand-rolling frequency propagating, manually construct BFI and use BFI::distributeMass.
Adds a final step where frequencies are rounded up to at least one, to distinguish between unreachable and almost never executed blocks. This matches BFI behavior.
Note that we need to still track whether frequencies are from edges with unknown profile info.
---
Patch is 28.42 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/226841.diff
4 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.cpp (+47-46)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll (+31-43)
- (modified) llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll (+64-6)
- (modified) llvm/test/Transforms/LoopVectorize/replicate-region-branch-weights.ll (+18-18)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index a56689848efdf..c7d551aa332f8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -16,6 +16,7 @@
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -1250,59 +1251,59 @@ getSuccessorProbabilities(const VPBasicBlock *VPBB) {
});
}
-/// Returns \p Freq scaled by \p Prob, rounding up to 1 instead of 0 to keep a
-/// rarely executed block distinguishable from an unreachable one.
-static BlockFrequency scaleKeepingNonZero(BlockFrequency Freq,
- BranchProbability Prob) {
- BlockFrequency Scaled = Freq * Prob;
- if (Scaled == BlockFrequency() && Freq != BlockFrequency() && !Prob.isZero())
- return BlockFrequency(1);
- return Scaled;
-}
-
DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
vputils::computeExecutionFrequencies(ArrayRef<VPBasicBlock *> Blocks) {
+ using BFIBase = BlockFrequencyInfoImplBase;
assert(!Blocks.empty() && "expected at least the header block");
- // Push each block's frequency along its outgoing edges. Blocks is in reverse
- // post-order and forms a DAG with the backedge from the latch (the last
- // block) ignored, so a block's frequency is final by the time it is visited.
- DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
- Frequencies;
- Frequencies.reserve(Blocks.size());
- // The header (first block) always executes, the others start out unreachable.
- Frequencies[Blocks.front()].emplace(BlockFrequency(AlwaysExecutesFreq),
- false);
- for (VPBasicBlock *VPBB : Blocks.drop_front())
- Frequencies[VPBB].emplace(BlockFrequency(), false);
-
- for (VPBasicBlock *VPBB : Blocks) {
- std::optional<VPExecutionFrequency> Src = Frequencies.at(VPBB);
+ // Distribute the header's frequency using BFI. Nodes for blocks are numbered
+ // in reverse post-order. Edges leaving Blocks, i.e. a plain CFG's edges to
+ // the middle block or to an exit block, exit to a node outside the loop.
+ BFIBase BFI;
+ BFIBase::BlockNode Header(0), Outside(Blocks.size());
+ BFIBase::LoopData &Loop = BFI.Loops.emplace_back(nullptr, Header);
+ DenseMap<const VPBlockBase *, BFIBase::BlockNode> Nodes;
+ for (auto [Idx, VPBB] : enumerate(Blocks)) {
+ Nodes[VPBB] = BFIBase::BlockNode(Idx);
+ BFI.Working.emplace_back(BFIBase::BlockNode(Idx)).Loop = &Loop;
+ }
+ BFI.Working.emplace_back(Outside);
+ BFI.Working[Header.Index].getMass() = BFIBase::BlockMass(AlwaysExecutesFreq);
+
+ // Keep track nodes reached via an edge without branch weighs or with
+ // estimated ones
+ SmallVector<bool> IsUnknown(Blocks.size()), IsEstimated(Blocks.size());
+ for (auto [Idx, VPBB] : enumerate(Blocks)) {
+ BFIBase::BlockNode Node(Idx);
auto *Term = dyn_cast_if_present<VPInstruction>(VPBB->getTerminator());
bool TermIsEstimated = Term && Term->hasEstimatedBranchWeights();
- for (const auto &[Succ, EdgeProb] : getSuccessorProbabilities(VPBB)) {
- // Ignore the backedge to the header (already treated as always
- // executing).
- if (Succ == Blocks.front())
- continue;
- // Ignore edges leaving Blocks, i.e. a plain CFG's edges to the middle
- // block or to an exit block.
- auto It = Frequencies.find(Succ);
- if (It == Frequencies.end())
- continue;
- std::optional<VPExecutionFrequency> &SuccFreq = It->second;
- // An unknown edge or predecessor poisons the successor.
- if (!Src || EdgeProb.isUnknown() || !SuccFreq) {
- SuccFreq = std::nullopt;
- continue;
+ BFIBase::Distribution Dist;
+ bool HasProbs = true;
+ for (const auto &[Succ, Prob] : getSuccessorProbabilities(VPBB)) {
+ BFIBase::BlockNode SuccNode = Nodes.lookup_or(Succ, Outside);
+ if (SuccNode != Header && SuccNode != Outside) {
+ IsUnknown[SuccNode.Index] |= IsUnknown[Idx] || Prob.isUnknown();
+ IsEstimated[SuccNode.Index] |= IsEstimated[Idx] || TermIsEstimated;
}
- // The sum can only exceed AlwaysExecutesFreq by rounding.
- BlockFrequency NewFreq =
- std::min(BlockFrequency(AlwaysExecutesFreq),
- SuccFreq->Freq + scaleKeepingNonZero(Src->Freq, EdgeProb));
- bool NewIsEstimated =
- SuccFreq->IsEstimated || Src->IsEstimated || TermIsEstimated;
- SuccFreq.emplace(NewFreq, NewIsEstimated);
+ HasProbs &= !Prob.isUnknown();
+ if (!Prob.isUnknown())
+ BFI.addToDist(Dist, &Loop, Node, SuccNode,
+ getWeightFromBranchProb(Prob));
}
+ if (HasProbs)
+ BFI.distributeMass(Node, &Loop, Dist);
+ }
+
+ // Round frequencies up to at least 1, so all edges are reached with a
+ // non-zero frequency, to distinguish rarely executed blocks from unreachable
+ // ones. blocks distinguishable from unreachable ones.
+ DenseMap<const VPBasicBlock *, std::optional<VPExecutionFrequency>>
+ Frequencies;
+ for (auto [Idx, VPBB] : enumerate(Blocks)) {
+ std::optional<VPExecutionFrequency> &Freq = Frequencies[VPBB];
+ if (IsUnknown[Idx])
+ continue;
+ uint64_t Mass = BFI.Working[Idx].getMass().getMass();
+ Freq.emplace(BlockFrequency(std::max<uint64_t>(Mass, 1)), IsEstimated[Idx]);
}
return Frequencies;
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll b/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
index a99a536578d6f..27a86f1123160 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/execution-frequencies-match-bfi.ll
@@ -78,8 +78,6 @@ define void @single_pred_zero_weight(ptr noalias %a, ptr noalias %b, ptr noalias
; %if.then 2^-31 ~ 4.66e-10
; %latch 1 = 1
;
-; TODO: VPlan currently records a frequency of 0 for %if.then.
-;
; BFI-LABEL: block-frequency-info: single_pred_zero_weight
; BFI-NEXT: - entry: float = 1.0,
; BFI-NEXT: - loop: float = 1000.0,
@@ -99,8 +97,8 @@ define void @single_pred_zero_weight(ptr noalias %a, ptr noalias %b, ptr noalias
; VPLAN-NEXT: Successor(s): if.then, latch
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
-; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>{{$}}
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>{{$}}
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 4294967296 (4.657E-8%))
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a> (!vplan.execution.frequency 4294967296 (4.657E-8%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -143,8 +141,6 @@ define void @single_pred_zero_weight_sibling(ptr noalias %a, ptr noalias %b, ptr
; %if.then 1 - 2^-31 ~ 1 - 4.66e-10
; %latch 1 = 1
;
-; TODO: VPlan currently records %if.then as always executing.
-;
; BFI-LABEL: block-frequency-info: single_pred_zero_weight_sibling
; BFI-NEXT: - entry: float = 1.0,
; BFI-NEXT: - loop: float = 1000.0, int = 18014398509481984
@@ -164,8 +160,8 @@ define void @single_pred_zero_weight_sibling(ptr noalias %a, ptr noalias %b, ptr
; VPLAN-NEXT: Successor(s): if.then, latch
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
-; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv>{{$}}
-; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a>{{$}}
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 9223372032559808512 (100%))
+; VPLAN-NEXT: EMIT store ir<%i>, ir<%gep.a> (!vplan.execution.frequency 9223372032559808512 (100%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -459,13 +455,13 @@ define void @switch_common_dest(ptr noalias %a, ptr noalias %b, ptr noalias %c,
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: if.then:
-; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 3458764513820540928 (37.5%))
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a> (!vplan.execution.frequency 3458764513820540928 (37.5%))
+; VPLAN-NEXT: EMIT ir<%gep.a> = getelementptr inbounds ir<%a>, ir<%iv> (!vplan.execution.frequency 3458764514357411840 (37.5%))
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a> (!vplan.execution.frequency 3458764514357411840 (37.5%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: default:
-; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv> (!vplan.execution.frequency 4611686018427387904 (50%))
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c> (!vplan.execution.frequency 4611686018427387904 (50%))
+; VPLAN-NEXT: EMIT ir<%gep.c> = getelementptr inbounds ir<%c>, ir<%iv> (!vplan.execution.frequency 4611686017890516992 (50%))
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.c> (!vplan.execution.frequency 4611686017890516992 (50%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
@@ -1093,9 +1089,6 @@ define void @switch_join_always(ptr noalias %a, ptr noalias %idx) {
; %join 1 = 1
; %latch 1 = 1
;
-; TODO: %join and %latch are currently recorded as executing slightly less
-; often, as the rounded probabilities do not add up to 1.
-;
; BFI-LABEL: block-frequency-info: switch_join_always
; BFI-NEXT: - entry: float = 1.0,
; BFI-NEXT: - loop: float = 1000.0,
@@ -1111,18 +1104,18 @@ define void @switch_join_always(ptr noalias %a, ptr noalias %idx) {
; VPLAN-NEXT: Successor(s): join
; VPLAN-EMPTY:
; VPLAN-NEXT: case.1:
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a> (!vplan.execution.frequency 1317624575466405888 (14.29%))
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a> (!vplan.execution.frequency 1317624575670928140 (14.29%))
; VPLAN-NEXT: Successor(s): join
; VPLAN-EMPTY:
; VPLAN-NEXT: join:
-; VPLAN-NEXT: EMIT ir<%add> = add ir<%i>, ir<10> (!vplan.execution.frequency 9223372032559808512 (100%))
-; VPLAN-NEXT: EMIT store ir<%add>, ir<%gep.a> (!vplan.execution.frequency 9223372032559808512 (100%))
+; VPLAN-NEXT: EMIT ir<%add> = add ir<%i>, ir<10>{{$}}
+; VPLAN-NEXT: EMIT store ir<%add>, ir<%gep.a>{{$}}
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
-; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1> (!vplan.execution.frequency 9223372032559808512 (100%))
-; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024> (!vplan.execution.frequency 9223372032559808512 (100%))
-; VPLAN-NEXT: EMIT branch-on-cond ir<%ec> (!prof {1, 999}, !vplan.execution.frequency 9223372032559808512 (100%))
+; VPLAN-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>{{$}}
+; VPLAN-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<1024>{{$}}
+; VPLAN-NEXT: EMIT branch-on-cond ir<%ec> (!prof {1, 999}){{$}}
; VPLAN-NEXT: Successor(s): middle.block, loop
;
entry:
@@ -1306,9 +1299,6 @@ define void @rarely_executed_chain(ptr noalias %a, ptr noalias %idx) {
; %if.e 2^-64 ~ 5.42e-20 (clamped up from 2^-65)
; %latch 1 = 1
;
-; TODO: %if.a and the blocks it reaches are currently recorded as never
-; executing.
-;
; BFI-LABEL: block-frequency-info: rarely_executed_chain
; BFI-NEXT: - entry: float = 1.0,
; BFI-NEXT: - loop: float = 1000.0,
@@ -1322,28 +1312,28 @@ define void @rarely_executed_chain(ptr noalias %a, ptr noalias %idx) {
;
; VPLAN-LABEL: VPlan for loop in 'rarely_executed_chain'
; VPLAN: if.a:
-; VPLAN-NEXT: EMIT ir<%c.1> = icmp sgt ir<%i>, ir<10>{{$}}
-; VPLAN-NEXT: EMIT branch-on-cond ir<%c.1> (!prof {1000, 0}){{$}}
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp sgt ir<%i>, ir<10> (!vplan.execution.frequency 4294967296 (4.657E-8%))
+; VPLAN-NEXT: EMIT branch-on-cond ir<%c.1> (!prof {1000, 0}, !vplan.execution.frequency 4294967296 (4.657E-8%))
; VPLAN-NEXT: Successor(s): latch, if.b
; VPLAN-EMPTY:
; VPLAN-NEXT: if.b:
-; VPLAN-NEXT: EMIT ir<%c.2> = icmp sgt ir<%i>, ir<20>{{$}}
-; VPLAN-NEXT: EMIT branch-on-cond ir<%c.2> (!prof {1, 1}){{$}}
+; VPLAN-NEXT: EMIT ir<%c.2> = icmp sgt ir<%i>, ir<20> (!vplan.execution.frequency 2 (2.168E-17%))
+; VPLAN-NEXT: EMIT branch-on-cond ir<%c.2> (!prof {1, 1}, !vplan.execution.frequency 2 (2.168E-17%))
; VPLAN-NEXT: Successor(s): if.c, latch
; VPLAN-EMPTY:
; VPLAN-NEXT: if.c:
-; VPLAN-NEXT: EMIT ir<%c.3> = icmp sgt ir<%i>, ir<30>{{$}}
-; VPLAN-NEXT: EMIT branch-on-cond ir<%c.3> (!prof {1, 1}){{$}}
+; VPLAN-NEXT: EMIT ir<%c.3> = icmp sgt ir<%i>, ir<30> (!vplan.execution.frequency 1 (1.084E-17%))
+; VPLAN-NEXT: EMIT branch-on-cond ir<%c.3> (!prof {1, 1}, !vplan.execution.frequency 1 (1.084E-17%))
; VPLAN-NEXT: Successor(s): latch, if.d
; VPLAN-EMPTY:
; VPLAN-NEXT: if.d:
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>{{$}}
-; VPLAN-NEXT: EMIT ir<%c.4> = icmp sgt ir<%i>, ir<40>{{$}}
-; VPLAN-NEXT: EMIT branch-on-cond ir<%c.4> (!prof {1, 1}){{$}}
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a> (!vplan.execution.frequency 1 (1.084E-17%))
+; VPLAN-NEXT: EMIT ir<%c.4> = icmp sgt ir<%i>, ir<40> (!vplan.execution.frequency 1 (1.084E-17%))
+; VPLAN-NEXT: EMIT branch-on-cond ir<%c.4> (!prof {1, 1}, !vplan.execution.frequency 1 (1.084E-17%))
; VPLAN-NEXT: Successor(s): if.e, latch
; VPLAN-EMPTY:
; VPLAN-NEXT: if.e:
-; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.a>{{$}}
+; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.a> (!vplan.execution.frequency 1 (1.084E-17%))
; VPLAN-NEXT: Successor(s): latch
;
entry:
@@ -1397,8 +1387,6 @@ define void @nested_zero_weight_siblings(ptr noalias %a, ptr noalias %idx) {
; %then.2 (1 - 2^-31)^3 ~ 1 - 1.40e-9
; %latch 1 = 1
;
-; TODO: %then.0, %then.1 and %then.2 are currently recorded as always executing.
-;
; BFI-LABEL: block-frequency-info: nested_zero_weight_siblings
; BFI-NEXT: - entry: float = 1.0,
; BFI-NEXT: - loop: float = 1000.0, int = 18014398509481984
@@ -1410,19 +1398,19 @@ define void @nested_zero_weight_siblings(ptr noalias %a, ptr noalias %idx) {
;
; VPLAN-LABEL: VPlan for loop in 'nested_zero_weight_siblings'
; VPLAN: then.0:
-; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.a>{{$}}
-; VPLAN-NEXT: EMIT ir<%c.1> = icmp sgt ir<%i>, ir<10>{{$}}
-; VPLAN-NEXT: EMIT branch-on-cond ir<%c.1> (!prof {1000, 0}){{$}}
+; VPLAN-NEXT: EMIT store ir<0>, ir<%gep.a> (!vplan.execution.frequency 9223372032559808512 (100%))
+; VPLAN-NEXT: EMIT ir<%c.1> = icmp sgt ir<%i>, ir<10> (!vplan.execution.frequency 9223372032559808512 (100%))
+; VPLAN-NEXT: EMIT branch-on-cond ir<%c.1> (!prof {1000, 0}, !vplan.execution.frequency 9223372032559808512 (100%))
; VPLAN-NEXT: Successor(s): then.1, latch
; VPLAN-EMPTY:
; VPLAN-NEXT: then.1:
-; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a>{{$}}
-; VPLAN-NEXT: EMIT ir<%c.2> = icmp sgt ir<%i>, ir<20>{{$}}
-; VPLAN-NEXT: EMIT branch-on-cond ir<%c.2> (!prof {1000, 0}){{$}}
+; VPLAN-NEXT: EMIT store ir<1>, ir<%gep.a> (!vplan.execution.frequency 9223372028264841218 (100%))
+; VPLAN-NEXT: EMIT ir<%c.2> = icmp sgt ir<%i>, ir<20> (!vplan.execution.frequency 9223372028264841218 (100%))
+; VPLAN-NEXT: EMIT branch-on-cond ir<%c.2> (!prof {1000, 0}, !vplan.execution.frequency 9223372028264841218 (100%))
; VPLAN-NEXT: Successor(s): then.2, latch
; VPLAN-EMPTY:
; VPLAN-NEXT: then.2:
-; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.a>{{$}}
+; VPLAN-NEXT: EMIT store ir<2>, ir<%gep.a> (!vplan.execution.frequency 9223372023969873925 (100%))
; VPLAN-NEXT: Successor(s): latch
; VPLAN-EMPTY:
; VPLAN-NEXT: latch:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
index 730b7fa2404bb..44997f8eeb540 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/predicated-instruction-cost.ll
@@ -305,22 +305,80 @@ define void @predicated_sdiv_zero_weight(ptr noalias %a, ptr noalias %b, i64 %n)
; CHECK-LABEL: define void @predicated_sdiv_zero_weight(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
+; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]], !prof [[PROF4:![0-9]+]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = sdiv i32 1000, [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: store i32 [[TMP5]], ptr [[TMP6]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]], !prof [[PROF4]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = sdiv i32 1000, [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP10]]
+; CHECK-NEXT: store i32 [[TMP9]], ptr [[TMP11]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE2]]
+; CHECK: [[PRED_STORE_CONTINUE2]]:
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2
+; CHECK-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]], !prof [[PROF4]]
+; CHECK: [[PRED_STORE_IF3]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 2
+; CHECK-NEXT: [[TMP14:%.*]] = sdiv i32 1000, [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP15]]
+; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP16]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE4]]
+; CHECK: [[PRED_STORE_CONTINUE4]]:
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3
+; CHECK-NEXT: br i1 [[TMP17]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]], !prof [[PROF4]]
+; CHECK: [[PRED_STORE_IF5]]:
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i64 3
+; CHECK-NEXT: [[TMP19:%.*]] = sdiv i32 1000, [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[IV]], 3
+; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP20]]
+; CHECK-NEXT: store i32 [[TMP19]], ptr [[TMP21]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE6]]
+; CHECK: [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHE...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/226841
More information about the llvm-commits
mailing list