[llvm-branch-commits] [llvm] [GVN] Preserve vectorization opportunities when PREing loop loads (PR #227069)
Tomas Matheson via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Sep 28 10:57:52 PDT 2026
https://github.com/tommat01 created https://github.com/llvm/llvm-project/pull/227069
Loop load PRE can replace an invariant-address load with a loop-carried PHI and conditional reload after a may-alias store. That scalar recurrence can prevent vectorization even when runtime alias checks could disambiguate the original accesses. Subsequent full unrolling then expands scalar code.
Conservatively preserve the header load in innermost loops whose clobber is a conditional may-alias store through a varying pointer. Keep existing PRE behavior for invariant-address clobbers, known aliasing, calls, ordered memory operations, and loops with vectorization disabled or completed. This is an opportunity heuristic, not a vectorization legality proof.
>From fd7df11fb45b29f7a737960e5330960de9082cd6 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson at arm.com>
Date: Mon, 28 Sep 2026 16:41:29 +0100
Subject: [PATCH] [GVN] Preserve vectorization opportunities when PREing loop
loads
Loop load PRE can replace an invariant-address load with a loop-carried
PHI and conditional reload after a may-alias store. That scalar recurrence
can prevent vectorization even when runtime alias checks could disambiguate
the original accesses. Subsequent full unrolling then expands scalar code.
Conservatively preserve the header load in innermost loops whose clobber
is a conditional may-alias store through a varying pointer. Keep existing
PRE behavior for invariant-address clobbers, known aliasing, calls, ordered
memory operations, and loops with vectorization disabled or completed.
This is an opportunity heuristic, not a vectorization legality proof.
---
llvm/lib/Transforms/Scalar/GVN.cpp | 55 ++
.../GVN/PRE/loop-load-pre-vectorization.ll | 582 ++++++++++++++++++
2 files changed, 637 insertions(+)
create mode 100644 llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index bf7115106fa6a..72fd33fd68665 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -48,6 +48,7 @@
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugLoc.h"
+#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/InstrTypes.h"
@@ -73,6 +74,7 @@
#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/SSAUpdater.h"
#include "llvm/Transforms/Utils/VNCoercion.h"
#include <algorithm>
@@ -1964,6 +1966,49 @@ bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
return true;
}
+// Loop load PRE replaces an invariant-address load with a loop-carried PHI.
+// When its clobber is a conditional store through a varying pointer, this can
+// prevent vectorization that could otherwise disambiguate the accesses using
+// runtime checks. Keep the load in such loops until the vectorizer has run.
+static bool shouldPreserveLoadForVectorization(LoadInst *Load, const Loop *L,
+ BasicBlock *Clobber,
+ AAResults &AA) {
+ if (!L->isInnermost() || (hasVectorizeTransformation(L) & TM_Disable) ||
+ !VectorType::isValidElementType(Load->getType()))
+ return false;
+
+ // Do not penalize PRE in loops with calls or ordered memory operations,
+ // where runtime memory checks alone cannot enable vectorization.
+ for (BasicBlock *BB : L->blocks())
+ for (Instruction &I : *BB) {
+ if (isa<CallBase>(I))
+ return false;
+ if (auto *LI = dyn_cast<LoadInst>(&I); LI && !LI->isSimple())
+ return false;
+ if (I.mayWriteToMemory()) {
+ auto *SI = dyn_cast<StoreInst>(&I);
+ if (!SI || !SI->isSimple())
+ return false;
+ }
+ }
+
+ bool HasMayAliasStore = false;
+ for (Instruction &I : *Clobber) {
+ auto *SI = dyn_cast<StoreInst>(&I);
+ if (!SI)
+ continue;
+ AliasResult Alias =
+ AA.alias(MemoryLocation::get(Load), MemoryLocation::get(SI));
+ if (Alias == AliasResult::NoAlias)
+ continue;
+ if (Alias != AliasResult::MayAlias ||
+ L->isLoopInvariant(SI->getPointerOperand()))
+ return false;
+ HasMayAliasStore = true;
+ }
+ return HasMayAliasStore;
+}
+
bool GVNPass::performLoopLoadPRE(LoadInst *Load,
AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks) {
@@ -2029,6 +2074,16 @@ bool GVNPass::performLoopLoadPRE(LoadInst *Load,
if (LoadPtr->canBeFreed())
return false;
+ if (shouldPreserveLoadForVectorization(Load, L, LoopBlock, *AA)) {
+ ORE->emit([&]() {
+ return OptimizationRemarkAnalysis(DEBUG_TYPE,
+ "PreserveLoadForVectorization", Load)
+ << "preserving loop-header load to avoid a loop-carried "
+ "dependency";
+ });
+ return false;
+ }
+
// TODO: Support critical edge splitting if blocker has more than 1 successor.
MapVector<BasicBlock *, Value *> AvailableLoads;
AvailableLoads[LoopBlock] = LoadPtr;
diff --git a/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll b/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
new file mode 100644
index 0000000000000..59fa095a2fdf8
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
@@ -0,0 +1,582 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; REQUIRES: aarch64-registered-target
+; RUN: opt -passes='loop-simplify,lcssa,gvn' -S %s | FileCheck %s
+; RUN: opt -mtriple=aarch64 -passes='loop-simplify,lcssa,gvn,loop-vectorize' -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=VEC
+;
+; Loop load PRE must not introduce a scalar recurrence for a conditional
+; may-alias store through a loop-varying pointer. Runtime alias checks can
+; disambiguate the original accesses when the loop is vectorized.
+;
+
+; Preserve the load so runtime alias checks can enable vectorization.
+define void @may_alias_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @may_alias_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[BOUND:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @may_alias_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*:]]
+; VEC-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; VEC: [[VECTOR_MEMCHECK]]:
+; VEC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i64 128
+; VEC-NEXT: [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[LIMIT]], i64 8
+; VEC-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]
+; VEC-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[LIMIT]], [[SCEVGEP]]
+; VEC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; VEC-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC: [[VECTOR_PH]]:
+; VEC-NEXT: [[TMP0:%.*]] = load double, ptr [[LIMIT]], align 8, !alias.scope [[META0:![0-9]+]]
+; VEC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP0]], i64 0
+; VEC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; VEC-NEXT: [[TMP1:%.*]] = getelementptr double, ptr [[IN]], i64 [[INDEX]]
+; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
+; VEC-NEXT: [[TMP2:%.*]] = fcmp olt <2 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; VEC-NEXT: [[TMP3:%.*]] = fsub <2 x double> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
+; VEC-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[OUT]], i64 [[INDEX]]
+; VEC-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; VEC-NEXT: store double [[TMP6]], ptr [[TMP5]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC: [[PRED_STORE_CONTINUE]]:
+; VEC-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
+; VEC-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; VEC: [[PRED_STORE_IF2]]:
+; VEC-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT: [[TMP9:%.*]] = getelementptr double, ptr [[OUT]], i64 [[TMP8]]
+; VEC-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; VEC-NEXT: store double [[TMP10]], ptr [[TMP9]], align 8, !alias.scope [[META3]], !noalias [[META0]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; VEC: [[PRED_STORE_CONTINUE3]]:
+; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VEC-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; VEC-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VEC: [[MIDDLE_BLOCK]]:
+; VEC-NEXT: br label %[[EXIT:.*]]
+; VEC: [[SCALAR_PH]]:
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[BOUND:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Keep PRE when the clobber also has an invariant address.
+define void @invariant_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @invariant_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @invariant_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[OUT]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %out, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Keep PRE across calls that memory versioning cannot disambiguate.
+define void @call_clobber(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @call_clobber(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: call void @clobber(ptr [[Q]])
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @call_clobber(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: call void @clobber(ptr [[Q]])
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ call void @clobber(ptr %q)
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Keep existing PRE behavior in loops with ordered memory operations.
+define void @volatile_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @volatile_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store volatile double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @volatile_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store volatile double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store volatile double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Known aliasing is not a runtime disambiguation opportunity.
+define void @must_alias_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @must_alias_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE]], %[[ENTRY]] ], [ [[BOUND1:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND1]] = phi double [ [[DIFF]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @must_alias_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE]], %[[ENTRY]] ], [ [[BOUND1:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND1]] = phi double [ [[DIFF]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %limit, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Honor an explicit request to disable vectorization.
+define void @vectorization_disabled(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @vectorization_disabled(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @vectorization_disabled(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
+; Do not inhibit scalar cleanup after vectorization.
+define void @already_vectorized(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @already_vectorized(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @already_vectorized(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop, !llvm.loop !2
+exit:
+ ret void
+}
+
+declare void @clobber(ptr) nofree
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.disable"}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.isvectorized", i32 1}
More information about the llvm-branch-commits
mailing list