[llvm-branch-commits] [clang] [llvm] [GVN] Preserve vectorization opportunities when PREing loop loads (PR #227069)
Tomas Matheson via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Oct 1 04:04:44 PDT 2026
https://github.com/tommat01 updated https://github.com/llvm/llvm-project/pull/227069
>From 707faced01da1f6505e8559a0ef012e726a197e4 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson at arm.com>
Date: Mon, 28 Sep 2026 16:41:29 +0100
Subject: [PATCH 1/2] [GVN] Preserve vectorization opportunities when PREing
loop loads
Loop load PRE can replace an invariant-address load with a loop-carried
PHI and conditional reload after a may-alias store. That scalar recurrence
can prevent vectorization even when runtime alias checks could disambiguate
the original accesses. Subsequent full unrolling then expands scalar code.
Conservatively preserve the header load in innermost loops whose clobber
is a conditional may-alias store through a varying pointer. Keep existing
PRE behavior for invariant-address clobbers, known aliasing, calls, ordered
memory operations, and loops with vectorization disabled or completed.
This is an opportunity heuristic, not a vectorization legality proof.
---
llvm/lib/Transforms/Scalar/GVN.cpp | 55 ++
.../GVN/PRE/loop-load-pre-vectorization.ll | 582 ++++++++++++++++++
2 files changed, 637 insertions(+)
create mode 100644 llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index b8a44882af867..7ff5015818895 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -48,6 +48,7 @@
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugLoc.h"
+#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/InstrTypes.h"
@@ -73,6 +74,7 @@
#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/SSAUpdater.h"
#include "llvm/Transforms/Utils/VNCoercion.h"
#include <algorithm>
@@ -1970,6 +1972,49 @@ bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
return true;
}
+// Loop load PRE replaces an invariant-address load with a loop-carried PHI.
+// When its clobber is a conditional store through a varying pointer, this can
+// prevent vectorization that could otherwise disambiguate the accesses using
+// runtime checks. Keep the load in such loops until the vectorizer has run.
+static bool shouldPreserveLoadForVectorization(LoadInst *Load, const Loop *L,
+ BasicBlock *Clobber,
+ AAResults &AA) {
+ if (!L->isInnermost() || (hasVectorizeTransformation(L) & TM_Disable) ||
+ !VectorType::isValidElementType(Load->getType()))
+ return false;
+
+ // Do not penalize PRE in loops with calls or ordered memory operations,
+ // where runtime memory checks alone cannot enable vectorization.
+ for (BasicBlock *BB : L->blocks())
+ for (Instruction &I : *BB) {
+ if (isa<CallBase>(I))
+ return false;
+ if (auto *LI = dyn_cast<LoadInst>(&I); LI && !LI->isSimple())
+ return false;
+ if (I.mayWriteToMemory()) {
+ auto *SI = dyn_cast<StoreInst>(&I);
+ if (!SI || !SI->isSimple())
+ return false;
+ }
+ }
+
+ bool HasMayAliasStore = false;
+ for (Instruction &I : *Clobber) {
+ auto *SI = dyn_cast<StoreInst>(&I);
+ if (!SI)
+ continue;
+ AliasResult Alias =
+ AA.alias(MemoryLocation::get(Load), MemoryLocation::get(SI));
+ if (Alias == AliasResult::NoAlias)
+ continue;
+ if (Alias != AliasResult::MayAlias ||
+ L->isLoopInvariant(SI->getPointerOperand()))
+ return false;
+ HasMayAliasStore = true;
+ }
+ return HasMayAliasStore;
+}
+
bool GVNPass::performLoopLoadPRE(LoadInst *Load,
AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks) {
@@ -2035,6 +2080,16 @@ bool GVNPass::performLoopLoadPRE(LoadInst *Load,
if (LoadPtr->canBeFreed())
return false;
+ if (shouldPreserveLoadForVectorization(Load, L, LoopBlock, *AA)) {
+ ORE->emit([&]() {
+ return OptimizationRemarkAnalysis(DEBUG_TYPE,
+ "PreserveLoadForVectorization", Load)
+ << "preserving loop-header load to avoid a loop-carried "
+ "dependency";
+ });
+ return false;
+ }
+
// TODO: Support critical edge splitting if blocker has more than 1 successor.
MapVector<BasicBlock *, Value *> AvailableLoads;
AvailableLoads[LoopBlock] = LoadPtr;
diff --git a/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll b/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
new file mode 100644
index 0000000000000..59fa095a2fdf8
--- /dev/null
+++ b/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
@@ -0,0 +1,582 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; REQUIRES: aarch64-registered-target
+; RUN: opt -passes='loop-simplify,lcssa,gvn' -S %s | FileCheck %s
+; RUN: opt -mtriple=aarch64 -passes='loop-simplify,lcssa,gvn,loop-vectorize' -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=VEC
+;
+; Loop load PRE must not introduce a scalar recurrence for a conditional
+; may-alias store through a loop-varying pointer. Runtime alias checks can
+; disambiguate the original accesses when the loop is vectorized.
+;
+
+; Preserve the load so runtime alias checks can enable vectorization.
+define void @may_alias_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @may_alias_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[BOUND:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @may_alias_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*:]]
+; VEC-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
+; VEC: [[VECTOR_MEMCHECK]]:
+; VEC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i64 128
+; VEC-NEXT: [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[LIMIT]], i64 8
+; VEC-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]
+; VEC-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[LIMIT]], [[SCEVGEP]]
+; VEC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; VEC-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VEC: [[VECTOR_PH]]:
+; VEC-NEXT: [[TMP0:%.*]] = load double, ptr [[LIMIT]], align 8, !alias.scope [[META0:![0-9]+]]
+; VEC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP0]], i64 0
+; VEC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; VEC-NEXT: br label %[[VECTOR_BODY:.*]]
+; VEC: [[VECTOR_BODY]]:
+; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE3:.*]] ]
+; VEC-NEXT: [[TMP1:%.*]] = getelementptr double, ptr [[IN]], i64 [[INDEX]]
+; VEC-NEXT: [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP1]], align 8
+; VEC-NEXT: [[TMP2:%.*]] = fcmp olt <2 x double> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; VEC-NEXT: [[TMP3:%.*]] = fsub <2 x double> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
+; VEC-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
+; VEC-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VEC: [[PRED_STORE_IF]]:
+; VEC-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[OUT]], i64 [[INDEX]]
+; VEC-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; VEC-NEXT: store double [[TMP6]], ptr [[TMP5]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; VEC: [[PRED_STORE_CONTINUE]]:
+; VEC-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP2]], i64 1
+; VEC-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3]]
+; VEC: [[PRED_STORE_IF2]]:
+; VEC-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
+; VEC-NEXT: [[TMP9:%.*]] = getelementptr double, ptr [[OUT]], i64 [[TMP8]]
+; VEC-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; VEC-NEXT: store double [[TMP10]], ptr [[TMP9]], align 8, !alias.scope [[META3]], !noalias [[META0]]
+; VEC-NEXT: br label %[[PRED_STORE_CONTINUE3]]
+; VEC: [[PRED_STORE_CONTINUE3]]:
+; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VEC-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
+; VEC-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; VEC: [[MIDDLE_BLOCK]]:
+; VEC-NEXT: br label %[[EXIT:.*]]
+; VEC: [[SCALAR_PH]]:
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[NEXT:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[BOUND:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Keep PRE when the clobber also has an invariant address.
+define void @invariant_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @invariant_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @invariant_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[OUT]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %out, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Keep PRE across calls that memory versioning cannot disambiguate.
+define void @call_clobber(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @call_clobber(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: call void @clobber(ptr [[Q]])
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @call_clobber(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: call void @clobber(ptr [[Q]])
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ call void @clobber(ptr %q)
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Keep existing PRE behavior in loops with ordered memory operations.
+define void @volatile_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @volatile_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store volatile double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @volatile_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store volatile double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store volatile double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Known aliasing is not a runtime disambiguation opportunity.
+define void @must_alias_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @must_alias_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE]], %[[ENTRY]] ], [ [[BOUND1:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND1]] = phi double [ [[DIFF]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @must_alias_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE]], %[[ENTRY]] ], [ [[BOUND1:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND1]] = phi double [ [[DIFF]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %limit, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
+; Honor an explicit request to disable vectorization.
+define void @vectorization_disabled(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @vectorization_disabled(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @vectorization_disabled(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop, !llvm.loop !0
+exit:
+ ret void
+}
+
+; Do not inhibit scalar cleanup after vectorization.
+define void @already_vectorized(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @already_vectorized(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @already_vectorized(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %q = getelementptr double, ptr %out, i64 %iv
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop, !llvm.loop !2
+exit:
+ ret void
+}
+
+declare void @clobber(ptr) nofree
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.disable"}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.isvectorized", i32 1}
>From 4eec78b0f0506eb4d3414a5e4716cafa0f18e4aa Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson at arm.com>
Date: Mon, 28 Sep 2026 19:12:28 +0100
Subject: [PATCH 2/2] - GVN now respects pipeline vectorization settings. -
Loop-access analysis rejects unsuitable indirect stores, retaining PRE.
---
.../test/CodeGen/gvn-vectorization-pipeline.c | 14 +
llvm/include/llvm/Transforms/Scalar/GVN.h | 8 +
llvm/lib/Passes/PassBuilder.cpp | 2 +
llvm/lib/Passes/PassBuilderPipelines.cpp | 6 +-
llvm/lib/Passes/PassRegistry.def | 3 +-
llvm/lib/Transforms/Scalar/GVN.cpp | 33 +-
llvm/test/Other/new-pm-print-pipeline.ll | 3 +
.../GVN/PRE/loop-load-pre-vectorization.ll | 296 +++++++++++++++++-
8 files changed, 354 insertions(+), 11 deletions(-)
create mode 100644 clang/test/CodeGen/gvn-vectorization-pipeline.c
diff --git a/clang/test/CodeGen/gvn-vectorization-pipeline.c b/clang/test/CodeGen/gvn-vectorization-pipeline.c
new file mode 100644
index 0000000000000..a05f0e713ef07
--- /dev/null
+++ b/clang/test/CodeGen/gvn-vectorization-pipeline.c
@@ -0,0 +1,14 @@
+// RUN: %clang -O2 -S -emit-llvm -o /dev/null -mllvm -print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -O3 -S -emit-llvm -o /dev/null -mllvm -print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -O2 -fno-vectorize -S -emit-llvm -o /dev/null -mllvm -print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=DISABLED --implicit-check-not=preserve-vectorization
+// RUN: %clang -O3 -fno-vectorize -S -emit-llvm -o /dev/null -mllvm -print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=DISABLED --implicit-check-not=preserve-vectorization
+// RUN: %clang -O2 -flto=thin -S -emit-llvm -o /dev/null -mllvm -print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -O2 -flto=full -S -emit-llvm -o /dev/null -mllvm -print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=ENABLED
+// RUN: %clang -O2 -fno-vectorize -flto=thin -S -emit-llvm -o /dev/null -mllvm -print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=DISABLED --implicit-check-not=preserve-vectorization
+// RUN: %clang -O2 -fno-vectorize -flto=full -S -emit-llvm -o /dev/null -mllvm -print-pipeline-passes %s 2>&1 | FileCheck %s --check-prefix=DISABLED --implicit-check-not=preserve-vectorization
+
+// Preserve vectorization opportunities only when loop vectorization is enabled
+// later in the compilation, including vectorization deferred until ThinLTO.
+// ENABLED: gvn<preserve-vectorization;>
+// DISABLED: gvn<>
+void f(void) {}
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index c59d830bb7016..beb8d33ccd1ea 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -79,8 +79,16 @@ struct GVNOptions {
std::optional<bool> AllowMemDep;
std::optional<bool> AllowMemorySSA;
+ /// Preserve opportunities for vectorization later in the compilation.
+ bool PreserveVectorization = false;
+
GVNOptions() = default;
+ GVNOptions &setPreserveVectorization(bool Enable) {
+ PreserveVectorization = Enable;
+ return *this;
+ }
+
/// Enables or disables PRE of scalars in GVN.
GVNOptions &setScalarPRE(bool ScalarPRE) {
AllowScalarPRE = ScalarPRE;
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 30a6f75a1b86d..8d8bfb6d50fc8 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -1430,6 +1430,8 @@ Expected<GVNOptions> parseGVNOptions(StringRef Params) {
Result.setScalarPRE(Enable);
} else if (ParamName == "load-pre") {
Result.setLoadPRE(Enable);
+ } else if (ParamName == "preserve-vectorization") {
+ Result.setPreserveVectorization(Enable);
} else if (ParamName == "split-backedge-load-pre") {
Result.setLoadPRESplitBackedge(Enable);
} else if (ParamName == "memdep") {
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index bd37fffc8e991..f6a03eb64c2f4 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -786,7 +786,8 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
if (RunNewGVN)
FPM.addPass(NewGVNPass());
else
- FPM.addPass(GVNPass());
+ FPM.addPass(
+ GVNPass(GVNOptions().setPreserveVectorization(PTO.LoopVectorization)));
// Sparse conditional constant propagation.
// FIXME: It isn't clear why we do this *after* loop passes rather than
@@ -2302,7 +2303,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
if (RunNewGVN)
MainFPM.addPass(NewGVNPass());
else
- MainFPM.addPass(GVNPass());
+ MainFPM.addPass(
+ GVNPass(GVNOptions().setPreserveVectorization(PTO.LoopVectorization)));
// Remove dead memcpy()'s.
MainFPM.addPass(MemCpyOptPass());
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index af4ce5029551d..b2b1031c54727 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -615,7 +615,8 @@ FUNCTION_PASS_WITH_PARAMS(
"gvn", "GVNPass", [](GVNOptions Opts) { return GVNPass(Opts); },
parseGVNOptions,
"no-scalar-pre;scalar-pre;no-load-pre;load-pre;no-split-backedge-load-pre;"
- "split-backedge-load-pre;no-memdep;memdep;no-memoryssa;memoryssa")
+ "split-backedge-load-pre;no-memdep;memdep;no-memoryssa;memoryssa;"
+ "preserve-vectorization;no-preserve-vectorization")
FUNCTION_PASS_WITH_PARAMS(
"hardware-loops", "HardwareLoopsPass",
[](HardwareLoopOptions Opts) { return HardwareLoopsPass(Opts); },
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 7ff5015818895..45bb214f7afa8 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -34,6 +34,7 @@
#include "llvm/Analysis/InstructionPrecedenceTracking.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryDependenceAnalysis.h"
@@ -41,6 +42,7 @@
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/PHITransAddr.h"
+#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
@@ -934,6 +936,8 @@ void GVNPass::printPipeline(
OS, MapClassName2PassName);
OS << '<';
+ if (Options.PreserveVectorization)
+ OS << "preserve-vectorization;";
if (Options.AllowScalarPRE != std::nullopt)
OS << (*Options.AllowScalarPRE ? "" : "no-") << "scalar-pre;";
if (Options.AllowLoadPRE != std::nullopt)
@@ -1976,9 +1980,11 @@ bool GVNPass::performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
// When its clobber is a conditional store through a varying pointer, this can
// prevent vectorization that could otherwise disambiguate the accesses using
// runtime checks. Keep the load in such loops until the vectorizer has run.
-static bool shouldPreserveLoadForVectorization(LoadInst *Load, const Loop *L,
- BasicBlock *Clobber,
- AAResults &AA) {
+static bool
+shouldPreserveLoadForVectorization(LoadInst *Load, Loop *L, BasicBlock *Clobber,
+ AAResults &AA, const TargetLibraryInfo &TLI,
+ AssumptionCache &AC, DominatorTree &DT,
+ LoopInfo &LI) {
if (!L->isInnermost() || (hasVectorizeTransformation(L) & TM_Disable) ||
!VectorType::isValidElementType(Load->getType()))
return false;
@@ -2012,13 +2018,26 @@ static bool shouldPreserveLoadForVectorization(LoadInst *Load, const Loop *L,
return false;
HasMayAliasStore = true;
}
- return HasMayAliasStore;
+ if (!HasMayAliasStore)
+ return false;
+
+ // A varying pointer need not have a checkable range (e.g. out[indices[iv]]).
+ // Ask the same memory analysis used by the vectorizer before giving up PRE.
+ // GVN mutates the IR without maintaining ScalarEvolution, so use fresh,
+ // short-lived analyses only after the inexpensive candidate checks above.
+ TargetLibraryInfo LocalTLI(TLI);
+ ScalarEvolution SE(*Load->getFunction(), LocalTLI, AC, DT, LI);
+ LoopAccessInfo LAI(L, &SE, /*TTI=*/nullptr, &TLI, &AA, &DT, &LI, &AC);
+ return LAI.canVectorizeMemory() &&
+ !LAI.hasStoreStoreDependenceInvolvingLoopInvariantAddress() &&
+ !LAI.hasLoadStoreDependenceInvolvingLoopInvariantAddress() &&
+ LAI.getRuntimePointerChecking()->Need;
}
bool GVNPass::performLoopLoadPRE(LoadInst *Load,
AvailValInBlkVect &ValuesPerBlock,
UnavailBlkVect &UnavailableBlocks) {
- const Loop *L = LI->getLoopFor(Load->getParent());
+ Loop *L = LI->getLoopFor(Load->getParent());
// TODO: Generalize to other loop blocks that dominate the latch.
if (!L || L->getHeader() != Load->getParent())
return false;
@@ -2080,7 +2099,9 @@ bool GVNPass::performLoopLoadPRE(LoadInst *Load,
if (LoadPtr->canBeFreed())
return false;
- if (shouldPreserveLoadForVectorization(Load, L, LoopBlock, *AA)) {
+ if (Options.PreserveVectorization &&
+ shouldPreserveLoadForVectorization(Load, L, LoopBlock, *AA, *TLI, *AC,
+ *DT, *LI)) {
ORE->emit([&]() {
return OptimizationRemarkAnalysis(DEBUG_TYPE,
"PreserveLoadForVectorization", Load)
diff --git a/llvm/test/Other/new-pm-print-pipeline.ll b/llvm/test/Other/new-pm-print-pipeline.ll
index a8fcf113e49b2..650b745c45ce1 100644
--- a/llvm/test/Other/new-pm-print-pipeline.ll
+++ b/llvm/test/Other/new-pm-print-pipeline.ll
@@ -126,3 +126,6 @@
; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='function(infer-address-spaces,infer-address-spaces<assume-default-is-flat-addrspace>)' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-39
; CHECK-39: function(infer-address-spaces,infer-address-spaces<assume-default-is-flat-addrspace>)
+
+; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='gvn<preserve-vectorization>,gvn<no-preserve-vectorization>' < %s | FileCheck %s --match-full-lines --check-prefix=GVN-VECTORIZATION
+; GVN-VECTORIZATION: {{.*}}gvn<preserve-vectorization;>,gvn<>{{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll b/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
index 59fa095a2fdf8..aa3626422b666 100644
--- a/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
+++ b/llvm/test/Transforms/GVN/PRE/loop-load-pre-vectorization.ll
@@ -1,7 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; REQUIRES: aarch64-registered-target
-; RUN: opt -passes='loop-simplify,lcssa,gvn' -S %s | FileCheck %s
-; RUN: opt -mtriple=aarch64 -passes='loop-simplify,lcssa,gvn,loop-vectorize' -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=VEC
+; RUN: opt -passes='loop-simplify,lcssa,gvn<preserve-vectorization>' -S %s | FileCheck %s
+; RUN: opt -mtriple=aarch64 -passes='loop-simplify,lcssa,gvn<preserve-vectorization>,loop-vectorize' -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s --check-prefix=VEC
+; RUN: opt -passes='loop-simplify,lcssa,gvn' -S %s | FileCheck %s --check-prefix=DEFAULT
;
; Loop load PRE must not introduce a scalar recurrence for a conditional
; may-alias store through a loop-varying pointer. Runtime alias checks can
@@ -98,6 +99,32 @@ define void @may_alias_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenc
; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
+; DEFAULT-LABEL: define void @may_alias_store(
+; DEFAULT-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; DEFAULT-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; DEFAULT-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; DEFAULT-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; DEFAULT: [[STORE]]:
+; DEFAULT-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; DEFAULT-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; DEFAULT-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; DEFAULT-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LATCH]]
+; DEFAULT: [[LATCH]]:
+; DEFAULT-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; DEFAULT-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
entry:
br label %loop
loop:
@@ -120,6 +147,116 @@ exit:
ret void
}
+; Indirect stores lack the address bounds required for runtime disambiguation.
+define void @indirect_store(ptr %out, ptr noalias %indices, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
+; CHECK-LABEL: define void @indirect_store(
+; CHECK-SAME: ptr [[OUT:%.*]], ptr noalias [[INDICES:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; CHECK-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; CHECK: [[STORE]]:
+; CHECK-NEXT: [[IDXP:%.*]] = getelementptr i64, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[IDX:%.*]] = load i64, ptr [[IDXP]], align 8
+; CHECK-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IDX]]
+; CHECK-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; CHECK-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; CHECK-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; CHECK-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; VEC-LABEL: define void @indirect_store(
+; VEC-SAME: ptr [[OUT:%.*]], ptr noalias [[INDICES:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; VEC-NEXT: [[ENTRY:.*]]:
+; VEC-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LOOP:.*]]
+; VEC: [[LOOP]]:
+; VEC-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; VEC-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; VEC-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; VEC-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; VEC-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; VEC-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; VEC: [[STORE]]:
+; VEC-NEXT: [[IDXP:%.*]] = getelementptr i64, ptr [[INDICES]], i64 [[IV]]
+; VEC-NEXT: [[IDX:%.*]] = load i64, ptr [[IDXP]], align 8
+; VEC-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IDX]]
+; VEC-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; VEC-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; VEC-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; VEC-NEXT: br label %[[LATCH]]
+; VEC: [[LATCH]]:
+; VEC-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; VEC-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; VEC-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; VEC-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; VEC: [[EXIT]]:
+; VEC-NEXT: ret void
+;
+; DEFAULT-LABEL: define void @indirect_store(
+; DEFAULT-SAME: ptr [[OUT:%.*]], ptr noalias [[INDICES:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; DEFAULT-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; DEFAULT-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; DEFAULT-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; DEFAULT: [[STORE]]:
+; DEFAULT-NEXT: [[IDXP:%.*]] = getelementptr i64, ptr [[INDICES]], i64 [[IV]]
+; DEFAULT-NEXT: [[IDX:%.*]] = load i64, ptr [[IDXP]], align 8
+; DEFAULT-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IDX]]
+; DEFAULT-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; DEFAULT-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; DEFAULT-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LATCH]]
+; DEFAULT: [[LATCH]]:
+; DEFAULT-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; DEFAULT-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %next, %latch ]
+ %p = getelementptr double, ptr %in, i64 %iv
+ %value = load double, ptr %p, align 8
+ %bound = load double, ptr %limit, align 8
+ %cmp = fcmp olt double %value, %bound
+ br i1 %cmp, label %store, label %latch
+store:
+ %idxp = getelementptr i64, ptr %indices, i64 %iv
+ %idx = load i64, ptr %idxp, align 8
+ %q = getelementptr double, ptr %out, i64 %idx
+ %diff = fsub double %bound, %value
+ store double %diff, ptr %q, align 8
+ br label %latch
+latch:
+ %next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %next, 16
+ br i1 %done, label %exit, label %loop
+exit:
+ ret void
+}
+
; Keep PRE when the clobber also has an invariant address.
define void @invariant_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceable(8) %limit) {
; CHECK-LABEL: define void @invariant_store(
@@ -174,6 +311,32 @@ define void @invariant_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferenc
; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
+; DEFAULT-LABEL: define void @invariant_store(
+; DEFAULT-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; DEFAULT-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; DEFAULT-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; DEFAULT-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; DEFAULT: [[STORE]]:
+; DEFAULT-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; DEFAULT-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; DEFAULT-NEXT: store double [[DIFF]], ptr [[OUT]], align 8
+; DEFAULT-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LATCH]]
+; DEFAULT: [[LATCH]]:
+; DEFAULT-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; DEFAULT-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
entry:
br label %loop
loop:
@@ -250,6 +413,32 @@ define void @call_clobber(ptr %out, ptr noalias %in, ptr nofreeobj dereferenceab
; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
+; DEFAULT-LABEL: define void @call_clobber(
+; DEFAULT-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; DEFAULT-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; DEFAULT-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; DEFAULT-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; DEFAULT: [[STORE]]:
+; DEFAULT-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; DEFAULT-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; DEFAULT-NEXT: call void @clobber(ptr [[Q]])
+; DEFAULT-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LATCH]]
+; DEFAULT: [[LATCH]]:
+; DEFAULT-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; DEFAULT-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
entry:
br label %loop
loop:
@@ -326,6 +515,32 @@ define void @volatile_store(ptr %out, ptr noalias %in, ptr nofreeobj dereference
; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
+; DEFAULT-LABEL: define void @volatile_store(
+; DEFAULT-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; DEFAULT-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; DEFAULT-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; DEFAULT-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; DEFAULT: [[STORE]]:
+; DEFAULT-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; DEFAULT-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; DEFAULT-NEXT: store volatile double [[DIFF]], ptr [[Q]], align 8
+; DEFAULT-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LATCH]]
+; DEFAULT: [[LATCH]]:
+; DEFAULT-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; DEFAULT-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
entry:
br label %loop
loop:
@@ -400,6 +615,31 @@ define void @must_alias_store(ptr %out, ptr noalias %in, ptr nofreeobj dereferen
; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
+; DEFAULT-LABEL: define void @must_alias_store(
+; DEFAULT-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE]], %[[ENTRY]] ], [ [[BOUND1:%.*]], %[[LATCH:.*]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; DEFAULT-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; DEFAULT-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; DEFAULT-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; DEFAULT: [[STORE]]:
+; DEFAULT-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; DEFAULT-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; DEFAULT-NEXT: store double [[DIFF]], ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LATCH]]
+; DEFAULT: [[LATCH]]:
+; DEFAULT-NEXT: [[BOUND1]] = phi double [ [[DIFF]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; DEFAULT-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
entry:
br label %loop
loop:
@@ -476,6 +716,32 @@ define void @vectorization_disabled(ptr %out, ptr noalias %in, ptr nofreeobj der
; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
+; DEFAULT-LABEL: define void @vectorization_disabled(
+; DEFAULT-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; DEFAULT-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; DEFAULT-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; DEFAULT-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; DEFAULT: [[STORE]]:
+; DEFAULT-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; DEFAULT-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; DEFAULT-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; DEFAULT-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LATCH]]
+; DEFAULT: [[LATCH]]:
+; DEFAULT-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; DEFAULT-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
entry:
br label %loop
loop:
@@ -552,6 +818,32 @@ define void @already_vectorized(ptr %out, ptr noalias %in, ptr nofreeobj derefer
; VEC: [[EXIT]]:
; VEC-NEXT: ret void
;
+; DEFAULT-LABEL: define void @already_vectorized(
+; DEFAULT-SAME: ptr [[OUT:%.*]], ptr noalias [[IN:%.*]], ptr nofreeobj dereferenceable(8) [[LIMIT:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*]]:
+; DEFAULT-NEXT: [[BOUND_PRE1:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LOOP:.*]]
+; DEFAULT: [[LOOP]]:
+; DEFAULT-NEXT: [[BOUND:%.*]] = phi double [ [[BOUND_PRE1]], %[[ENTRY]] ], [ [[BOUND2:%.*]], %[[LATCH:.*]] ]
+; DEFAULT-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LATCH]] ]
+; DEFAULT-NEXT: [[P:%.*]] = getelementptr double, ptr [[IN]], i64 [[IV]]
+; DEFAULT-NEXT: [[VALUE:%.*]] = load double, ptr [[P]], align 8
+; DEFAULT-NEXT: [[CMP:%.*]] = fcmp olt double [[VALUE]], [[BOUND]]
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[STORE:.*]], label %[[LATCH]]
+; DEFAULT: [[STORE]]:
+; DEFAULT-NEXT: [[Q:%.*]] = getelementptr double, ptr [[OUT]], i64 [[IV]]
+; DEFAULT-NEXT: [[DIFF:%.*]] = fsub double [[BOUND]], [[VALUE]]
+; DEFAULT-NEXT: store double [[DIFF]], ptr [[Q]], align 8
+; DEFAULT-NEXT: [[BOUND_PRE:%.*]] = load double, ptr [[LIMIT]], align 8
+; DEFAULT-NEXT: br label %[[LATCH]]
+; DEFAULT: [[LATCH]]:
+; DEFAULT-NEXT: [[BOUND2]] = phi double [ [[BOUND_PRE]], %[[STORE]] ], [ [[BOUND]], %[[LOOP]] ]
+; DEFAULT-NEXT: [[NEXT]] = add nuw nsw i64 [[IV]], 1
+; DEFAULT-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXT]], 16
+; DEFAULT-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP2:![0-9]+]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
entry:
br label %loop
loop:
More information about the llvm-branch-commits
mailing list