[llvm] 4cfbf8e - [ValueTracking] Conservative NoSync check prevents vectorization (#199002)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 2 02:40:13 PDT 2026
Author: Kshitij Paranjape
Date: 2026-06-02T09:40:07Z
New Revision: 4cfbf8ec82cb7e6f06c3f382a1b4ec082bd42b18
URL: https://github.com/llvm/llvm-project/commit/4cfbf8ec82cb7e6f06c3f382a1b4ec082bd42b18
DIFF: https://github.com/llvm/llvm-project/commit/4cfbf8ec82cb7e6f06c3f382a1b4ec082bd42b18.diff
LOG: [ValueTracking] Conservative NoSync check prevents vectorization (#199002)
Conservative nosync check in ValueTracking.cpp returns false causing
potentially faulting load preventing vectorization. Instead check if any
instructions betweenAssume Instruction and Ctx Instruction are synchronizing.
Fixes #180180.
Added:
Modified:
llvm/include/llvm/Analysis/ValueTracking.h
llvm/lib/Analysis/ValueTracking.cpp
llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index b2f664a9c9c0d..684c22fdab2e4 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -639,8 +639,7 @@ inline bool isValidAssumeForContext(const Instruction *I,
}
/// Returns true, if no instruction between \p Assume and \p CtxI may free
-/// memory and the function is marked as NoSync. The latter ensures the current
-/// function cannot arrange for another thread to free on its behalf.
+/// (including through synchronization).
LLVM_ABI bool willNotFreeBetween(const Instruction *Assume,
const Instruction *CtxI);
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 7b1550a10f134..8ff5b750203cc 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -704,22 +704,20 @@ bool llvm::isValidAssumeForContext(const Instruction *Inv,
bool llvm::willNotFreeBetween(const Instruction *Assume,
const Instruction *CtxI) {
// Helper to check if there are any calls in the range that may free memory.
- auto hasNoFreeCalls = [](auto Range) {
+ auto hasNoFreeInRange = [](auto Range) {
for (const auto &[Idx, I] : enumerate(Range)) {
if (Idx > MaxInstrsToCheckForFree)
return false;
- if (const auto *CB = dyn_cast<CallBase>(&I))
+
+ if (auto *CB = dyn_cast<CallBase>(&I)) {
if (!CB->hasFnAttr(Attribute::NoFree))
return false;
+ } else if (I.maySynchronize())
+ return false;
}
return true;
};
- // Make sure the current function cannot arrange for another thread to free on
- // its behalf.
- if (!CtxI->getFunction()->hasNoSync())
- return false;
-
// Handle cross-block case: CtxI in a successor of Assume's block.
const BasicBlock *CtxBB = CtxI->getParent();
const BasicBlock *AssumeBB = Assume->getParent();
@@ -728,7 +726,7 @@ bool llvm::willNotFreeBetween(const Instruction *Assume,
if (CtxBB->getSinglePredecessor() != AssumeBB)
return false;
- if (!hasNoFreeCalls(make_range(CtxBB->begin(), CtxIter)))
+ if (!hasNoFreeInRange(make_range(CtxBB->begin(), CtxIter)))
return false;
CtxIter = AssumeBB->end();
@@ -740,7 +738,7 @@ bool llvm::willNotFreeBetween(const Instruction *Assume,
// Check if there are any calls between Assume and CtxIter that may free
// memory.
- return hasNoFreeCalls(make_range(Assume->getIterator(), CtxIter));
+ return hasNoFreeInRange(make_range(Assume->getIterator(), CtxIter));
}
// TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but
diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
index fe758c3ed21e3..894960a74dd34 100644
--- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
@@ -1307,30 +1307,13 @@ define void @deref_assumption_in_header_constant_trip_count_nofree_via_context_b
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK: [[PRED_LOAD_IF]]:
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
-; CHECK: [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
-; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
-; CHECK: [[PRED_LOAD_IF1]]:
-; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i64 1
-; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK: [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP13]], <2 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -1456,4 +1439,150 @@ exit:
ret void
}
+define void @atomicVectorizingConditionallyTest(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @atomicVectorizingConditionallyTest(
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ]
+; CHECK-NEXT: [[ATOMIC:%.*]] = load atomic ptr, ptr [[A]] seq_cst, align 8
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP10]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP13]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ]
+ %atomic = load atomic ptr, ptr %a seq_cst, align 8 ; Atomic Instruction
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.a = getelementptr i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %l.b = load i32, ptr %gep.b, align 4
+ %c.1 = icmp sge i32 %l.b, 0
+ br i1 %c.1, label %loop.latch, label %loop.then
+
+loop.then:
+ %l.a = load i32, ptr %gep.a, align 4
+ br label %loop.latch
+
+loop.latch:
+ %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ]
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %merge, ptr %gep.c, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop.header
+
+exit:
+ ret void
+}
+
+define void @FenceVectorizingConditionallyTest(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) {
+; CHECK-LABEL: define void @FenceVectorizingConditionallyTest(
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ]
+; CHECK-NEXT: fence seq_cst
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK: [[PRED_LOAD_IF]]:
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
+; CHECK: [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1
+; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_IF1]]:
+; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP10]], i64 1
+; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK: [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP13]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ]
+ fence seq_cst ; Fence Instruction
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.a = getelementptr i32, ptr %a, i64 %iv
+ %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+ %l.b = load i32, ptr %gep.b, align 4
+ %c.1 = icmp sge i32 %l.b, 0
+ br i1 %c.1, label %loop.latch, label %loop.then
+
+loop.then:
+ %l.a = load i32, ptr %gep.a, align 4
+ br label %loop.latch
+
+loop.latch:
+ %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ]
+ %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+ store i32 %merge, ptr %gep.c, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1000
+ br i1 %ec, label %exit, label %loop.header
+
+exit:
+ ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
index 13d17b28709a3..9d6be3c9cc5e5 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
@@ -1090,3 +1090,237 @@ found:
exit:
ret i32 0
}
+
+define i64 @VectorizingWithoutNoSyncAttributeTest(ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) %v, i32 noundef %n) {
+; CHECK-LABEL: define i64 @VectorizingWithoutNoSyncAttributeTest(
+; CHECK-SAME: ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[V:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[L_V:%.*]] = load ptr, ptr [[V]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_V]], i64 4) ]
+; CHECK-NEXT: [[PI_L_V:%.*]] = ptrtoint ptr [[L_V]] to i64
+; CHECK-NEXT: [[GEP_V:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8
+; CHECK-NEXT: [[L_GEP_V:%.*]] = load ptr, ptr [[GEP_V]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_GEP_V]], i64 4) ]
+; CHECK-NEXT: [[PI_L_GEP_V:%.*]] = ptrtoint ptr [[L_GEP_V]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub i64 [[PI_L_GEP_V]], [[PI_L_V]]
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[L_V]], i64 [[SUB]]) ]
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_V]], i64 4) ]
+; CHECK-NEXT: [[C_1:%.*]] = icmp eq ptr [[L_V]], [[L_GEP_V]]
+; CHECK-NEXT: br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[PI_L_GEP_V]], -4
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[PI_L_V]]
+; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP4]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP6]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[C_2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[C_2]], label %[[LOOP_EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: br i1 [[TMP10]], label %[[LOOP_COMP:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[LOOP_COMP]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[LOOP_COMP1:.*]], label %[[SCALAR_PH]]
+; CHECK: [[LOOP_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 false)
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = shl i64 [[TMP12]], 2
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP13]]
+; CHECK-NEXT: br label %[[LOOP_EARLY_EXIT1:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[LOOP_COMP]] ], [ [[L_V]], %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[MERGE:%.*]] = phi ptr [ [[GEP_MERGE:%.*]], %[[LOOP_LATCH1:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[MERGE]], align 4
+; CHECK-NEXT: [[C_4:%.*]] = icmp slt i32 [[VAL]], [[N]]
+; CHECK-NEXT: br i1 [[C_4]], label %[[LOOP_EARLY_EXIT1]], label %[[LOOP_LATCH1]]
+; CHECK: [[LOOP_EARLY_EXIT1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[MERGE]], %[[LOOP_HEADER1]] ], [ [[TMP14]], %[[LOOP_EARLY_EXIT]] ]
+; CHECK-NEXT: [[PI_IV:%.*]] = ptrtoint ptr [[IV]] to i64
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[LOOP_LATCH1]]:
+; CHECK-NEXT: [[GEP_MERGE]] = getelementptr inbounds nuw i8, ptr [[MERGE]], i64 4
+; CHECK-NEXT: [[C_3:%.*]] = icmp eq ptr [[GEP_MERGE]], [[L_GEP_V]]
+; CHECK-NEXT: br i1 [[C_3]], label %[[LOOP_COMP1]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[LOOP_COMP1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[MER:%.*]] = phi i64 [ [[PI_L_GEP_V]], %[[ENTRY]] ], [ [[PI_IV]], %[[LOOP_EARLY_EXIT1]] ], [ [[PI_L_GEP_V]], %[[LOOP_COMP1]] ]
+; CHECK-NEXT: ret i64 [[MER]]
+;
+entry:
+ %l.v = load ptr, ptr %v, align 8
+ call void @llvm.assume(i1 true) [ "align"(ptr %l.v, i64 4) ]
+ %pi.l.v = ptrtoint ptr %l.v to i64
+ %gep.v = getelementptr inbounds nuw i8, ptr %v, i64 8
+ %l.gep.v = load ptr, ptr %gep.v, align 8
+ call void @llvm.assume(i1 true) [ "align"(ptr %l.gep.v, i64 4) ]
+ %pi.l.gep.v = ptrtoint ptr %l.gep.v to i64
+ %sub = sub i64 %pi.l.gep.v, %pi.l.v
+ call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %l.v, i64 %sub) ]
+ call void @llvm.assume(i1 true) [ "align"(ptr %l.v, i64 4) ]
+ %c.1 = icmp eq ptr %l.v, %l.gep.v
+ br i1 %c.1, label %exit, label %loop.preheader
+
+loop.preheader:
+ br label %loop.header
+
+loop.header:
+ %iv = phi ptr [ %gep.merge, %loop.latch ], [ %l.v, %loop.preheader ]
+ %val = load i32, ptr %iv, align 4
+ %c.2 = icmp slt i32 %val, %n
+ br i1 %c.2, label %loop.early.exit, label %loop.latch
+
+loop.early.exit:
+ %iv.1 = phi ptr [ %iv, %loop.header ]
+ %pi.iv = ptrtoint ptr %iv.1 to i64
+ br label %exit
+
+loop.latch:
+ %gep.merge = getelementptr inbounds nuw i8, ptr %iv, i64 4
+ %c.3 = icmp eq ptr %gep.merge, %l.gep.v
+ br i1 %c.3, label %loop.comp, label %loop.header
+
+loop.comp:
+ br label %exit
+
+exit:
+ %mer = phi i64 [ %pi.l.gep.v, %entry ], [ %pi.iv, %loop.early.exit ], [ %pi.l.gep.v, %loop.comp ]
+ ret i64 %mer
+}
+
+define i64 @volatileNotVectorizingTest(ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) %v, i32 noundef %n) {
+; CHECK-LABEL: define i64 @volatileNotVectorizingTest(
+; CHECK-SAME: ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[V:%.*]], i32 noundef [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[L_V:%.*]] = load ptr, ptr [[V]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_V]], i64 4) ]
+; CHECK-NEXT: [[PI_L_V:%.*]] = ptrtoint ptr [[L_V]] to i64
+; CHECK-NEXT: [[GEP_V:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8
+; CHECK-NEXT: [[L_GEP_V:%.*]] = load ptr, ptr [[GEP_V]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_GEP_V]], i64 4) ]
+; CHECK-NEXT: [[PI_L_GEP_V:%.*]] = ptrtoint ptr [[L_GEP_V]] to i64
+; CHECK-NEXT: [[SUB:%.*]] = sub i64 [[PI_L_GEP_V]], [[PI_L_V]]
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[L_V]], i64 [[SUB]]) ]
+; CHECK-NEXT: [[VOLATILE:%.*]] = load volatile ptr, ptr [[V]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_V]], i64 4) ]
+; CHECK-NEXT: [[C_1:%.*]] = icmp eq ptr [[L_V]], [[L_GEP_V]]
+; CHECK-NEXT: br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
+; CHECK: [[LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[PI_L_GEP_V]], -4
+; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[PI_L_V]]
+; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP4]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP6]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[LOOP_COMP:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 false)
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = shl i64 [[TMP12]], 2
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP13]]
+; CHECK-NEXT: br label %[[LOOP_EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[L_V]], %[[LOOP_PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[MERGE:%.*]] = phi ptr [ [[GEP_MERGE:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[MERGE]], align 4
+; CHECK-NEXT: [[C_2:%.*]] = icmp slt i32 [[VAL]], [[N]]
+; CHECK-NEXT: br i1 [[C_2]], label %[[LOOP_EARLY_EXIT]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_EARLY_EXIT]]:
+; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[MERGE]], %[[LOOP_HEADER1]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: [[PI_IV:%.*]] = ptrtoint ptr [[IV]] to i64
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[GEP_MERGE]] = getelementptr inbounds nuw i8, ptr [[MERGE]], i64 4
+; CHECK-NEXT: [[C_3:%.*]] = icmp eq ptr [[GEP_MERGE]], [[L_GEP_V]]
+; CHECK-NEXT: br i1 [[C_3]], label %[[LOOP_COMP]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[LOOP_COMP]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[MER:%.*]] = phi i64 [ [[PI_L_GEP_V]], %[[ENTRY]] ], [ [[PI_IV]], %[[LOOP_EARLY_EXIT]] ], [ [[PI_L_GEP_V]], %[[LOOP_COMP]] ]
+; CHECK-NEXT: ret i64 [[MER]]
+;
+entry:
+ %l.v = load ptr, ptr %v, align 8
+ call void @llvm.assume(i1 true) [ "align"(ptr %l.v, i64 4) ]
+ %pi.l.v = ptrtoint ptr %l.v to i64
+ %gep.v = getelementptr inbounds nuw i8, ptr %v, i64 8
+ %l.gep.v = load ptr, ptr %gep.v, align 8
+ call void @llvm.assume(i1 true) [ "align"(ptr %l.gep.v, i64 4) ]
+ %pi.l.gep.v = ptrtoint ptr %l.gep.v to i64
+ %sub = sub i64 %pi.l.gep.v, %pi.l.v
+ call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %l.v, i64 %sub) ]
+ %volatile = load volatile ptr, ptr %v, align 8 ; Volatile Instruction
+ call void @llvm.assume(i1 true) [ "align"(ptr %l.v, i64 4) ]
+ %c.1 = icmp eq ptr %l.v, %l.gep.v
+ br i1 %c.1, label %exit, label %loop.preheader
+
+loop.preheader:
+ br label %loop.header
+
+loop.header:
+ %iv = phi ptr [ %gep.merge, %loop.latch ], [ %l.v, %loop.preheader ]
+ %val = load i32, ptr %iv, align 4
+ %c.2 = icmp slt i32 %val, %n
+ br i1 %c.2, label %loop.early.exit, label %loop.latch
+
+loop.early.exit:
+ %iv.1 = phi ptr [ %iv, %loop.header ]
+ %pi.iv = ptrtoint ptr %iv.1 to i64
+ br label %exit
+
+loop.latch:
+ %gep.merge = getelementptr inbounds nuw i8, ptr %iv, i64 4
+ %c.3 = icmp eq ptr %gep.merge, %l.gep.v
+ br i1 %c.3, label %loop.comp, label %loop.header
+
+loop.comp:
+ br label %exit
+
+exit:
+ %mer = phi i64 [ %pi.l.gep.v, %entry ], [ %pi.iv, %loop.early.exit ], [ %pi.l.gep.v, %loop.comp ]
+ ret i64 %mer
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll b/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll
index b8c999d700aa7..693da976d1752 100644
--- a/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll
@@ -4,14 +4,10 @@
define i64 @align_deref_align(i1 %c, ptr %p) {
; CHECK-LABEL: define i64 @align_deref_align(
; CHECK-SAME: i1 [[C:%.*]], ptr [[P:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 8), "align"(ptr [[P]], i64 8) ]
-; CHECK-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF]]:
; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[V]], %[[IF]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[RES:%.*]] = select i1 [[C]], i64 [[V]], i64 0
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
@@ -52,17 +48,12 @@ exit:
define i64 @assume_deref_align2(i1 %c1, i32 %x, ptr %p) {
; CHECK-LABEL: define i64 @assume_deref_align2(
; CHECK-SAME: i1 [[C1:%.*]], i32 [[X:%.*]], ptr [[P:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 8), "align"(ptr [[P]], i64 8) ]
-; CHECK-NEXT: br i1 [[C1]], label %[[IF1:.*]], label %[[EXIT:.*]]
-; CHECK: [[IF1]]:
; CHECK-NEXT: [[C2:%.*]] = icmp ugt i32 [[X]], 10
-; CHECK-NEXT: br i1 [[C2]], label %[[IF2:.*]], label %[[EXIT]]
-; CHECK: [[IF2]]:
; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[V]], %[[IF2]] ], [ 1, %[[IF1]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[C2]], i64 [[V]], i64 1
+; CHECK-NEXT: [[RES:%.*]] = select i1 [[C1]], i64 [[SPEC_SELECT]], i64 0
; CHECK-NEXT: ret i64 [[RES]]
;
entry:
More information about the llvm-commits
mailing list