[llvm] [LoopIdiomVectorize] Bail when vectorization is disabled (PR #181142)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 12 05:56:08 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Hari Limaye (hazzlim)
<details>
<summary>Changes</summary>
Bail on vectorizing a loop in LoopIdiomVectorize when the loop carries
hints that indicate vectorization is disabled.
This means that LoopIdiomVectorize will now respect vectorize(disable)
loop hints.
---
Full diff: https://github.com/llvm/llvm-project/pull/181142.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp (+30-4)
- (added) llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll (+196)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
index 53129e2e5fbba..b2108732c6c8f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
@@ -67,6 +67,7 @@
#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
@@ -74,6 +75,7 @@
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
using namespace llvm;
using namespace PatternMatch;
@@ -123,6 +125,9 @@ class LoopIdiomVectorize {
const TargetTransformInfo *TTI;
const DataLayout *DL;
+ /// Interface to emit optimization remarks.
+ OptimizationRemarkEmitter &ORE;
+
// Blocks that will be used for inserting vectorized code.
BasicBlock *EndBlock = nullptr;
BasicBlock *VectorLoopPreheaderBlock = nullptr;
@@ -133,9 +138,9 @@ class LoopIdiomVectorize {
public:
LoopIdiomVectorize(LoopIdiomVectorizeStyle S, unsigned VF, DominatorTree *DT,
LoopInfo *LI, const TargetTransformInfo *TTI,
- const DataLayout *DL)
- : VectorizeStyle(S), ByteCompareVF(VF), DT(DT), LI(LI), TTI(TTI), DL(DL) {
- }
+ const DataLayout *DL, OptimizationRemarkEmitter &ORE)
+ : VectorizeStyle(S), ByteCompareVF(VF), DT(DT), LI(LI), TTI(TTI), DL(DL),
+ ORE(ORE) {}
bool run(Loop *L);
@@ -199,7 +204,9 @@ PreservedAnalyses LoopIdiomVectorizePass::run(Loop &L, LoopAnalysisManager &AM,
if (ByteCmpVF.getNumOccurrences())
BCVF = ByteCmpVF;
- LoopIdiomVectorize LIV(VecStyle, BCVF, &AR.DT, &AR.LI, &AR.TTI, DL);
+ OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
+
+ LoopIdiomVectorize LIV(VecStyle, BCVF, &AR.DT, &AR.LI, &AR.TTI, DL, ORE);
if (!LIV.run(&L))
return PreservedAnalyses::all();
@@ -219,6 +226,14 @@ bool LoopIdiomVectorize::run(Loop *L) {
if (DisableAll || F.hasOptSize())
return false;
+ // Bail if vectorization is disabled on loop.
+ LoopVectorizeHints Hints(L, /*InterleaveOnlyWhenForced=*/true, ORE);
+ if (!Hints.allowVectorization(&F, L, /*VectorizeOnlyWhenForced=*/false)) {
+ LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << F.getName()
+ << " due to vectorization hints\n");
+ return false;
+ }
+
if (F.hasFnAttribute(Attribute::NoImplicitFloat)) {
LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << F.getName()
<< " due to its NoImplicitFloat attribute");
@@ -1004,6 +1019,17 @@ bool LoopIdiomVectorize::recognizeFindFirstByte() {
return false;
auto *InnerLoop = CurLoop->getSubLoops().front();
+ Function &F = *InnerLoop->getHeader()->getParent();
+
+ // Bail if vectorization is disabled on inner loop.
+ LoopVectorizeHints Hints(InnerLoop, /*InterleaveOnlyWhenForced=*/true, ORE);
+ if (!Hints.allowVectorization(&F, InnerLoop,
+ /*VectorizeOnlyWhenForced=*/false)) {
+ LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << F.getName()
+ << " due to vectorization hints\n");
+ return false;
+ }
+
PHINode *IndPhi = dyn_cast<PHINode>(&Header->front());
if (!IndPhi || IndPhi->getNumIncomingValues() != 2)
return false;
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
new file mode 100644
index 0000000000000..8906392d2c35b
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
@@ -0,0 +1,196 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -S < %s | FileCheck %s
+
+define i32 @compare_bytes_simple_hint(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
+; CHECK-LABEL: define i32 @compare_bytes_simple_hint(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[MISMATCH_LOOP_PRE:.*]]:
+; CHECK-NEXT: br label %[[MISMATCH_LOOP:.*]]
+; CHECK: [[MISMATCH_LOOP]]:
+; CHECK-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], %[[MISMATCH_LOOP_PRE]] ], [ [[INC:%.*]], %[[WHILE_BODY:.*]] ]
+; CHECK-NEXT: [[INC]] = add i32 [[LEN_ADDR]], 1
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[INC]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[CMP_NOT2:%.*]] = icmp eq i8 [[TMP45]], [[TMP46]]
+; CHECK-NEXT: br i1 [[CMP_NOT2]], label %[[MISMATCH_LOOP]], label %[[WHILE_END]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[WHILE_END]]:
+; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[WHILE_BODY]] ], [ [[INC]], %[[MISMATCH_LOOP]] ]
+; CHECK-NEXT: [[EXTRA_PHI:%.*]] = phi i32 [ [[EXTRA]], %[[WHILE_BODY]] ], [ [[EXTRA]], %[[MISMATCH_LOOP]] ]
+; CHECK-NEXT: [[RES:%.*]] = add i32 [[INC_LCSSA]], [[EXTRA_PHI]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %while.cond
+
+while.cond:
+ %len.addr = phi i32 [ %len, %entry ], [ %inc, %while.body ]
+ %inc = add i32 %len.addr, 1
+ %cmp.not = icmp eq i32 %inc, %n
+ br i1 %cmp.not, label %while.end, label %while.body
+
+while.body:
+ %idxprom = zext i32 %inc to i64
+ %arrayidx = getelementptr inbounds i8, ptr %a, i64 %idxprom
+ %0 = load i8, ptr %arrayidx
+ %arrayidx2 = getelementptr inbounds i8, ptr %b, i64 %idxprom
+ %1 = load i8, ptr %arrayidx2
+ %cmp.not2 = icmp eq i8 %0, %1
+ br i1 %cmp.not2, label %while.cond, label %while.end, !llvm.loop !0
+
+while.end:
+ %inc.lcssa = phi i32 [ %inc, %while.body ], [ %inc, %while.cond ]
+ %extra.phi = phi i32 [ %extra, %while.body ], [ %extra, %while.cond ]
+ %res = add i32 %inc.lcssa, %extra.phi
+ ret i32 %res
+}
+
+define ptr @find_first_of_i8_outer_loop_hint(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER:.*]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
+; CHECK: [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; CHECK: [[MATCH_CHECK]]:
+; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
+; CHECK: [[SEARCH_CHECK]]:
+; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT:.*]], label %[[HEADER]], !llvm.loop [[LOOP0]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
+; CHECK: [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT]] ]
+; CHECK-NEXT: ret ptr [[RES]]
+;
+entry:
+ %search_test = icmp eq ptr %search_start, %search_end
+ %needle_test = icmp eq ptr %needle_start, %needle_end
+ %combined_test = or i1 %search_test, %needle_test
+ br i1 %combined_test, label %exit, label %header
+
+header:
+ %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+ %search_load = load i8, ptr %search_ptr, align 1
+ br label %match_check
+
+needle_check:
+ %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+ %needle_cmp = icmp eq ptr %needle_next, %needle_end
+ br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+ %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+ %needle_load = load i8, ptr %needle_ptr, align 1
+ %match_cmp = icmp eq i8 %search_load, %needle_load
+ br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+ %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+ %search_cmp = icmp eq ptr %search_next, %search_end
+ br i1 %search_cmp, label %exit, label %header, !llvm.loop !0
+
+exit:
+ %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+ ret ptr %res
+}
+
+define ptr @find_first_of_i8_inner_loop_hint(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER:.*]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
+; CHECK: [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]], !llvm.loop [[LOOP0]]
+; CHECK: [[MATCH_CHECK]]:
+; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
+; CHECK: [[SEARCH_CHECK]]:
+; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT:.*]], label %[[HEADER]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
+; CHECK: [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT]] ]
+; CHECK-NEXT: ret ptr [[RES]]
+;
+entry:
+ %search_test = icmp eq ptr %search_start, %search_end
+ %needle_test = icmp eq ptr %needle_start, %needle_end
+ %combined_test = or i1 %search_test, %needle_test
+ br i1 %combined_test, label %exit, label %header
+
+header:
+ %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+ %search_load = load i8, ptr %search_ptr, align 1
+ br label %match_check
+
+needle_check:
+ %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+ %needle_cmp = icmp eq ptr %needle_next, %needle_end
+ br i1 %needle_cmp, label %search_check, label %match_check, !llvm.loop !0
+
+match_check:
+ %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+ %needle_load = load i8, ptr %needle_ptr, align 1
+ %match_cmp = icmp eq i8 %search_load, %needle_load
+ br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+ %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+ %search_cmp = icmp eq ptr %search_next, %search_end
+ br i1 %search_cmp, label %exit, label %header
+
+exit:
+ %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+ ret ptr %res
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 1}
+
+attributes #0 = { "target-features"="+sve2" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.vectorize.width", i32 1}
+;.
``````````
</details>
https://github.com/llvm/llvm-project/pull/181142
More information about the llvm-commits
mailing list