[llvm] [LoopIdiomVectorize] Bail when vectorization is disabled (PR #181142)
Hari Limaye via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 19 10:07:53 PST 2026
https://github.com/hazzlim updated https://github.com/llvm/llvm-project/pull/181142
>From 45ae3ec18dfaf8aade7b192c52105abcd6ff6c96 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Thu, 12 Feb 2026 12:27:19 +0000
Subject: [PATCH 1/7] Add test
---
.../LoopIdiom/AArch64/disable-vectorize.ll | 374 ++++++++++++++++++
1 file changed, 374 insertions(+)
create mode 100644 llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
new file mode 100644
index 0000000000000..184d3a242ebae
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
@@ -0,0 +1,374 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -S < %s | FileCheck %s
+
+define i32 @compare_bytes_simple_hint(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
+; CHECK-LABEL: define i32 @compare_bytes_simple_hint(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[LEN]], 1
+; CHECK-NEXT: br label %[[MISMATCH_MIN_IT_CHECK:.*]]
+; CHECK: [[MISMATCH_MIN_IT_CHECK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i32 [[TMP0]], [[N]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MISMATCH_MEM_CHECK:.*]], label %[[MISMATCH_LOOP_PRE:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK: [[MISMATCH_MEM_CHECK]]:
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
+; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP8]] to i64
+; CHECK-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP9]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = lshr i64 [[TMP7]], 12
+; CHECK-NEXT: [[TMP13:%.*]] = lshr i64 [[TMP10]], 12
+; CHECK-NEXT: [[TMP14:%.*]] = lshr i64 [[TMP6]], 12
+; CHECK-NEXT: [[TMP15:%.*]] = lshr i64 [[TMP11]], 12
+; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT: [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MISMATCH_LOOP_PRE]], label %[[MISMATCH_VEC_LOOP_PREHEADER:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK: [[MISMATCH_VEC_LOOP_PREHEADER]]:
+; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
+; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
+; CHECK-NEXT: br label %[[MISMATCH_VEC_LOOP:.*]]
+; CHECK: [[MISMATCH_VEC_LOOP]]:
+; CHECK-NEXT: [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], %[[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], %[[MISMATCH_VEC_LOOP_INC:.*]] ]
+; CHECK-NEXT: [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], %[[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], %[[MISMATCH_VEC_LOOP_INC]] ]
+; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT: [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP22]], <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
+; CHECK-NEXT: [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP24]], <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT: [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
+; CHECK-NEXT: [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
+; CHECK-NEXT: [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
+; CHECK-NEXT: br i1 [[TMP28]], label %[[MISMATCH_VEC_LOOP_FOUND:.*]], label %[[MISMATCH_VEC_LOOP_INC]]
+; CHECK: [[MISMATCH_VEC_LOOP_INC]]:
+; CHECK-NEXT: [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
+; CHECK-NEXT: [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
+; CHECK-NEXT: br i1 [[TMP31]], label %[[MISMATCH_VEC_LOOP]], label %[[MISMATCH_END:.*]]
+; CHECK: [[MISMATCH_VEC_LOOP_FOUND]]:
+; CHECK-NEXT: [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], %[[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT: [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], %[[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT: [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], %[[MISMATCH_VEC_LOOP]] ]
+; CHECK-NEXT: [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
+; CHECK-NEXT: [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
+; CHECK-NEXT: [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
+; CHECK-NEXT: [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
+; CHECK-NEXT: [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
+; CHECK-NEXT: br label %[[MISMATCH_END]]
+; CHECK: [[MISMATCH_LOOP_PRE]]:
+; CHECK-NEXT: br label %[[MISMATCH_LOOP:.*]]
+; CHECK: [[MISMATCH_LOOP]]:
+; CHECK-NEXT: [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], %[[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], %[[MISMATCH_LOOP_INC:.*]] ]
+; CHECK-NEXT: [[TMP37:%.*]] = zext i32 [[MISMATCH_INDEX]] to i64
+; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP37]]
+; CHECK-NEXT: [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
+; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP37]]
+; CHECK-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
+; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i8 [[TMP39]], [[TMP41]]
+; CHECK-NEXT: br i1 [[TMP42]], label %[[MISMATCH_LOOP_INC]], label %[[MISMATCH_END]]
+; CHECK: [[MISMATCH_LOOP_INC]]:
+; CHECK-NEXT: [[TMP43]] = add i32 [[MISMATCH_INDEX]], 1
+; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
+; CHECK-NEXT: br i1 [[TMP44]], label %[[MISMATCH_END]], label %[[MISMATCH_LOOP]]
+; CHECK: [[MISMATCH_END]]:
+; CHECK-NEXT: [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], %[[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], %[[MISMATCH_LOOP]] ], [ [[N]], %[[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], %[[MISMATCH_VEC_LOOP_FOUND]] ]
+; CHECK-NEXT: br i1 true, label %[[BYTE_COMPARE:.*]], label %[[WHILE_COND:.*]]
+; CHECK: [[WHILE_COND]]:
+; CHECK-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], %[[MISMATCH_END]] ], [ [[MISMATCH_RESULT]], %[[WHILE_BODY:.*]] ]
+; CHECK-NEXT: [[INC:%.*]] = add i32 [[LEN_ADDR]], 1
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[MISMATCH_RESULT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[MISMATCH_RESULT]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[CMP_NOT2:%.*]] = icmp eq i8 [[TMP45]], [[TMP46]]
+; CHECK-NEXT: br i1 [[CMP_NOT2]], label %[[WHILE_COND]], label %[[WHILE_END]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[BYTE_COMPARE]]:
+; CHECK-NEXT: br label %[[WHILE_END]]
+; CHECK: [[WHILE_END]]:
+; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[MISMATCH_RESULT]], %[[WHILE_BODY]] ], [ [[MISMATCH_RESULT]], %[[WHILE_COND]] ], [ [[MISMATCH_RESULT]], %[[BYTE_COMPARE]] ]
+; CHECK-NEXT: [[EXTRA_PHI:%.*]] = phi i32 [ [[EXTRA]], %[[WHILE_BODY]] ], [ [[EXTRA]], %[[WHILE_COND]] ], [ [[EXTRA]], %[[BYTE_COMPARE]] ]
+; CHECK-NEXT: [[RES:%.*]] = add i32 [[INC_LCSSA]], [[EXTRA_PHI]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %while.cond
+
+while.cond:
+ %len.addr = phi i32 [ %len, %entry ], [ %inc, %while.body ]
+ %inc = add i32 %len.addr, 1
+ %cmp.not = icmp eq i32 %inc, %n
+ br i1 %cmp.not, label %while.end, label %while.body
+
+while.body:
+ %idxprom = zext i32 %inc to i64
+ %arrayidx = getelementptr inbounds i8, ptr %a, i64 %idxprom
+ %0 = load i8, ptr %arrayidx
+ %arrayidx2 = getelementptr inbounds i8, ptr %b, i64 %idxprom
+ %1 = load i8, ptr %arrayidx2
+ %cmp.not2 = icmp eq i8 %0, %1
+ br i1 %cmp.not2, label %while.cond, label %while.end, !llvm.loop !0
+
+while.end:
+ %inc.lcssa = phi i32 [ %inc, %while.body ], [ %inc, %while.cond ]
+ %extra.phi = phi i32 [ %extra, %while.body ], [ %extra, %while.cond ]
+ %res = add i32 %inc.lcssa, %extra.phi
+ ret i32 %res
+}
+
+define ptr @find_first_of_i8_outer_loop_hint(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK: [[HEADER_PREHEADER]]:
+; CHECK-NEXT: br label %[[MEM_CHECK:.*]]
+; CHECK: [[MEM_CHECK]]:
+; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
+; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
+; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
+; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
+; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
+; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
+; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
+; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
+; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
+; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
+; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
+; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF1]]
+; CHECK: [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64
+; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]])
+; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]]
+; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]]
+; CHECK: [[MATCH_CHECK_VEC]]:
+; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64
+; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]])
+; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]]
+; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
+; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
+; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
+; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
+; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
+; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]]
+; CHECK: [[CALCULATE_MATCH]]:
+; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
+; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
+; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK: [[NEEDLE_CHECK_VEC]]:
+; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]]
+; CHECK: [[SEARCH_CHECK_VEC]]:
+; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK: [[SCALAR_PREHEADER]]:
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
+; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
+; CHECK: [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; CHECK: [[MATCH_CHECK]]:
+; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
+; CHECK: [[SEARCH_CHECK]]:
+; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]], !llvm.loop [[LOOP2]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: ret ptr [[RES]]
+;
+entry:
+ %search_test = icmp eq ptr %search_start, %search_end
+ %needle_test = icmp eq ptr %needle_start, %needle_end
+ %combined_test = or i1 %search_test, %needle_test
+ br i1 %combined_test, label %exit, label %header
+
+header:
+ %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+ %search_load = load i8, ptr %search_ptr, align 1
+ br label %match_check
+
+needle_check:
+ %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+ %needle_cmp = icmp eq ptr %needle_next, %needle_end
+ br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+ %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+ %needle_load = load i8, ptr %needle_ptr, align 1
+ %match_cmp = icmp eq i8 %search_load, %needle_load
+ br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+ %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+ %search_cmp = icmp eq ptr %search_next, %search_end
+ br i1 %search_cmp, label %exit, label %header, !llvm.loop !0
+
+exit:
+ %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+ ret ptr %res
+}
+
+define ptr @find_first_of_i8_inner_loop_hint(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK: [[HEADER_PREHEADER]]:
+; CHECK-NEXT: br label %[[MEM_CHECK:.*]]
+; CHECK: [[MEM_CHECK]]:
+; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
+; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
+; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
+; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
+; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
+; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
+; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
+; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
+; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
+; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
+; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
+; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF1]]
+; CHECK: [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64
+; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]])
+; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]]
+; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]]
+; CHECK: [[MATCH_CHECK_VEC]]:
+; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64
+; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]])
+; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]]
+; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
+; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
+; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
+; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
+; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
+; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
+; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]]
+; CHECK: [[CALCULATE_MATCH]]:
+; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
+; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
+; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
+; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]]
+; CHECK: [[NEEDLE_CHECK_VEC]]:
+; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]]
+; CHECK: [[SEARCH_CHECK_VEC]]:
+; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK: [[SCALAR_PREHEADER]]:
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
+; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
+; CHECK: [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]], !llvm.loop [[LOOP2]]
+; CHECK: [[MATCH_CHECK]]:
+; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
+; CHECK: [[SEARCH_CHECK]]:
+; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: ret ptr [[RES]]
+;
+entry:
+ %search_test = icmp eq ptr %search_start, %search_end
+ %needle_test = icmp eq ptr %needle_start, %needle_end
+ %combined_test = or i1 %search_test, %needle_test
+ br i1 %combined_test, label %exit, label %header
+
+header:
+ %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+ %search_load = load i8, ptr %search_ptr, align 1
+ br label %match_check
+
+needle_check:
+ %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+ %needle_cmp = icmp eq ptr %needle_next, %needle_end
+ br i1 %needle_cmp, label %search_check, label %match_check, !llvm.loop !0
+
+match_check:
+ %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+ %needle_load = load i8, ptr %needle_ptr, align 1
+ %match_cmp = icmp eq i8 %search_load, %needle_load
+ br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+ %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+ %search_cmp = icmp eq ptr %search_next, %search_end
+ br i1 %search_cmp, label %exit, label %header
+
+exit:
+ %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+ ret ptr %res
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.width", i32 1}
+
+attributes #0 = { "target-features"="+sve2" }
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 99, i32 1}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 10, i32 90}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+; CHECK: [[META3]] = !{!"llvm.loop.vectorize.width", i32 1}
+;.
>From 8ac2d6631b57e617aaa4d7327c2e0f84b4f14974 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Thu, 12 Feb 2026 12:27:29 +0000
Subject: [PATCH 2/7] [LoopIdiomVectorize] Bail when vectorization is disabled
Bail on vectorizing a loop in LoopIdiomVectorize when the loop carries
hints that indicate vectorization is disabled.
This means that LoopIdiomVectorize will now respect vectorize(disable)
loop hints.
---
.../Vectorize/LoopIdiomVectorize.cpp | 34 ++-
.../LoopIdiom/AArch64/disable-vectorize.ll | 244 +++---------------
2 files changed, 63 insertions(+), 215 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
index 53129e2e5fbba..b2108732c6c8f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
@@ -67,6 +67,7 @@
#include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
@@ -74,6 +75,7 @@
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
using namespace llvm;
using namespace PatternMatch;
@@ -123,6 +125,9 @@ class LoopIdiomVectorize {
const TargetTransformInfo *TTI;
const DataLayout *DL;
+ /// Interface to emit optimization remarks.
+ OptimizationRemarkEmitter &ORE;
+
// Blocks that will be used for inserting vectorized code.
BasicBlock *EndBlock = nullptr;
BasicBlock *VectorLoopPreheaderBlock = nullptr;
@@ -133,9 +138,9 @@ class LoopIdiomVectorize {
public:
LoopIdiomVectorize(LoopIdiomVectorizeStyle S, unsigned VF, DominatorTree *DT,
LoopInfo *LI, const TargetTransformInfo *TTI,
- const DataLayout *DL)
- : VectorizeStyle(S), ByteCompareVF(VF), DT(DT), LI(LI), TTI(TTI), DL(DL) {
- }
+ const DataLayout *DL, OptimizationRemarkEmitter &ORE)
+ : VectorizeStyle(S), ByteCompareVF(VF), DT(DT), LI(LI), TTI(TTI), DL(DL),
+ ORE(ORE) {}
bool run(Loop *L);
@@ -199,7 +204,9 @@ PreservedAnalyses LoopIdiomVectorizePass::run(Loop &L, LoopAnalysisManager &AM,
if (ByteCmpVF.getNumOccurrences())
BCVF = ByteCmpVF;
- LoopIdiomVectorize LIV(VecStyle, BCVF, &AR.DT, &AR.LI, &AR.TTI, DL);
+ OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
+
+ LoopIdiomVectorize LIV(VecStyle, BCVF, &AR.DT, &AR.LI, &AR.TTI, DL, ORE);
if (!LIV.run(&L))
return PreservedAnalyses::all();
@@ -219,6 +226,14 @@ bool LoopIdiomVectorize::run(Loop *L) {
if (DisableAll || F.hasOptSize())
return false;
+ // Bail if vectorization is disabled on loop.
+ LoopVectorizeHints Hints(L, /*InterleaveOnlyWhenForced=*/true, ORE);
+ if (!Hints.allowVectorization(&F, L, /*VectorizeOnlyWhenForced=*/false)) {
+ LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << F.getName()
+ << " due to vectorization hints\n");
+ return false;
+ }
+
if (F.hasFnAttribute(Attribute::NoImplicitFloat)) {
LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << F.getName()
<< " due to its NoImplicitFloat attribute");
@@ -1004,6 +1019,17 @@ bool LoopIdiomVectorize::recognizeFindFirstByte() {
return false;
auto *InnerLoop = CurLoop->getSubLoops().front();
+ Function &F = *InnerLoop->getHeader()->getParent();
+
+ // Bail if vectorization is disabled on inner loop.
+ LoopVectorizeHints Hints(InnerLoop, /*InterleaveOnlyWhenForced=*/true, ORE);
+ if (!Hints.allowVectorization(&F, InnerLoop,
+ /*VectorizeOnlyWhenForced=*/false)) {
+ LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << F.getName()
+ << " due to vectorization hints\n");
+ return false;
+ }
+
PHINode *IndPhi = dyn_cast<PHINode>(&Header->front());
if (!IndPhi || IndPhi->getNumIncomingValues() != 2)
return false;
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
index 184d3a242ebae..8906392d2c35b 100644
--- a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
@@ -4,98 +4,24 @@
define i32 @compare_bytes_simple_hint(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
; CHECK-LABEL: define i32 @compare_bytes_simple_hint(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[LEN]], 1
-; CHECK-NEXT: br label %[[MISMATCH_MIN_IT_CHECK:.*]]
-; CHECK: [[MISMATCH_MIN_IT_CHECK]]:
-; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[N]] to i64
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i32 [[TMP0]], [[N]]
-; CHECK-NEXT: br i1 [[TMP3]], label %[[MISMATCH_MEM_CHECK:.*]], label %[[MISMATCH_LOOP_PRE:.*]], !prof [[PROF0:![0-9]+]]
-; CHECK: [[MISMATCH_MEM_CHECK]]:
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP1]]
-; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
-; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[TMP4]] to i64
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP10:%.*]] = ptrtoint ptr [[TMP8]] to i64
-; CHECK-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP9]] to i64
-; CHECK-NEXT: [[TMP12:%.*]] = lshr i64 [[TMP7]], 12
-; CHECK-NEXT: [[TMP13:%.*]] = lshr i64 [[TMP10]], 12
-; CHECK-NEXT: [[TMP14:%.*]] = lshr i64 [[TMP6]], 12
-; CHECK-NEXT: [[TMP15:%.*]] = lshr i64 [[TMP11]], 12
-; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i64 [[TMP12]], [[TMP13]]
-; CHECK-NEXT: [[TMP17:%.*]] = icmp ne i64 [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]]
-; CHECK-NEXT: br i1 [[TMP18]], label %[[MISMATCH_LOOP_PRE]], label %[[MISMATCH_VEC_LOOP_PREHEADER:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK: [[MISMATCH_VEC_LOOP_PREHEADER]]:
-; CHECK-NEXT: [[TMP19:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[TMP2]])
-; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP21:%.*]] = mul nuw nsw i64 [[TMP20]], 16
-; CHECK-NEXT: br label %[[MISMATCH_VEC_LOOP:.*]]
-; CHECK: [[MISMATCH_VEC_LOOP]]:
-; CHECK-NEXT: [[MISMATCH_VEC_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP19]], %[[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP30:%.*]], %[[MISMATCH_VEC_LOOP_INC:.*]] ]
-; CHECK-NEXT: [[MISMATCH_VEC_INDEX:%.*]] = phi i64 [ [[TMP1]], %[[MISMATCH_VEC_LOOP_PREHEADER]] ], [ [[TMP29:%.*]], %[[MISMATCH_VEC_LOOP_INC]] ]
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[MISMATCH_VEC_INDEX]]
-; CHECK-NEXT: [[TMP23:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP22]], <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[MISMATCH_VEC_INDEX]]
-; CHECK-NEXT: [[TMP25:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP24]], <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT: [[TMP26:%.*]] = icmp ne <vscale x 16 x i8> [[TMP23]], [[TMP25]]
-; CHECK-NEXT: [[TMP27:%.*]] = select <vscale x 16 x i1> [[MISMATCH_VEC_LOOP_PRED]], <vscale x 16 x i1> [[TMP26]], <vscale x 16 x i1> zeroinitializer
-; CHECK-NEXT: [[TMP28:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP27]])
-; CHECK-NEXT: br i1 [[TMP28]], label %[[MISMATCH_VEC_LOOP_FOUND:.*]], label %[[MISMATCH_VEC_LOOP_INC]]
-; CHECK: [[MISMATCH_VEC_LOOP_INC]]:
-; CHECK-NEXT: [[TMP29]] = add nuw nsw i64 [[MISMATCH_VEC_INDEX]], [[TMP21]]
-; CHECK-NEXT: [[TMP30]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP29]], i64 [[TMP2]])
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <vscale x 16 x i1> [[TMP30]], i64 0
-; CHECK-NEXT: br i1 [[TMP31]], label %[[MISMATCH_VEC_LOOP]], label %[[MISMATCH_END:.*]]
-; CHECK: [[MISMATCH_VEC_LOOP_FOUND]]:
-; CHECK-NEXT: [[MISMATCH_VEC_FOUND_PRED:%.*]] = phi <vscale x 16 x i1> [ [[TMP27]], %[[MISMATCH_VEC_LOOP]] ]
-; CHECK-NEXT: [[MISMATCH_VEC_LAST_LOOP_PRED:%.*]] = phi <vscale x 16 x i1> [ [[MISMATCH_VEC_LOOP_PRED]], %[[MISMATCH_VEC_LOOP]] ]
-; CHECK-NEXT: [[MISMATCH_VEC_FOUND_INDEX:%.*]] = phi i64 [ [[MISMATCH_VEC_INDEX]], %[[MISMATCH_VEC_LOOP]] ]
-; CHECK-NEXT: [[TMP32:%.*]] = and <vscale x 16 x i1> [[MISMATCH_VEC_LAST_LOOP_PRED]], [[MISMATCH_VEC_FOUND_PRED]]
-; CHECK-NEXT: [[TMP33:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
-; CHECK-NEXT: [[TMP34:%.*]] = zext i32 [[TMP33]] to i64
-; CHECK-NEXT: [[TMP35:%.*]] = add nuw nsw i64 [[MISMATCH_VEC_FOUND_INDEX]], [[TMP34]]
-; CHECK-NEXT: [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
-; CHECK-NEXT: br label %[[MISMATCH_END]]
-; CHECK: [[MISMATCH_LOOP_PRE]]:
+; CHECK-NEXT: [[MISMATCH_LOOP_PRE:.*]]:
; CHECK-NEXT: br label %[[MISMATCH_LOOP:.*]]
; CHECK: [[MISMATCH_LOOP]]:
-; CHECK-NEXT: [[MISMATCH_INDEX:%.*]] = phi i32 [ [[TMP0]], %[[MISMATCH_LOOP_PRE]] ], [ [[TMP43:%.*]], %[[MISMATCH_LOOP_INC:.*]] ]
-; CHECK-NEXT: [[TMP37:%.*]] = zext i32 [[MISMATCH_INDEX]] to i64
-; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP37]]
-; CHECK-NEXT: [[TMP39:%.*]] = load i8, ptr [[TMP38]], align 1
-; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP37]]
-; CHECK-NEXT: [[TMP41:%.*]] = load i8, ptr [[TMP40]], align 1
-; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i8 [[TMP39]], [[TMP41]]
-; CHECK-NEXT: br i1 [[TMP42]], label %[[MISMATCH_LOOP_INC]], label %[[MISMATCH_END]]
-; CHECK: [[MISMATCH_LOOP_INC]]:
-; CHECK-NEXT: [[TMP43]] = add i32 [[MISMATCH_INDEX]], 1
-; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i32 [[TMP43]], [[N]]
-; CHECK-NEXT: br i1 [[TMP44]], label %[[MISMATCH_END]], label %[[MISMATCH_LOOP]]
-; CHECK: [[MISMATCH_END]]:
-; CHECK-NEXT: [[MISMATCH_RESULT:%.*]] = phi i32 [ [[N]], %[[MISMATCH_LOOP_INC]] ], [ [[MISMATCH_INDEX]], %[[MISMATCH_LOOP]] ], [ [[N]], %[[MISMATCH_VEC_LOOP_INC]] ], [ [[TMP36]], %[[MISMATCH_VEC_LOOP_FOUND]] ]
-; CHECK-NEXT: br i1 true, label %[[BYTE_COMPARE:.*]], label %[[WHILE_COND:.*]]
-; CHECK: [[WHILE_COND]]:
-; CHECK-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], %[[MISMATCH_END]] ], [ [[MISMATCH_RESULT]], %[[WHILE_BODY:.*]] ]
-; CHECK-NEXT: [[INC:%.*]] = add i32 [[LEN_ADDR]], 1
-; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[MISMATCH_RESULT]], [[N]]
+; CHECK-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], %[[MISMATCH_LOOP_PRE]] ], [ [[INC:%.*]], %[[WHILE_BODY:.*]] ]
+; CHECK-NEXT: [[INC]] = add i32 [[LEN_ADDR]], 1
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
; CHECK: [[WHILE_BODY]]:
-; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[MISMATCH_RESULT]] to i64
+; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[INC]] to i64
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDXPROM]]
; CHECK-NEXT: [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IDXPROM]]
; CHECK-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
; CHECK-NEXT: [[CMP_NOT2:%.*]] = icmp eq i8 [[TMP45]], [[TMP46]]
-; CHECK-NEXT: br i1 [[CMP_NOT2]], label %[[WHILE_COND]], label %[[WHILE_END]], !llvm.loop [[LOOP2:![0-9]+]]
-; CHECK: [[BYTE_COMPARE]]:
-; CHECK-NEXT: br label %[[WHILE_END]]
+; CHECK-NEXT: br i1 [[CMP_NOT2]], label %[[MISMATCH_LOOP]], label %[[WHILE_END]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[WHILE_END]]:
-; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[MISMATCH_RESULT]], %[[WHILE_BODY]] ], [ [[MISMATCH_RESULT]], %[[WHILE_COND]] ], [ [[MISMATCH_RESULT]], %[[BYTE_COMPARE]] ]
-; CHECK-NEXT: [[EXTRA_PHI:%.*]] = phi i32 [ [[EXTRA]], %[[WHILE_BODY]] ], [ [[EXTRA]], %[[WHILE_COND]] ], [ [[EXTRA]], %[[BYTE_COMPARE]] ]
+; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[WHILE_BODY]] ], [ [[INC]], %[[MISMATCH_LOOP]] ]
+; CHECK-NEXT: [[EXTRA_PHI:%.*]] = phi i32 [ [[EXTRA]], %[[WHILE_BODY]] ], [ [[EXTRA]], %[[MISMATCH_LOOP]] ]
; CHECK-NEXT: [[RES:%.*]] = add i32 [[INC_LCSSA]], [[EXTRA_PHI]]
; CHECK-NEXT: ret i32 [[RES]]
;
@@ -130,63 +56,12 @@ define ptr @find_first_of_i8_outer_loop_hint(ptr %search_start, ptr %search_end,
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
-; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
-; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
-; CHECK: [[HEADER_PREHEADER]]:
-; CHECK-NEXT: br label %[[MEM_CHECK:.*]]
-; CHECK: [[MEM_CHECK]]:
-; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
-; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
-; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
-; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
-; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
-; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
-; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
-; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
-; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
-; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
-; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
-; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
-; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF1]]
-; CHECK: [[FIND_FIRST_VEC_HEADER]]:
-; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64
-; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]])
-; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]]
-; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]]
-; CHECK: [[MATCH_CHECK_VEC]]:
-; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64
-; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]])
-; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]]
-; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
-; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
-; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
-; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
-; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
-; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]]
-; CHECK: [[CALCULATE_MATCH]]:
-; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
-; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
-; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
-; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
-; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]]
-; CHECK: [[NEEDLE_CHECK_VEC]]:
-; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]]
-; CHECK: [[SEARCH_CHECK_VEC]]:
-; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
-; CHECK: [[SCALAR_PREHEADER]]:
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER:.*]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
; CHECK-NEXT: br label %[[HEADER:.*]]
; CHECK: [[HEADER]]:
-; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[EXIT_LOOPEXIT1]] ]
; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
; CHECK: [[NEEDLE_CHECK:.*]]:
@@ -197,18 +72,18 @@ define ptr @find_first_of_i8_outer_loop_hint(ptr %search_start, ptr %search_end,
; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
-; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
+; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
; CHECK: [[SEARCH_CHECK]]:
; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
-; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]], !llvm.loop [[LOOP2]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT:.*]], label %[[HEADER]], !llvm.loop [[LOOP0]]
; CHECK: [[EXIT_LOOPEXIT]]:
-; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT_LOOPEXIT1]]:
-; CHECK-NEXT: br label %[[EXIT]]
+; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
+; CHECK: [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT]] ]
; CHECK-NEXT: ret ptr [[RES]]
;
entry:
@@ -249,85 +124,34 @@ define ptr @find_first_of_i8_inner_loop_hint(ptr %search_start, ptr %search_end,
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
-; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
-; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
-; CHECK: [[HEADER_PREHEADER]]:
-; CHECK-NEXT: br label %[[MEM_CHECK:.*]]
-; CHECK: [[MEM_CHECK]]:
-; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64
-; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64
-; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64
-; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64
-; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16)
-; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12
-; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12
-; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12
-; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12
-; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]]
-; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]]
-; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]]
-; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF1]]
-; CHECK: [[FIND_FIRST_VEC_HEADER]]:
-; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64
-; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]])
-; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]]
-; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]]
-; CHECK: [[MATCH_CHECK_VEC]]:
-; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64
-; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]])
-; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]]
-; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer)
-; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0
-; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0
-; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
-; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]]
-; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0)
-; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]])
-; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]])
-; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]]
-; CHECK: [[CALCULATE_MATCH]]:
-; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ]
-; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ]
-; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true)
-; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]]
-; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]]
-; CHECK: [[NEEDLE_CHECK_VEC]]:
-; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]]
-; CHECK: [[SEARCH_CHECK_VEC]]:
-; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]]
-; CHECK: [[SCALAR_PREHEADER]]:
+; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER:.*]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
; CHECK-NEXT: br label %[[HEADER:.*]]
; CHECK: [[HEADER]]:
-; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ]
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[EXIT_LOOPEXIT1]] ]
; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
; CHECK: [[NEEDLE_CHECK:.*]]:
; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
-; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]], !llvm.loop [[LOOP2]]
+; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]], !llvm.loop [[LOOP0]]
; CHECK: [[MATCH_CHECK]]:
; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
-; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]]
+; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
; CHECK: [[SEARCH_CHECK]]:
; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
-; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT:.*]], label %[[HEADER]]
; CHECK: [[EXIT_LOOPEXIT]]:
-; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ]
-; CHECK-NEXT: br label %[[EXIT]]
-; CHECK: [[EXIT_LOOPEXIT1]]:
-; CHECK-NEXT: br label %[[EXIT]]
+; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
+; CHECK: [[FIND_FIRST_VEC_HEADER]]:
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT]] ]
; CHECK-NEXT: ret ptr [[RES]]
;
entry:
@@ -367,8 +191,6 @@ exit:
attributes #0 = { "target-features"="+sve2" }
;.
-; CHECK: [[PROF0]] = !{!"branch_weights", i32 99, i32 1}
-; CHECK: [[PROF1]] = !{!"branch_weights", i32 10, i32 90}
-; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
-; CHECK: [[META3]] = !{!"llvm.loop.vectorize.width", i32 1}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.vectorize.width", i32 1}
;.
>From 4102dc56360daae39afd8bf60e49710fe0e85934 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 18 Feb 2026 22:13:12 +0000
Subject: [PATCH 3/7] Improve debug
---
llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
index b2108732c6c8f..9f88f7d3da8a5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
@@ -229,7 +229,7 @@ bool LoopIdiomVectorize::run(Loop *L) {
// Bail if vectorization is disabled on loop.
LoopVectorizeHints Hints(L, /*InterleaveOnlyWhenForced=*/true, ORE);
if (!Hints.allowVectorization(&F, L, /*VectorizeOnlyWhenForced=*/false)) {
- LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << F.getName()
+ LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << L->getName()
<< " due to vectorization hints\n");
return false;
}
@@ -1025,7 +1025,8 @@ bool LoopIdiomVectorize::recognizeFindFirstByte() {
LoopVectorizeHints Hints(InnerLoop, /*InterleaveOnlyWhenForced=*/true, ORE);
if (!Hints.allowVectorization(&F, InnerLoop,
/*VectorizeOnlyWhenForced=*/false)) {
- LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on " << F.getName()
+ LLVM_DEBUG(dbgs() << DEBUG_TYPE << " is disabled on inner loop "
+ << InnerLoop->getName()
<< " due to vectorization hints\n");
return false;
}
>From c06fd1495c8ce0a183681f1f49651b778b5828c8 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 18 Feb 2026 22:20:45 +0000
Subject: [PATCH 4/7] Add tests for llvm.loop.vectorize.enable
---
.../LoopIdiom/AArch64/disable-vectorize.ll | 255 +++++++++++++++---
1 file changed, 222 insertions(+), 33 deletions(-)
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
index 8906392d2c35b..9061c16837be1 100644
--- a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
@@ -1,27 +1,27 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -S < %s | FileCheck %s
-define i32 @compare_bytes_simple_hint(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
-; CHECK-LABEL: define i32 @compare_bytes_simple_hint(
+define i32 @compare_bytes_simple_hint_width_1(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
+; CHECK-LABEL: define i32 @compare_bytes_simple_hint_width_1(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[MISMATCH_LOOP_PRE:.*]]:
-; CHECK-NEXT: br label %[[MISMATCH_LOOP:.*]]
-; CHECK: [[MISMATCH_LOOP]]:
-; CHECK-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], %[[MISMATCH_LOOP_PRE]] ], [ [[INC:%.*]], %[[WHILE_BODY:.*]] ]
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[WHILE_COND:.*]]
+; CHECK: [[WHILE_COND]]:
+; CHECK-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], %[[ENTRY]] ], [ [[INC:%.*]], %[[WHILE_BODY:.*]] ]
; CHECK-NEXT: [[INC]] = add i32 [[LEN_ADDR]], 1
; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
; CHECK: [[WHILE_BODY]]:
; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[INC]] to i64
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDXPROM]]
-; CHECK-NEXT: [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IDXPROM]]
-; CHECK-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-NEXT: [[CMP_NOT2:%.*]] = icmp eq i8 [[TMP45]], [[TMP46]]
-; CHECK-NEXT: br i1 [[CMP_NOT2]], label %[[MISMATCH_LOOP]], label %[[WHILE_END]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[CMP_NOT2:%.*]] = icmp eq i8 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br i1 [[CMP_NOT2]], label %[[WHILE_COND]], label %[[WHILE_END]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[WHILE_END]]:
-; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[WHILE_BODY]] ], [ [[INC]], %[[MISMATCH_LOOP]] ]
-; CHECK-NEXT: [[EXTRA_PHI:%.*]] = phi i32 [ [[EXTRA]], %[[WHILE_BODY]] ], [ [[EXTRA]], %[[MISMATCH_LOOP]] ]
+; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[WHILE_BODY]] ], [ [[INC]], %[[WHILE_COND]] ]
+; CHECK-NEXT: [[EXTRA_PHI:%.*]] = phi i32 [ [[EXTRA]], %[[WHILE_BODY]] ], [ [[EXTRA]], %[[WHILE_COND]] ]
; CHECK-NEXT: [[RES:%.*]] = add i32 [[INC_LCSSA]], [[EXTRA_PHI]]
; CHECK-NEXT: ret i32 [[RES]]
;
@@ -50,18 +50,67 @@ while.end:
ret i32 %res
}
-define ptr @find_first_of_i8_outer_loop_hint(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
-; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint(
+define i32 @compare_bytes_simple_hint_disable(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
+; CHECK-LABEL: define i32 @compare_bytes_simple_hint_disable(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[WHILE_COND:.*]]
+; CHECK: [[WHILE_COND]]:
+; CHECK-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], %[[ENTRY]] ], [ [[INC:%.*]], %[[WHILE_BODY:.*]] ]
+; CHECK-NEXT: [[INC]] = add i32 [[LEN_ADDR]], 1
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT: br i1 [[CMP_NOT]], label %[[WHILE_END:.*]], label %[[WHILE_BODY]]
+; CHECK: [[WHILE_BODY]]:
+; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[INC]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-NEXT: [[CMP_NOT2:%.*]] = icmp eq i8 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br i1 [[CMP_NOT2]], label %[[WHILE_COND]], label %[[WHILE_END]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK: [[WHILE_END]]:
+; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[WHILE_BODY]] ], [ [[INC]], %[[WHILE_COND]] ]
+; CHECK-NEXT: [[EXTRA_PHI:%.*]] = phi i32 [ [[EXTRA]], %[[WHILE_BODY]] ], [ [[EXTRA]], %[[WHILE_COND]] ]
+; CHECK-NEXT: [[RES:%.*]] = add i32 [[INC_LCSSA]], [[EXTRA_PHI]]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+entry:
+ br label %while.cond
+
+while.cond:
+ %len.addr = phi i32 [ %len, %entry ], [ %inc, %while.body ]
+ %inc = add i32 %len.addr, 1
+ %cmp.not = icmp eq i32 %inc, %n
+ br i1 %cmp.not, label %while.end, label %while.body
+
+while.body:
+ %idxprom = zext i32 %inc to i64
+ %arrayidx = getelementptr inbounds i8, ptr %a, i64 %idxprom
+ %0 = load i8, ptr %arrayidx
+ %arrayidx2 = getelementptr inbounds i8, ptr %b, i64 %idxprom
+ %1 = load i8, ptr %arrayidx2
+ %cmp.not2 = icmp eq i8 %0, %1
+ br i1 %cmp.not2, label %while.cond, label %while.end, !llvm.loop !2
+
+while.end:
+ %inc.lcssa = phi i32 [ %inc, %while.body ], [ %inc, %while.cond ]
+ %extra.phi = phi i32 [ %extra, %while.body ], [ %extra, %while.cond ]
+ %res = add i32 %inc.lcssa, %extra.phi
+ ret i32 %res
+}
+
+define ptr @find_first_of_i8_outer_loop_hint_width_1(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint_width_1(
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
-; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER:.*]], label %[[EXIT_LOOPEXIT1:.*]]
-; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK: [[HEADER_PREHEADER]]:
; CHECK-NEXT: br label %[[HEADER:.*]]
; CHECK: [[HEADER]]:
-; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
; CHECK: [[NEEDLE_CHECK:.*]]:
@@ -76,14 +125,14 @@ define ptr @find_first_of_i8_outer_loop_hint(ptr %search_start, ptr %search_end,
; CHECK: [[SEARCH_CHECK]]:
; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
-; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT:.*]], label %[[HEADER]], !llvm.loop [[LOOP0]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]], !llvm.loop [[LOOP0]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
-; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
-; CHECK: [[FIND_FIRST_VEC_HEADER]]:
-; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
; CHECK-NEXT: ret ptr [[RES]]
;
entry:
@@ -118,18 +167,86 @@ exit:
ret ptr %res
}
-define ptr @find_first_of_i8_inner_loop_hint(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
-; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint(
+define ptr @find_first_of_i8_outer_loop_hint_disable(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint_disable(
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
-; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
-; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER:.*]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK: [[HEADER_PREHEADER]]:
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
+; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
+; CHECK: [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]]
+; CHECK: [[MATCH_CHECK]]:
+; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
+; CHECK: [[SEARCH_CHECK]]:
+; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]], !llvm.loop [[LOOP2]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: ret ptr [[RES]]
+;
+entry:
+ %search_test = icmp eq ptr %search_start, %search_end
+ %needle_test = icmp eq ptr %needle_start, %needle_end
+ %combined_test = or i1 %search_test, %needle_test
+ br i1 %combined_test, label %exit, label %header
+
+header:
+ %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+ %search_load = load i8, ptr %search_ptr, align 1
+ br label %match_check
+
+needle_check:
+ %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+ %needle_cmp = icmp eq ptr %needle_next, %needle_end
+ br i1 %needle_cmp, label %search_check, label %match_check
+
+match_check:
+ %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+ %needle_load = load i8, ptr %needle_ptr, align 1
+ %match_cmp = icmp eq i8 %search_load, %needle_load
+ br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+ %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+ %search_cmp = icmp eq ptr %search_next, %search_end
+ br i1 %search_cmp, label %exit, label %header, !llvm.loop !2
+
+exit:
+ %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+ ret ptr %res
+}
+
+define ptr @find_first_of_i8_inner_loop_hint_width_1(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint_width_1(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK: [[HEADER_PREHEADER]]:
; CHECK-NEXT: br label %[[HEADER:.*]]
; CHECK: [[HEADER]]:
-; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
; CHECK: [[NEEDLE_CHECK:.*]]:
@@ -144,14 +261,14 @@ define ptr @find_first_of_i8_inner_loop_hint(ptr %search_start, ptr %search_end,
; CHECK: [[SEARCH_CHECK]]:
; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
-; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT:.*]], label %[[HEADER]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]]
; CHECK: [[EXIT_LOOPEXIT]]:
; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
-; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: br label %[[FIND_FIRST_VEC_HEADER]]
-; CHECK: [[FIND_FIRST_VEC_HEADER]]:
-; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
; CHECK-NEXT: ret ptr [[RES]]
;
entry:
@@ -186,11 +303,83 @@ exit:
ret ptr %res
}
+define ptr @find_first_of_i8_inner_loop_hint_disable(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
+; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint_disable(
+; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
+; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]]
+; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]]
+; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]]
+; CHECK: [[HEADER_PREHEADER]]:
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ]
+; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1
+; CHECK-NEXT: br label %[[MATCH_CHECK:.*]]
+; CHECK: [[NEEDLE_CHECK:.*]]:
+; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1
+; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]]
+; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]], !llvm.loop [[LOOP2]]
+; CHECK: [[MATCH_CHECK]]:
+; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ]
+; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1
+; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]]
+; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]]
+; CHECK: [[SEARCH_CHECK]]:
+; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1
+; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]]
+; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ]
+; CHECK-NEXT: ret ptr [[RES]]
+;
+entry:
+ %search_test = icmp eq ptr %search_start, %search_end
+ %needle_test = icmp eq ptr %needle_start, %needle_end
+ %combined_test = or i1 %search_test, %needle_test
+ br i1 %combined_test, label %exit, label %header
+
+header:
+ %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ]
+ %search_load = load i8, ptr %search_ptr, align 1
+ br label %match_check
+
+needle_check:
+ %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1
+ %needle_cmp = icmp eq ptr %needle_next, %needle_end
+ br i1 %needle_cmp, label %search_check, label %match_check, !llvm.loop !2
+
+match_check:
+ %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ]
+ %needle_load = load i8, ptr %needle_ptr, align 1
+ %match_cmp = icmp eq i8 %search_load, %needle_load
+ br i1 %match_cmp, label %exit, label %needle_check
+
+search_check:
+ %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1
+ %search_cmp = icmp eq ptr %search_next, %search_end
+ br i1 %search_cmp, label %exit, label %header
+
+exit:
+ %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ]
+ ret ptr %res
+}
+
!0 = distinct !{!0, !1}
!1 = !{!"llvm.loop.vectorize.width", i32 1}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.enable", i1 false}
attributes #0 = { "target-features"="+sve2" }
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.vectorize.width", i32 1}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+; CHECK: [[META3]] = !{!"llvm.loop.vectorize.enable", i1 false}
;.
>From 3363378a402acea060b259be4ec511fda3d9d20a Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Thu, 19 Feb 2026 00:28:42 +0000
Subject: [PATCH 5/7] Try to used cached ORE
---
llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
index 9f88f7d3da8a5..d3a91fc5b1326 100644
--- a/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopIdiomVectorize.cpp
@@ -204,9 +204,17 @@ PreservedAnalyses LoopIdiomVectorizePass::run(Loop &L, LoopAnalysisManager &AM,
if (ByteCmpVF.getNumOccurrences())
BCVF = ByteCmpVF;
- OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
+ Function &F = *L.getHeader()->getParent();
+ auto &FAMP = AM.getResult<FunctionAnalysisManagerLoopProxy>(L, AR);
+ auto *ORE = FAMP.getCachedResult<OptimizationRemarkEmitterAnalysis>(F);
+
+ std::optional<OptimizationRemarkEmitter> ORELocal;
+ if (!ORE) {
+ ORELocal.emplace(&F);
+ ORE = &*ORELocal;
+ }
- LoopIdiomVectorize LIV(VecStyle, BCVF, &AR.DT, &AR.LI, &AR.TTI, DL, ORE);
+ LoopIdiomVectorize LIV(VecStyle, BCVF, &AR.DT, &AR.LI, &AR.TTI, DL, *ORE);
if (!LIV.run(&L))
return PreservedAnalyses::all();
>From bd5fdfdfe77901f27670ce874853285af0305cb2 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Thu, 19 Feb 2026 17:11:11 +0000
Subject: [PATCH 6/7] Add remarks to test
---
.../Transforms/LoopIdiom/AArch64/disable-vectorize.ll | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
index 9061c16837be1..38f997497c1a8 100644
--- a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
@@ -1,8 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -S < %s | FileCheck %s
+; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -pass-remarks-analysis=loop-vectorize -S 2>&1 %s | FileCheck %s
define i32 @compare_bytes_simple_hint_width_1(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
; CHECK-LABEL: define i32 @compare_bytes_simple_hint_width_1(
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[WHILE_COND:.*]]
@@ -52,6 +53,7 @@ while.end:
define i32 @compare_bytes_simple_hint_disable(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
; CHECK-LABEL: define i32 @compare_bytes_simple_hint_disable(
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[WHILE_COND:.*]]
@@ -101,6 +103,7 @@ while.end:
define ptr @find_first_of_i8_outer_loop_hint_width_1(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint_width_1(
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
@@ -169,6 +172,7 @@ exit:
define ptr @find_first_of_i8_outer_loop_hint_disable(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint_disable(
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
@@ -237,6 +241,7 @@ exit:
define ptr @find_first_of_i8_inner_loop_hint_width_1(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint_width_1(
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
@@ -305,6 +310,7 @@ exit:
define ptr @find_first_of_i8_inner_loop_hint_disable(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint_disable(
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
>From 61a73b4ebfe8ae98e22cdd2bd77e2dfd88dfc3cb Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Thu, 19 Feb 2026 18:07:15 +0000
Subject: [PATCH 7/7] Fix remark CHECK lines
---
.../Transforms/LoopIdiom/AArch64/disable-vectorize.ll | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
index 38f997497c1a8..875f45ebd951c 100644
--- a/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
+++ b/llvm/test/Transforms/LoopIdiom/AArch64/disable-vectorize.ll
@@ -1,9 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -pass-remarks-analysis=loop-vectorize -S 2>&1 %s | FileCheck %s
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
+; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
+
define i32 @compare_bytes_simple_hint_width_1(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
; CHECK-LABEL: define i32 @compare_bytes_simple_hint_width_1(
-; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[WHILE_COND:.*]]
@@ -53,7 +57,6 @@ while.end:
define i32 @compare_bytes_simple_hint_disable(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) {
; CHECK-LABEL: define i32 @compare_bytes_simple_hint_disable(
-; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[LEN:%.*]], i32 [[EXTRA:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[WHILE_COND:.*]]
@@ -103,7 +106,6 @@ while.end:
define ptr @find_first_of_i8_outer_loop_hint_width_1(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint_width_1(
-; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
@@ -172,7 +174,6 @@ exit:
define ptr @find_first_of_i8_outer_loop_hint_disable(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_outer_loop_hint_disable(
-; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
@@ -241,7 +242,6 @@ exit:
define ptr @find_first_of_i8_inner_loop_hint_width_1(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint_width_1(
-; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
@@ -310,7 +310,6 @@ exit:
define ptr @find_first_of_i8_inner_loop_hint_disable(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 {
; CHECK-LABEL: define ptr @find_first_of_i8_inner_loop_hint_disable(
-; CHECK: remark: <unknown>:0:0: loop not vectorized: vectorization and interleaving are explicitly disabled, or the loop has already been vectorized
; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]]
More information about the llvm-commits
mailing list