[llvm] [LV] Regen some tests with UTC. NFC (PR #202960)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 05:55:01 PDT 2026
https://github.com/lukel97 created https://github.com/llvm/llvm-project/pull/202960
None
>From 6ff52e936ca8b7c31e12f942aa4a564c55ff95ef Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 10 Jun 2026 20:53:19 +0800
Subject: [PATCH] [LV] Regen some tests with UTC. NFC
---
.../ARM/tail-folding-counting-down.ll | 574 ++++++++++++++++--
.../VPlan/RISCV/vplan-riscv-vector-reverse.ll | 101 +--
.../LoopVectorize/bzip_reverse_loops.ll | 94 ++-
3 files changed, 654 insertions(+), 115 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
index 1a1fcd6f535b7..3fbfa32b6fccf 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 6
; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s --check-prefixes=COMMON,DEFAULT
; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=must-fold-tail -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER
; RUN: opt < %s -passes=loop-vectorize -tail-predication=enabled -tail-folding-policy=prefer-fold-tail -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER
@@ -14,14 +15,38 @@ target triple = "thumbv8.1m.main-arm-unknown-eabihf"
; }
;
define void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_loopguard(
-; COMMON: vector.body:
-
-; CHECK-TF: %[[INDEX:.*]] = phi i32
-; CHECK-TF: %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %[[INDEX]], i32 %N)
-; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
-; CHECK-TF: llvm.masked.load.v16i8.p0(ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask
-; CHECK-TF: llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr align 1 %{{.*}}, <16 x i1> %active.lane.mask)
+; COMMON-LABEL: define void @sgt_loopguard(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
+; COMMON-NEXT: br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; COMMON: [[WHILE_BODY_PREHEADER]]:
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 15
+; COMMON-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 16
+; COMMON-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
+; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; COMMON-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP0]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; COMMON-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMMON-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; COMMON: [[WHILE_END_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[WHILE_END]]
+; COMMON: [[WHILE_END]]:
+; COMMON-NEXT: ret void
+;
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %while.body.preheader, label %while.end
@@ -55,11 +80,38 @@ while.end:
; No loop-guard: we need one for this to be valid.
;
define void @sgt_no_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_no_loopguard(
-; COMMON: vector.body:
-; CHECK-TF: masked.load
-; CHECK-TF: masked.load
-; CHECK-TF: masked.store
+; COMMON-LABEL: define void @sgt_no_loopguard(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
+; COMMON-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[N]], i32 1)
+; COMMON-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 15
+; COMMON-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 16
+; COMMON-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[TMP1]])
+; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[TMP2:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; COMMON-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP2]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; COMMON-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMMON-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; COMMON: [[WHILE_END_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[WHILE_END:.*]]
+; COMMON: [[WHILE_END]]:
+; COMMON-NEXT: ret void
+;
entry:
br label %while.body
@@ -87,11 +139,46 @@ while.end:
}
define void @sgt_extra_use_cmp(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_extra_use_cmp(
-; COMMON: vector.body:
-; CHECK-TF: masked.load
-; CHECK-TF: masked.load
-; CHECK-TF: masked.store
+; COMMON-LABEL: define void @sgt_extra_use_cmp(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[TMP0:%.*]] = add i32 [[N]], 1
+; COMMON-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[N]], i32 1)
+; COMMON-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 3
+; COMMON-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4
+; COMMON-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; COMMON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0
+; COMMON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; COMMON-NEXT: [[TMP2:%.*]] = sub nsw <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[TMP1]])
+; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[NEXT_GEP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
+; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[NEXT_GEP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
+; COMMON-NEXT: [[TMP3:%.*]] = add <4 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; COMMON-NEXT: [[TMP4:%.*]] = icmp sle <4 x i32> [[VEC_IND]], splat (i32 1)
+; COMMON-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x i8> [[WIDE_MASKED_LOAD3]], <4 x i8> [[WIDE_MASKED_LOAD]]
+; COMMON-NEXT: [[TMP6:%.*]] = add <4 x i8> [[TMP3]], [[TMP5]]
+; COMMON-NEXT: call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP6]], ptr align 1 [[NEXT_GEP]], <4 x i1> [[ACTIVE_LANE_MASK]])
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMMON-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 -4)
+; COMMON-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMMON-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; COMMON: [[WHILE_END_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[WHILE_END:.*]]
+; COMMON: [[WHILE_END]]:
+; COMMON-NEXT: ret void
+;
entry:
br label %while.body
@@ -121,11 +208,35 @@ while.end:
}
define void @sgt_const_tripcount(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_const_tripcount(
-; COMMON: vector.body:
-; CHECK-TF: masked.load
-; CHECK-TF: masked.load
-; CHECK-TF: masked.store
+; COMMON-LABEL: define void @sgt_const_tripcount(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
+; COMMON-NEXT: br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; COMMON: [[WHILE_BODY_PREHEADER]]:
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 2049)
+; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; COMMON-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP0]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; COMMON-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], 2064
+; COMMON-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; COMMON: [[WHILE_END_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[WHILE_END]]
+; COMMON: [[WHILE_END]]:
+; COMMON-NEXT: ret void
+;
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %while.body.preheader, label %while.end
@@ -157,8 +268,30 @@ while.end:
}
define void @sgt_no_guard_0_startval(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_no_guard_0_startval(
-; COMMON-NOT: vector.body:
+; COMMON-LABEL: define void @sgt_no_guard_0_startval(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT: [[ENTRY:.*]]:
+; COMMON-NEXT: br label %[[WHILE_BODY:.*]]
+; COMMON: [[WHILE_BODY]]:
+; COMMON-NEXT: [[N_ADDR_09:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ 0, %[[ENTRY]] ]
+; COMMON-NEXT: [[C_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], %[[WHILE_BODY]] ], [ [[C]], %[[ENTRY]] ]
+; COMMON-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], %[[WHILE_BODY]] ], [ [[B]], %[[ENTRY]] ]
+; COMMON-NEXT: [[A_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[A]], %[[ENTRY]] ]
+; COMMON-NEXT: [[DEC]] = add nsw i32 [[N_ADDR_09]], -1
+; COMMON-NEXT: [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_06]], i32 1
+; COMMON-NEXT: [[TMP0:%.*]] = load i8, ptr [[A_ADDR_06]], align 1
+; COMMON-NEXT: [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 1
+; COMMON-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR_07]], align 1
+; COMMON-NEXT: [[ADD:%.*]] = add i8 [[TMP1]], [[TMP0]]
+; COMMON-NEXT: [[INCDEC_PTR4]] = getelementptr inbounds i8, ptr [[C_ADDR_08]], i32 1
+; COMMON-NEXT: store i8 [[ADD]], ptr [[C_ADDR_08]], align 1
+; COMMON-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N_ADDR_09]], 1
+; COMMON-NEXT: br i1 [[CMP]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT:.*]]
+; COMMON: [[WHILE_END_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[WHILE_END:.*]]
+; COMMON: [[WHILE_END]]:
+; COMMON-NEXT: ret void
+;
entry:
br label %while.body
@@ -186,11 +319,40 @@ while.end:
}
define void @sgt_step_minus_two(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_step_minus_two(
-; COMMON: vector.body:
-; CHECK-TF: masked.load
-; CHECK-TF: masked.load
-; CHECK-TF: masked.store
+; COMMON-LABEL: define void @sgt_step_minus_two(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
+; COMMON-NEXT: br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; COMMON: [[WHILE_BODY_PREHEADER]]:
+; COMMON-NEXT: [[TMP0:%.*]] = lshr i32 [[N]], 1
+; COMMON-NEXT: [[TMP1:%.*]] = add nuw i32 [[TMP0]], 1
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 15
+; COMMON-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 16
+; COMMON-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[TMP1]])
+; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[TMP2:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; COMMON-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP2]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
+; COMMON-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; COMMON-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMMON-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; COMMON: [[WHILE_END_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[WHILE_END]]
+; COMMON: [[WHILE_END]]:
+; COMMON-NEXT: ret void
+;
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %while.body.preheader, label %while.end
@@ -222,8 +384,33 @@ while.end:
}
define void @sgt_step_not_constant(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N, i32 %S) #0 {
-; COMMON-LABEL: @sgt_step_not_constant(
-; COMMON-NOT: vector.body:
+; COMMON-LABEL: define void @sgt_step_not_constant(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]], i32 [[S:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
+; COMMON-NEXT: br i1 [[CMP5]], label %[[WHILE_BODY_PREHEADER:.*]], label %[[WHILE_END:.*]]
+; COMMON: [[WHILE_BODY_PREHEADER]]:
+; COMMON-NEXT: br label %[[WHILE_BODY:.*]]
+; COMMON: [[WHILE_BODY]]:
+; COMMON-NEXT: [[N_ADDR_09:%.*]] = phi i32 [ [[DEC:%.*]], %[[WHILE_BODY]] ], [ [[N]], %[[WHILE_BODY_PREHEADER]] ]
+; COMMON-NEXT: [[C_ADDR_08:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], %[[WHILE_BODY]] ], [ [[C]], %[[WHILE_BODY_PREHEADER]] ]
+; COMMON-NEXT: [[B_ADDR_07:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], %[[WHILE_BODY]] ], [ [[B]], %[[WHILE_BODY_PREHEADER]] ]
+; COMMON-NEXT: [[A_ADDR_06:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], %[[WHILE_BODY]] ], [ [[A]], %[[WHILE_BODY_PREHEADER]] ]
+; COMMON-NEXT: [[DEC]] = add nsw i32 [[N_ADDR_09]], [[S]]
+; COMMON-NEXT: [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[A_ADDR_06]], i32 1
+; COMMON-NEXT: [[TMP0:%.*]] = load i8, ptr [[A_ADDR_06]], align 1
+; COMMON-NEXT: [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[B_ADDR_07]], i32 1
+; COMMON-NEXT: [[TMP1:%.*]] = load i8, ptr [[B_ADDR_07]], align 1
+; COMMON-NEXT: [[ADD:%.*]] = add i8 [[TMP1]], [[TMP0]]
+; COMMON-NEXT: [[INCDEC_PTR4]] = getelementptr inbounds i8, ptr [[C_ADDR_08]], i32 1
+; COMMON-NEXT: store i8 [[ADD]], ptr [[C_ADDR_08]], align 1
+; COMMON-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N_ADDR_09]], 1
+; COMMON-NEXT: br i1 [[CMP]], label %[[WHILE_BODY]], label %[[WHILE_END_LOOPEXIT:.*]]
+; COMMON: [[WHILE_END_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[WHILE_END]]
+; COMMON: [[WHILE_END]]:
+; COMMON-NEXT: ret void
+;
entry:
%cmp5 = icmp sgt i32 %N, 0
br i1 %cmp5, label %while.body.preheader, label %while.end
@@ -255,8 +442,38 @@ while.end:
}
define void @icmp_eq(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, ptr noalias nocapture %C, i32 %N) #0 {
-; COMMON-LABEL: @icmp_eq
-; COMMON: vector.body:
+; COMMON-LABEL: define void @icmp_eq(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[CMP6:%.*]] = icmp eq i32 [[N]], 0
+; COMMON-NEXT: br i1 [[CMP6]], label %[[WHILE_END:.*]], label %[[WHILE_BODY_PREHEADER:.*]]
+; COMMON: [[WHILE_BODY_PREHEADER]]:
+; COMMON-NEXT: br label %[[VECTOR_PH:.*]]
+; COMMON: [[VECTOR_PH]]:
+; COMMON-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 15
+; COMMON-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 16
+; COMMON-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMMON: [[VECTOR_BODY]]:
+; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
+; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
+; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
+; COMMON-NEXT: [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
+; COMMON-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP0]], ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]])
+; COMMON-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 16
+; COMMON-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMMON-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMMON: [[MIDDLE_BLOCK]]:
+; COMMON-NEXT: br label %[[WHILE_END_LOOPEXIT:.*]]
+; COMMON: [[WHILE_END_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[WHILE_END]]
+; COMMON: [[WHILE_END]]:
+; COMMON-NEXT: ret void
+;
entry:
%cmp6 = icmp eq i32 %N, 0
br i1 %cmp6, label %while.end, label %while.body.preheader
@@ -295,20 +512,123 @@ while.end:
; c[i] = a[i] + b[i];
; }
;
-define void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_for_loop(
-; COMMON: vector.body:
-; CHECK-PREFER: masked.load
-; CHECK-PREFER: masked.load
-; CHECK-PREFER: masked.store
-;
; TODO: if tail-predication is requested, tail-folding isn't triggered because
; the profitability check returns "Different strides found, can't tail-predicate",
; investigate this.
+define void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
+; DEFAULT-LABEL: define void @sgt_for_loop(
+; DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; DEFAULT-NEXT: [[ENTRY:.*:]]
+; DEFAULT-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
+; DEFAULT-NEXT: br i1 [[CMP5]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
+; DEFAULT: [[FOR_BODY_PREHEADER]]:
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 16
+; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; DEFAULT: [[VECTOR_PH]]:
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
+; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; DEFAULT-NEXT: [[TMP0:%.*]] = sub i32 [[N]], [[N_VEC]]
+; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; DEFAULT: [[VECTOR_BODY]]:
+; DEFAULT-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[TMP1:%.*]] = sub i32 [[N]], [[INDEX]]
+; DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
+; DEFAULT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15
+; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; DEFAULT-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]]
+; DEFAULT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15
+; DEFAULT-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; DEFAULT-NEXT: [[REVERSE2:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD1]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; DEFAULT-NEXT: [[TMP6:%.*]] = add <16 x i8> [[REVERSE2]], [[REVERSE]]
+; DEFAULT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]]
+; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15
+; DEFAULT-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; DEFAULT-NEXT: store <16 x i8> [[REVERSE3]], ptr [[TMP8]], align 1
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; DEFAULT-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; DEFAULT: [[MIDDLE_BLOCK]]:
+; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
+; DEFAULT: [[SCALAR_PH]]:
+;
+; CHECK-PREFER-LABEL: define void @sgt_for_loop(
+; CHECK-PREFER-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-PREFER-NEXT: [[ENTRY:.*:]]
+; CHECK-PREFER-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-PREFER-NEXT: br i1 [[CMP5]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_END:.*]]
+; CHECK-PREFER: [[FOR_BODY_PREHEADER]]:
+; CHECK-PREFER-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-PREFER: [[VECTOR_PH]]:
+; CHECK-PREFER-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 15
+; CHECK-PREFER-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 16
+; CHECK-PREFER-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-PREFER-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-PREFER: [[VECTOR_BODY]]:
+; CHECK-PREFER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PREFER-NEXT: [[TMP0:%.*]] = sub i32 [[N]], [[INDEX]]
+; CHECK-PREFER-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
+; CHECK-PREFER-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP0]]
+; CHECK-PREFER-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 -15
+; CHECK-PREFER-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[REVERSE]], <16 x i8> poison)
+; CHECK-PREFER-NEXT: [[REVERSE1:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-PREFER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP0]]
+; CHECK-PREFER-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP3]], i32 -15
+; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[REVERSE]], <16 x i8> poison)
+; CHECK-PREFER-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD2]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-PREFER-NEXT: [[TMP5:%.*]] = add <16 x i8> [[REVERSE3]], [[REVERSE1]]
+; CHECK-PREFER-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP0]]
+; CHECK-PREFER-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 -15
+; CHECK-PREFER-NEXT: [[REVERSE4:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-PREFER-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[REVERSE4]], ptr align 1 [[TMP7]], <16 x i1> [[REVERSE]])
+; CHECK-PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-PREFER-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-PREFER-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-PREFER: [[MIDDLE_BLOCK]]:
+; CHECK-PREFER-NEXT: br label %[[FOR_END_LOOPEXIT:.*]]
+; CHECK-PREFER: [[FOR_END_LOOPEXIT]]:
+; CHECK-PREFER-NEXT: br label %[[FOR_END]]
+; CHECK-PREFER: [[FOR_END]]:
+; CHECK-PREFER-NEXT: ret void
;
-; CHECK-ENABLE-TP-NOT: masked.load
-; CHECK-ENABLE-TP-NOT: masked.load
-; CHECK-ENABLE-TP-NOT: masked.store
+; CHECK-ENABLE-TP-LABEL: define void @sgt_for_loop(
+; CHECK-ENABLE-TP-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ENABLE-TP-NEXT: [[ENTRY:.*:]]
+; CHECK-ENABLE-TP-NEXT: [[CMP5:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-ENABLE-TP-NEXT: br i1 [[CMP5]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_END:label %.*]]
+; CHECK-ENABLE-TP: [[FOR_BODY_PREHEADER]]:
+; CHECK-ENABLE-TP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 16
+; CHECK-ENABLE-TP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-ENABLE-TP: [[VECTOR_PH]]:
+; CHECK-ENABLE-TP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 16
+; CHECK-ENABLE-TP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; CHECK-ENABLE-TP-NEXT: [[TMP0:%.*]] = sub i32 [[N]], [[N_VEC]]
+; CHECK-ENABLE-TP-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-ENABLE-TP: [[VECTOR_BODY]]:
+; CHECK-ENABLE-TP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ENABLE-TP-NEXT: [[TMP1:%.*]] = sub i32 [[N]], [[INDEX]]
+; CHECK-ENABLE-TP-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
+; CHECK-ENABLE-TP-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15
+; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-ENABLE-TP-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-ENABLE-TP-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]]
+; CHECK-ENABLE-TP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15
+; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-ENABLE-TP-NEXT: [[REVERSE2:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD1]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-ENABLE-TP-NEXT: [[TMP6:%.*]] = add <16 x i8> [[REVERSE2]], [[REVERSE]]
+; CHECK-ENABLE-TP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]]
+; CHECK-ENABLE-TP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15
+; CHECK-ENABLE-TP-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-ENABLE-TP-NEXT: store <16 x i8> [[REVERSE3]], ptr [[TMP8]], align 1
+; CHECK-ENABLE-TP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-ENABLE-TP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ENABLE-TP-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-ENABLE-TP: [[MIDDLE_BLOCK]]:
+; CHECK-ENABLE-TP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; CHECK-ENABLE-TP-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-ENABLE-TP: [[SCALAR_PH]]:
;
entry:
%cmp5 = icmp sgt i32 %N, 0
@@ -335,21 +655,129 @@ for.end:
}
define void @sgt_for_loop_i64(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_for_loop_i64(
-; COMMON: vector.body:
+; DEFAULT-LABEL: define void @sgt_for_loop_i64(
+; DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; DEFAULT-NEXT: [[ENTRY:.*:]]
+; DEFAULT-NEXT: [[CMP14:%.*]] = icmp sgt i32 [[N]], 0
+; DEFAULT-NEXT: br i1 [[CMP14]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_COND_CLEANUP:label %.*]]
+; DEFAULT: [[FOR_BODY_PREHEADER]]:
+; DEFAULT-NEXT: [[CONV16:%.*]] = zext i32 [[N]] to i64
+; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[CONV16]], 16
+; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; DEFAULT: [[VECTOR_PH]]:
+; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[CONV16]], 16
+; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i64 [[CONV16]], [[N_MOD_VF]]
+; DEFAULT-NEXT: [[TMP0:%.*]] = sub i64 [[CONV16]], [[N_VEC]]
+; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
+; DEFAULT: [[VECTOR_BODY]]:
+; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT: [[TMP1:%.*]] = sub i64 [[CONV16]], [[INDEX]]
+; DEFAULT-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; DEFAULT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP2]]
+; DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 -15
+; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; DEFAULT-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; DEFAULT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP2]]
+; DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -15
+; DEFAULT-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; DEFAULT-NEXT: [[REVERSE2:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD1]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; DEFAULT-NEXT: [[TMP7:%.*]] = add <16 x i8> [[REVERSE2]], [[REVERSE]]
+; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP2]]
+; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 -15
+; DEFAULT-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; DEFAULT-NEXT: store <16 x i8> [[REVERSE3]], ptr [[TMP9]], align 1
+; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; DEFAULT-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; DEFAULT: [[MIDDLE_BLOCK]]:
+; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[CONV16]], [[N_VEC]]
+; DEFAULT-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
+; DEFAULT: [[SCALAR_PH]]:
;
-; CHECK-PREFER: masked.load
-; CHECK-PREFER: masked.load
-; CHECK-PREFER: masked.store
+; CHECK-PREFER-LABEL: define void @sgt_for_loop_i64(
+; CHECK-PREFER-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-PREFER-NEXT: [[ENTRY:.*:]]
+; CHECK-PREFER-NEXT: [[CMP14:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-PREFER-NEXT: br i1 [[CMP14]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-PREFER: [[FOR_BODY_PREHEADER]]:
+; CHECK-PREFER-NEXT: [[CONV16:%.*]] = zext i32 [[N]] to i64
+; CHECK-PREFER-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-PREFER: [[VECTOR_PH]]:
+; CHECK-PREFER-NEXT: [[N_RND_UP:%.*]] = add i64 [[CONV16]], 15
+; CHECK-PREFER-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16
+; CHECK-PREFER-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-PREFER-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-PREFER: [[VECTOR_BODY]]:
+; CHECK-PREFER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PREFER-NEXT: [[TMP0:%.*]] = sub i64 [[CONV16]], [[INDEX]]
+; CHECK-PREFER-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX]], i64 [[CONV16]])
+; CHECK-PREFER-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-PREFER-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
+; CHECK-PREFER-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -15
+; CHECK-PREFER-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP3]], <16 x i1> [[REVERSE]], <16 x i8> poison)
+; CHECK-PREFER-NEXT: [[REVERSE1:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-PREFER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]]
+; CHECK-PREFER-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i32 -15
+; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[REVERSE]], <16 x i8> poison)
+; CHECK-PREFER-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD2]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-PREFER-NEXT: [[TMP6:%.*]] = add <16 x i8> [[REVERSE3]], [[REVERSE1]]
+; CHECK-PREFER-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]]
+; CHECK-PREFER-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 -15
+; CHECK-PREFER-NEXT: [[REVERSE4:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-PREFER-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[REVERSE4]], ptr align 1 [[TMP8]], <16 x i1> [[REVERSE]])
+; CHECK-PREFER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-PREFER-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-PREFER-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-PREFER: [[MIDDLE_BLOCK]]:
+; CHECK-PREFER-NEXT: br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
+; CHECK-PREFER: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-PREFER-NEXT: br label %[[FOR_COND_CLEANUP]]
+; CHECK-PREFER: [[FOR_COND_CLEANUP]]:
+; CHECK-PREFER-NEXT: ret void
+;
+; CHECK-ENABLE-TP-LABEL: define void @sgt_for_loop_i64(
+; CHECK-ENABLE-TP-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-ENABLE-TP-NEXT: [[ENTRY:.*:]]
+; CHECK-ENABLE-TP-NEXT: [[CMP14:%.*]] = icmp sgt i32 [[N]], 0
+; CHECK-ENABLE-TP-NEXT: br i1 [[CMP14]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_COND_CLEANUP:label %.*]]
+; CHECK-ENABLE-TP: [[FOR_BODY_PREHEADER]]:
+; CHECK-ENABLE-TP-NEXT: [[CONV16:%.*]] = zext i32 [[N]] to i64
+; CHECK-ENABLE-TP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[CONV16]], 16
+; CHECK-ENABLE-TP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-ENABLE-TP: [[VECTOR_PH]]:
+; CHECK-ENABLE-TP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[CONV16]], 16
+; CHECK-ENABLE-TP-NEXT: [[N_VEC:%.*]] = sub i64 [[CONV16]], [[N_MOD_VF]]
+; CHECK-ENABLE-TP-NEXT: [[TMP0:%.*]] = sub i64 [[CONV16]], [[N_VEC]]
+; CHECK-ENABLE-TP-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-ENABLE-TP: [[VECTOR_BODY]]:
+; CHECK-ENABLE-TP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-ENABLE-TP-NEXT: [[TMP1:%.*]] = sub i64 [[CONV16]], [[INDEX]]
+; CHECK-ENABLE-TP-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-ENABLE-TP-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP2]]
+; CHECK-ENABLE-TP-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 -15
+; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-ENABLE-TP-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-ENABLE-TP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP2]]
+; CHECK-ENABLE-TP-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -15
+; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-ENABLE-TP-NEXT: [[REVERSE2:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD1]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-ENABLE-TP-NEXT: [[TMP7:%.*]] = add <16 x i8> [[REVERSE2]], [[REVERSE]]
+; CHECK-ENABLE-TP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP2]]
+; CHECK-ENABLE-TP-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 -15
+; CHECK-ENABLE-TP-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-ENABLE-TP-NEXT: store <16 x i8> [[REVERSE3]], ptr [[TMP9]], align 1
+; CHECK-ENABLE-TP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-ENABLE-TP-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ENABLE-TP-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-ENABLE-TP: [[MIDDLE_BLOCK]]:
+; CHECK-ENABLE-TP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[CONV16]], [[N_VEC]]
+; CHECK-ENABLE-TP-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-ENABLE-TP: [[SCALAR_PH]]:
;
; With -disable-mve-tail-predication=false, the target hook returns
; "preferPredicateOverEpilogue: hardware-loop is not profitable."
; so here we don't expect the tail-folding. TODO: look into this.
-;
-; CHECK-ENABLE-TP-NOT: masked.load
-; CHECK-ENABLE-TP-NOT: masked.load
-; CHECK-ENABLE-TP-NOT: masked.store
-;
entry:
%cmp14 = icmp sgt i32 %N, 0
br i1 %cmp14, label %for.body.preheader, label %for.cond.cleanup
@@ -390,12 +818,36 @@ for.body:
; false for the inner-loop.
;
define void @sgt_nested_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 {
-; COMMON-LABEL: @sgt_nested_loop(
-; DEFAULT-NOT: vector.body:
-; CHECK-TF-NOT: masked.load
-; CHECK-TF-NOT: masked.load
-; CHECK-TF-NOT: masked.store
-; COMMON: }
+; COMMON-LABEL: define void @sgt_nested_loop(
+; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; COMMON-NEXT: [[ENTRY:.*:]]
+; COMMON-NEXT: [[CMP21:%.*]] = icmp sgt i32 [[N]], 0
+; COMMON-NEXT: br i1 [[CMP21]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; COMMON: [[FOR_BODY_PREHEADER]]:
+; COMMON-NEXT: br label %[[FOR_BODY:.*]]
+; COMMON: [[FOR_COND_LOOPEXIT:.*]]:
+; COMMON-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[ADD:%.*]], [[N]]
+; COMMON-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[FOR_BODY]]
+; COMMON: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; COMMON-NEXT: br label %[[FOR_COND_CLEANUP]]
+; COMMON: [[FOR_COND_CLEANUP]]:
+; COMMON-NEXT: ret void
+; COMMON: [[FOR_BODY]]:
+; COMMON-NEXT: [[I_022:%.*]] = phi i32 [ [[ADD]], %[[FOR_COND_LOOPEXIT]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; COMMON-NEXT: [[ADD]] = add nuw nsw i32 [[I_022]], 1
+; COMMON-NEXT: br label %[[FOR_BODY4:.*]]
+; COMMON: [[FOR_BODY4]]:
+; COMMON-NEXT: [[J_020:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[DEC:%.*]], %[[FOR_BODY4]] ]
+; COMMON-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[J_020]]
+; COMMON-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; COMMON-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[J_020]]
+; COMMON-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX5]], align 1
+; COMMON-NEXT: [[ADD7:%.*]] = add i8 [[TMP1]], [[TMP0]]
+; COMMON-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[J_020]]
+; COMMON-NEXT: store i8 [[ADD7]], ptr [[ARRAYIDX9]], align 1
+; COMMON-NEXT: [[DEC]] = add nsw i32 [[J_020]], -1
+; COMMON-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[J_020]], 1
+; COMMON-NEXT: br i1 [[CMP2]], label %[[FOR_BODY4]], label %[[FOR_COND_LOOPEXIT]]
;
entry:
%cmp21 = icmp sgt i32 %N, 0
@@ -437,3 +889,5 @@ attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+
!1 = distinct !{!1, !2}
!2 = !{!"llvm.loop.vectorize.width", i32 16}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-TF: {{.*}}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll
index 32765c53efdc8..51ef6e77d0b09 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; This is the loop in c++ being vectorize in this file with
;vector.reverse
; #pragma clang loop vectorize_width(4, scalable)
@@ -5,58 +6,74 @@
; a[i] = b[i] + 1.0;
; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v \
-; RUN: -debug-only=loop-vectorize -scalable-vectorization=on \
+; RUN: -vplan-print-after=printOptimizedVPlan -force-vector-width=4 -scalable-vectorization=on \
; RUN: -disable-output < %s 2>&1 | FileCheck %s
define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
-; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
-; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
-; CHECK-NEXT: vp<[[OTC:%.+]]> = original trip-count
+; CHECK-LABEL: VPlan for loop in 'vector_reverse_i64'
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 4},UF={1}' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: EMIT vp<[[OTC]]> = EXPAND SCEV (1 + (-1 * (1 umin %n))<nuw><nsw> + %n)
-; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 + (-1 * (1 umin %n))<nuw><nsw> + %n)
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
-; CHECK-NEXT: vector.ph:
-; CHECK-NEXT: Successor(s): vector loop
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
-; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[INDUCTION:%.+]]> = CANONICAL-IV
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
-; CHECK-NEXT: vector.body:
-; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[EVL_PHI:%.+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ vp<[[OTC]]>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
-; CHECK-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[EVL_PHI]]> * ir<-1>
-; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>, vp<[[EVL]]>
-; CHECK-NEXT: CLONE ir<[[IDX:%.+]]> = add nsw vp<[[SCALAR_STEPS]]>, ir<-1>
-; CHECK-NEXT: CLONE ir<[[IDX_PROM:%.+]]> = zext ir<[[IDX]]>
-; CHECK-NEXT: CLONE ir<[[ARRAY_IDX_B:%.+]]> = getelementptr inbounds ir<[[B:%.+]]>, ir<[[IDX_PROM]]>
-; CHECK-NEXT: vp<[[VEC_END_PTR_B:%.+]]> = vector-end-pointer ir<[[ARRAY_IDX_B]]>, vp<[[EVL]]>
-; CHECK-NEXT: WIDEN ir<[[LOAD_B:%.+]]> = vp.load vp<[[VEC_END_PTR_B]]>, vp<[[EVL]]>
-; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VAL_B:%.+]]> = call llvm.experimental.vp.reverse(ir<[[LOAD_B]]>, ir<true>, vp<[[EVL]]>)
-; CHECK-NEXT: WIDEN ir<[[ADD_RESULT:%.+]]> = add vp<[[VAL_B]]>, ir<1>
-; CHECK-NEXT: CLONE ir<[[ARRAY_IDX_A:%.+]]> = getelementptr inbounds ir<[[A:%.+]]>, ir<[[IDX_PROM]]>
-; CHECK-NEXT: vp<[[VEC_END_PTR_A:%.+]]> = vector-end-pointer ir<[[ARRAY_IDX_A]]>, vp<[[EVL]]>
-; CHECK-NEXT: WIDEN-INTRINSIC vp<[[STORE_VAL:%.+]]> = call llvm.experimental.vp.reverse(ir<[[ADD_RESULT]]>, ir<true>, vp<[[EVL]]>)
-; CHECK-NEXT: WIDEN vp.store vp<[[VEC_END_PTR_A]]>, vp<[[STORE_VAL]]>, vp<[[EVL]]>
-; CHECK-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[EVL]]>, vp<[[EVL_PHI]]>
-; CHECK-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[EVL]]>
-; CHECK-NEXT: EMIT vp<[[INDEX_NEXT:%.+]]> = add vp<[[INDUCTION]]>, vp<[[VFxUF]]>
-; CHECK-NEXT: EMIT branch-on-count vp<[[INDEX_NEXT]]>, vp<[[VTC]]>
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-; CHECK-NEXT: Successor(s): middle.block
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: CURRENT-ITERATION-PHI vp<[[VP5:%[0-9]+]]> = phi ir<0>, vp<%current.iteration.next>
+; CHECK-NEXT: EMIT-SCALAR vp<%avl> = phi [ vp<[[VP3]]>, vector.ph ], [ vp<%avl.next>, vector.body ]
+; CHECK-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<%avl>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<%n> + vp<[[VP5]]> * ir<-1>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<-1>, vp<%evl>
+; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[VP7]]>, ir<-1>
+; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-end-pointer ir<%arrayidx>, vp<%evl>
+; CHECK-NEXT: WIDEN ir<%0> = vp.load vp<[[VP8]]>, vp<%evl>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP9:%[0-9]+]]> = call llvm.experimental.vp.reverse(ir<%0>, ir<true>, vp<%evl>)
+; CHECK-NEXT: WIDEN ir<%add9> = add vp<[[VP9]]>, ir<1>
+; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-end-pointer ir<%arrayidx3>, vp<%evl>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP11:%[0-9]+]]> = call llvm.experimental.vp.reverse(ir<%add9>, ir<true>, vp<%evl>)
+; CHECK-NEXT: WIDEN vp.store vp<[[VP10]]>, vp<[[VP11]]>, vp<%evl>
+; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl>
+; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
-; CHECK-NEXT: middle.block:
-; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>
; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<for.cond.cleanup>:
-; CHECK-NEXT: No successors
+; CHECK-NEXT: ir-bb<for.cond.cleanup>:
+; CHECK-NEXT: No successors
; CHECK-EMPTY:
-; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: Successor(s): ir-bb<for.body>
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: Successor(s): ir-bb<for.body>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.body>:
+; CHECK-NEXT: IR %indvars.iv = phi i32 [ %n, %entry ], [ %indvars.iv.next, %for.body ] (extra operand: ir<%n> from scalar.ph)
+; CHECK-NEXT: IR %i.0.in8 = phi i32 [ %n, %entry ], [ %i.0, %for.body ] (extra operand: ir<%n> from scalar.ph)
+; CHECK-NEXT: IR %i.0 = add nsw i32 %i.0.in8, -1
+; CHECK-NEXT: IR %idxprom = zext i32 %i.0 to i64
+; CHECK-NEXT: IR %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom
+; CHECK-NEXT: IR %0 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT: IR %add9 = add i32 %0, 1
+; CHECK-NEXT: IR %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom
+; CHECK-NEXT: IR store i32 %add9, ptr %arrayidx3, align 4
+; CHECK-NEXT: IR %cmp = icmp ugt i32 %indvars.iv, 1
+; CHECK-NEXT: IR %indvars.iv.next = add nsw i32 %indvars.iv, -1
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll b/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll
index bb99a5363373e..abe970ea303cf 100644
--- a/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/bzip_reverse_loops.ll
@@ -1,14 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 6
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S -enable-if-conversion | FileCheck %s
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
-;CHECK: fc
-;CHECK: load <4 x i16>
-;CHECK-NEXT: shufflevector <4 x i16>
-;CHECK: select <4 x i1>
-;CHECK: store <4 x i16>
-;CHECK: ret
define void @fc(ptr nocapture %p, i32 %n, i32 %size) {
+; CHECK-LABEL: define void @fc(
+; CHECK-SAME: ptr captures(none) [[P:%.*]], i32 [[N:%.*]], i32 [[SIZE:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[N]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[N_VEC]], -2
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[SIZE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[INDEX]], -2
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[NEXT_GEP]], i64 -1
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[TMP8]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP9]], align 2
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP10:%.*]] = zext <4 x i16> [[REVERSE]] to <4 x i32>
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP12:%.*]] = sub <4 x i32> [[TMP10]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP13:%.*]] = trunc <4 x i32> [[TMP12]] to <4 x i16>
+; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x i16> zeroinitializer, <4 x i16> [[TMP13]]
+; CHECK-NEXT: [[REVERSE1:%.*]] = shufflevector <4 x i16> [[PREDPHI]], <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: store <4 x i16> [[REVERSE1]], ptr [[TMP9]], align 2
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[DO_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
entry:
br label %do.body
@@ -37,14 +73,46 @@ do.end:
ret void
}
-;CHECK: example1
-;CHECK: load <4 x i32>
-;CHECK-NEXT: shufflevector <4 x i32>
-;CHECK: sub nsw <4 x i32>
-;CHECK: select <4 x i1>
-;CHECK: store <4 x i32>
-;CHECK: ret
define void @example1(ptr nocapture %a, i32 %n, i32 %wsize) {
+; CHECK-LABEL: define void @example1(
+; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[N:%.*]], i32 [[WSIZE:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[N]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[N_VEC]], -4
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[WSIZE]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[INDEX]], -4
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i64 -1
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 -3
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 4
+; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP10:%.*]] = icmp slt <4 x i32> [[REVERSE]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP11:%.*]] = sub nsw <4 x i32> [[REVERSE]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> zeroinitializer, <4 x i32> [[TMP11]]
+; CHECK-NEXT: [[REVERSE1:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: store <4 x i32> [[REVERSE1]], ptr [[TMP9]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[DO_END:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
entry:
br label %do.body
More information about the llvm-commits
mailing list