[llvm] [NFC][VPlan] Add extra tests for VPlan-based stride speculation (PR #218795)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 21:03:19 PDT 2026
================
@@ -3828,4 +4100,358 @@ exit:
ret void
}
+; Used to crash because `getSymbolicMaxBackedgeTakenCount` is only available in
+; PSE but not SE.
+define void @stride_mv_predicated_btc(ptr noalias %p.out, ptr %p, i32 %M, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @stride_mv_predicated_btc(
+; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[M:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 0)
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add nuw i32 [[TMP0]], 1
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 0)
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = trunc i32 [[SMAX]] to i16
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = add i16 1, [[TMP3]]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = icmp slt i16 [[TMP4]], 1
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[SMAX]], 65535
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = and i32 [[TMP1]], 3
+; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP8]]
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = trunc i32 [[N_VEC]] to i16
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[VEC_IND]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[TMP10]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP12]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP11]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP13]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[I:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[I_EXT:%.*]] = sext i16 [[I]] to i64
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[I_EXT]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 4
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[I_EXT]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 4
+; COMPARE-NO-MV-NEXT: [[I_NEXT]] = add i16 [[I]], 1
+; COMPARE-NO-MV-NEXT: [[I_NEXT_EXT:%.*]] = sext i16 [[I_NEXT]] to i32
+; COMPARE-NO-MV-NEXT: [[EC:%.*]] = icmp sle i32 [[I_NEXT_EXT]], [[M]]
+; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP53:![0-9]+]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @stride_mv_predicated_btc(
+; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[M:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 0)
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add nuw i32 [[TMP0]], 1
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[M]], i32 0)
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = trunc i32 [[SMAX]] to i16
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = add i16 1, [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp slt i16 [[TMP4]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[SMAX]], 65535
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = and i32 [[TMP1]], 3
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP8]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = trunc i32 [[N_VEC]] to i16
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[VEC_IND]] to <4 x i64>
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP10]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = mul <4 x i64> [[TMP10]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP12]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i64> poison)
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP11]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP13]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 4)
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP69:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[I:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[I_EXT:%.*]] = sext i16 [[I]] to i64
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[I_EXT]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 4
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[I_EXT]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 4
+; COMPARE-LAA-MV-NEXT: [[I_NEXT]] = add i16 [[I]], 1
+; COMPARE-LAA-MV-NEXT: [[I_NEXT_EXT:%.*]] = sext i16 [[I_NEXT]] to i32
+; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp sle i32 [[I_NEXT_EXT]], [[M]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP70:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i16 [ 0, %entry ], [ %iv.next, %header ]
+ %iv.ext = sext i16 %iv to i64
+ %idx = mul i64 %iv.ext, %stride
+ %gep.ld = getelementptr i64, ptr %p, i64 %idx
+ %ld = load i64, ptr %gep.ld
+ %gep.st = getelementptr i64, ptr %p.out, i64 %iv.ext
+ store i64 %ld, ptr %gep.st
+ %iv.next = add i16 %iv, 1
+ %iv.next.ext = sext i16 %iv.next to i32
+ %ec = icmp sle i32 %iv.next.ext, %M
+ br i1 %ec, label %header, label %exit
+
+exit:
+ ret void
+}
+
+define void @known_non_unit_via_range_attr(ptr noalias %out, ptr %p, i64 range(i64 4, 8) %stride) {
+; COMPARE-NO-MV-LABEL: define void @known_non_unit_via_range_attr(
+; COMPARE-NO-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], i64 range(i64 4, 8) [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP1]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-NO-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @known_non_unit_via_range_attr(
+; COMPARE-LAA-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], i64 range(i64 4, 8) [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[VECTOR_PH:.*]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store i32 [[L]], ptr [[TMP1]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[EXIT:.*]], label %[[VECTOR_BODY]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+ %l = load i32, ptr %gep, align 4
+ %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %l, ptr %gep.out, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+declare void @llvm.assume(i1 noundef)
+
+; TODO: We shouldn't speculate `%stride == 1` as it's known to be false via assume.
+define void @known_non_unit_via_assume(ptr noalias %out, ptr %p, i64 %stride) {
+; COMPARE-NO-MV-LABEL: define void @known_non_unit_via_assume(
+; COMPARE-NO-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[NON_UNIT:%.*]] = icmp uge i64 [[STRIDE]], 4
+; COMPARE-NO-MV-NEXT: call void @llvm.assume(i1 [[NON_UNIT]])
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[STRIDES_CHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-NO-MV-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[LOOP:.*]]
+; COMPARE-NO-MV: [[LOOP]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @known_non_unit_via_assume(
+; COMPARE-LAA-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[NON_UNIT:%.*]] = icmp uge i64 [[STRIDE]], 4
+; COMPARE-LAA-MV-NEXT: call void @llvm.assume(i1 [[NON_UNIT]])
+; COMPARE-LAA-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-LAA-MV: [[STRIDES_CHECK]]:
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP0]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP2]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP71:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[LOOP:.*]]
+; COMPARE-LAA-MV: [[LOOP]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; COMPARE-LAA-MV-NEXT: [[GEP_OUT:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i32 [[L]], ptr [[GEP_OUT]], align 4
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP72:![0-9]+]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+entry:
+ %non_unit = icmp uge i64 %stride, 4
+ call void @llvm.assume(i1 %non_unit)
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, %stride
+ %gep = getelementptr inbounds i32, ptr %p, i64 %idx
+ %l = load i32, ptr %gep, align 4
+ %gep.out = getelementptr inbounds i32, ptr %out, i64 %iv
+ store i32 %l, ptr %gep.out, align 4
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1024
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The loaded stride has range metadata excluding unit stride.
+define void @known_non_unit_via_load_range(ptr noalias %out, ptr %p,
+; COMPARE-NO-MV-LABEL: define void @known_non_unit_via_load_range(
+; COMPARE-NO-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], ptr [[STRIDE_PTR:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDE:%.*]] = load i64, ptr [[STRIDE_PTR]], align 8, !range [[RNG56:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], <4 x i64> [[TMP0]]
+; COMPARE-NO-MV-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP1]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-NO-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[EXIT]]:
+; COMPARE-NO-MV-NEXT: ret void
+;
+; COMPARE-LAA-MV-LABEL: define void @known_non_unit_via_load_range(
+; COMPARE-LAA-MV-SAME: ptr noalias [[OUT:%.*]], ptr [[P:%.*]], ptr [[STRIDE_PTR:%.*]]) {
+; COMPARE-LAA-MV-NEXT: [[VECTOR_PH:.*]]:
+; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = load i64, ptr [[STRIDE_PTR]], align 8, !range [[RNG73:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-LAA-MV: [[VECTOR_BODY]]:
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[INDEX]], [[STRIDE]]
+; COMPARE-LAA-MV-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store i32 [[L]], ptr [[TMP1]], align 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[EXIT:.*]], label %[[VECTOR_BODY]]
+; COMPARE-LAA-MV: [[EXIT]]:
+; COMPARE-LAA-MV-NEXT: ret void
+;
+ ptr %stride.ptr) {
----------------
lukel97 wrote:
Same here
https://github.com/llvm/llvm-project/pull/218795
More information about the llvm-commits
mailing list