[llvm] [LAA] Add stencil group merging to reduce runtime pointer checks (PR #187252)
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 05:26:53 PDT 2026
================
@@ -0,0 +1,2951 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=force -disable-output %s 2>&1 | FileCheck --check-prefixes=CHECK,MERGE %s
+; RUN: opt -passes='print<access-info>' -stencil-runtime-check-merge=off -disable-output %s 2>&1 | FileCheck --check-prefixes=CHECK,NOMERGE %s
+
+;; Test 1: Basic stencil merge with one runtime stride.
+;; 6 loads from %a at byte offsets {-3*cdj, -2*cdj, -cdj, +cdj, +2*cdj, +3*cdj}
+;; relative to base = %a + 8*iv. Store to %out.
+;; With merge: all 6 loads form 1 merged group with bounds spanning [-3*cdj, +3*cdj]
+;; relative to base, producing 1 runtime check and 1 stride predicate (cdj > 0).
+;; Without merge: 6 separate groups (each load alone), 6 checks, no predicates.
+define void @stencil_merge_single_stride(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'stencil_merge_single_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_single_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (24 + (3 * %cdj) + %a) High: (-24 + (3 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + (3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + (2 * %cdj) + %a) High: (-24 + (2 * %cdj) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (24 + (-2 * %cdj) + %a) High: (-24 + (8 * %n) + (-2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (24 + (-3 * %cdj) + %a) High: (-24 + (8 * %n) + (-3 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-3 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %neg3cdj = mul nsw i64 %cdj, -3
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %neg3cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+ %v1 = load double, ptr %p1, align 8
+
+ %negcdj = sub nsw i64 0, %cdj
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v3 = load double, ptr %p3, align 8
+
+ %pos2cdj = mul nsw i64 %cdj, 2
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos2cdj
+ %v4 = load double, ptr %p4, align 8
+
+ %pos3cdj = mul nsw i64 %cdj, 3
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %pos3cdj
+ %v5 = load double, ptr %p5, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+ %s3 = fadd double %s2, %v4
+ %s4 = fadd double %s3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 2: Constant offsets only — existing grouping handles this.
+;; 4 loads from %a at constant byte offsets {-16, -8, +8, +16}. Store to %out.
+;; The standard grouping algorithm merges these into 1 group (constant SCEV diffs).
+;; Both with and without flag: 1 check, 2 groups, no predicates.
+define void @constant_offsets_only(ptr %a, ptr %out, i64 %n) {
+; CHECK-LABEL: 'constant_offsets_only'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 16
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 8
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; CHECK-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(48 + %a),+,8}<nuw><%loop>
+; CHECK-NEXT: Member: {(40 + %a),+,8}<nuw><%loop>
+; CHECK-NEXT: Member: {(24 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Member: {(16 + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %p0 = getelementptr inbounds i8, ptr %base, i64 -16
+ %v0 = load double, ptr %p0, align 8
+
+ %p1 = getelementptr inbounds i8, ptr %base, i64 -8
+ %v1 = load double, ptr %p1, align 8
+
+ %p2 = getelementptr inbounds i8, ptr %base, i64 8
+ %v2 = load double, ptr %p2, align 8
+
+ %p3 = getelementptr inbounds i8, ptr %base, i64 16
+ %v3 = load double, ptr %p3, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+ %s2 = fadd double %s1, %v3
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s2, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 3: Cost model rejection — only 2 loads with runtime stride.
+;; Merging saves 1 check but costs 1 predicate = net 0 saving -> skip.
+;; Same result with and without the flag: 2 checks, 3 groups, no predicates.
+define void @cost_model_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; CHECK-LABEL: 'cost_model_rejection'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; CHECK-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; CHECK-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v0 = load double, ptr %p0, align 8
+
+ %v1 = load double, ptr %base, align 8
+
+ %s = fadd double %v0, %v1
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 4: Invariant + strided reads from same base -> different access ranges.
+;; A strided read {%a,+,8} and an invariant read from %a have different
+;; access ranges (8*n vs 8), so merging must NOT combine them.
+;; Same result with and without flag: 2 checks, 3 separate groups.
+define void @different_steps_no_merge(ptr %a, ptr %out, i64 %n) {
+; CHECK-LABEL: 'different_steps_no_merge'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: ptr %a
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * %n) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %a High: ((8 * %n) + %a))
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %a High: (8 + %a))
+; CHECK-NEXT: Member: %a
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+; GRP0: store to %out (write, different DepSet):
+; GRP1: strided read from %a (step=8):
+; GRP2: invariant read from %a (step=0, different from GRP1):
+entry:
+ %cmp = icmp sgt i64 %n, 2
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+
+ ; Invariant read from %a (step = 0, different from strided)
+ %v.inv = load double, ptr %a, align 8
+
+ ; Strided read: {%a,+,8}
+ %gep.a = getelementptr inbounds double, ptr %a, i64 %iv
+ %v.strided = load double, ptr %gep.a, align 8
+
+ ; Store to %out (different DepSet, triggers runtime checks)
+ %sum = fadd double %v.strided, %v.inv
+ %gep.out = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %sum, ptr %gep.out, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 5: A member with no stride term.
+;; 3 loads from %a at offsets {0, -cdj, -2*cdj}. The base load has no cdj
+;; term at all, so its cdj coefficient counts as 0. The candidate comparison
+;; must see that 0: the base member (+0) is the high bound, above -cdj and
+;; -2*cdj, even though it never mentions cdj.
+define void @coefficient_clamping_regression(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'coefficient_clamping_regression'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-2 * %cdj) + %a) High: (-16 + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(16 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'coefficient_clamping_regression'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + (-2 * %cdj) + %a) High: (-16 + (8 * %n) + (-2 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-2 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + (-1 * %cdj) + %a) High: (-16 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; load at base + 0 (no stride offset -- this is the base member)
+ %v0 = load double, ptr %base, align 8
+
+ ; load at base - cdj
+ %negcdj = sub nsw i64 0, %cdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v1 = load double, ptr %p1, align 8
+
+ ; load at base - 2*cdj
+ %neg2cdj = mul nsw i64 %cdj, -2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %neg2cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 6: Predicated accesses are rejected from stencil merging.
+;; Models the dilateKernel pattern (llvm-test-suite MicroBenchmarks/ImageProcessing/Dilate):
+;; 3 loads at {-cdj, 0, +cdj} where the -cdj and +cdj loads are conditional.
+;; Predicated loads have overapproximated SCEV bounds; merging would widen
+;; them further, causing false runtime overlap detection.
+;; Both modes: 3 checks (groups stay separate), no predicates.
+define void @predicated_access_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c1, i1 %c2) {
+; CHECK-LABEL: 'predicated_access_rejection'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * %n) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((-1 * %cdj) + %a) High: ((8 * %n) + (-1 * %cdj) + %a))
+; CHECK-NEXT: Member: {((-1 * %cdj) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (%cdj + %a) High: ((8 * %n) + %cdj + %a))
+; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: %a High: ((8 * %n) + %a))
+; CHECK-NEXT: Member: {%a,+,8}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+; 3 checks: each temp group separately vs %out (not merged due to predication):
+; 4 groups: 1 for %out, 3 separate temp groups:
+entry:
+ %cmp = icmp sgt i64 %n, 0
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; Unconditional load at base + 0
+ %v0 = load double, ptr %base, align 8
+
+ ; Conditional load at base + cdj (predicated block)
+ br i1 %c1, label %if.then1, label %if.end1
+
+if.then1:
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+ br label %if.end1
+
+if.end1:
+ %m1 = phi double [ %v1, %if.then1 ], [ %v0, %loop ]
+
+ ; Conditional load at base - cdj (predicated block)
+ %negcdj = sub nsw i64 0, %cdj
+ br i1 %c2, label %if.then2, label %if.end2
+
+if.then2:
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v2 = load double, ptr %p2, align 8
+ br label %if.end2
+
+if.end2:
+ %m2 = phi double [ %v2, %if.then2 ], [ %m1, %if.end1 ]
+
+ %s = fadd double %m1, %m2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, %n
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 7: Write pointer in a group prevents stencil merging.
+;; 2 reads from %a at offsets {0, +cdj} and 1 write to %a at offset {+2*cdj}.
+;; All three share the same base %a (same DepSet). The write prevents merging.
+;; Both modes: groups stay separate.
+define void @write_in_group_rejection(ptr %a, ptr %out, i64 %n, i64 %cdj) {
+; CHECK-LABEL: 'write_in_group_rejection'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP1:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; CHECK-NEXT: Check 4:
+; CHECK-NEXT: Comparing group GRP1:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: (16 + (2 * %cdj) + %a) High: (-16 + (2 * %cdj) + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(16 + (2 * %cdj) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; CHECK-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; CHECK-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: (16 + %cdj + %a) High: (-16 + (8 * %n) + %cdj + %a))
+; CHECK-NEXT: Member: {(16 + %cdj + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+; 5 checks: write group vs all others, %out vs the two read groups:
+; 4 groups: GRP0 (write to %a+2*cdj), GRP1 (%out), GRP2 (read %a+0), GRP3 (read %a+cdj):
+; Both modes produce identical output (write prevents merging).
+entry:
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; Read at base + 0
+ %v0 = load double, ptr %base, align 8
+
+ ; Read at base + cdj
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v1 = load double, ptr %p1, align 8
+
+ ; Write at base + 2*cdj (same DepSet as reads — prevents merging)
+ %cdj2 = mul nsw i64 %cdj, 2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj2
+ %s = fadd double %v0, %v1
+ store double %s, ptr %p2, align 8
+
+ ; Store to %out (different DepSet)
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 8: Shared stride predicate deduplication across two DepSets.
+;; Two arrays %a and %b each read at offsets {-cdj, 0, +cdj}. Store to %out.
+;; Both DepSets share the same stride %cdj, so only 1 predicate (cdj > 0)
+;; should be emitted, not 2.
+;; With merge: 2 merged groups + 1 %out group = 3 groups, 2 checks, 1 predicate.
+;; Without merge: 3 groups per array + 1 %out = 7 groups, 6 checks, no predicates.
+define void @shared_stride_predicate_dedup(ptr %a, ptr %b, ptr %out, i64 %n, i64 %cdj) {
+; MERGE-LABEL: 'shared_stride_predicate_dedup'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
+; MERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
+; MERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+; MERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; MERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + %cdj + %b))
+; MERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %cdj sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'shared_stride_predicate_dedup'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %basea = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (24 + %out) High: (-24 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(24 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %a) High: (-24 + (8 * %n) + %cdj + %a))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (24 + %a) High: (-24 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(24 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %a) High: (-24 + (8 * %n) + (-1 * %cdj) + %a))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (24 + %cdj + %b) High: (-24 + (8 * %n) + %cdj + %b))
+; NOMERGE-NEXT: Member: {(24 + %cdj + %b),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (24 + %b) High: (-24 + (8 * %n) + %b))
+; NOMERGE-NEXT: Member: {(24 + %b),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (24 + (-1 * %cdj) + %b) High: (-24 + (8 * %n) + (-1 * %cdj) + %b))
+; NOMERGE-NEXT: Member: {(24 + (-1 * %cdj) + %b),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 6
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 3, %entry ], [ %iv.next, %loop ]
+
+ ; Reads from %a at {-cdj, 0, +cdj}
+ %basea = getelementptr inbounds double, ptr %a, i64 %iv
+ %negcdj = sub nsw i64 0, %cdj
+ %pa0 = getelementptr inbounds i8, ptr %basea, i64 %negcdj
+ %va0 = load double, ptr %pa0, align 8
+ %va1 = load double, ptr %basea, align 8
+ %pa2 = getelementptr inbounds i8, ptr %basea, i64 %cdj
+ %va2 = load double, ptr %pa2, align 8
+
+ ; Reads from %b at {-cdj, 0, +cdj}
+ %baseb = getelementptr inbounds double, ptr %b, i64 %iv
+ %pb0 = getelementptr inbounds i8, ptr %baseb, i64 %negcdj
+ %vb0 = load double, ptr %pb0, align 8
+ %vb1 = load double, ptr %baseb, align 8
+ %pb2 = getelementptr inbounds i8, ptr %baseb, i64 %cdj
+ %vb2 = load double, ptr %pb2, align 8
+
+ %s0 = fadd double %va0, %va1
+ %s1 = fadd double %s0, %va2
+ %s2 = fadd double %s1, %vb0
+ %s3 = fadd double %s2, %vb1
+ %s4 = fadd double %s3, %vb2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 3
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+
+;; Test 9: Known-positive stride skips predicate emission.
+;; The stride is smax(%cdj_in, 1), which SCEV can prove is > 0.
+;; 3 loads from %a at offsets {-stride, 0, +stride}. Store to %out.
+;; With merge: 1 merged group, 1 check, NO predicates (stride known positive).
+;; Without merge: 3 separate groups + 1 %out, 3 checks, no predicates.
+define void @known_positive_stride_no_predicate(ptr %a, ptr %out, i64 %n, i64 %cdj_in) {
+; MERGE-LABEL: 'known_positive_stride_no_predicate'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: (16 + (-1 * (1 smax %cdj_in))<nsw> + %a) High: (-16 + (8 * %n) + (1 smax %cdj_in) + %a))
+; MERGE-NEXT: Member: {(16 + (1 smax %cdj_in) + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; MERGE-NEXT: Member: {(16 + (-1 * (1 smax %cdj_in))<nsw> + %a),+,8}<nw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'known_positive_stride_no_predicate'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (16 + %out) High: (-16 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(16 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (16 + (1 smax %cdj_in) + %a) High: (-16 + (8 * %n) + (1 smax %cdj_in) + %a))
+; NOMERGE-NEXT: Member: {(16 + (1 smax %cdj_in) + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (16 + %a) High: (-16 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(16 + %a),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (16 + (-1 * (1 smax %cdj_in))<nsw> + %a) High: (-16 + (8 * %n) + (-1 * (1 smax %cdj_in))<nsw> + %a))
+; NOMERGE-NEXT: Member: {(16 + (-1 * (1 smax %cdj_in))<nsw> + %a),+,8}<nw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cdj = call i64 @llvm.smax.i64(i64 %cdj_in, i64 1)
+ %cmp = icmp sgt i64 %n, 4
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 2, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; load at base - stride
+ %negcdj = sub nsw i64 0, %cdj
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %negcdj
+ %v0 = load double, ptr %p0, align 8
+
+ ; load at base
+ %v1 = load double, ptr %base, align 8
+
+ ; load at base + stride
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %cdj
+ %v2 = load double, ptr %p2, align 8
+
+ %s0 = fadd double %v0, %v1
+ %s1 = fadd double %s0, %v2
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %s1, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 2
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Test 10: loop-invariant pointer is computed in the preheader.
+;; %inv.ptr = %a + %cdj is invariant, so its GEP is outside the loop. The
+;; runtime offset %cdj keeps it in a separate group from the strided {%a,+,1}
+;; read, so we reach the predicated-access check. We must look at the load,
+;; which is in the loop, not at the pointer, which is in the preheader -
+;; otherwise we crash.
+define void @invariant_pointer_in_preheader(ptr %a, ptr %out, i64 %cdj) {
+; CHECK-LABEL: 'invariant_pointer_in_preheader'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: (64 + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; CHECK-NEXT: Member: (%cdj + %a)
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %a High: (8 + %a))
+; CHECK-NEXT: Member: {%a,+,1}<nuw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %inv.ptr = getelementptr inbounds i8, ptr %a, i64 %cdj
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sp = getelementptr inbounds i8, ptr %a, i64 %iv
+ %sv = load i8, ptr %sp
+ %sv64 = zext i8 %sv to i64
+ %iv8 = load i64, ptr %inv.ptr
+ %sum = add i64 %sv64, %iv8
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 8
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+
+;; Test 11: predicated access whose address is computed in the header.
+;; The stride GEPs are in the header, which dominates the latch, but the loads
+;; are in a conditional block guarded by %c. So the accesses are predicated and
+;; must not be merged. We must look at the load block, which is conditional,
+;; not at the GEP block, which is the header. Both modes: groups stay separate.
+define void @predicated_access_hoisted_address(ptr %a, ptr %out, i64 %n, i64 %cdj, i1 %c) {
+; CHECK-LABEL: 'predicated_access_hoisted_address'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP4:
+; CHECK-NEXT: %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: ((8 * %n) + %out))
+; CHECK-NEXT: Member: {%out,+,8}<%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((16 * %cdj) + %a) High: ((8 * %n) + (16 * %cdj) + %a))
+; CHECK-NEXT: Member: {((16 * %cdj) + %a),+,8}<%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: ((-8 * %cdj) + %a) High: ((8 * %n) + (-8 * %cdj) + %a))
+; CHECK-NEXT: Member: {((-8 * %cdj) + %a),+,8}<%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: ((8 * %cdj) + %a) High: ((8 * %n) + (8 * %cdj) + %a))
+; CHECK-NEXT: Member: {((8 * %cdj) + %a),+,8}<%loop>
+; CHECK-NEXT: Group GRP4:
+; CHECK-NEXT: (Low: %a High: ((8 * %n) + %a))
+; CHECK-NEXT: Member: {%a,+,8}<%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+ %p0 = getelementptr inbounds i64, ptr %a, i64 %iv
+ %v0 = load i64, ptr %p0
+ %i1 = add i64 %iv, %cdj
+ %p1 = getelementptr inbounds i64, ptr %a, i64 %i1
+ %i2 = sub i64 %iv, %cdj
+ %p2 = getelementptr inbounds i64, ptr %a, i64 %i2
+ %i3 = add i64 %i1, %cdj
+ %p3 = getelementptr inbounds i64, ptr %a, i64 %i3
+ br i1 %c, label %then, label %backedge
+then:
+ %v1 = load i64, ptr %p1
+ %v2 = load i64, ptr %p2
+ %v3 = load i64, ptr %p3
+ %t = add i64 %v1, %v2
+ %u = add i64 %t, %v3
+ br label %backedge
+backedge:
+ %vp = phi i64 [ %u, %then ], [ 0, %loop ]
+ %sum = add i64 %v0, %vp
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+;; Test 12: equal access ranges but different recurrence steps -> no merge.
+;; In a single-iteration loop every access spans just its element size, so the
+;; ranges are equal even though the steps differ (8 vs 16).
+define void @equal_range_different_step_no_merge(ptr %a, ptr %out, i64 %cdj) {
+; CHECK-LABEL: 'equal_range_different_step_no_merge'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %op = getelementptr inbounds i64, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %out High: (8 + %out))
+; CHECK-NEXT: Member: {%out,+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: %a High: (8 + %a))
+; CHECK-NEXT: Member: {%a,+,16}<nuw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (%cdj + %a) High: (8 + %cdj + %a))
+; CHECK-NEXT: Member: {(%cdj + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %s8 = mul i64 %iv, 8
+ %b0 = getelementptr inbounds i8, ptr %a, i64 %s8
+ %p0 = getelementptr inbounds i8, ptr %b0, i64 %cdj
+ %v0 = load i64, ptr %p0
+ %s16 = mul i64 %iv, 16
+ %p1 = getelementptr inbounds i8, ptr %a, i64 %s16
+ %v1 = load i64, ptr %p1
+ %sum = add i64 %v0, %v1
+ %op = getelementptr inbounds i64, ptr %out, i64 %iv
+ store i64 %sum, ptr %op
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 1
+ br i1 %ec, label %exit, label %loop
+exit:
+ ret void
+}
+
+;; Test 13: %gep Start/End expressions are min/max expressions.
+;; getMinusSCEV for such expressions can produce SCEVCouldNotCompute.
+define void @stencil_merge_range_could_not_compute(ptr %p, ptr %q, i64 %m) {
+; CHECK-LABEL: 'stencil_merge_range_could_not_compute'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: ptr %q
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %gep = getelementptr i8, ptr %p, i64 %off
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: ptr %q
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: ptr %p
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: %q High: (1 + %q))
+; CHECK-NEXT: Member: %q
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: ((1 + %m + %p) umin %p) High: (1 + ((1 + %m + %p) umax %p)))
+; CHECK-NEXT: Member: {%p,+,(1 + %m)}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: %p High: (1 + %p))
+; CHECK-NEXT: Member: %p
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %step = add i64 %m, 1
+ br label %loop
+
+loop:
+ %off = phi i64 [ 0, %entry ], [ %off.next, %loop ]
+ %i = phi i1 [ false, %entry ], [ true, %loop ]
+ %v0 = load i8, ptr %p
+ %gep = getelementptr i8, ptr %p, i64 %off
+ %v1 = load i8, ptr %gep
+ %s = add i8 %v0, %v1
+ store i8 %s, ptr %q
+ %off.next = add i64 %off, %step
+ br i1 %i, label %done, label %loop
+
+done:
+ ret void
+}
+
+declare i64 @llvm.smax.i64(i64, i64)
+
+;; Test 14: A stride that is itself a sum (s1 + s2).
+;; s1 = smax(s1in, 1), s2 = smax(s2in, 1) (known positive); 3 loads from %p at
+;; offsets {0, s1, s1 + s2}, store to %q.
+;; One member offset is (-1 * (s1 + s2)). decomposeStencilOffset distributes
+;; the -1, so that member is keyed on s1 and s2 like the others. It then
+;; defines the low bound alone: Low is plain %p with no umin. Both strides
+;; are known positive, so no predicate is emitted.
+;; Checks: 3 before the merge, 1 after.
+define void @stencil_merge_summed_stride(ptr %p, ptr %q, i64 %s1in, i64 %s2in, i64 %n) {
+; MERGE-LABEL: 'stencil_merge_summed_stride'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; MERGE-NEXT: Member: {%q,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %p High: (1 + (8 * %n) + (1 smax %s1in) + (1 smax %s2in) + %p))
+; MERGE-NEXT: Member: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {((1 smax %s1in) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {%p,+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((1 smax %s1in) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_summed_stride'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; NOMERGE-NEXT: Member: {%q,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + (1 smax %s2in) + %p) High: (1 + (8 * %n) + (1 smax %s1in) + (1 smax %s2in) + %p))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((1 smax %s1in) + %p) High: (1 + (8 * %n) + (1 smax %s1in) + %p))
+; NOMERGE-NEXT: Member: {((1 smax %s1in) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: %p High: (1 + (8 * %n) + %p))
+; NOMERGE-NEXT: Member: {%p,+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((1 smax %s1in) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((1 smax %s1in) + (1 smax %s2in) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %s1 = call i64 @llvm.smax.i64(i64 %s1in, i64 1)
+ %s2 = call i64 @llvm.smax.i64(i64 %s2in, i64 1)
+ %s12 = add i64 %s1, %s2
+ %p1 = getelementptr i8, ptr %p, i64 %s1
+ %p2 = getelementptr i8, ptr %p, i64 %s12
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, 8
+ %a0 = getelementptr i8, ptr %p, i64 %idx
+ %v0 = load i8, ptr %a0
+ %a1 = getelementptr i8, ptr %p1, i64 %idx
+ %v1 = load i8, ptr %a1
+ %a2 = getelementptr i8, ptr %p2, i64 %idx
+ %v2 = load i8, ptr %a2
+ %s01 = add i8 %v0, %v1
+ %s = add i8 %s01, %v2
+ %aq = getelementptr i8, ptr %q, i64 %idx
+ store i8 %s, ptr %aq
+ %iv.next = add i64 %iv, 1
+ %c = icmp eq i64 %iv, %n
+ br i1 %c, label %done, label %loop
+
+done:
+ ret void
+}
+
+
+;; Test 15: A member that uses two strides at once.
+;; 6 loads from %a: +0, -5*s1, +5*s1, -5*s2, +5*s2, and the mixed
+;; +5*s1+5*s2. Store to %out.
+;; Low is a umin over the members at -5*s1 and -5*s2: which one is lower
+;; depends on s1 and s2. High is the End of the +5*s1+5*s2 member, because it
+;; is always above every other member. No umax.
+define void @stencil_merge_mixed_member(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %s2) {
+; MERGE-LABEL: 'stencil_merge_mixed_member'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %mixed
+; MERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos5s2
+; MERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %neg5s2
+; MERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %pos5s1
+; MERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg5s1
+; MERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; MERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: ((32 + (-5 * %s2) + (ptrtoaddr ptr %a to i64)) umin (32 + (-5 * %s1) + (ptrtoaddr ptr %a to i64))) High: (-32 + (5 * %s1) + (5 * %s2) + (8 * %n) + %a))
+; MERGE-NEXT: Member: {(32 + (5 * %s1) + (5 * %s2) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (5 * %s2) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (-5 * %s2) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (5 * %s1) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + (-5 * %s1) + %a),+,8}<nw><%loop>
+; MERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_mixed_member'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %mixed
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %pos5s2
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %neg5s2
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %pos5s1
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %neg5s1
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; NOMERGE-NEXT: Against group GRP6:
+; NOMERGE-NEXT: %base = getelementptr inbounds double, ptr %a, i64 %iv
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; NOMERGE-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: (32 + (5 * %s1) + (5 * %s2) + %a) High: (-32 + (5 * %s1) + (5 * %s2) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + (5 * %s1) + (5 * %s2) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: (32 + (5 * %s2) + %a) High: (-32 + (5 * %s2) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + (5 * %s2) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: (32 + (-5 * %s2) + %a) High: (-32 + (8 * %n) + (-5 * %s2) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-5 * %s2) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: (32 + (5 * %s1) + %a) High: (-32 + (5 * %s1) + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + (5 * %s1) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: (32 + (-5 * %s1) + %a) High: (-32 + (8 * %n) + (-5 * %s1) + %a))
+; NOMERGE-NEXT: Member: {(32 + (-5 * %s1) + %a),+,8}<nw><%loop>
+; NOMERGE-NEXT: Group GRP6:
+; NOMERGE-NEXT: (Low: (32 + %a) High: (-32 + (8 * %n) + %a))
+; NOMERGE-NEXT: Member: {(32 + %a),+,8}<nuw><%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ ; +0 (the base member)
+ %v0 = load double, ptr %base, align 8
+
+ ; -5*s1
+ %neg5s1 = mul nsw i64 %s1, -5
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %neg5s1
+ %v1 = load double, ptr %p1, align 8
+
+ ; +5*s1
+ %pos5s1 = mul nsw i64 %s1, 5
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %pos5s1
+ %v2 = load double, ptr %p2, align 8
+
+ ; -5*s2
+ %neg5s2 = mul nsw i64 %s2, -5
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %neg5s2
+ %v3 = load double, ptr %p3, align 8
+
+ ; +5*s2
+ %pos5s2 = mul nsw i64 %s2, 5
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %pos5s2
+ %v4 = load double, ptr %p4, align 8
+
+ ; +5*s1 + 5*s2 (the mixed member)
+ %mixed = add nsw i64 %pos5s1, %pos5s2
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %mixed
+ %v5 = load double, ptr %p5, align 8
+
+ %t0 = fadd double %v0, %v1
+ %t1 = fadd double %t0, %v2
+ %t2 = fadd double %t1, %v3
+ %t3 = fadd double %t2, %v4
+ %t4 = fadd double %t3, %v5
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t4, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Test 16: Too many candidate members - the cost model rejects the merge.
+;; 8 loads from %a at offsets {-s1, +s1, -s2, +s2, -s3, +s3, -s4, +s4}.
+;; The strides are unrelated, so all four -s_i members can be the lowest
+;; address and all four +s_i members can be the highest. The merged bounds
+;; would need a 4-way umin and a 4-way umax.
+;; Cost: 8 checks before. After: 1 check + 4 stride predicates + 3 umin
+;; operands + 3 umax operands = 11. 11 >= 8, so no merge, even with
+;; -stencil-runtime-check-merge=force.
+define void @too_many_candidates_no_merge(ptr %a, ptr %out, i64 %n, i64 %s1, i64 %s2, i64 %s3, i64 %s4) {
+; CHECK-LABEL: 'too_many_candidates_no_merge'
+; CHECK-NEXT: loop:
+; CHECK-NEXT: Memory dependences are safe with run-time checks
+; CHECK-NEXT: Dependences:
+; CHECK-NEXT: Run-time memory checks:
+; CHECK-NEXT: Check 0:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP1:
+; CHECK-NEXT: %p7 = getelementptr inbounds i8, ptr %base, i64 %s4
+; CHECK-NEXT: Check 1:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP2:
+; CHECK-NEXT: %p6 = getelementptr inbounds i8, ptr %base, i64 %negs4
+; CHECK-NEXT: Check 2:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP3:
+; CHECK-NEXT: %p5 = getelementptr inbounds i8, ptr %base, i64 %s3
+; CHECK-NEXT: Check 3:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP4:
+; CHECK-NEXT: %p4 = getelementptr inbounds i8, ptr %base, i64 %negs3
+; CHECK-NEXT: Check 4:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP5:
+; CHECK-NEXT: %p3 = getelementptr inbounds i8, ptr %base, i64 %s2
+; CHECK-NEXT: Check 5:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP6:
+; CHECK-NEXT: %p2 = getelementptr inbounds i8, ptr %base, i64 %negs2
+; CHECK-NEXT: Check 6:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP7:
+; CHECK-NEXT: %p1 = getelementptr inbounds i8, ptr %base, i64 %s1
+; CHECK-NEXT: Check 7:
+; CHECK-NEXT: Comparing group GRP0:
+; CHECK-NEXT: %outp = getelementptr inbounds double, ptr %out, i64 %iv
+; CHECK-NEXT: Against group GRP8:
+; CHECK-NEXT: %p0 = getelementptr inbounds i8, ptr %base, i64 %negs1
+; CHECK-NEXT: Grouped accesses:
+; CHECK-NEXT: Group GRP0:
+; CHECK-NEXT: (Low: (32 + %out) High: (-32 + (8 * %n) + %out))
+; CHECK-NEXT: Member: {(32 + %out),+,8}<nuw><%loop>
+; CHECK-NEXT: Group GRP1:
+; CHECK-NEXT: (Low: (32 + %s4 + %a) High: (-32 + (8 * %n) + %s4 + %a))
+; CHECK-NEXT: Member: {(32 + %s4 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP2:
+; CHECK-NEXT: (Low: (32 + (-1 * %s4) + %a) High: (-32 + (8 * %n) + (-1 * %s4) + %a))
+; CHECK-NEXT: Member: {(32 + (-1 * %s4) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP3:
+; CHECK-NEXT: (Low: (32 + %s3 + %a) High: (-32 + (8 * %n) + %s3 + %a))
+; CHECK-NEXT: Member: {(32 + %s3 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP4:
+; CHECK-NEXT: (Low: (32 + (-1 * %s3) + %a) High: (-32 + (8 * %n) + (-1 * %s3) + %a))
+; CHECK-NEXT: Member: {(32 + (-1 * %s3) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP5:
+; CHECK-NEXT: (Low: (32 + %s2 + %a) High: (-32 + (8 * %n) + %s2 + %a))
+; CHECK-NEXT: Member: {(32 + %s2 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP6:
+; CHECK-NEXT: (Low: (32 + (-1 * %s2) + %a) High: (-32 + (8 * %n) + (-1 * %s2) + %a))
+; CHECK-NEXT: Member: {(32 + (-1 * %s2) + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP7:
+; CHECK-NEXT: (Low: (32 + %s1 + %a) High: (-32 + (8 * %n) + %s1 + %a))
+; CHECK-NEXT: Member: {(32 + %s1 + %a),+,8}<nw><%loop>
+; CHECK-NEXT: Group GRP8:
+; CHECK-NEXT: (Low: (32 + (-1 * %s1) + %a) High: (-32 + (8 * %n) + (-1 * %s1) + %a))
+; CHECK-NEXT: Member: {(32 + (-1 * %s1) + %a),+,8}<nw><%loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT: SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT: Expressions re-written:
+;
+entry:
+ %cmp = icmp sgt i64 %n, 8
+ br i1 %cmp, label %loop, label %exit
+
+loop:
+ %iv = phi i64 [ 4, %entry ], [ %iv.next, %loop ]
+ %base = getelementptr inbounds double, ptr %a, i64 %iv
+
+ %negs1 = sub nsw i64 0, %s1
+ %p0 = getelementptr inbounds i8, ptr %base, i64 %negs1
+ %v0 = load double, ptr %p0, align 8
+ %p1 = getelementptr inbounds i8, ptr %base, i64 %s1
+ %v1 = load double, ptr %p1, align 8
+
+ %negs2 = sub nsw i64 0, %s2
+ %p2 = getelementptr inbounds i8, ptr %base, i64 %negs2
+ %v2 = load double, ptr %p2, align 8
+ %p3 = getelementptr inbounds i8, ptr %base, i64 %s2
+ %v3 = load double, ptr %p3, align 8
+
+ %negs3 = sub nsw i64 0, %s3
+ %p4 = getelementptr inbounds i8, ptr %base, i64 %negs3
+ %v4 = load double, ptr %p4, align 8
+ %p5 = getelementptr inbounds i8, ptr %base, i64 %s3
+ %v5 = load double, ptr %p5, align 8
+
+ %negs4 = sub nsw i64 0, %s4
+ %p6 = getelementptr inbounds i8, ptr %base, i64 %negs4
+ %v6 = load double, ptr %p6, align 8
+ %p7 = getelementptr inbounds i8, ptr %base, i64 %s4
+ %v7 = load double, ptr %p7, align 8
+
+ %t0 = fadd double %v0, %v1
+ %t1 = fadd double %t0, %v2
+ %t2 = fadd double %t1, %v3
+ %t3 = fadd double %t2, %v4
+ %t4 = fadd double %t3, %v5
+ %t5 = fadd double %t4, %v6
+ %t6 = fadd double %t5, %v7
+
+ %outp = getelementptr inbounds double, ptr %out, i64 %iv
+ store double %t6, ptr %outp, align 8
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %sub = sub nsw i64 %n, 4
+ %cond = icmp slt i64 %iv.next, %sub
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+;; Test 17: One member's offset stays factored as (64 * (s1 + s2)).
+;; 4 loads from %p at byte offsets {0, 64*s1, 64*s2, 64*(s1+s2)}; the last
+;; offset is computed as (s1 + s2) * 64, and SCEV keeps the factored form.
+;; Stores to %q and %q2.
+;; The depth cap is not reached here, so the 64 is multiplied into the sum
+;; and the member is keyed on s1 and s2 like the others. That is enough to
+;; prove it is the highest member and %p is the lowest. So the merged bounds
+;; are simple: Low is %p, High is the End of the 64*(s1+s2) member, no umin
+;; or umax, and only two predicates, s1 > 0 and s2 > 0.
+;; Two stores, so the cost model accepts the merge: 8 checks before, 4 after.
+define void @stencil_merge_factored_diagonal(ptr %p, ptr %q, ptr %q2, i64 %s1, i64 %s2, i64 %n) {
+; MERGE-LABEL: 'stencil_merge_factored_diagonal'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %a3 = getelementptr i8, ptr %p3, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %a3 = getelementptr i8, ptr %p3, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; MERGE-NEXT: Member: {%q,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; MERGE-NEXT: Member: {%q2,+,8}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %p High: (1 + (8 * %n) + (64 * (%s1 + %s2)) + %p))
+; MERGE-NEXT: Member: {((64 * (%s1 + %s2)) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {((64 * %s2) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {%p,+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * %s2) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * (%s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-NEXT: Compare predicate: %s2 sgt) 0
+; MERGE-EMPTY:
+; MERGE-NEXT: Expressions re-written:
+;
+; NOMERGE-LABEL: 'stencil_merge_factored_diagonal'
+; NOMERGE-NEXT: loop:
+; NOMERGE-NEXT: Memory dependences are safe with run-time checks
+; NOMERGE-NEXT: Dependences:
+; NOMERGE-NEXT: Run-time memory checks:
+; NOMERGE-NEXT: Check 0:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Check 1:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %a3 = getelementptr i8, ptr %p3, i64 %idx
+; NOMERGE-NEXT: Check 2:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 3:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 4:
+; NOMERGE-NEXT: Comparing group GRP0:
+; NOMERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Check 5:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP2:
+; NOMERGE-NEXT: %a3 = getelementptr i8, ptr %p3, i64 %idx
+; NOMERGE-NEXT: Check 6:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP3:
+; NOMERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; NOMERGE-NEXT: Check 7:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP4:
+; NOMERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; NOMERGE-NEXT: Check 8:
+; NOMERGE-NEXT: Comparing group GRP1:
+; NOMERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; NOMERGE-NEXT: Against group GRP5:
+; NOMERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; NOMERGE-NEXT: Grouped accesses:
+; NOMERGE-NEXT: Group GRP0:
+; NOMERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; NOMERGE-NEXT: Member: {%q,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP1:
+; NOMERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; NOMERGE-NEXT: Member: {%q2,+,8}<%loop>
+; NOMERGE-NEXT: Group GRP2:
+; NOMERGE-NEXT: (Low: ((64 * (%s1 + %s2)) + %p) High: (1 + (8 * %n) + (64 * (%s1 + %s2)) + %p))
+; NOMERGE-NEXT: Member: {((64 * (%s1 + %s2)) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP3:
+; NOMERGE-NEXT: (Low: ((64 * %s2) + %p) High: (1 + (8 * %n) + (64 * %s2) + %p))
+; NOMERGE-NEXT: Member: {((64 * %s2) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP4:
+; NOMERGE-NEXT: (Low: ((64 * %s1) + %p) High: (1 + (8 * %n) + (64 * %s1) + %p))
+; NOMERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; NOMERGE-NEXT: Group GRP5:
+; NOMERGE-NEXT: (Low: %p High: (1 + (8 * %n) + %p))
+; NOMERGE-NEXT: Member: {%p,+,8}<%loop>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; NOMERGE-NEXT: SCEV assumptions:
+; NOMERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * %s2) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-NEXT: {((64 * (%s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
+; NOMERGE-EMPTY:
+; NOMERGE-NEXT: Expressions re-written:
+;
+entry:
+ %pos64s1 = mul i64 %s1, 64
+ %pos64s2 = mul i64 %s2, 64
+ %s12 = add i64 %s1, %s2
+ %diag = mul i64 %s12, 64
+ %p1 = getelementptr i8, ptr %p, i64 %pos64s1
+ %p2 = getelementptr i8, ptr %p, i64 %pos64s2
+ %p3 = getelementptr i8, ptr %p, i64 %diag
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %idx = mul i64 %iv, 8
+ %a0 = getelementptr i8, ptr %p, i64 %idx
+ %v0 = load i8, ptr %a0
+ %a1 = getelementptr i8, ptr %p1, i64 %idx
+ %v1 = load i8, ptr %a1
+ %a2 = getelementptr i8, ptr %p2, i64 %idx
+ %v2 = load i8, ptr %a2
+ %a3 = getelementptr i8, ptr %p3, i64 %idx
+ %v3 = load i8, ptr %a3
+ %s01 = add i8 %v0, %v1
+ %s23 = add i8 %v2, %v3
+ %s = add i8 %s01, %s23
+ %aq = getelementptr i8, ptr %q, i64 %idx
+ store i8 %s, ptr %aq
+ %aq2 = getelementptr i8, ptr %q2, i64 %idx
+ store i8 %s, ptr %aq2
+ %iv.next = add i64 %iv, 1
+ %c = icmp eq i64 %iv, %n
+ br i1 %c, label %done, label %loop
+
+done:
+ ret void
+}
+
+;; Test 18: A term at the depth cap stays one stride key.
+;; 3 loads from %p at byte offsets {0, 64*s1, 8 + 64*(s1 + s2 + 4*s3)}.
+;; Stores to %q, %q2 and %q3.
+;; In the last offset, 4*s3 sits at depth 3, the cap. It is not split into
+;; 4 times s3. The keys are s1, s2 and (4 * s3), each with a > 0 predicate,
+;; and the group still merges.
+;; Three stores, so the cost model accepts the merge: 9 checks before, 6 after.
+define void @stencil_merge_depth_cap(ptr %p, ptr %q, ptr %q2, ptr %q3, i64 %s1, i64 %s2, i64 %s3, i64 %n) {
+; MERGE-LABEL: 'stencil_merge_depth_cap'
+; MERGE-NEXT: loop:
+; MERGE-NEXT: Memory dependences are safe with run-time checks
+; MERGE-NEXT: Dependences:
+; MERGE-NEXT: Run-time memory checks:
+; MERGE-NEXT: Check 0:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Check 1:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Check 2:
+; MERGE-NEXT: Comparing group GRP0:
+; MERGE-NEXT: %aq = getelementptr i8, ptr %q, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Check 3:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Check 4:
+; MERGE-NEXT: Comparing group GRP1:
+; MERGE-NEXT: %aq2 = getelementptr i8, ptr %q2, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Check 5:
+; MERGE-NEXT: Comparing group GRP2:
+; MERGE-NEXT: %aq3 = getelementptr i8, ptr %q3, i64 %idx
+; MERGE-NEXT: Against group GRP3:
+; MERGE-NEXT: %a0 = getelementptr i8, ptr %p, i64 %idx
+; MERGE-NEXT: %a1 = getelementptr i8, ptr %p1, i64 %idx
+; MERGE-NEXT: %a2 = getelementptr i8, ptr %p2, i64 %idx
+; MERGE-NEXT: Grouped accesses:
+; MERGE-NEXT: Group GRP0:
+; MERGE-NEXT: (Low: %q High: (1 + (8 * %n) + %q))
+; MERGE-NEXT: Member: {%q,+,8}<%loop>
+; MERGE-NEXT: Group GRP1:
+; MERGE-NEXT: (Low: %q2 High: (1 + (8 * %n) + %q2))
+; MERGE-NEXT: Member: {%q2,+,8}<%loop>
+; MERGE-NEXT: Group GRP2:
+; MERGE-NEXT: (Low: %q3 High: (1 + (8 * %n) + %q3))
+; MERGE-NEXT: Member: {%q3,+,8}<%loop>
+; MERGE-NEXT: Group GRP3:
+; MERGE-NEXT: (Low: %p High: (9 + (8 * %n) + (64 * ((4 * %s3) + %s1 + %s2)) + %p))
+; MERGE-NEXT: Member: {%p,+,8}<%loop>
+; MERGE-NEXT: Member: {((64 * %s1) + %p),+,8}<%loop>
+; MERGE-NEXT: Member: {(8 + (64 * ((4 * %s3) + %s1 + %s2)) + %p),+,8}<%loop>
+; MERGE-EMPTY:
+; MERGE-NEXT: Non vectorizable stores to invariant address were not found in loop.
+; MERGE-NEXT: SCEV assumptions:
+; MERGE-NEXT: {%q,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q2,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%q3,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {(8 + (64 * ((4 * %s3) + %s1 + %s2)) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {((64 * %s1) + %p),+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: {%p,+,8}<%loop> Added Flags: <nusw>
+; MERGE-NEXT: Compare predicate: %s1 sgt) 0
+; MERGE-NEXT: Compare predicate: (4 * %s3) sgt) 0
----------------
david-arm wrote:
This is interesting. Suppose the term was actually (-4 * %s3), then we'd require this term to be > 0, which would mean that %s3 is negative! So the depth limit actually permits negative strides by accident. If %s3 was also caught separately in another term we'd have a contradiction, i.e.
%s3 sgt 0
(-4 * %s3) sgt 0
or
(8 * %s3) sgt 0
(-4 * %s3) sgt 0
then we would never enter the vector loop. It's not necessarily incorrect, but it is pointless. :) I guess we'd probably end up vectorising the loop, then a subsequent pass would just delete it. I'm not sure if there is much you can do about it though. The depth limit is there for a good reason to limit pathological cases.
https://github.com/llvm/llvm-project/pull/187252
More information about the llvm-commits
mailing list