[llvm] 4df1376 - [SLP] Add tests for affine loop address computations (NFC) (#226907)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 04:23:40 PDT 2026
Author: Tim Besard
Date: 2026-09-28T11:23:33Z
New Revision: 4df13761d078a2f3c10c4ed7fab033df116b6cdc
URL: https://github.com/llvm/llvm-project/commit/4df13761d078a2f3c10c4ed7fab033df116b6cdc
DIFF: https://github.com/llvm/llvm-project/commit/4df13761d078a2f3c10c4ed7fab033df116b6cdc.diff
LOG: [SLP] Add tests for affine loop address computations (NFC) (#226907)
Precommit tests for #226220: related affine address recurrences in a
loop whose index arithmetic SLP currently vectorizes and extracts per
lane for scalar loads and stores, which LSR would otherwise turn into
pointer increments.
`X86/strength-reducible-address.ll` has Haswell and skylake-avx512 RUN
lines. It covers strided loads, constant-offset GEPs, and a load/store
at the same address. The negative cases are loop-loaded offsets,
escaping addresses, and offsets computed in the preheader. The checks
are generated with current main.
Added:
llvm/test/Transforms/SLPVectorizer/X86/strength-reducible-address.ll
Modified:
Removed:
################################################################################
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/strength-reducible-address.ll b/llvm/test/Transforms/SLPVectorizer/X86/strength-reducible-address.ll
new file mode 100644
index 0000000000000..553b472868edb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/strength-reducible-address.ll
@@ -0,0 +1,696 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell < %s | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; The addresses are affine recurrences of the loop that
diff er by loop-invariant
+; amounts. Keep their index arithmetic scalar, so loop strength reduction can
+; replace it with pointer increments, instead of vectorizing it and extracting
+; every lane. With AVX-512, the loads are gathered together with their
+; addresses, which is not affected.
+
+define i32 @strided_gather(ptr %base, ptr %dims, i64 %n) {
+; AVX2-LABEL: define i32 @strided_gather(
+; AVX2-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; AVX2-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; AVX2-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; AVX2-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], <i64 1, i64 2, i64 3, i64 4>
+; AVX2-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX2-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[TMP3]], [[TMP6]]
+; AVX2-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP1]], [[TMP7]]
+; AVX2-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; AVX2-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; AVX2-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; AVX2-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; AVX2-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; AVX2-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; AVX2-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; AVX2-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; AVX2-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; AVX2-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; AVX2-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; AVX2-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; AVX2-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; AVX2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[L0]], i64 0
+; AVX2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[L1]], i64 1
+; AVX2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[L2]], i64 2
+; AVX2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[L3]], i64 3
+; AVX2-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[V3]])
+; AVX2-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; AVX2-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; AVX2-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX2: [[EXIT]]:
+; AVX2-NEXT: ret i32 [[ACC_NEXT]]
+;
+; AVX512-LABEL: define i32 @strided_gather(
+; AVX512-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; AVX512-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; AVX512-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; AVX512-NEXT: [[TMP0:%.*]] = insertelement <4 x ptr> poison, ptr [[BASE]], i64 0
+; AVX512-NEXT: [[TMP1:%.*]] = shufflevector <4 x ptr> [[TMP0]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: br label %[[LOOP:.*]]
+; AVX512: [[LOOP]]:
+; AVX512-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP7]], <i64 1, i64 2, i64 3, i64 4>
+; AVX512-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX512-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[TMP5]], [[TMP8]]
+; AVX512-NEXT: [[TMP10:%.*]] = add <4 x i64> [[TMP3]], [[TMP9]]
+; AVX512-NEXT: [[TMP11:%.*]] = shl <4 x i64> [[TMP10]], splat (i64 2)
+; AVX512-NEXT: [[TMP12:%.*]] = getelementptr i8, <4 x ptr> [[TMP1]], <4 x i64> [[TMP11]]
+; AVX512-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP12]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; AVX512-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP13]])
+; AVX512-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; AVX512-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; AVX512-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512: [[EXIT]]:
+; AVX512-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ %stride = load i64, ptr %dims, align 8
+ %off.ptr = getelementptr i8, ptr %dims, i64 8
+ %off = load i64, ptr %off.ptr, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %i1 = add i64 %iv, 1
+ %i2 = add i64 %iv, 2
+ %i3 = add i64 %iv, 3
+ %iv.next = add i64 %iv, 4
+ %m0 = mul i64 %stride, %i1
+ %m1 = mul i64 %stride, %i2
+ %m2 = mul i64 %stride, %i3
+ %m3 = mul i64 %stride, %iv.next
+ %a0 = add i64 %off, %m0
+ %a1 = add i64 %off, %m1
+ %a2 = add i64 %off, %m2
+ %a3 = add i64 %off, %m3
+ %s0 = shl i64 %a0, 2
+ %s1 = shl i64 %a1, 2
+ %s2 = shl i64 %a2, 2
+ %s3 = shl i64 %a3, 2
+ %p0 = getelementptr i8, ptr %base, i64 %s0
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %p2 = getelementptr i8, ptr %base, i64 %s2
+ %p3 = getelementptr i8, ptr %base, i64 %s3
+ %l0 = load i32, ptr %p0, align 4
+ %l1 = load i32, ptr %p1, align 4
+ %l2 = load i32, ptr %p2, align 4
+ %l3 = load i32, ptr %p3, align 4
+ %v0 = insertelement <4 x i32> poison, i32 %l0, i64 0
+ %v1 = insertelement <4 x i32> %v0, i32 %l1, i64 1
+ %v2 = insertelement <4 x i32> %v1, i32 %l2, i64 2
+ %v3 = insertelement <4 x i32> %v2, i32 %l3, i64 3
+ %r = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v3)
+ %acc.next = add i32 %acc, %r
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; Constant-offset GEPs between the addresses and the loads do not prevent this.
+; The loads are not gathered with AVX-512 here, so the index arithmetic would
+; be vectorized despite the extracts.
+
+define i32 @strided_gather_offset(ptr %base, ptr %dims, i64 %n) {
+; CHECK-LABEL: define i32 @strided_gather_offset(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; CHECK-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; CHECK-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], <i64 1, i64 2, i64 3, i64 4>
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP1]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; CHECK-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; CHECK-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; CHECK-NEXT: [[Q0:%.*]] = getelementptr i8, ptr [[P0]], i64 -4
+; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[Q0]], align 4
+; CHECK-NEXT: [[Q1:%.*]] = getelementptr i8, ptr [[P1]], i64 -4
+; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[Q1]], align 4
+; CHECK-NEXT: [[Q2:%.*]] = getelementptr i8, ptr [[P2]], i64 -4
+; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[Q2]], align 4
+; CHECK-NEXT: [[Q3:%.*]] = getelementptr i8, ptr [[P3]], i64 -4
+; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[Q3]], align 4
+; CHECK-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[L0]], i64 0
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[L1]], i64 1
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[L2]], i64 2
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[L3]], i64 3
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[V3]])
+; CHECK-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ %stride = load i64, ptr %dims, align 8
+ %off.ptr = getelementptr i8, ptr %dims, i64 8
+ %off = load i64, ptr %off.ptr, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %i1 = add i64 %iv, 1
+ %i2 = add i64 %iv, 2
+ %i3 = add i64 %iv, 3
+ %iv.next = add i64 %iv, 4
+ %m0 = mul i64 %stride, %i1
+ %m1 = mul i64 %stride, %i2
+ %m2 = mul i64 %stride, %i3
+ %m3 = mul i64 %stride, %iv.next
+ %a0 = add i64 %off, %m0
+ %a1 = add i64 %off, %m1
+ %a2 = add i64 %off, %m2
+ %a3 = add i64 %off, %m3
+ %s0 = shl i64 %a0, 2
+ %s1 = shl i64 %a1, 2
+ %s2 = shl i64 %a2, 2
+ %s3 = shl i64 %a3, 2
+ %p0 = getelementptr i8, ptr %base, i64 %s0
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %p2 = getelementptr i8, ptr %base, i64 %s2
+ %p3 = getelementptr i8, ptr %base, i64 %s3
+ %q0 = getelementptr i8, ptr %p0, i64 -4
+ %l0 = load i32, ptr %q0, align 4
+ %q1 = getelementptr i8, ptr %p1, i64 -4
+ %l1 = load i32, ptr %q1, align 4
+ %q2 = getelementptr i8, ptr %p2, i64 -4
+ %l2 = load i32, ptr %q2, align 4
+ %q3 = getelementptr i8, ptr %p3, i64 -4
+ %l3 = load i32, ptr %q3, align 4
+ %v0 = insertelement <4 x i32> poison, i32 %l0, i64 0
+ %v1 = insertelement <4 x i32> %v0, i32 %l1, i64 1
+ %v2 = insertelement <4 x i32> %v1, i32 %l2, i64 2
+ %v3 = insertelement <4 x i32> %v2, i32 %l3, i64 3
+ %r = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v3)
+ %acc.next = add i32 %acc, %r
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; Stores, and a load and a store of the same address, are handled the same.
+
+define void @strided_update(ptr %base, ptr %dims, i64 %n) {
+; CHECK-LABEL: define void @strided_update(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; CHECK-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; CHECK-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], <i64 1, i64 2, i64 3, i64 4>
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP1]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; CHECK-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; CHECK-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; CHECK-NEXT: [[U0:%.*]] = add i32 [[L0]], 1
+; CHECK-NEXT: store i32 [[U0]], ptr [[P0]], align 4
+; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; CHECK-NEXT: [[U1:%.*]] = add i32 [[L1]], 1
+; CHECK-NEXT: store i32 [[U1]], ptr [[P1]], align 4
+; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; CHECK-NEXT: [[U2:%.*]] = add i32 [[L2]], 1
+; CHECK-NEXT: store i32 [[U2]], ptr [[P2]], align 4
+; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; CHECK-NEXT: [[U3:%.*]] = add i32 [[L3]], 1
+; CHECK-NEXT: store i32 [[U3]], ptr [[P3]], align 4
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %stride = load i64, ptr %dims, align 8
+ %off.ptr = getelementptr i8, ptr %dims, i64 8
+ %off = load i64, ptr %off.ptr, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %i1 = add i64 %iv, 1
+ %i2 = add i64 %iv, 2
+ %i3 = add i64 %iv, 3
+ %iv.next = add i64 %iv, 4
+ %m0 = mul i64 %stride, %i1
+ %m1 = mul i64 %stride, %i2
+ %m2 = mul i64 %stride, %i3
+ %m3 = mul i64 %stride, %iv.next
+ %a0 = add i64 %off, %m0
+ %a1 = add i64 %off, %m1
+ %a2 = add i64 %off, %m2
+ %a3 = add i64 %off, %m3
+ %s0 = shl i64 %a0, 2
+ %s1 = shl i64 %a1, 2
+ %s2 = shl i64 %a2, 2
+ %s3 = shl i64 %a3, 2
+ %p0 = getelementptr i8, ptr %base, i64 %s0
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %p2 = getelementptr i8, ptr %base, i64 %s2
+ %p3 = getelementptr i8, ptr %base, i64 %s3
+ %l0 = load i32, ptr %p0, align 4
+ %u0 = add i32 %l0, 1
+ store i32 %u0, ptr %p0, align 4
+ %l1 = load i32, ptr %p1, align 4
+ %u1 = add i32 %l1, 1
+ store i32 %u1, ptr %p1, align 4
+ %l2 = load i32, ptr %p2, align 4
+ %u2 = add i32 %l2, 1
+ store i32 %u2, ptr %p2, align 4
+ %l3 = load i32, ptr %p3, align 4
+ %u3 = add i32 %l3, 1
+ store i32 %u3, ptr %p3, align 4
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The offset is loaded in the loop, so the addresses are not recurrences of
+; the loop and the index arithmetic is vectorized as before.
+
+define i32 @variant_offset(ptr %base, ptr %dims, i64 %n) {
+; AVX2-LABEL: define i32 @variant_offset(
+; AVX2-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; AVX2-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; AVX2-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; AVX2-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP4:%.*]] = add <4 x i64> [[TMP3]], <i64 1, i64 2, i64 3, i64 4>
+; AVX2-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX2-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[TMP1]], [[TMP4]]
+; AVX2-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP7]], [[TMP5]]
+; AVX2-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; AVX2-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; AVX2-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; AVX2-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; AVX2-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; AVX2-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; AVX2-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; AVX2-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; AVX2-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; AVX2-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; AVX2-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; AVX2-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; AVX2-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; AVX2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[L0]], i64 0
+; AVX2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[L1]], i64 1
+; AVX2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[L2]], i64 2
+; AVX2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[L3]], i64 3
+; AVX2-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[V3]])
+; AVX2-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; AVX2-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; AVX2-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX2: [[EXIT]]:
+; AVX2-NEXT: ret i32 [[ACC_NEXT]]
+;
+; AVX512-LABEL: define i32 @variant_offset(
+; AVX512-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; AVX512-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; AVX512-NEXT: [[TMP0:%.*]] = insertelement <4 x ptr> poison, ptr [[BASE]], i64 0
+; AVX512-NEXT: [[TMP1:%.*]] = shufflevector <4 x ptr> [[TMP0]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: br label %[[LOOP:.*]]
+; AVX512: [[LOOP]]:
+; AVX512-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; AVX512-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], <i64 1, i64 2, i64 3, i64 4>
+; AVX512-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX512-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[TMP3]], [[TMP6]]
+; AVX512-NEXT: [[TMP8:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; AVX512-NEXT: [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP10:%.*]] = add <4 x i64> [[TMP9]], [[TMP7]]
+; AVX512-NEXT: [[TMP11:%.*]] = shl <4 x i64> [[TMP10]], splat (i64 2)
+; AVX512-NEXT: [[TMP12:%.*]] = getelementptr i8, <4 x ptr> [[TMP1]], <4 x i64> [[TMP11]]
+; AVX512-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP12]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; AVX512-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP13]])
+; AVX512-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; AVX512-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; AVX512-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512: [[EXIT]]:
+; AVX512-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ %stride = load i64, ptr %dims, align 8
+ %off.ptr = getelementptr i8, ptr %dims, i64 8
+
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %off = load i64, ptr %off.ptr, align 8
+ %i1 = add i64 %iv, 1
+ %i2 = add i64 %iv, 2
+ %i3 = add i64 %iv, 3
+ %iv.next = add i64 %iv, 4
+ %m0 = mul i64 %stride, %i1
+ %m1 = mul i64 %stride, %i2
+ %m2 = mul i64 %stride, %i3
+ %m3 = mul i64 %stride, %iv.next
+ %a0 = add i64 %off, %m0
+ %a1 = add i64 %off, %m1
+ %a2 = add i64 %off, %m2
+ %a3 = add i64 %off, %m3
+ %s0 = shl i64 %a0, 2
+ %s1 = shl i64 %a1, 2
+ %s2 = shl i64 %a2, 2
+ %s3 = shl i64 %a3, 2
+ %p0 = getelementptr i8, ptr %base, i64 %s0
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %p2 = getelementptr i8, ptr %base, i64 %s2
+ %p3 = getelementptr i8, ptr %base, i64 %s3
+ %l0 = load i32, ptr %p0, align 4
+ %l1 = load i32, ptr %p1, align 4
+ %l2 = load i32, ptr %p2, align 4
+ %l3 = load i32, ptr %p3, align 4
+ %v0 = insertelement <4 x i32> poison, i32 %l0, i64 0
+ %v1 = insertelement <4 x i32> %v0, i32 %l1, i64 1
+ %v2 = insertelement <4 x i32> %v1, i32 %l2, i64 2
+ %v3 = insertelement <4 x i32> %v2, i32 %l3, i64 3
+ %r = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v3)
+ %acc.next = add i32 %acc, %r
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; One of the addresses is also passed to a call, so the index arithmetic is
+; vectorized as before.
+
+define i32 @escaping_address(ptr %base, ptr %dims, i64 %n) {
+; CHECK-LABEL: define i32 @escaping_address(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; CHECK-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; CHECK-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], <i64 1, i64 2, i64 3, i64 4>
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP1]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; CHECK-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; CHECK-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; CHECK-NEXT: call void @use(ptr [[P0]])
+; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; CHECK-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[L0]], i64 0
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[L1]], i64 1
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[L2]], i64 2
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[L3]], i64 3
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[V3]])
+; CHECK-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ %stride = load i64, ptr %dims, align 8
+ %off.ptr = getelementptr i8, ptr %dims, i64 8
+ %off = load i64, ptr %off.ptr, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %i1 = add i64 %iv, 1
+ %i2 = add i64 %iv, 2
+ %i3 = add i64 %iv, 3
+ %iv.next = add i64 %iv, 4
+ %m0 = mul i64 %stride, %i1
+ %m1 = mul i64 %stride, %i2
+ %m2 = mul i64 %stride, %i3
+ %m3 = mul i64 %stride, %iv.next
+ %a0 = add i64 %off, %m0
+ %a1 = add i64 %off, %m1
+ %a2 = add i64 %off, %m2
+ %a3 = add i64 %off, %m3
+ %s0 = shl i64 %a0, 2
+ %s1 = shl i64 %a1, 2
+ %s2 = shl i64 %a2, 2
+ %s3 = shl i64 %a3, 2
+ %p0 = getelementptr i8, ptr %base, i64 %s0
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %p2 = getelementptr i8, ptr %base, i64 %s2
+ %p3 = getelementptr i8, ptr %base, i64 %s3
+ %l0 = load i32, ptr %p0, align 4
+ call void @use(ptr %p0)
+ %l1 = load i32, ptr %p1, align 4
+ %l2 = load i32, ptr %p2, align 4
+ %l3 = load i32, ptr %p3, align 4
+ %v0 = insertelement <4 x i32> poison, i32 %l0, i64 0
+ %v1 = insertelement <4 x i32> %v0, i32 %l1, i64 1
+ %v2 = insertelement <4 x i32> %v1, i32 %l2, i64 2
+ %v3 = insertelement <4 x i32> %v2, i32 %l3, i64 3
+ %r = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v3)
+ %acc.next = add i32 %acc, %r
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; The offsets are computed before the loop, where LSR does not remove them, so
+; they are left to the cost model. Skipping the in-loop bundle lets them be
+; vectorized as once-used seeds.
+
+define i32 @preheader_offsets(ptr %base, ptr %offs, i64 %n) {
+; AVX2-LABEL: define i32 @preheader_offsets(
+; AVX2-SAME: ptr [[BASE:%.*]], ptr [[OFFS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: [[X1_PTR:%.*]] = getelementptr i8, ptr [[OFFS]], i64 8
+; AVX2-NEXT: [[X2_PTR:%.*]] = getelementptr i8, ptr [[OFFS]], i64 16
+; AVX2-NEXT: [[X3_PTR:%.*]] = getelementptr i8, ptr [[OFFS]], i64 24
+; AVX2-NEXT: [[X0:%.*]] = load i64, ptr [[OFFS]], align 8
+; AVX2-NEXT: [[X1:%.*]] = load i64, ptr [[X1_PTR]], align 8
+; AVX2-NEXT: [[X2:%.*]] = load i64, ptr [[X2_PTR]], align 8
+; AVX2-NEXT: [[X3:%.*]] = load i64, ptr [[X3_PTR]], align 8
+; AVX2-NEXT: [[Y0_PTR:%.*]] = getelementptr i8, ptr [[OFFS]], i64 32
+; AVX2-NEXT: [[Y1_PTR:%.*]] = getelementptr i8, ptr [[OFFS]], i64 40
+; AVX2-NEXT: [[Y2_PTR:%.*]] = getelementptr i8, ptr [[OFFS]], i64 48
+; AVX2-NEXT: [[Y3_PTR:%.*]] = getelementptr i8, ptr [[OFFS]], i64 56
+; AVX2-NEXT: [[Y0:%.*]] = load i64, ptr [[Y0_PTR]], align 8
+; AVX2-NEXT: [[Y1:%.*]] = load i64, ptr [[Y1_PTR]], align 8
+; AVX2-NEXT: [[Y2:%.*]] = load i64, ptr [[Y2_PTR]], align 8
+; AVX2-NEXT: [[Y3:%.*]] = load i64, ptr [[Y3_PTR]], align 8
+; AVX2-NEXT: [[T0:%.*]] = add i64 [[X0]], [[Y0]]
+; AVX2-NEXT: [[T1:%.*]] = add i64 [[X1]], [[Y1]]
+; AVX2-NEXT: [[T2:%.*]] = add i64 [[X2]], [[Y2]]
+; AVX2-NEXT: [[T3:%.*]] = add i64 [[X3]], [[Y3]]
+; AVX2-NEXT: [[O0:%.*]] = shl i64 [[T0]], 2
+; AVX2-NEXT: [[O1:%.*]] = shl i64 [[T1]], 2
+; AVX2-NEXT: [[O2:%.*]] = shl i64 [[T2]], 2
+; AVX2-NEXT: [[O3:%.*]] = shl i64 [[T3]], 2
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[A0:%.*]] = add i64 [[IV]], [[O0]]
+; AVX2-NEXT: [[A1:%.*]] = add i64 [[IV]], [[O1]]
+; AVX2-NEXT: [[A2:%.*]] = add i64 [[IV]], [[O2]]
+; AVX2-NEXT: [[A3:%.*]] = add i64 [[IV]], [[O3]]
+; AVX2-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[A0]]
+; AVX2-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[A1]]
+; AVX2-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[A2]]
+; AVX2-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[A3]]
+; AVX2-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; AVX2-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; AVX2-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; AVX2-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; AVX2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[L0]], i64 0
+; AVX2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[L1]], i64 1
+; AVX2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[L2]], i64 2
+; AVX2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[L3]], i64 3
+; AVX2-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[V3]])
+; AVX2-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; AVX2-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX2-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; AVX2-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX2: [[EXIT]]:
+; AVX2-NEXT: ret i32 [[ACC_NEXT]]
+;
+; AVX512-LABEL: define i32 @preheader_offsets(
+; AVX512-SAME: ptr [[BASE:%.*]], ptr [[OFFS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: [[Y0_PTR:%.*]] = getelementptr i8, ptr [[OFFS]], i64 32
+; AVX512-NEXT: [[TMP0:%.*]] = load <4 x i64>, ptr [[OFFS]], align 8
+; AVX512-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[Y0_PTR]], align 8
+; AVX512-NEXT: [[TMP2:%.*]] = add <4 x i64> [[TMP0]], [[TMP1]]
+; AVX512-NEXT: [[TMP3:%.*]] = shl <4 x i64> [[TMP2]], splat (i64 2)
+; AVX512-NEXT: [[TMP4:%.*]] = insertelement <4 x ptr> poison, ptr [[BASE]], i64 0
+; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <4 x ptr> [[TMP4]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: br label %[[LOOP:.*]]
+; AVX512: [[LOOP]]:
+; AVX512-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP7]], [[TMP3]]
+; AVX512-NEXT: [[TMP9:%.*]] = getelementptr i8, <4 x ptr> [[TMP5]], <4 x i64> [[TMP8]]
+; AVX512-NEXT: [[TMP10:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP9]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; AVX512-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP10]])
+; AVX512-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; AVX512-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX512-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; AVX512-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512: [[EXIT]]:
+; AVX512-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ %x1.ptr = getelementptr i8, ptr %offs, i64 8
+ %x2.ptr = getelementptr i8, ptr %offs, i64 16
+ %x3.ptr = getelementptr i8, ptr %offs, i64 24
+ %x0 = load i64, ptr %offs, align 8
+ %x1 = load i64, ptr %x1.ptr, align 8
+ %x2 = load i64, ptr %x2.ptr, align 8
+ %x3 = load i64, ptr %x3.ptr, align 8
+ %y0.ptr = getelementptr i8, ptr %offs, i64 32
+ %y1.ptr = getelementptr i8, ptr %offs, i64 40
+ %y2.ptr = getelementptr i8, ptr %offs, i64 48
+ %y3.ptr = getelementptr i8, ptr %offs, i64 56
+ %y0 = load i64, ptr %y0.ptr, align 8
+ %y1 = load i64, ptr %y1.ptr, align 8
+ %y2 = load i64, ptr %y2.ptr, align 8
+ %y3 = load i64, ptr %y3.ptr, align 8
+ %t0 = add i64 %x0, %y0
+ %t1 = add i64 %x1, %y1
+ %t2 = add i64 %x2, %y2
+ %t3 = add i64 %x3, %y3
+ %o0 = shl i64 %t0, 2
+ %o1 = shl i64 %t1, 2
+ %o2 = shl i64 %t2, 2
+ %o3 = shl i64 %t3, 2
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %a0 = add i64 %iv, %o0
+ %a1 = add i64 %iv, %o1
+ %a2 = add i64 %iv, %o2
+ %a3 = add i64 %iv, %o3
+ %p0 = getelementptr i8, ptr %base, i64 %a0
+ %p1 = getelementptr i8, ptr %base, i64 %a1
+ %p2 = getelementptr i8, ptr %base, i64 %a2
+ %p3 = getelementptr i8, ptr %base, i64 %a3
+ %l0 = load i32, ptr %p0, align 4
+ %l1 = load i32, ptr %p1, align 4
+ %l2 = load i32, ptr %p2, align 4
+ %l3 = load i32, ptr %p3, align 4
+ %v0 = insertelement <4 x i32> poison, i32 %l0, i64 0
+ %v1 = insertelement <4 x i32> %v0, i32 %l1, i64 1
+ %v2 = insertelement <4 x i32> %v1, i32 %l2, i64 2
+ %v3 = insertelement <4 x i32> %v2, i32 %l3, i64 3
+ %r = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v3)
+ %acc.next = add i32 %acc, %r
+ %iv.next = add i64 %iv, 4
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+declare void @use(ptr)
More information about the llvm-commits
mailing list