[llvm] [SLP] Add tests for affine loop address computations (NFC) (PR #226907)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 00:40:13 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Tim Besard (maleadt)
<details>
<summary>Changes</summary>
Precommit tests for #<!-- -->226220: related affine address recurrences in a loop whose index arithmetic SLP currently vectorizes and extracts per lane for scalar loads and stores, which LSR would otherwise turn into pointer increments.
`X86/strength-reducible-address.ll` has Haswell and skylake-avx512 RUN lines. It covers strided loads, constant-offset GEPs, and a load/store at the same address. The negative cases are loop-loaded offsets, escaping addresses, and offsets computed in the preheader. The checks are generated with current main.
---
Patch is 35.76 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/226907.diff
1 Files Affected:
- (added) llvm/test/Transforms/SLPVectorizer/X86/strength-reducible-address.ll (+696)
``````````diff
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/strength-reducible-address.ll b/llvm/test/Transforms/SLPVectorizer/X86/strength-reducible-address.ll
new file mode 100644
index 0000000000000..553b472868edb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/strength-reducible-address.ll
@@ -0,0 +1,696 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell < %s | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; The addresses are affine recurrences of the loop that differ by loop-invariant
+; amounts. Keep their index arithmetic scalar, so loop strength reduction can
+; replace it with pointer increments, instead of vectorizing it and extracting
+; every lane. With AVX-512, the loads are gathered together with their
+; addresses, which is not affected.
+
+define i32 @strided_gather(ptr %base, ptr %dims, i64 %n) {
+; AVX2-LABEL: define i32 @strided_gather(
+; AVX2-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; AVX2-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; AVX2-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; AVX2-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], <i64 1, i64 2, i64 3, i64 4>
+; AVX2-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX2-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[TMP3]], [[TMP6]]
+; AVX2-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP1]], [[TMP7]]
+; AVX2-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; AVX2-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; AVX2-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; AVX2-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; AVX2-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; AVX2-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; AVX2-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; AVX2-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; AVX2-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; AVX2-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; AVX2-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; AVX2-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; AVX2-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; AVX2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[L0]], i64 0
+; AVX2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[L1]], i64 1
+; AVX2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[L2]], i64 2
+; AVX2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[L3]], i64 3
+; AVX2-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[V3]])
+; AVX2-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; AVX2-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; AVX2-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX2: [[EXIT]]:
+; AVX2-NEXT: ret i32 [[ACC_NEXT]]
+;
+; AVX512-LABEL: define i32 @strided_gather(
+; AVX512-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; AVX512-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; AVX512-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; AVX512-NEXT: [[TMP0:%.*]] = insertelement <4 x ptr> poison, ptr [[BASE]], i64 0
+; AVX512-NEXT: [[TMP1:%.*]] = shufflevector <4 x ptr> [[TMP0]], <4 x ptr> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: br label %[[LOOP:.*]]
+; AVX512: [[LOOP]]:
+; AVX512-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX512-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP7]], <i64 1, i64 2, i64 3, i64 4>
+; AVX512-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX512-NEXT: [[TMP9:%.*]] = mul <4 x i64> [[TMP5]], [[TMP8]]
+; AVX512-NEXT: [[TMP10:%.*]] = add <4 x i64> [[TMP3]], [[TMP9]]
+; AVX512-NEXT: [[TMP11:%.*]] = shl <4 x i64> [[TMP10]], splat (i64 2)
+; AVX512-NEXT: [[TMP12:%.*]] = getelementptr i8, <4 x ptr> [[TMP1]], <4 x i64> [[TMP11]]
+; AVX512-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP12]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; AVX512-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP13]])
+; AVX512-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; AVX512-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; AVX512-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512: [[EXIT]]:
+; AVX512-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ %stride = load i64, ptr %dims, align 8
+ %off.ptr = getelementptr i8, ptr %dims, i64 8
+ %off = load i64, ptr %off.ptr, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %i1 = add i64 %iv, 1
+ %i2 = add i64 %iv, 2
+ %i3 = add i64 %iv, 3
+ %iv.next = add i64 %iv, 4
+ %m0 = mul i64 %stride, %i1
+ %m1 = mul i64 %stride, %i2
+ %m2 = mul i64 %stride, %i3
+ %m3 = mul i64 %stride, %iv.next
+ %a0 = add i64 %off, %m0
+ %a1 = add i64 %off, %m1
+ %a2 = add i64 %off, %m2
+ %a3 = add i64 %off, %m3
+ %s0 = shl i64 %a0, 2
+ %s1 = shl i64 %a1, 2
+ %s2 = shl i64 %a2, 2
+ %s3 = shl i64 %a3, 2
+ %p0 = getelementptr i8, ptr %base, i64 %s0
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %p2 = getelementptr i8, ptr %base, i64 %s2
+ %p3 = getelementptr i8, ptr %base, i64 %s3
+ %l0 = load i32, ptr %p0, align 4
+ %l1 = load i32, ptr %p1, align 4
+ %l2 = load i32, ptr %p2, align 4
+ %l3 = load i32, ptr %p3, align 4
+ %v0 = insertelement <4 x i32> poison, i32 %l0, i64 0
+ %v1 = insertelement <4 x i32> %v0, i32 %l1, i64 1
+ %v2 = insertelement <4 x i32> %v1, i32 %l2, i64 2
+ %v3 = insertelement <4 x i32> %v2, i32 %l3, i64 3
+ %r = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v3)
+ %acc.next = add i32 %acc, %r
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; Constant-offset GEPs between the addresses and the loads do not prevent this.
+; The loads are not gathered with AVX-512 here, so the index arithmetic would
+; be vectorized despite the extracts.
+
+define i32 @strided_gather_offset(ptr %base, ptr %dims, i64 %n) {
+; CHECK-LABEL: define i32 @strided_gather_offset(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; CHECK-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; CHECK-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], <i64 1, i64 2, i64 3, i64 4>
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP1]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; CHECK-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; CHECK-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; CHECK-NEXT: [[Q0:%.*]] = getelementptr i8, ptr [[P0]], i64 -4
+; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[Q0]], align 4
+; CHECK-NEXT: [[Q1:%.*]] = getelementptr i8, ptr [[P1]], i64 -4
+; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[Q1]], align 4
+; CHECK-NEXT: [[Q2:%.*]] = getelementptr i8, ptr [[P2]], i64 -4
+; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[Q2]], align 4
+; CHECK-NEXT: [[Q3:%.*]] = getelementptr i8, ptr [[P3]], i64 -4
+; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[Q3]], align 4
+; CHECK-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[L0]], i64 0
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[L1]], i64 1
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[L2]], i64 2
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[L3]], i64 3
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[V3]])
+; CHECK-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[R]]
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i32 [[ACC_NEXT]]
+;
+entry:
+ %stride = load i64, ptr %dims, align 8
+ %off.ptr = getelementptr i8, ptr %dims, i64 8
+ %off = load i64, ptr %off.ptr, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ]
+ %i1 = add i64 %iv, 1
+ %i2 = add i64 %iv, 2
+ %i3 = add i64 %iv, 3
+ %iv.next = add i64 %iv, 4
+ %m0 = mul i64 %stride, %i1
+ %m1 = mul i64 %stride, %i2
+ %m2 = mul i64 %stride, %i3
+ %m3 = mul i64 %stride, %iv.next
+ %a0 = add i64 %off, %m0
+ %a1 = add i64 %off, %m1
+ %a2 = add i64 %off, %m2
+ %a3 = add i64 %off, %m3
+ %s0 = shl i64 %a0, 2
+ %s1 = shl i64 %a1, 2
+ %s2 = shl i64 %a2, 2
+ %s3 = shl i64 %a3, 2
+ %p0 = getelementptr i8, ptr %base, i64 %s0
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %p2 = getelementptr i8, ptr %base, i64 %s2
+ %p3 = getelementptr i8, ptr %base, i64 %s3
+ %q0 = getelementptr i8, ptr %p0, i64 -4
+ %l0 = load i32, ptr %q0, align 4
+ %q1 = getelementptr i8, ptr %p1, i64 -4
+ %l1 = load i32, ptr %q1, align 4
+ %q2 = getelementptr i8, ptr %p2, i64 -4
+ %l2 = load i32, ptr %q2, align 4
+ %q3 = getelementptr i8, ptr %p3, i64 -4
+ %l3 = load i32, ptr %q3, align 4
+ %v0 = insertelement <4 x i32> poison, i32 %l0, i64 0
+ %v1 = insertelement <4 x i32> %v0, i32 %l1, i64 1
+ %v2 = insertelement <4 x i32> %v1, i32 %l2, i64 2
+ %v3 = insertelement <4 x i32> %v2, i32 %l3, i64 3
+ %r = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v3)
+ %acc.next = add i32 %acc, %r
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret i32 %acc.next
+}
+
+; Stores, and a load and a store of the same address, are handled the same.
+
+define void @strided_update(ptr %base, ptr %dims, i64 %n) {
+; CHECK-LABEL: define void @strided_update(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; CHECK-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; CHECK-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i64> [[TMP5]], <i64 1, i64 2, i64 3, i64 4>
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = mul <4 x i64> [[TMP3]], [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP1]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; CHECK-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; CHECK-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; CHECK-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; CHECK-NEXT: [[U0:%.*]] = add i32 [[L0]], 1
+; CHECK-NEXT: store i32 [[U0]], ptr [[P0]], align 4
+; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; CHECK-NEXT: [[U1:%.*]] = add i32 [[L1]], 1
+; CHECK-NEXT: store i32 [[U1]], ptr [[P1]], align 4
+; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; CHECK-NEXT: [[U2:%.*]] = add i32 [[L2]], 1
+; CHECK-NEXT: store i32 [[U2]], ptr [[P2]], align 4
+; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; CHECK-NEXT: [[U3:%.*]] = add i32 [[L3]], 1
+; CHECK-NEXT: store i32 [[U3]], ptr [[P3]], align 4
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %stride = load i64, ptr %dims, align 8
+ %off.ptr = getelementptr i8, ptr %dims, i64 8
+ %off = load i64, ptr %off.ptr, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %i1 = add i64 %iv, 1
+ %i2 = add i64 %iv, 2
+ %i3 = add i64 %iv, 3
+ %iv.next = add i64 %iv, 4
+ %m0 = mul i64 %stride, %i1
+ %m1 = mul i64 %stride, %i2
+ %m2 = mul i64 %stride, %i3
+ %m3 = mul i64 %stride, %iv.next
+ %a0 = add i64 %off, %m0
+ %a1 = add i64 %off, %m1
+ %a2 = add i64 %off, %m2
+ %a3 = add i64 %off, %m3
+ %s0 = shl i64 %a0, 2
+ %s1 = shl i64 %a1, 2
+ %s2 = shl i64 %a2, 2
+ %s3 = shl i64 %a3, 2
+ %p0 = getelementptr i8, ptr %base, i64 %s0
+ %p1 = getelementptr i8, ptr %base, i64 %s1
+ %p2 = getelementptr i8, ptr %base, i64 %s2
+ %p3 = getelementptr i8, ptr %base, i64 %s3
+ %l0 = load i32, ptr %p0, align 4
+ %u0 = add i32 %l0, 1
+ store i32 %u0, ptr %p0, align 4
+ %l1 = load i32, ptr %p1, align 4
+ %u1 = add i32 %l1, 1
+ store i32 %u1, ptr %p1, align 4
+ %l2 = load i32, ptr %p2, align 4
+ %u2 = add i32 %l2, 1
+ store i32 %u2, ptr %p2, align 4
+ %l3 = load i32, ptr %p3, align 4
+ %u3 = add i32 %l3, 1
+ store i32 %u3, ptr %p3, align 4
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+; The offset is loaded in the loop, so the addresses are not recurrences of
+; the loop and the index arithmetic is vectorized as before.
+
+define i32 @variant_offset(ptr %base, ptr %dims, i64 %n) {
+; AVX2-LABEL: define i32 @variant_offset(
+; AVX2-SAME: ptr [[BASE:%.*]], ptr [[DIMS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: [[STRIDE:%.*]] = load i64, ptr [[DIMS]], align 8
+; AVX2-NEXT: [[OFF_PTR:%.*]] = getelementptr i8, ptr [[DIMS]], i64 8
+; AVX2-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[OFF:%.*]] = load i64, ptr [[OFF_PTR]], align 8
+; AVX2-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[IV]], i64 0
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP4:%.*]] = add <4 x i64> [[TMP3]], <i64 1, i64 2, i64 3, i64 4>
+; AVX2-NEXT: [[IV_NEXT]] = add i64 [[IV]], 4
+; AVX2-NEXT: [[TMP5:%.*]] = mul <4 x i64> [[TMP1]], [[TMP4]]
+; AVX2-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[OFF]], i64 0
+; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP8:%.*]] = add <4 x i64> [[TMP7]], [[TMP5]]
+; AVX2-NEXT: [[TMP9:%.*]] = shl <4 x i64> [[TMP8]], splat (i64 2)
+; AVX2-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP9]], i64 0
+; AVX2-NEXT: [[P0:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP10]]
+; AVX2-NEXT: [[TMP11:%.*]] = extractelement <4 x i64> [[TMP9]], i64 1
+; AVX2-NEXT: [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP11]]
+; AVX2-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP9]], i64 2
+; AVX2-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP12]]
+; AVX2-NEXT: [[TMP13:%.*]] = extractelement <4 x i64> [[TMP9]], i64 3
+; AVX2-NEXT: [[P3:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP13]]
+; AVX2-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; AVX2-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; AVX2-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; AVX2-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; AVX2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[L0]], i64 0
+; AVX2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[L1]], i64 1
+; AVX2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[L2]], i64 2
+; AVX2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[L3]], i64 3
+; AVX2-NEXT: [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/226907
More information about the llvm-commits
mailing list