[llvm-branch-commits] [llvm] [llubi] Add support for vp.strided.load and vp.strided.store (PR #241715)
Luke Lau via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 11 23:02:41 PST 2026
https://github.com/lukel97 created https://github.com/llvm/llvm-project/pull/241715
Building on top of https://github.com/llvm/llvm-project/pull/241710, these are defined in terms of vp.gather and vp.scatter
Assisted-by: Opus 5.5
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From 46c80cf16fbafe6d5a5d46de975ed8a97c38dc8f Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 12 Nov 2026 15:00:26 +0800
Subject: [PATCH] [llubi] Add support for vp.strided.load and vp.strided.store
Building on top of https://github.com/llvm/llvm-project/pull/241710, these are defined in terms of vp.gather and vp.scatter
Assisted-by: Opus 5.5
---
llvm/test/tools/llubi/intr_vp_strided_load.ll | 37 +++++++++
.../tools/llubi/intr_vp_strided_load_ub.ll | 83 +++++++++++++++++++
.../test/tools/llubi/intr_vp_strided_store.ll | 47 +++++++++++
.../tools/llubi/intr_vp_strided_store_ub.ll | 72 ++++++++++++++++
llvm/tools/llubi/lib/Interpreter.cpp | 37 +++++++++
5 files changed, 276 insertions(+)
create mode 100644 llvm/test/tools/llubi/intr_vp_strided_load.ll
create mode 100644 llvm/test/tools/llubi/intr_vp_strided_load_ub.ll
create mode 100644 llvm/test/tools/llubi/intr_vp_strided_store.ll
create mode 100644 llvm/test/tools/llubi/intr_vp_strided_store_ub.ll
diff --git a/llvm/test/tools/llubi/intr_vp_strided_load.ll b/llvm/test/tools/llubi/intr_vp_strided_load.ll
new file mode 100644
index 0000000000000..5a4386a6a1930
--- /dev/null
+++ b/llvm/test/tools/llubi/intr_vp_strided_load.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llubi_test_checks.py UTC_ARGS: --version 6
+; RUN: llubi --verbose --vscale=2 < %s 2>&1 | FileCheck %s
+
+ at a = global [8 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7]
+
+define void @main() {
+ %load_all = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 8, <4 x i1> splat (i1 true), i32 4)
+ %load_evl = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 8, <4 x i1> splat (i1 true), i32 2)
+ %load_mask = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 8, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, i32 3)
+
+ ; The stride is signed, and is sign extended to the index width.
+ %end = getelementptr i32, ptr @a, i64 7
+ %load_reverse = call <4 x i32> @llvm.experimental.vp.strided.load(ptr %end, i64 -4, <4 x i1> splat (i1 true), i32 4)
+ %load_reverse_i32 = call <4 x i32> @llvm.experimental.vp.strided.load(ptr %end, i32 -8, <4 x i1> splat (i1 true), i32 4)
+ %load_splat = call <4 x i32> @llvm.experimental.vp.strided.load(ptr %end, i64 0, <4 x i1> splat (i1 true), i32 4)
+
+ ; Lanes at or past %evl are disabled even if their mask lane is poison, so
+ ; their addresses may be out of bounds.
+ %load_tail = call <4 x i32> @llvm.experimental.vp.strided.load(ptr %end, i64 4, <4 x i1> <i1 true, i1 poison, i1 poison, i1 poison>, i32 1)
+ %load_none = call <4 x i32> @llvm.experimental.vp.strided.load(ptr poison, i64 poison, <4 x i1> splat (i1 true), i32 0)
+
+ %load_scalable = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 4, <vscale x 2 x i1> splat (i1 true), i32 3)
+ ret void
+}
+; CHECK: Entering function: main
+; CHECK-NEXT: %load_all = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 8, <4 x i1> splat (i1 true), i32 4) => { i32 0, i32 2, i32 4, i32 6 }
+; CHECK-NEXT: %load_evl = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 8, <4 x i1> splat (i1 true), i32 2) => { i32 0, i32 2, poison, poison }
+; CHECK-NEXT: %load_mask = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 8, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, i32 3) => { i32 0, poison, i32 4, poison }
+; CHECK-NEXT: %end = getelementptr i32, ptr @a, i64 7 => ptr 0x2C [@a + 28]
+; CHECK-NEXT: %load_reverse = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr %end, i64 -4, <4 x i1> splat (i1 true), i32 4) => { i32 7, i32 6, i32 5, i32 4 }
+; CHECK-NEXT: %load_reverse_i32 = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i32(ptr %end, i32 -8, <4 x i1> splat (i1 true), i32 4) => { i32 7, i32 5, i32 3, i32 1 }
+; CHECK-NEXT: %load_splat = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr %end, i64 0, <4 x i1> splat (i1 true), i32 4) => { i32 7, i32 7, i32 7, i32 7 }
+; CHECK-NEXT: %load_tail = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr %end, i64 4, <4 x i1> <i1 true, i1 poison, i1 poison, i1 poison>, i32 1) => { i32 7, poison, poison, poison }
+; CHECK-NEXT: %load_none = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr poison, i64 poison, <4 x i1> splat (i1 true), i32 0) => { poison, poison, poison, poison }
+; CHECK-NEXT: %load_scalable = call <vscale x 2 x i32> @llvm.experimental.vp.strided.load.nxv2i32.p0.i64(ptr @a, i64 4, <vscale x 2 x i1> splat (i1 true), i32 3) => { i32 0, i32 1, i32 2, poison }
+; CHECK-NEXT: ret void
+; CHECK-NEXT: Exiting function: main
diff --git a/llvm/test/tools/llubi/intr_vp_strided_load_ub.ll b/llvm/test/tools/llubi/intr_vp_strided_load_ub.ll
new file mode 100644
index 0000000000000..268cf1825dd3b
--- /dev/null
+++ b/llvm/test/tools/llubi/intr_vp_strided_load_ub.ll
@@ -0,0 +1,83 @@
+; RUN: not llubi --verbose --entry-function=evl_too_large < %s 2>&1 | FileCheck %s --check-prefix=EVL
+; RUN: not llubi --verbose --entry-function=poison_evl < %s 2>&1 | FileCheck %s --check-prefix=POISON-EVL
+; RUN: not llubi --verbose --entry-function=poison_mask_lane < %s 2>&1 | FileCheck %s --check-prefix=POISON-MASK-LANE
+; RUN: not llubi --verbose --entry-function=poison_stride < %s 2>&1 | FileCheck %s --check-prefix=POISON-STRIDE
+; RUN: not llubi --verbose --entry-function=out_of_bounds < %s 2>&1 | FileCheck %s --check-prefix=OOB
+; RUN: not llubi --verbose --entry-function=misaligned_stride < %s 2>&1 | FileCheck %s --check-prefix=MISALIGNED-STRIDE
+; RUN: not llubi --verbose --entry-function=align_attr < %s 2>&1 | FileCheck %s --check-prefix=ALIGN-ATTR
+
+ at a = global [8 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7]
+
+define void @evl_too_large() {
+; EVL: Entering function: evl_too_large
+; EVL-NEXT: Stacktrace:
+; EVL-NEXT: #0 %r = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 4, <4 x i1> splat (i1 true), i32 5) at @evl_too_large <stdin>:{{[0-9]+}}
+; EVL-NEXT: Immediate UB detected: llvm.experimental.vp.strided.load with explicit vector length 5 greater than the number of elements 4.
+; EVL-NEXT: error: Execution of function 'evl_too_large' failed.
+ %r = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 4, <4 x i1> splat (i1 true), i32 5)
+ ret void
+}
+
+define void @poison_evl() {
+; POISON-EVL: Entering function: poison_evl
+; POISON-EVL-NEXT: Stacktrace:
+; POISON-EVL-NEXT: #0 %r = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 4, <4 x i1> splat (i1 true), i32 poison) at @poison_evl <stdin>:{{[0-9]+}}
+; POISON-EVL-NEXT: Immediate UB detected: llvm.experimental.vp.strided.load with poison explicit vector length.
+; POISON-EVL-NEXT: error: Execution of function 'poison_evl' failed.
+ %r = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 4, <4 x i1> splat (i1 true), i32 poison)
+ ret void
+}
+
+define void @poison_mask_lane() {
+; POISON-MASK-LANE: Entering function: poison_mask_lane
+; POISON-MASK-LANE-NEXT: Stacktrace:
+; POISON-MASK-LANE-NEXT: #0 %r = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 4, <4 x i1> <i1 true, i1 poison, i1 false, i1 false>, i32 2) at @poison_mask_lane <stdin>:{{[0-9]+}}
+; POISON-MASK-LANE-NEXT: Immediate UB detected: Poison mask lane in llvm.experimental.vp.strided.load.
+; POISON-MASK-LANE-NEXT: error: Execution of function 'poison_mask_lane' failed.
+ %r = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 4, <4 x i1> <i1 true, i1 poison, i1 false, i1 false>, i32 2)
+ ret void
+}
+
+; A poison stride makes every lane's pointer poison.
+define void @poison_stride() {
+; POISON-STRIDE: Entering function: poison_stride
+; POISON-STRIDE-NEXT: Stacktrace:
+; POISON-STRIDE-NEXT: #0 %r = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 poison, <4 x i1> splat (i1 true), i32 1) at @poison_stride <stdin>:{{[0-9]+}}
+; POISON-STRIDE-NEXT: Immediate UB detected: Invalid memory access with a poison pointer.
+; POISON-STRIDE-NEXT: error: Execution of function 'poison_stride' failed.
+ %r = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 poison, <4 x i1> splat (i1 true), i32 1)
+ ret void
+}
+
+define void @out_of_bounds() {
+; OOB: Entering function: out_of_bounds
+; OOB-NEXT: Stacktrace:
+; OOB-NEXT: #0 %r = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 16, <4 x i1> splat (i1 true), i32 3) at @out_of_bounds <stdin>:{{[0-9]+}}
+; OOB-NEXT: Immediate UB detected: Memory access is out of bounds. Accessed size: 4, Address: 0x30, Object base: 0x10, Object size: 32.
+; OOB-NEXT: error: Execution of function 'out_of_bounds' failed.
+ %r = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 16, <4 x i1> splat (i1 true), i32 3)
+ ret void
+}
+
+; Without an align attribute each lane's pointer must have the ABI alignment of
+; i32, so a stride of 2 makes lane 1's pointer poison.
+define void @misaligned_stride() {
+; MISALIGNED-STRIDE: Entering function: misaligned_stride
+; MISALIGNED-STRIDE-NEXT: Stacktrace:
+; MISALIGNED-STRIDE-NEXT: #0 %r = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr @a, i64 2, <4 x i1> splat (i1 true), i32 2) at @misaligned_stride <stdin>:{{[0-9]+}}
+; MISALIGNED-STRIDE-NEXT: Immediate UB detected: Invalid memory access with a poison pointer.
+; MISALIGNED-STRIDE-NEXT: error: Execution of function 'misaligned_stride' failed.
+ %r = call <4 x i32> @llvm.experimental.vp.strided.load(ptr @a, i64 2, <4 x i1> splat (i1 true), i32 2)
+ ret void
+}
+
+; The align attribute on the base pointer applies to every lane's pointer.
+define void @align_attr() {
+; ALIGN-ATTR: Entering function: align_attr
+; ALIGN-ATTR-NEXT: Stacktrace:
+; ALIGN-ATTR-NEXT: #0 %r = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 8 @a, i64 4, <4 x i1> splat (i1 true), i32 2) at @align_attr <stdin>:{{[0-9]+}}
+; ALIGN-ATTR-NEXT: Immediate UB detected: Invalid memory access with a poison pointer.
+; ALIGN-ATTR-NEXT: error: Execution of function 'align_attr' failed.
+ %r = call <4 x i32> @llvm.experimental.vp.strided.load(ptr align 8 @a, i64 4, <4 x i1> splat (i1 true), i32 2)
+ ret void
+}
diff --git a/llvm/test/tools/llubi/intr_vp_strided_store.ll b/llvm/test/tools/llubi/intr_vp_strided_store.ll
new file mode 100644
index 0000000000000..0c10b2520f49b
--- /dev/null
+++ b/llvm/test/tools/llubi/intr_vp_strided_store.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llubi_test_checks.py UTC_ARGS: --version 6
+; RUN: llubi --verbose --vscale=2 < %s 2>&1 | FileCheck %s
+
+ at a = global [8 x i32] zeroinitializer
+ at c = constant [4 x i32] [i32 0, i32 1, i32 2, i32 3]
+
+define void @main() {
+ call void @llvm.experimental.vp.strided.store(<4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr @a, i64 8, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, i32 3)
+ %after_partial = load <8 x i32>, ptr @a, align 4
+
+ ; The stride is signed.
+ %end = getelementptr i32, ptr @a, i64 7
+ call void @llvm.experimental.vp.strided.store(<4 x i32> <i32 10, i32 11, i32 12, i32 13>, ptr %end, i64 -4, <4 x i1> splat (i1 true), i32 4)
+ %after_reverse = load <8 x i32>, ptr @a, align 4
+
+ ; With a zero stride the lanes are stored in order, so the last one wins.
+ %p1 = getelementptr i32, ptr @a, i64 1
+ call void @llvm.experimental.vp.strided.store(<4 x i32> <i32 20, i32 21, i32 22, i32 23>, ptr %p1, i64 0, <4 x i1> splat (i1 true), i32 4)
+ %after_splat = load <8 x i32>, ptr @a, align 4
+
+ ; Lanes at or past %evl are disabled even if their mask lane is poison.
+ call void @llvm.experimental.vp.strided.store(<4 x i32> splat (i32 30), ptr %end, i64 4, <4 x i1> <i1 true, i1 poison, i1 poison, i1 poison>, i32 1)
+ %after_poison_tail = load <8 x i32>, ptr @a, align 4
+
+ ; Disabled lanes are not accessed, so constant memory is fine.
+ call void @llvm.experimental.vp.strided.store(<4 x i32> poison, ptr @c, i64 4, <4 x i1> splat (i1 true), i32 0)
+
+ call void @llvm.experimental.vp.strided.store(<vscale x 2 x i32> splat (i32 40), ptr @a, i64 8, <vscale x 2 x i1> splat (i1 true), i32 3)
+ %after_scalable = load <8 x i32>, ptr @a, align 4
+ ret void
+}
+; CHECK: Entering function: main
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr @a, i64 8, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, i32 3)
+; CHECK-NEXT: %after_partial = load <8 x i32>, ptr @a, align 4 => { i32 1, i32 0, i32 0, i32 0, i32 3, i32 0, i32 0, i32 0 }
+; CHECK-NEXT: %end = getelementptr i32, ptr @a, i64 7 => ptr 0x2C [@a + 28]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> <i32 10, i32 11, i32 12, i32 13>, ptr %end, i64 -4, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: %after_reverse = load <8 x i32>, ptr @a, align 4 => { i32 1, i32 0, i32 0, i32 0, i32 13, i32 12, i32 11, i32 10 }
+; CHECK-NEXT: %p1 = getelementptr i32, ptr @a, i64 1 => ptr 0x14 [@a + 4]
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> <i32 20, i32 21, i32 22, i32 23>, ptr %p1, i64 0, <4 x i1> splat (i1 true), i32 4)
+; CHECK-NEXT: %after_splat = load <8 x i32>, ptr @a, align 4 => { i32 1, i32 23, i32 0, i32 0, i32 13, i32 12, i32 11, i32 10 }
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> splat (i32 30), ptr %end, i64 4, <4 x i1> <i1 true, i1 poison, i1 poison, i1 poison>, i32 1)
+; CHECK-NEXT: %after_poison_tail = load <8 x i32>, ptr @a, align 4 => { i32 1, i32 23, i32 0, i32 0, i32 13, i32 12, i32 11, i32 30 }
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> poison, ptr @c, i64 4, <4 x i1> splat (i1 true), i32 0)
+; CHECK-NEXT: call void @llvm.experimental.vp.strided.store.nxv2i32.p0.i64(<vscale x 2 x i32> splat (i32 40), ptr @a, i64 8, <vscale x 2 x i1> splat (i1 true), i32 3)
+; CHECK-NEXT: %after_scalable = load <8 x i32>, ptr @a, align 4 => { i32 40, i32 23, i32 40, i32 0, i32 40, i32 12, i32 11, i32 30 }
+; CHECK-NEXT: ret void
+; CHECK-NEXT: Exiting function: main
diff --git a/llvm/test/tools/llubi/intr_vp_strided_store_ub.ll b/llvm/test/tools/llubi/intr_vp_strided_store_ub.ll
new file mode 100644
index 0000000000000..1a0ead27e1c2d
--- /dev/null
+++ b/llvm/test/tools/llubi/intr_vp_strided_store_ub.ll
@@ -0,0 +1,72 @@
+; RUN: not llubi --verbose --entry-function=evl_too_large < %s 2>&1 | FileCheck %s --check-prefix=EVL
+; RUN: not llubi --verbose --entry-function=poison_evl < %s 2>&1 | FileCheck %s --check-prefix=POISON-EVL
+; RUN: not llubi --verbose --entry-function=poison_mask_lane < %s 2>&1 | FileCheck %s --check-prefix=POISON-MASK-LANE
+; RUN: not llubi --verbose --entry-function=poison_stride < %s 2>&1 | FileCheck %s --check-prefix=POISON-STRIDE
+; RUN: not llubi --verbose --entry-function=constant_mem < %s 2>&1 | FileCheck %s --check-prefix=CONSTANT-MEM
+; RUN: not llubi --verbose --entry-function=misaligned_stride < %s 2>&1 | FileCheck %s --check-prefix=MISALIGNED-STRIDE
+
+ at a = global [8 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7]
+ at c = constant [4 x i32] [i32 0, i32 1, i32 2, i32 3]
+
+define void @evl_too_large() {
+; EVL: Entering function: evl_too_large
+; EVL-NEXT: Stacktrace:
+; EVL-NEXT: #0 call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> zeroinitializer, ptr @a, i64 4, <4 x i1> splat (i1 true), i32 5) at @evl_too_large <stdin>:{{[0-9]+}}
+; EVL-NEXT: Immediate UB detected: llvm.experimental.vp.strided.store with explicit vector length 5 greater than the number of elements 4.
+; EVL-NEXT: error: Execution of function 'evl_too_large' failed.
+ call void @llvm.experimental.vp.strided.store(<4 x i32> zeroinitializer, ptr @a, i64 4, <4 x i1> splat (i1 true), i32 5)
+ ret void
+}
+
+define void @poison_evl() {
+; POISON-EVL: Entering function: poison_evl
+; POISON-EVL-NEXT: Stacktrace:
+; POISON-EVL-NEXT: #0 call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> zeroinitializer, ptr @a, i64 4, <4 x i1> splat (i1 true), i32 poison) at @poison_evl <stdin>:{{[0-9]+}}
+; POISON-EVL-NEXT: Immediate UB detected: llvm.experimental.vp.strided.store with poison explicit vector length.
+; POISON-EVL-NEXT: error: Execution of function 'poison_evl' failed.
+ call void @llvm.experimental.vp.strided.store(<4 x i32> zeroinitializer, ptr @a, i64 4, <4 x i1> splat (i1 true), i32 poison)
+ ret void
+}
+
+define void @poison_mask_lane() {
+; POISON-MASK-LANE: Entering function: poison_mask_lane
+; POISON-MASK-LANE-NEXT: Stacktrace:
+; POISON-MASK-LANE-NEXT: #0 call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> zeroinitializer, ptr @a, i64 4, <4 x i1> <i1 true, i1 poison, i1 false, i1 false>, i32 2) at @poison_mask_lane <stdin>:{{[0-9]+}}
+; POISON-MASK-LANE-NEXT: Immediate UB detected: Poison mask lane in llvm.experimental.vp.strided.store.
+; POISON-MASK-LANE-NEXT: error: Execution of function 'poison_mask_lane' failed.
+ call void @llvm.experimental.vp.strided.store(<4 x i32> zeroinitializer, ptr @a, i64 4, <4 x i1> <i1 true, i1 poison, i1 false, i1 false>, i32 2)
+ ret void
+}
+
+; A poison stride makes every lane's pointer poison.
+define void @poison_stride() {
+; POISON-STRIDE: Entering function: poison_stride
+; POISON-STRIDE-NEXT: Stacktrace:
+; POISON-STRIDE-NEXT: #0 call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> zeroinitializer, ptr @a, i64 poison, <4 x i1> splat (i1 true), i32 1) at @poison_stride <stdin>:{{[0-9]+}}
+; POISON-STRIDE-NEXT: Immediate UB detected: Invalid memory access with a poison pointer.
+; POISON-STRIDE-NEXT: error: Execution of function 'poison_stride' failed.
+ call void @llvm.experimental.vp.strided.store(<4 x i32> zeroinitializer, ptr @a, i64 poison, <4 x i1> splat (i1 true), i32 1)
+ ret void
+}
+
+define void @constant_mem() {
+; CONSTANT-MEM: Entering function: constant_mem
+; CONSTANT-MEM-NEXT: Stacktrace:
+; CONSTANT-MEM-NEXT: #0 call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> zeroinitializer, ptr @c, i64 4, <4 x i1> splat (i1 true), i32 1) at @constant_mem <stdin>:{{[0-9]+}}
+; CONSTANT-MEM-NEXT: Immediate UB detected: Try to write to a constant memory object: ptr 0x34 [@c].
+; CONSTANT-MEM-NEXT: error: Execution of function 'constant_mem' failed.
+ call void @llvm.experimental.vp.strided.store(<4 x i32> zeroinitializer, ptr @c, i64 4, <4 x i1> splat (i1 true), i32 1)
+ ret void
+}
+
+; Without an align attribute each lane's pointer must have the ABI alignment of
+; i32, so a stride of 2 makes lane 1's pointer poison.
+define void @misaligned_stride() {
+; MISALIGNED-STRIDE: Entering function: misaligned_stride
+; MISALIGNED-STRIDE-NEXT: Stacktrace:
+; MISALIGNED-STRIDE-NEXT: #0 call void @llvm.experimental.vp.strided.store.v4i32.p0.i64(<4 x i32> zeroinitializer, ptr @a, i64 2, <4 x i1> splat (i1 true), i32 2) at @misaligned_stride <stdin>:{{[0-9]+}}
+; MISALIGNED-STRIDE-NEXT: Immediate UB detected: Invalid memory access with a poison pointer.
+; MISALIGNED-STRIDE-NEXT: error: Execution of function 'misaligned_stride' failed.
+ call void @llvm.experimental.vp.strided.store(<4 x i32> zeroinitializer, ptr @a, i64 2, <4 x i1> splat (i1 true), i32 2)
+ ret void
+}
diff --git a/llvm/tools/llubi/lib/Interpreter.cpp b/llvm/tools/llubi/lib/Interpreter.cpp
index 06e8d93d3a099..b8af4fef93e97 100644
--- a/llvm/tools/llubi/lib/Interpreter.cpp
+++ b/llvm/tools/llubi/lib/Interpreter.cpp
@@ -1075,6 +1075,35 @@ class InstExecutor : public InstVisitor<InstExecutor, void>,
std::move(Data), Mode, IsStore);
}
+ /// Implements llvm.experimental.vp.strided.{load,store} as a vp.gather or
+ /// vp.scatter of the pointers <Ptr, Ptr + Stride, Ptr + 2 * Stride, ...>.
+ /// \p Stride is signed and the arithmetic wraps in the pointer index type. An
+ /// align attribute on the base pointer applies to every lane's pointer.
+ AnyValue callVPStridedLoadStoreIntrinsic(CallBase &CB, const AnyValue &Ptr,
+ const AnyValue &Stride,
+ const AnyValue &Mask,
+ const AnyValue &EVL, AnyValue Data,
+ bool IsStore) {
+ const unsigned IndexBW =
+ DL.getIndexTypeSizeInBits(CB.getArgOperand(IsStore ? 1 : 0)->getType());
+ std::vector<AnyValue> LanePtrs(Mask.asAggregate().size(),
+ AnyValue::poison());
+ if (!Stride.isPoison()) {
+ const APInt StrideIdx = Stride.asInteger().sextOrTrunc(IndexBW);
+ for (auto [I, LanePtr] : enumerate(LanePtrs)) {
+ AnyValue AccumulatedOffset = AnyValue::poison();
+ LanePtr = Ctx.computePtrAdd(Ptr, StrideIdx * APInt(IndexBW, I),
+ GEPNoWrapFlags::none(), AccumulatedOffset);
+ }
+ }
+ AnyValue Ptrs(std::move(LanePtrs));
+ if (MaybeAlign Alignment = CB.getParamAlign(IsStore ? 1 : 0))
+ applyAlignAttr(Ptrs, *Alignment);
+ return callVPLoadStoreIntrinsic(CB, std::move(Ptrs), Mask, EVL,
+ std::move(Data), MaskedAddrMode::PerLane,
+ IsStore);
+ }
+
/// Implements llvm.matrix.column.major.store. Column C of \p Data is stored
/// as a <Rows x ElemTy> vector at an inbounds nuw GEP of C * \p Stride
/// elements from \p Ptr.
@@ -2086,6 +2115,14 @@ class InstExecutor : public InstVisitor<InstExecutor, void>,
return callVPLoadStoreIntrinsic(CB, Args[1], Args[2], Args[3], Args[0],
MaskedAddrMode::PerLane,
/*IsStore=*/true);
+ case Intrinsic::experimental_vp_strided_load:
+ return callVPStridedLoadStoreIntrinsic(
+ CB, Args[0], Args[1], Args[2], Args[3],
+ AnyValue::getPoisonValue(Ctx, RetTy), /*IsStore=*/false);
+ case Intrinsic::experimental_vp_strided_store:
+ return callVPStridedLoadStoreIntrinsic(CB, Args[1], Args[2], Args[3],
+ Args[4], Args[0],
+ /*IsStore=*/true);
case Intrinsic::matrix_transpose: {
const auto &Vec = Args[0].asAggregate();
const unsigned Rows =
More information about the llvm-branch-commits
mailing list