[clang] [llvm] [AArch64] Add unprivileged non-temporal pairs LD/St (PR #240875)
Martin Wehking via cfe-commits
cfe-commits at lists.llvm.org
Wed Nov 11 01:02:23 PST 2026
https://github.com/MartinWehking updated https://github.com/llvm/llvm-project/pull/240875
>From b7e9541efe57d348f1c0d62b83c86ca5667da522 Mon Sep 17 00:00:00 2001
From: Martin Wehking <martin.wehking at arm.com>
Date: Thu, 5 Nov 2026 11:48:50 +0000
Subject: [PATCH 1/2] [AArch64] Add unprivileged non-temporal pairs LD/St
Add the intrinsics proposed in https://github.com/ARM-software/acle/pull/471.
These are vldtnq_*_x2 and vsttnq_*_x2
Lower them to their corresponding AArch64 load/store instructions:
LDTNP and STTNP
Add also new IR intrinsics for them.
Tests created by codex.
---
clang/include/clang/Basic/arm_neon.td | 6 +
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 36 ++
clang/test/CodeGen/AArch64/neon-lsui.c | 597 ++++++++++++++++++++
llvm/include/llvm/IR/IntrinsicsAArch64.td | 4 +
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 16 +
llvm/test/CodeGen/AArch64/neon-lsui.ll | 287 ++++++++++
6 files changed, 946 insertions(+)
create mode 100644 clang/test/CodeGen/AArch64/neon-lsui.c
create mode 100644 llvm/test/CodeGen/AArch64/neon-lsui.ll
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index b74d3805e950a9..e824709adcc968 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -2245,3 +2245,9 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8,neon" in {
def FSCALE_V128 : WInst<"vscale", "..(.S)", "QdQfQh">;
def FSCALE_V64 : WInst<"vscale", "(.q)(.q)(.qS)", "fh">;
}
+
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "lsui,neon" in {
+ // unprivileged non-temporal pair load/stores
+ def VLDTN_X2 : SInst<"vldtn_x2", "2(c*!)", "QcQsQiQlQUcQUsQUiQUlQfQhQdQbQm">;
+ def VSTTN_X2 : SInst<"vsttn_x2", "v(*)(2!)", "QcQsQiQlQUcQUsQUiQUlQfQhQdQbQm">;
+}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 5d4b87b75abe7b..80523226c45dc9 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -6848,6 +6848,42 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld4_lane");
return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
}
+ case NEON::BI__builtin_neon_vldtnq_s8_x2:
+ case NEON::BI__builtin_neon_vldtnq_s16_x2:
+ case NEON::BI__builtin_neon_vldtnq_s32_x2:
+ case NEON::BI__builtin_neon_vldtnq_s64_x2:
+ case NEON::BI__builtin_neon_vldtnq_u8_x2:
+ case NEON::BI__builtin_neon_vldtnq_u16_x2:
+ case NEON::BI__builtin_neon_vldtnq_u32_x2:
+ case NEON::BI__builtin_neon_vldtnq_u64_x2:
+ case NEON::BI__builtin_neon_vldtnq_f16_x2:
+ case NEON::BI__builtin_neon_vldtnq_bf16_x2:
+ case NEON::BI__builtin_neon_vldtnq_f32_x2:
+ case NEON::BI__builtin_neon_vldtnq_f64_x2:
+ case NEON::BI__builtin_neon_vldtnq_mf8_x2: {
+ llvm::Type *Tys[2] = {VTy, Ops[1]->getType()};
+ Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_vldtn_x2, Tys);
+ Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vldtn_x2");
+ return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
+ }
+ case NEON::BI__builtin_neon_vsttnq_s8_x2:
+ case NEON::BI__builtin_neon_vsttnq_s16_x2:
+ case NEON::BI__builtin_neon_vsttnq_s32_x2:
+ case NEON::BI__builtin_neon_vsttnq_s64_x2:
+ case NEON::BI__builtin_neon_vsttnq_u8_x2:
+ case NEON::BI__builtin_neon_vsttnq_u16_x2:
+ case NEON::BI__builtin_neon_vsttnq_u32_x2:
+ case NEON::BI__builtin_neon_vsttnq_u64_x2:
+ case NEON::BI__builtin_neon_vsttnq_f16_x2:
+ case NEON::BI__builtin_neon_vsttnq_bf16_x2:
+ case NEON::BI__builtin_neon_vsttnq_f32_x2:
+ case NEON::BI__builtin_neon_vsttnq_f64_x2:
+ case NEON::BI__builtin_neon_vsttnq_mf8_x2: {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
+ llvm::Type *Tys[2] = {VTy, Ops[2]->getType()};
+ return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_vsttn_x2, Tys),
+ Ops, "");
+ }
case NEON::BI__builtin_neon_vst2_v:
case NEON::BI__builtin_neon_vst2q_v: {
std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
diff --git a/clang/test/CodeGen/AArch64/neon-lsui.c b/clang/test/CodeGen/AArch64/neon-lsui.c
new file mode 100644
index 00000000000000..e975598a1d496b
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/neon-lsui.c
@@ -0,0 +1,597 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +v9.6a \
+// RUN: -target-feature +lsui -O2 \
+// RUN: -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-LE
+// RUN: %clang_cc1 -triple aarch64_be-none-linux-gnu -target-feature +v9.6a \
+// RUN: -target-feature +lsui -O2 \
+// RUN: -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BE
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+// Signed and unsigned variants share IR types. mf8 uses the byte-vector type.
+
+// CHECK-LE-LABEL: define dso_local %struct.int8x16x2_t @test_vldtnq_s8_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldtn.x2.v16i8.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT8X16X2_T:%.*]] poison, <16 x i8> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_INT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.int8x16x2_t @test_vldtnq_s8_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldtn.x2.v16i8.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VLDTN_X2_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <16 x i8> [[VLDTN_X2_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT8X16X2_T:%.*]] poison, <16 x i8> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_INT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+int8x16x2_t test_vldtnq_s8_x2(const int8_t *ptr) {
+ return vldtnq_s8_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.int16x8x2_t @test_vldtnq_s16_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldtn.x2.v8i16.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT16X8X2_T:%.*]] poison, <8 x i16> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i16> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_INT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.int16x8x2_t @test_vldtnq_s16_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldtn.x2.v8i16.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[VLDTN_X2_FCA_0_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <8 x i16> [[VLDTN_X2_FCA_1_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT16X8X2_T:%.*]] poison, <8 x i16> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i16> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_INT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+int16x8x2_t test_vldtnq_s16_x2(const int16_t *ptr) {
+ return vldtnq_s16_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.int32x4x2_t @test_vldtnq_s32_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldtn.x2.v4i32.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT32X4X2_T:%.*]] poison, <4 x i32> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x i32> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_INT32X4X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.int32x4x2_t @test_vldtnq_s32_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldtn.x2.v4i32.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VLDTN_X2_FCA_0_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <4 x i32> [[VLDTN_X2_FCA_1_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT32X4X2_T:%.*]] poison, <4 x i32> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x i32> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_INT32X4X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+int32x4x2_t test_vldtnq_s32_x2(const int32_t *ptr) {
+ return vldtnq_s32_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.int64x2x2_t @test_vldtnq_s64_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldtn.x2.v2i64.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT64X2X2_T:%.*]] poison, <2 x i64> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x i64> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_INT64X2X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.int64x2x2_t @test_vldtnq_s64_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldtn.x2.v2i64.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[VLDTN_X2_FCA_0_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <2 x i64> [[VLDTN_X2_FCA_1_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT64X2X2_T:%.*]] poison, <2 x i64> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x i64> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_INT64X2X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+int64x2x2_t test_vldtnq_s64_x2(const int64_t *ptr) {
+ return vldtnq_s64_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.uint8x16x2_t @test_vldtnq_u8_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldtn.x2.v16i8.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT8X16X2_T:%.*]] poison, <16 x i8> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_UINT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.uint8x16x2_t @test_vldtnq_u8_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldtn.x2.v16i8.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VLDTN_X2_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <16 x i8> [[VLDTN_X2_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT8X16X2_T:%.*]] poison, <16 x i8> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_UINT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+uint8x16x2_t test_vldtnq_u8_x2(const uint8_t *ptr) {
+ return vldtnq_u8_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.uint16x8x2_t @test_vldtnq_u16_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldtn.x2.v8i16.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT16X8X2_T:%.*]] poison, <8 x i16> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i16> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_UINT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.uint16x8x2_t @test_vldtnq_u16_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldtn.x2.v8i16.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[VLDTN_X2_FCA_0_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <8 x i16> [[VLDTN_X2_FCA_1_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT16X8X2_T:%.*]] poison, <8 x i16> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i16> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_UINT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+uint16x8x2_t test_vldtnq_u16_x2(const uint16_t *ptr) {
+ return vldtnq_u16_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.uint32x4x2_t @test_vldtnq_u32_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldtn.x2.v4i32.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT32X4X2_T:%.*]] poison, <4 x i32> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x i32> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_UINT32X4X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.uint32x4x2_t @test_vldtnq_u32_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldtn.x2.v4i32.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VLDTN_X2_FCA_0_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <4 x i32> [[VLDTN_X2_FCA_1_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT32X4X2_T:%.*]] poison, <4 x i32> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x i32> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_UINT32X4X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+uint32x4x2_t test_vldtnq_u32_x2(const uint32_t *ptr) {
+ return vldtnq_u32_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.uint64x2x2_t @test_vldtnq_u64_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldtn.x2.v2i64.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT64X2X2_T:%.*]] poison, <2 x i64> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x i64> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_UINT64X2X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.uint64x2x2_t @test_vldtnq_u64_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldtn.x2.v2i64.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[VLDTN_X2_FCA_0_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <2 x i64> [[VLDTN_X2_FCA_1_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT64X2X2_T:%.*]] poison, <2 x i64> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x i64> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_UINT64X2X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+uint64x2x2_t test_vldtnq_u64_x2(const uint64_t *ptr) {
+ return vldtnq_u64_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.float16x8x2_t @test_vldtnq_f16_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <8 x half>, <8 x half> } @llvm.aarch64.neon.vldtn.x2.v8f16.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x half>, <8 x half> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x half>, <8 x half> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T:%.*]] poison, <8 x half> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x half> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_FLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.float16x8x2_t @test_vldtnq_f16_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <8 x half>, <8 x half> } @llvm.aarch64.neon.vldtn.x2.v8f16.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x half>, <8 x half> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x half>, <8 x half> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x half> [[VLDTN_X2_FCA_0_EXTRACT]], <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <8 x half> [[VLDTN_X2_FCA_1_EXTRACT]], <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T:%.*]] poison, <8 x half> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x half> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_FLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+float16x8x2_t test_vldtnq_f16_x2(const float16_t *ptr) {
+ return vldtnq_f16_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.bfloat16x8x2_t @test_vldtnq_bf16_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.vldtn.x2.v8bf16.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.bfloat16x8x2_t @test_vldtnq_bf16_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.vldtn.x2.v8bf16.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x bfloat> [[VLDTN_X2_FCA_0_EXTRACT]], <8 x bfloat> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <8 x bfloat> [[VLDTN_X2_FCA_1_EXTRACT]], <8 x bfloat> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+bfloat16x8x2_t test_vldtnq_bf16_x2(const bfloat16_t *ptr) {
+ return vldtnq_bf16_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.float32x4x2_t @test_vldtnq_f32_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.aarch64.neon.vldtn.x2.v4f32.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x float>, <4 x float> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x float>, <4 x float> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT32X4X2_T:%.*]] poison, <4 x float> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x float> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_FLOAT32X4X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.float32x4x2_t @test_vldtnq_f32_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.aarch64.neon.vldtn.x2.v4f32.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x float>, <4 x float> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x float>, <4 x float> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x float> [[VLDTN_X2_FCA_0_EXTRACT]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <4 x float> [[VLDTN_X2_FCA_1_EXTRACT]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT32X4X2_T:%.*]] poison, <4 x float> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x float> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_FLOAT32X4X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+float32x4x2_t test_vldtnq_f32_x2(const float32_t *ptr) {
+ return vldtnq_f32_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.float64x2x2_t @test_vldtnq_f64_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <2 x double>, <2 x double> } @llvm.aarch64.neon.vldtn.x2.v2f64.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x double>, <2 x double> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x double>, <2 x double> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT64X2X2_T:%.*]] poison, <2 x double> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x double> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_FLOAT64X2X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.float64x2x2_t @test_vldtnq_f64_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <2 x double>, <2 x double> } @llvm.aarch64.neon.vldtn.x2.v2f64.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x double>, <2 x double> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x double>, <2 x double> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x double> [[VLDTN_X2_FCA_0_EXTRACT]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <2 x double> [[VLDTN_X2_FCA_1_EXTRACT]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT64X2X2_T:%.*]] poison, <2 x double> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x double> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_FLOAT64X2X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+float64x2x2_t test_vldtnq_f64_x2(const float64_t *ptr) {
+ return vldtnq_f64_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vldtnq_mf8_x2(
+// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: [[VLDTN_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldtn.x2.v16i8.p0(ptr [[PTR]])
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 0
+// CHECK-LE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 1
+// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLDTN_X2_FCA_0_EXTRACT]], 0, 0
+// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLDTN_X2_FCA_1_EXTRACT]], 0, 1
+// CHECK-LE-NEXT: ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+// CHECK-BE-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vldtnq_mf8_x2(
+// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[VLDTN_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldtn.x2.v16i8.p0(ptr [[PTR]])
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 0
+// CHECK-BE-NEXT: [[VLDTN_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDTN_X2]], 1
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VLDTN_X2_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <16 x i8> [[VLDTN_X2_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[SHUFFLE]], 0, 0
+// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[SHUFFLE9]], 0, 1
+// CHECK-BE-NEXT: ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vldtnq_mf8_x2(const mfloat8_t *ptr) {
+ return vldtnq_mf8_x2(ptr);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_s8_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <16 x i8> noundef [[FIRST:%.*]], <16 x i8> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v16i8.p0(<16 x i8> [[FIRST]], <16 x i8> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_s8_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <16 x i8> noundef [[FIRST:%.*]], <16 x i8> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[FIRST]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <16 x i8> [[SECOND]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v16i8.p0(<16 x i8> [[SHUFFLE]], <16 x i8> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_s8_x2(int8_t *ptr, int8x16_t first, int8x16_t second) {
+ int8x16x2_t value = {{first, second}};
+ vsttnq_s8_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_s16_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <8 x i16> noundef [[FIRST:%.*]], <8 x i16> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v8i16.p0(<8 x i16> [[FIRST]], <8 x i16> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_s16_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <8 x i16> noundef [[FIRST:%.*]], <8 x i16> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[FIRST]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <8 x i16> [[SECOND]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v8i16.p0(<8 x i16> [[SHUFFLE]], <8 x i16> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_s16_x2(int16_t *ptr, int16x8_t first, int16x8_t second) {
+ int16x8x2_t value = {{first, second}};
+ vsttnq_s16_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_s32_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <4 x i32> noundef [[FIRST:%.*]], <4 x i32> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v4i32.p0(<4 x i32> [[FIRST]], <4 x i32> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_s32_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <4 x i32> noundef [[FIRST:%.*]], <4 x i32> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FIRST]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <4 x i32> [[SECOND]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v4i32.p0(<4 x i32> [[SHUFFLE]], <4 x i32> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_s32_x2(int32_t *ptr, int32x4_t first, int32x4_t second) {
+ int32x4x2_t value = {{first, second}};
+ vsttnq_s32_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_s64_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <2 x i64> noundef [[FIRST:%.*]], <2 x i64> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v2i64.p0(<2 x i64> [[FIRST]], <2 x i64> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_s64_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <2 x i64> noundef [[FIRST:%.*]], <2 x i64> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[FIRST]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <2 x i64> [[SECOND]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v2i64.p0(<2 x i64> [[SHUFFLE]], <2 x i64> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_s64_x2(int64_t *ptr, int64x2_t first, int64x2_t second) {
+ int64x2x2_t value = {{first, second}};
+ vsttnq_s64_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_u8_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <16 x i8> noundef [[FIRST:%.*]], <16 x i8> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v16i8.p0(<16 x i8> [[FIRST]], <16 x i8> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_u8_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <16 x i8> noundef [[FIRST:%.*]], <16 x i8> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[FIRST]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <16 x i8> [[SECOND]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v16i8.p0(<16 x i8> [[SHUFFLE]], <16 x i8> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_u8_x2(uint8_t *ptr, uint8x16_t first, uint8x16_t second) {
+ uint8x16x2_t value = {{first, second}};
+ vsttnq_u8_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_u16_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <8 x i16> noundef [[FIRST:%.*]], <8 x i16> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v8i16.p0(<8 x i16> [[FIRST]], <8 x i16> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_u16_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <8 x i16> noundef [[FIRST:%.*]], <8 x i16> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[FIRST]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <8 x i16> [[SECOND]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v8i16.p0(<8 x i16> [[SHUFFLE]], <8 x i16> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_u16_x2(uint16_t *ptr, uint16x8_t first, uint16x8_t second) {
+ uint16x8x2_t value = {{first, second}};
+ vsttnq_u16_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_u32_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <4 x i32> noundef [[FIRST:%.*]], <4 x i32> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v4i32.p0(<4 x i32> [[FIRST]], <4 x i32> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_u32_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <4 x i32> noundef [[FIRST:%.*]], <4 x i32> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FIRST]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <4 x i32> [[SECOND]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v4i32.p0(<4 x i32> [[SHUFFLE]], <4 x i32> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_u32_x2(uint32_t *ptr, uint32x4_t first, uint32x4_t second) {
+ uint32x4x2_t value = {{first, second}};
+ vsttnq_u32_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_u64_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <2 x i64> noundef [[FIRST:%.*]], <2 x i64> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v2i64.p0(<2 x i64> [[FIRST]], <2 x i64> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_u64_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <2 x i64> noundef [[FIRST:%.*]], <2 x i64> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[FIRST]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <2 x i64> [[SECOND]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v2i64.p0(<2 x i64> [[SHUFFLE]], <2 x i64> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_u64_x2(uint64_t *ptr, uint64x2_t first, uint64x2_t second) {
+ uint64x2x2_t value = {{first, second}};
+ vsttnq_u64_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_f16_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <8 x half> noundef [[FIRST:%.*]], <8 x half> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v8f16.p0(<8 x half> [[FIRST]], <8 x half> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_f16_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <8 x half> noundef [[FIRST:%.*]], <8 x half> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[TMP0:%.*]] = shufflevector <8 x half> [[FIRST]], <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[TMP1:%.*]] = shufflevector <8 x half> [[SECOND]], <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v8f16.p0(<8 x half> [[TMP0]], <8 x half> [[TMP1]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_f16_x2(float16_t *ptr, float16x8_t first, float16x8_t second) {
+ float16x8x2_t value = {{first, second}};
+ vsttnq_f16_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_bf16_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <8 x bfloat> noundef [[FIRST:%.*]], <8 x bfloat> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v8bf16.p0(<8 x bfloat> [[FIRST]], <8 x bfloat> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_bf16_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <8 x bfloat> noundef [[FIRST:%.*]], <8 x bfloat> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[TMP0:%.*]] = shufflevector <8 x bfloat> [[FIRST]], <8 x bfloat> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[TMP1:%.*]] = shufflevector <8 x bfloat> [[SECOND]], <8 x bfloat> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v8bf16.p0(<8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_bf16_x2(bfloat16_t *ptr, bfloat16x8_t first, bfloat16x8_t second) {
+ bfloat16x8x2_t value = {{first, second}};
+ vsttnq_bf16_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_f32_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <4 x float> noundef [[FIRST:%.*]], <4 x float> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v4f32.p0(<4 x float> [[FIRST]], <4 x float> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_f32_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <4 x float> noundef [[FIRST:%.*]], <4 x float> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[TMP0:%.*]] = shufflevector <4 x float> [[FIRST]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[SECOND]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v4f32.p0(<4 x float> [[TMP0]], <4 x float> [[TMP1]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_f32_x2(float32_t *ptr, float32x4_t first, float32x4_t second) {
+ float32x4x2_t value = {{first, second}};
+ vsttnq_f32_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_f64_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <2 x double> noundef [[FIRST:%.*]], <2 x double> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v2f64.p0(<2 x double> [[FIRST]], <2 x double> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_f64_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <2 x double> noundef [[FIRST:%.*]], <2 x double> noundef [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[TMP0:%.*]] = shufflevector <2 x double> [[FIRST]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[SECOND]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v2f64.p0(<2 x double> [[TMP0]], <2 x double> [[TMP1]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_f64_x2(float64_t *ptr, float64x2_t first, float64x2_t second) {
+ float64x2x2_t value = {{first, second}};
+ vsttnq_f64_x2(ptr, value);
+}
+
+// CHECK-LE-LABEL: define dso_local void @test_vsttnq_mf8_x2(
+// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <16 x i8> [[FIRST:%.*]], <16 x i8> [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-LE-NEXT: [[ENTRY:.*:]]
+// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v16i8.p0(<16 x i8> [[FIRST]], <16 x i8> [[SECOND]], ptr [[PTR]])
+// CHECK-LE-NEXT: ret void
+//
+// CHECK-BE-LABEL: define dso_local void @test_vsttnq_mf8_x2(
+// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], <16 x i8> [[FIRST:%.*]], <16 x i8> [[SECOND:%.*]]) local_unnamed_addr #[[ATTR2]] {
+// CHECK-BE-NEXT: [[ENTRY:.*:]]
+// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[FIRST]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <16 x i8> [[SECOND]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vsttn.x2.v16i8.p0(<16 x i8> [[SHUFFLE]], <16 x i8> [[SHUFFLE10]], ptr [[PTR]])
+// CHECK-BE-NEXT: ret void
+//
+void test_vsttnq_mf8_x2(mfloat8_t *ptr, mfloat8x16_t first, mfloat8x16_t second) {
+ mfloat8x16x2_t value = {{first, second}};
+ vsttnq_mf8_x2(ptr, value);
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index d05f36c86b8191..4917c819c662fe 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -693,6 +693,10 @@ def int_aarch64_neon_st2lane : AdvSIMD_2Vec_Store_Lane_Intrinsic;
def int_aarch64_neon_st3lane : AdvSIMD_3Vec_Store_Lane_Intrinsic;
def int_aarch64_neon_st4lane : AdvSIMD_4Vec_Store_Lane_Intrinsic;
+// LSUI non-temporal pair loads and stores.
+def int_aarch64_neon_vldtn_x2 : AdvSIMD_2Vec_Load_Intrinsic;
+def int_aarch64_neon_vsttn_x2 : AdvSIMD_2Vec_Store_Intrinsic;
+
let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.".
class AdvSIMD_Tbl1_Intrinsic
: DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index e2639c95a004a6..7bfcbfd05f889a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3975,6 +3975,11 @@ defm LDNPD : LoadPairNoAlloc<0b01, 1, FPR64Op, simm7s8, "ldnp">;
defm LDNPQ : LoadPairNoAlloc<0b10, 1, FPR128Op, simm7s16, "ldnp">;
}
+multiclass LoadPairPats<ValueType VecTy, Intrinsic OpNode, Instruction Inst> {
+ def : Pat<([VecTy, VecTy] (OpNode GPR64sp:$Rn)),
+ ([VecTy, VecTy] (Inst GPR64sp:$Rn, 0))>;
+}
+
def : Pat<(AArch64ldp (am_indexed7s64 GPR64sp:$Rn, simm7s8:$offset)),
(LDPXi GPR64sp:$Rn, simm7s8:$offset)>;
@@ -4877,6 +4882,11 @@ defm STNPD : StorePairNoAlloc<0b01, 1, FPR64Op, simm7s8, "stnp">;
defm STNPQ : StorePairNoAlloc<0b10, 1, FPR128Op, simm7s16, "stnp">;
}
+multiclass StorePairPats<ValueType VecTy, Intrinsic OpNode, Instruction Inst> {
+ def : Pat<(OpNode (VecTy FPR128:$Rt1), (VecTy FPR128:$Rt2), GPR64sp:$Rn),
+ (Inst FPR128:$Rt1, FPR128:$Rt2, GPR64sp:$Rn, 0)>;
+}
+
// Armv9.6-a Load/store pair (FEAT_LSUI)
let Predicates = [HasLSUI] in {
defm LDTP : LoadPairOffset<0b11, 0, GPR64z, simm7s8, "ldtp">;
@@ -4902,6 +4912,12 @@ let Predicates = [HasLSUI, HasNEON] in {
defm STTPQ : StorePairOffset<0b11, 1, FPR128Op, simm7s16, "sttp">;
def STTPQpre : StorePairPreIdx<0b11, 1, FPR128Op, simm7s16, "sttp">;
def STTPQpost : StorePairPostIdx<0b11, 1, FPR128Op, simm7s16, "sttp">;
+
+ foreach VecTy = [v16i8, v8i16, v4i32, v2i64,
+ v8f16, v8bf16, v4f32, v2f64] in {
+ defm : LoadPairPats<VecTy, int_aarch64_neon_vldtn_x2, LDTNPQi>;
+ defm : StorePairPats<VecTy, int_aarch64_neon_vsttn_x2, STTNPQi>;
+ }
}
def : Pat<(AArch64stp GPR64z:$Rt, GPR64z:$Rt2, (am_indexed7s64 GPR64sp:$Rn, simm7s8:$offset)),
diff --git a/llvm/test/CodeGen/AArch64/neon-lsui.ll b/llvm/test/CodeGen/AArch64/neon-lsui.ll
new file mode 100644
index 00000000000000..713e511fcf8e78
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-lsui.ll
@@ -0,0 +1,287 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+v9.6a,+lsui -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-LE
+; RUN: llc -mtriple=aarch64_be-none-linux-gnu -mattr=+v9.6a,+lsui -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-BE
+
+define { <16 x i8>, <16 x i8> } @test_vldtn_v16i8(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v16i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v16i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %pair = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldtn.x2.v16i8.p0(ptr %a)
+ ret { <16 x i8>, <16 x i8> } %pair
+}
+
+define void @test_vsttn_v16i8(ptr %a, <16 x i8> %vec0, <16 x i8> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v16i8:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v16i8:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.16b, v1.16b
+; CHECK-BE-NEXT: rev64 v0.16b, v0.16b
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0]
+; CHECK-BE-NEXT: ret
+ call void @llvm.aarch64.neon.vsttn.x2.v16i8.p0(<16 x i8> %vec0, <16 x i8> %vec1, ptr %a)
+ ret void
+}
+
+
+define { <8 x i16>, <8 x i16> } @test_vldtn_v8i16(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v8i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v8i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %pair = call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldtn.x2.v8i16.p0(ptr %a)
+ ret { <8 x i16>, <8 x i16> } %pair
+}
+
+define void @test_vsttn_v8i16(ptr %a, <8 x i16> %vec0, <8 x i16> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v8i16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v8i16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0]
+; CHECK-BE-NEXT: ret
+ call void @llvm.aarch64.neon.vsttn.x2.v8i16.p0(<8 x i16> %vec0, <8 x i16> %vec1, ptr %a)
+ ret void
+}
+
+
+define { <4 x i32>, <4 x i32> } @test_vldtn_v4i32(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v4i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v4i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %pair = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldtn.x2.v4i32.p0(ptr %a)
+ ret { <4 x i32>, <4 x i32> } %pair
+}
+
+define void @test_vsttn_v4i32(ptr %a, <4 x i32> %vec0, <4 x i32> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v4i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v4i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0]
+; CHECK-BE-NEXT: ret
+ call void @llvm.aarch64.neon.vsttn.x2.v4i32.p0(<4 x i32> %vec0, <4 x i32> %vec1, ptr %a)
+ ret void
+}
+
+
+define { <2 x i64>, <2 x i64> } @test_vldtn_v2i64(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v2i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v2i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %pair = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldtn.x2.v2i64.p0(ptr %a)
+ ret { <2 x i64>, <2 x i64> } %pair
+}
+
+define void @test_vsttn_v2i64(ptr %a, <2 x i64> %vec0, <2 x i64> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v2i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v2i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0]
+; CHECK-BE-NEXT: ret
+ call void @llvm.aarch64.neon.vsttn.x2.v2i64.p0(<2 x i64> %vec0, <2 x i64> %vec1, ptr %a)
+ ret void
+}
+
+
+define { <8 x half>, <8 x half> } @test_vldtn_v8f16(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v8f16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v8f16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %pair = call { <8 x half>, <8 x half> } @llvm.aarch64.neon.vldtn.x2.v8f16.p0(ptr %a)
+ ret { <8 x half>, <8 x half> } %pair
+}
+
+define void @test_vsttn_v8f16(ptr %a, <8 x half> %vec0, <8 x half> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v8f16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v8f16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0]
+; CHECK-BE-NEXT: ret
+ call void @llvm.aarch64.neon.vsttn.x2.v8f16.p0(<8 x half> %vec0, <8 x half> %vec1, ptr %a)
+ ret void
+}
+
+define { <8 x bfloat>, <8 x bfloat> } @test_vldtn_v8bf16(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v8bf16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v8bf16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %pair = call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.vldtn.x2.v8bf16.p0(ptr %a)
+ ret { <8 x bfloat>, <8 x bfloat> } %pair
+}
+
+define void @test_vsttn_v8bf16(ptr %a, <8 x bfloat> %vec0, <8 x bfloat> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v8bf16:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v8bf16:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.8h, v1.8h
+; CHECK-BE-NEXT: rev64 v0.8h, v0.8h
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0]
+; CHECK-BE-NEXT: ret
+ call void @llvm.aarch64.neon.vsttn.x2.v8bf16.p0(<8 x bfloat> %vec0, <8 x bfloat> %vec1, ptr %a)
+ ret void
+}
+
+define { <4 x float>, <4 x float> } @test_vldtn_v4f32(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v4f32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v4f32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %pair = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.vldtn.x2.v4f32.p0(ptr %a)
+ ret { <4 x float>, <4 x float> } %pair
+}
+
+define void @test_vsttn_v4f32(ptr %a, <4 x float> %vec0, <4 x float> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v4f32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v4f32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0]
+; CHECK-BE-NEXT: ret
+ call void @llvm.aarch64.neon.vsttn.x2.v4f32.p0(<4 x float> %vec0, <4 x float> %vec1, ptr %a)
+ ret void
+}
+
+define { <2 x double>, <2 x double> } @test_vldtn_v2f64(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v2f64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v2f64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0]
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %pair = call { <2 x double>, <2 x double> } @llvm.aarch64.neon.vldtn.x2.v2f64.p0(ptr %a)
+ ret { <2 x double>, <2 x double> } %pair
+}
+
+define void @test_vsttn_v2f64(ptr %a, <2 x double> %vec0, <2 x double> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v2f64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v2f64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0]
+; CHECK-BE-NEXT: ret
+ call void @llvm.aarch64.neon.vsttn.x2.v2f64.p0(<2 x double> %vec0, <2 x double> %vec1, ptr %a)
+ ret void
+}
>From 71dff6ace94adb4d28a454dca35438bc76611ebd Mon Sep 17 00:00:00 2001
From: Martin Wehking <martin.wehking at arm.com>
Date: Wed, 11 Nov 2026 08:41:59 +0000
Subject: [PATCH 2/2] Add address matching
Theere could be technically e.g. a GEP instruction involved with an
offset. We need to match for that offset.
---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 9 +--
llvm/test/CodeGen/AArch64/neon-lsui.ll | 76 +++++++++++++++++++++
2 files changed, 81 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 7bfcbfd05f889a..5f8604824ca522 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3976,8 +3976,8 @@ defm LDNPQ : LoadPairNoAlloc<0b10, 1, FPR128Op, simm7s16, "ldnp">;
}
multiclass LoadPairPats<ValueType VecTy, Intrinsic OpNode, Instruction Inst> {
- def : Pat<([VecTy, VecTy] (OpNode GPR64sp:$Rn)),
- ([VecTy, VecTy] (Inst GPR64sp:$Rn, 0))>;
+ def : Pat<([VecTy, VecTy] (OpNode (am_indexed7s128 GPR64sp:$Rn, simm7s16:$offset))),
+ ([VecTy, VecTy] (Inst GPR64sp:$Rn, simm7s16:$offset))>;
}
def : Pat<(AArch64ldp (am_indexed7s64 GPR64sp:$Rn, simm7s8:$offset)),
@@ -4883,8 +4883,9 @@ defm STNPQ : StorePairNoAlloc<0b10, 1, FPR128Op, simm7s16, "stnp">;
}
multiclass StorePairPats<ValueType VecTy, Intrinsic OpNode, Instruction Inst> {
- def : Pat<(OpNode (VecTy FPR128:$Rt1), (VecTy FPR128:$Rt2), GPR64sp:$Rn),
- (Inst FPR128:$Rt1, FPR128:$Rt2, GPR64sp:$Rn, 0)>;
+ def : Pat<(OpNode (VecTy FPR128:$Rt1), (VecTy FPR128:$Rt2),
+ (am_indexed7s128 GPR64sp:$Rn, simm7s16:$offset)),
+ (Inst FPR128:$Rt1, FPR128:$Rt2, GPR64sp:$Rn, simm7s16:$offset)>;
}
// Armv9.6-a Load/store pair (FEAT_LSUI)
diff --git a/llvm/test/CodeGen/AArch64/neon-lsui.ll b/llvm/test/CodeGen/AArch64/neon-lsui.ll
index 713e511fcf8e78..74ed9c916bb634 100644
--- a/llvm/test/CodeGen/AArch64/neon-lsui.ll
+++ b/llvm/test/CodeGen/AArch64/neon-lsui.ll
@@ -285,3 +285,79 @@ define void @test_vsttn_v2f64(ptr %a, <2 x double> %vec0, <2 x double> %vec1) {
call void @llvm.aarch64.neon.vsttn.x2.v2f64.p0(<2 x double> %vec0, <2 x double> %vec1, ptr %a)
ret void
}
+
+define { <4 x i32>, <4 x i32> } @test_vldtn_v4i32_offset_i32(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v4i32_offset_i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0, #32]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v4i32_offset_i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0, #32]
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i32 32
+ %pair = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldtn.x2.v4i32.p0(ptr %addr)
+ ret { <4 x i32>, <4 x i32> } %pair
+}
+
+define { <4 x i32>, <4 x i32> } @test_vldtn_v4i32_offset_i64(ptr %a) {
+; CHECK-LE-LABEL: test_vldtn_v4i32_offset_i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: ldtnp q0, q1, [x0, #32]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vldtn_v4i32_offset_i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: ldtnp q0, q1, [x0, #32]
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 32
+ %pair = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldtn.x2.v4i32.p0(ptr %addr)
+ ret { <4 x i32>, <4 x i32> } %pair
+}
+
+define void @test_vsttn_v4i32_offset_i32(ptr %a, <4 x i32> %vec0, <4 x i32> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v4i32_offset_i32:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0, #32]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v4i32_offset_i32:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0, #32]
+; CHECK-BE-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i32 32
+ call void @llvm.aarch64.neon.vsttn.x2.v4i32.p0(<4 x i32> %vec0, <4 x i32> %vec1, ptr %addr)
+ ret void
+}
+
+define void @test_vsttn_v4i32_offset_i64(ptr %a, <4 x i32> %vec0, <4 x i32> %vec1) {
+; CHECK-LE-LABEL: test_vsttn_v4i32_offset_i64:
+; CHECK-LE: // %bb.0:
+; CHECK-LE-NEXT: sttnp q0, q1, [x0, #32]
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_vsttn_v4i32_offset_i64:
+; CHECK-BE: // %bb.0:
+; CHECK-BE-NEXT: rev64 v1.4s, v1.4s
+; CHECK-BE-NEXT: rev64 v0.4s, v0.4s
+; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-BE-NEXT: sttnp q0, q1, [x0, #32]
+; CHECK-BE-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 32
+ call void @llvm.aarch64.neon.vsttn.x2.v4i32.p0(<4 x i32> %vec0, <4 x i32> %vec1, ptr %addr)
+ ret void
+}
More information about the cfe-commits
mailing list