[llvm] [AArch64][TTI][EarlyCSE] Add support for ld1xN and st1xN intrinsics (PR #198765)
Shanzhi Chen via llvm-commits
llvm-commits at lists.llvm.org
Wed May 20 05:27:02 PDT 2026
https://github.com/chenshanzhi created https://github.com/llvm/llvm-project/pull/198765
Handle ld1x2, ld1x3, ld1x4, st1x2, st1x3, st1x4 in:
- AArch64TTIImpl::getTgtMemIntrinsic
- AArch64TTIImpl::getOrCreateResultFromMemIntrinsic
This enables EarlyCSE to optimize these NEON load/store intrinsics.
To test the changes, a new testcase (intrinsics-1xN.ll) derived from llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll is added.
>From 0425d1eb0defed46a90875c75fcfce2eb168251c Mon Sep 17 00:00:00 2001
From: Shanzhi Chen <chenshanzhi at huawei.com>
Date: Wed, 20 May 2026 20:01:17 +0800
Subject: [PATCH] [AArch64][TTI][EarlyCSE] Add support for ld1xN and st1xN
intrinsics
Handle ld1x2, ld1x3, ld1x4, st1x2, st1x3, st1x4 in:
- AArch64TTIImpl::getTgtMemIntrinsic
- AArch64TTIImpl::getOrCreateResultFromMemIntrinsic
This enables EarlyCSE to optimize these NEON load/store intrinsics.
To test the changes, a new testcase (intrinsics-1xN.ll) derived from
llvm/test/Transforms/EarlyCSE/AArch64/intrinsics.ll is added.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 31 +-
.../AArch64/AArch64TargetTransformInfo.h | 6 -
.../EarlyCSE/AArch64/intrinsics-1xN.ll | 365 ++++++++++++++++++
3 files changed, 393 insertions(+), 9 deletions(-)
create mode 100644 llvm/test/Transforms/EarlyCSE/AArch64/intrinsics-1xN.ll
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 3685177328f12..e2d6efd9707d3 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5542,6 +5542,9 @@ Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
switch (Inst->getIntrinsicID()) {
default:
return nullptr;
+ case Intrinsic::aarch64_neon_st1x2:
+ case Intrinsic::aarch64_neon_st1x3:
+ case Intrinsic::aarch64_neon_st1x4:
case Intrinsic::aarch64_neon_st2:
case Intrinsic::aarch64_neon_st3:
case Intrinsic::aarch64_neon_st4: {
@@ -5564,6 +5567,9 @@ Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
}
return Res;
}
+ case Intrinsic::aarch64_neon_ld1x2:
+ case Intrinsic::aarch64_neon_ld1x3:
+ case Intrinsic::aarch64_neon_ld1x4:
case Intrinsic::aarch64_neon_ld2:
case Intrinsic::aarch64_neon_ld3:
case Intrinsic::aarch64_neon_ld4:
@@ -5578,6 +5584,9 @@ bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
switch (Inst->getIntrinsicID()) {
default:
break;
+ case Intrinsic::aarch64_neon_ld1x2:
+ case Intrinsic::aarch64_neon_ld1x3:
+ case Intrinsic::aarch64_neon_ld1x4:
case Intrinsic::aarch64_neon_ld2:
case Intrinsic::aarch64_neon_ld3:
case Intrinsic::aarch64_neon_ld4:
@@ -5585,6 +5594,9 @@ bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
Info.WriteMem = false;
Info.PtrVal = Inst->getArgOperand(0);
break;
+ case Intrinsic::aarch64_neon_st1x2:
+ case Intrinsic::aarch64_neon_st1x3:
+ case Intrinsic::aarch64_neon_st1x4:
case Intrinsic::aarch64_neon_st2:
case Intrinsic::aarch64_neon_st3:
case Intrinsic::aarch64_neon_st4:
@@ -5594,20 +5606,33 @@ bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
break;
}
+ // Use the ID of neon load as the "matching id".
switch (Inst->getIntrinsicID()) {
default:
return false;
+ case Intrinsic::aarch64_neon_ld1x2:
+ case Intrinsic::aarch64_neon_st1x2:
+ Info.MatchingId = Intrinsic::aarch64_neon_ld1x2;
+ break;
+ case Intrinsic::aarch64_neon_ld1x3:
+ case Intrinsic::aarch64_neon_st1x3:
+ Info.MatchingId = Intrinsic::aarch64_neon_ld1x3;
+ break;
+ case Intrinsic::aarch64_neon_ld1x4:
+ case Intrinsic::aarch64_neon_st1x4:
+ Info.MatchingId = Intrinsic::aarch64_neon_ld1x4;
+ break;
case Intrinsic::aarch64_neon_ld2:
case Intrinsic::aarch64_neon_st2:
- Info.MatchingId = VECTOR_LDST_TWO_ELEMENTS;
+ Info.MatchingId = Intrinsic::aarch64_neon_ld2;
break;
case Intrinsic::aarch64_neon_ld3:
case Intrinsic::aarch64_neon_st3:
- Info.MatchingId = VECTOR_LDST_THREE_ELEMENTS;
+ Info.MatchingId = Intrinsic::aarch64_neon_ld3;
break;
case Intrinsic::aarch64_neon_ld4:
case Intrinsic::aarch64_neon_st4:
- Info.MatchingId = VECTOR_LDST_FOUR_ELEMENTS;
+ Info.MatchingId = Intrinsic::aarch64_neon_ld4;
break;
}
return true;
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 8125b7ffcae43..200b4c9fba196 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -54,12 +54,6 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
const AArch64Subtarget *getST() const { return ST; }
const AArch64TargetLowering *getTLI() const { return TLI; }
- enum MemIntrinsicType {
- VECTOR_LDST_TWO_ELEMENTS,
- VECTOR_LDST_THREE_ELEMENTS,
- VECTOR_LDST_FOUR_ELEMENTS
- };
-
/// Given a add/sub/mul operation, detect a widening addl/subl/mull pattern
/// where both operands can be treated like extends. Returns the minimal type
/// needed to compute the operation.
diff --git a/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics-1xN.ll b/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics-1xN.ll
new file mode 100644
index 0000000000000..f8ce89a4871f4
--- /dev/null
+++ b/llvm/test/Transforms/EarlyCSE/AArch64/intrinsics-1xN.ll
@@ -0,0 +1,365 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -passes=early-cse -earlycse-debug-hash | FileCheck %s
+; RUN: opt < %s -S -mtriple=aarch64-none-linux-gnu -mattr=+neon -aa-pipeline=basic-aa -passes='early-cse<memssa>' -verify-analysis-invalidation | FileCheck %s
+
+define <4 x i32> @test_cse(ptr %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
+; CHECK-LABEL: define <4 x i32> @test_cse(
+; CHECK-SAME: ptr [[A:%.*]], [2 x <4 x i32>] [[S_COERCE:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[S_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 0
+; CHECK-NEXT: [[S_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 1
+; CHECK-NEXT: br label %[[FOR_COND:.*]]
+; CHECK: [[FOR_COND]]:
+; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY:.*]] ]
+; CHECK-NEXT: [[RES_0:%.*]] = phi <4 x i32> [ undef, %[[ENTRY]] ], [ [[CALL:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_0_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_1_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[S_COERCE_FCA_0_EXTRACT]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP2]], <4 x i32> [[S_COERCE_FCA_1_EXTRACT]], 1
+; CHECK-NEXT: call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+; CHECK-NEXT: [[CALL]] = call <4 x i32> @vaddq_s32(<4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_0_EXTRACT]])
+; CHECK-NEXT: [[INC]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT: br label %[[FOR_COND]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret <4 x i32> [[RES_0]]
+;
+entry:
+; Check that @llvm.aarch64.neon.ld1x2 is optimized away by Early CSE.
+ %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+ %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+ br label %for.cond
+
+for.cond: ; preds = %for.body, %entry
+ %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+ %cmp = icmp slt i32 %i.0, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %0 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+ %1 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+ %2 = bitcast <16 x i8> %0 to <4 x i32>
+ %3 = bitcast <16 x i8> %1 to <4 x i32>
+ call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> %2, <4 x i32> %3, ptr %a)
+ %vld1x2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr %a)
+ %vld1x2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2, 0
+ %vld1x2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2, 1
+ %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld1x2.fca.0.extract, <4 x i32> %vld1x2.fca.0.extract)
+ %inc = add nsw i32 %i.0, 1
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret <4 x i32> %res.0
+}
+
+define <4 x i32> @test_cse2(ptr %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
+; CHECK-LABEL: define <4 x i32> @test_cse2(
+; CHECK-SAME: ptr [[A:%.*]], [2 x <4 x i32>] [[S_COERCE:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[S_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 0
+; CHECK-NEXT: [[S_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 1
+; CHECK-NEXT: br label %[[FOR_COND:.*]]
+; CHECK: [[FOR_COND]]:
+; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY:.*]] ]
+; CHECK-NEXT: [[RES_0:%.*]] = phi <4 x i32> [ undef, %[[ENTRY]] ], [ [[CALL:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_0_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_1_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_0_EXTRACT]], ptr [[A]])
+; CHECK-NEXT: [[TMP2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[S_COERCE_FCA_0_EXTRACT]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP2]], <4 x i32> [[S_COERCE_FCA_1_EXTRACT]], 1
+; CHECK-NEXT: call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+; CHECK-NEXT: [[CALL]] = call <4 x i32> @vaddq_s32(<4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_0_EXTRACT]])
+; CHECK-NEXT: [[INC]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT: br label %[[FOR_COND]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret <4 x i32> [[RES_0]]
+;
+entry:
+; Check that the first @llvm.aarch64.neon.st1x2 is optimized away by Early CSE.
+ %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+ %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+ br label %for.cond
+
+for.cond: ; preds = %for.body, %entry
+ %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+ %cmp = icmp slt i32 %i.0, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %0 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+ %1 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+ %2 = bitcast <16 x i8> %0 to <4 x i32>
+ %3 = bitcast <16 x i8> %1 to <4 x i32>
+ call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> %2, <4 x i32> %2, ptr %a)
+ call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> %2, <4 x i32> %3, ptr %a)
+ %vld1x2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr %a)
+ %vld1x2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2, 0
+ %vld1x2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2, 1
+ %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld1x2.fca.0.extract, <4 x i32> %vld1x2.fca.0.extract)
+ %inc = add nsw i32 %i.0, 1
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret <4 x i32> %res.0
+}
+
+define <4 x i32> @test_cse3(ptr %a, [2 x <4 x i32>] %s.coerce, i32 %n) #0 {
+; CHECK-LABEL: define <4 x i32> @test_cse3(
+; CHECK-SAME: ptr [[A:%.*]], [2 x <4 x i32>] [[S_COERCE:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_COND:.*]]
+; CHECK: [[FOR_COND]]:
+; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY:.*]] ]
+; CHECK-NEXT: [[RES_0:%.*]] = phi <4 x i32> [ undef, %[[ENTRY]] ], [ [[CALL:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[VLD2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr [[A]])
+; CHECK-NEXT: [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLD2]], 0
+; CHECK-NEXT: [[CALL]] = call <4 x i32> @vaddq_s32(<4 x i32> [[VLD2_FCA_0_EXTRACT]], <4 x i32> [[VLD2_FCA_0_EXTRACT]])
+; CHECK-NEXT: [[INC]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT: br label %[[FOR_COND]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret <4 x i32> [[RES_0]]
+;
+entry:
+; Check that the first @llvm.aarch64.neon.ld1x2 is optimized away by Early CSE.
+ %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+ %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+ br label %for.cond
+
+for.cond: ; preds = %for.body, %entry
+ %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+ %cmp = icmp slt i32 %i.0, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %vld1x2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr %a)
+ %vld1x2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2, 0
+ %vld1x2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2, 1
+ %vld1x2.2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr %a)
+ %vld1x2.2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2.2, 0
+ %vld1x2.2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2.2, 1
+ %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld1x2.fca.0.extract, <4 x i32> %vld1x2.2.fca.0.extract)
+ %inc = add nsw i32 %i.0, 1
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret <4 x i32> %res.0
+}
+
+
+define <4 x i32> @test_nocse(ptr %a, ptr %b, [2 x <4 x i32>] %s.coerce, i32 %n) {
+; CHECK-LABEL: define <4 x i32> @test_nocse(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], [2 x <4 x i32>] [[S_COERCE:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[S_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 0
+; CHECK-NEXT: [[S_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 1
+; CHECK-NEXT: br label %[[FOR_COND:.*]]
+; CHECK: [[FOR_COND]]:
+; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY:.*]] ]
+; CHECK-NEXT: [[RES_0:%.*]] = phi <4 x i32> [ undef, %[[ENTRY]] ], [ [[CALL:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_0_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_1_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+; CHECK-NEXT: store i32 0, ptr [[B]], align 4
+; CHECK-NEXT: [[VLD2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr [[A]])
+; CHECK-NEXT: [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLD2]], 0
+; CHECK-NEXT: [[CALL]] = call <4 x i32> @vaddq_s32(<4 x i32> [[VLD2_FCA_0_EXTRACT]], <4 x i32> [[VLD2_FCA_0_EXTRACT]])
+; CHECK-NEXT: [[INC]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT: br label %[[FOR_COND]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret <4 x i32> [[RES_0]]
+;
+entry:
+; Check that the store prevents @llvm.aarch64.neon.ld1x2 from being optimized
+; away by Early CSE.
+ %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+ %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+ br label %for.cond
+
+for.cond: ; preds = %for.body, %entry
+ %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+ %cmp = icmp slt i32 %i.0, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %0 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+ %1 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+ %2 = bitcast <16 x i8> %0 to <4 x i32>
+ %3 = bitcast <16 x i8> %1 to <4 x i32>
+ call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> %2, <4 x i32> %3, ptr %a)
+ store i32 0, ptr %b, align 4
+ %vld1x2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr %a)
+ %vld1x2.fca.0.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2, 0
+ %vld1x2.fca.1.extract = extractvalue { <4 x i32>, <4 x i32> } %vld1x2, 1
+ %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld1x2.fca.0.extract, <4 x i32> %vld1x2.fca.0.extract)
+ %inc = add nsw i32 %i.0, 1
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret <4 x i32> %res.0
+}
+
+define <4 x i32> @test_nocse2(ptr %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
+; CHECK-LABEL: define <4 x i32> @test_nocse2(
+; CHECK-SAME: ptr [[A:%.*]], [2 x <4 x i32>] [[S_COERCE:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[S_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 0
+; CHECK-NEXT: [[S_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 1
+; CHECK-NEXT: br label %[[FOR_COND:.*]]
+; CHECK: [[FOR_COND]]:
+; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY:.*]] ]
+; CHECK-NEXT: [[RES_0:%.*]] = phi <4 x i32> [ undef, %[[ENTRY]] ], [ [[CALL:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_0_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_1_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+; CHECK-NEXT: [[VLD3:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x3.v4i32.p0(ptr [[A]])
+; CHECK-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLD3]], 0
+; CHECK-NEXT: [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLD3]], 2
+; CHECK-NEXT: [[CALL]] = call <4 x i32> @vaddq_s32(<4 x i32> [[VLD3_FCA_0_EXTRACT]], <4 x i32> [[VLD3_FCA_2_EXTRACT]])
+; CHECK-NEXT: [[INC]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT: br label %[[FOR_COND]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret <4 x i32> [[RES_0]]
+;
+entry:
+; Check that @llvm.aarch64.neon.ld1x3 is not optimized away by Early CSE due
+; to mismatch between st1x2 and ld1x3.
+ %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+ %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+ br label %for.cond
+
+for.cond: ; preds = %for.body, %entry
+ %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+ %cmp = icmp slt i32 %i.0, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %0 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+ %1 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+ %2 = bitcast <16 x i8> %0 to <4 x i32>
+ %3 = bitcast <16 x i8> %1 to <4 x i32>
+ call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> %2, <4 x i32> %3, ptr %a)
+ %vld1x3 = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x3.v4i32.p0(ptr %a)
+ %vld1x3.fca.0.extract = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %vld1x3, 0
+ %vld1x3.fca.2.extract = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %vld1x3, 2
+ %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld1x3.fca.0.extract, <4 x i32> %vld1x3.fca.2.extract)
+ %inc = add nsw i32 %i.0, 1
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret <4 x i32> %res.0
+}
+
+define <4 x i32> @test_nocse3(ptr %a, [2 x <4 x i32>] %s.coerce, i32 %n) {
+; CHECK-LABEL: define <4 x i32> @test_nocse3(
+; CHECK-SAME: ptr [[A:%.*]], [2 x <4 x i32>] [[S_COERCE:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[S_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 0
+; CHECK-NEXT: [[S_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[S_COERCE]], 1
+; CHECK-NEXT: br label %[[FOR_COND:.*]]
+; CHECK: [[FOR_COND]]:
+; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY:.*]] ]
+; CHECK-NEXT: [[RES_0:%.*]] = phi <4 x i32> [ undef, %[[ENTRY]] ], [ [[CALL:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_0_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[S_COERCE_FCA_1_EXTRACT]] to <16 x i8>
+; CHECK-NEXT: call void @llvm.aarch64.neon.st1x3.v4i32.p0(<4 x i32> [[S_COERCE_FCA_1_EXTRACT]], <4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_0_EXTRACT]], ptr [[A]])
+; CHECK-NEXT: call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> [[S_COERCE_FCA_0_EXTRACT]], <4 x i32> [[S_COERCE_FCA_0_EXTRACT]], ptr [[A]])
+; CHECK-NEXT: [[VLD3:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x3.v4i32.p0(ptr [[A]])
+; CHECK-NEXT: [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[VLD3]], 0
+; CHECK-NEXT: [[CALL]] = call <4 x i32> @vaddq_s32(<4 x i32> [[VLD3_FCA_0_EXTRACT]], <4 x i32> [[VLD3_FCA_0_EXTRACT]])
+; CHECK-NEXT: [[INC]] = add nsw i32 [[I_0]], 1
+; CHECK-NEXT: br label %[[FOR_COND]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: ret <4 x i32> [[RES_0]]
+;
+entry:
+; Check that @llvm.aarch64.neon.st1x3 is not optimized away by Early CSE due to
+; mismatch between st1x2 and st1x3.
+ %s.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %s.coerce, 0
+ %s.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %s.coerce, 1
+ br label %for.cond
+
+for.cond: ; preds = %for.body, %entry
+ %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %res.0 = phi <4 x i32> [ undef, %entry ], [ %call, %for.body ]
+ %cmp = icmp slt i32 %i.0, %n
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %0 = bitcast <4 x i32> %s.coerce.fca.0.extract to <16 x i8>
+ %1 = bitcast <4 x i32> %s.coerce.fca.1.extract to <16 x i8>
+ %2 = bitcast <16 x i8> %0 to <4 x i32>
+ %3 = bitcast <16 x i8> %1 to <4 x i32>
+ call void @llvm.aarch64.neon.st1x3.v4i32.p0(<4 x i32> %3, <4 x i32> %2, <4 x i32> %2, ptr %a)
+ call void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32> %2, <4 x i32> %2, ptr %a)
+ %vld1x3 = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x3.v4i32.p0(ptr %a)
+ %vld1x3.fca.0.extract = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %vld1x3, 0
+ %vld1x3.fca.1.extract = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } %vld1x3, 1
+ %call = call <4 x i32> @vaddq_s32(<4 x i32> %vld1x3.fca.0.extract, <4 x i32> %vld1x3.fca.0.extract)
+ %inc = add nsw i32 %i.0, 1
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret <4 x i32> %res.0
+}
+
+define void @test_ld1x4_st1x4_no_cse(ptr %p, <16 x i8> %A, <16 x i8> %B) {
+; CHECK-LABEL: define void @test_ld1x4_st1x4_no_cse(
+; CHECK-SAME: ptr [[P:%.*]], <16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[LD:%.*]] = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x4.v16i8.p0(ptr [[P]])
+; CHECK-NEXT: [[EXT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[LD]], 0
+; CHECK-NEXT: tail call void @llvm.aarch64.neon.st1x4.v16i8.p0(<16 x i8> [[EXT]], <16 x i8> [[A]], <16 x i8> [[B]], <16 x i8> zeroinitializer, ptr [[P]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %ld = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x4.v16i8.p0(ptr %p)
+ %ext = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %ld, 0
+ tail call void @llvm.aarch64.neon.st1x4.v16i8.p0(<16 x i8> %ext, <16 x i8> %A, <16 x i8> %B, <16 x i8> zeroinitializer, ptr %p)
+ ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.aarch64.neon.st1x2.v4i32.p0(<4 x i32>, <4 x i32>, ptr nocapture)
+
+; Function Attrs: nounwind
+declare void @llvm.aarch64.neon.st1x3.v4i32.p0(<4 x i32>, <4 x i32>, <4 x i32>, ptr nocapture)
+
+; Function Attrs: nounwind readonly
+declare { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x2.v4i32.p0(ptr)
+
+; Function Attrs: nounwind readonly
+declare { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld1x3.v4i32.p0(ptr)
+
+define internal fastcc <4 x i32> @vaddq_s32(<4 x i32> %__p0, <4 x i32> %__p1) {
+; CHECK-LABEL: define internal fastcc <4 x i32> @vaddq_s32(
+; CHECK-SAME: <4 x i32> [[__P0:%.*]], <4 x i32> [[__P1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ADD:%.*]] = add <4 x i32> [[__P0]], [[__P1]]
+; CHECK-NEXT: ret <4 x i32> [[ADD]]
+;
+entry:
+ %add = add <4 x i32> %__p0, %__p1
+ ret <4 x i32> %add
+}
More information about the llvm-commits
mailing list