[llvm] [LV] Teach the loop vectorizer llvm.sincos is trivially vectorizable (PR #128035)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 27 00:24:49 PST 2025
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/128035
>From d1e2507e1a2b6b63f42f89b3e832b8a229356cc0 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 20 Feb 2025 17:03:07 +0000
Subject: [PATCH 1/4] [LV] Teach the loop vectorizer llvm.sincos is trivially
vectorizable
---
llvm/lib/Analysis/VectorUtils.cpp | 2 +
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 +-
.../LoopVectorize/AArch64/llvm.sincos.ll | 170 ++++++++++++++++++
.../Transforms/Scalarizer/deinterleave2.ll | 17 ++
llvm/test/Transforms/Scalarizer/sincos.ll | 17 --
6 files changed, 193 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
create mode 100644 llvm/test/Transforms/Scalarizer/deinterleave2.ll
delete mode 100644 llvm/test/Transforms/Scalarizer/sincos.ll
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 53be7fc0bee9f..dcfd3d5a8bd6e 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -72,6 +72,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
case Intrinsic::atan2:
case Intrinsic::sin:
case Intrinsic::cos:
+ case Intrinsic::sincos:
case Intrinsic::tan:
case Intrinsic::sinh:
case Intrinsic::cosh:
@@ -185,6 +186,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
case Intrinsic::ucmp:
case Intrinsic::scmp:
return OpdIdx == -1 || OpdIdx == 0;
+ case Intrinsic::sincos:
case Intrinsic::is_fpclass:
case Intrinsic::vp_is_fpclass:
return OpdIdx == 0;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 226fc23888f02..d9d9a1fc08459 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2926,7 +2926,8 @@ LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
[&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
- dyn_cast<IntrinsicInst>(CI));
+ dyn_cast<IntrinsicInst>(CI),
+ InstructionCost::getInvalid(), TLI);
return TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 8a928fa71fab8..0e933fae1edc8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1176,7 +1176,8 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
FastMathFlags FMF = hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
IntrinsicCostAttributes CostAttrs(
VectorIntrinsicID, RetTy, Arguments, ParamTys, FMF,
- dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()));
+ dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()),
+ InstructionCost::getInvalid(), &Ctx.TLI);
return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll b/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
new file mode 100644
index 0000000000000..9c2b3a2f57f9c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "(:|sincos|extractvalue|store)" --version 5
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-gnu-linux -mcpu=neoverse-v1 -mattr=+sve < %s -S -o - -debug-only=loop-vectorize 2>%t.1 | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-gnu-linux -mcpu=neoverse-v1 -mattr=+sve -vector-library=ArmPL < %s -S -o - -debug-only=loop-vectorize 2>%t.2 | FileCheck %s --check-prefix=CHECK-ARMPL
+; RUN: cat %t.1 | FileCheck --check-prefix=CHECK-COST %s
+; RUN: cat %t.2 | FileCheck --check-prefix=CHECK-COST-ARMPL %s
+; REQUIRES: asserts
+
+; CHECK-COST-LABEL: sincos_f32
+; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
+; CHECK-COST: Cost of 26 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of 58 for VF 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+
+; CHECK-COST-ARMPL-LABEL: sincos_f32
+; CHECK-COST-ARMPL: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
+; CHECK-COST-ARMPL: Cost of 26 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of 12 for VF 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of Invalid for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of 13 for VF vscale x 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+
+define void @sincos_f32(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @sincos_f32(
+; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[VECTOR_PH:.*:]]
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[TMP3:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> [[WIDE_LOAD:%.*]])
+; CHECK: [[TMP4:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 0
+; CHECK: [[TMP5:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 1
+; CHECK: store <2 x float> [[TMP4]], ptr [[TMP7:%.*]], align 4
+; CHECK: store <2 x float> [[TMP5]], ptr [[TMP9:%.*]], align 4
+; CHECK: [[MIDDLE_BLOCK:.*:]]
+; CHECK: [[SCALAR_PH:.*:]]
+; CHECK: [[FOR_BODY:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
+; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
+; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
+; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
+; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
+; CHECK: [[EXIT:.*:]]
+;
+; CHECK-ARMPL-LABEL: define void @sincos_f32(
+; CHECK-ARMPL-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-ARMPL: [[ENTRY:.*:]]
+; CHECK-ARMPL: [[VECTOR_PH:.*:]]
+; CHECK-ARMPL: [[VECTOR_BODY:.*:]]
+; CHECK-ARMPL: [[TMP12:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.sincos.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD:%.*]])
+; CHECK-ARMPL: [[TMP13:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.sincos.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD1:%.*]])
+; CHECK-ARMPL: [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP12]], 0
+; CHECK-ARMPL: [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP13]], 0
+; CHECK-ARMPL: [[TMP16:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP12]], 1
+; CHECK-ARMPL: [[TMP17:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP13]], 1
+; CHECK-ARMPL: store <vscale x 4 x float> [[TMP14]], ptr [[TMP19:%.*]], align 4
+; CHECK-ARMPL: store <vscale x 4 x float> [[TMP15]], ptr [[TMP22:%.*]], align 4
+; CHECK-ARMPL: store <vscale x 4 x float> [[TMP16]], ptr [[TMP24:%.*]], align 4
+; CHECK-ARMPL: store <vscale x 4 x float> [[TMP17]], ptr [[TMP27:%.*]], align 4
+; CHECK-ARMPL: [[MIDDLE_BLOCK:.*:]]
+; CHECK-ARMPL: [[SCALAR_PH:.*:]]
+; CHECK-ARMPL: [[FOR_BODY:.*:]]
+; CHECK-ARMPL: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
+; CHECK-ARMPL: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
+; CHECK-ARMPL: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
+; CHECK-ARMPL: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
+; CHECK-ARMPL: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
+; CHECK-ARMPL: [[EXIT:.*:]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
+ %in_val = load float, ptr %arrayidx, align 4
+ %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
+ %extract_a = extractvalue { float, float } %call, 0
+ %extract_b = extractvalue { float, float } %call, 1
+ %arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
+ store float %extract_a, ptr %arrayidx2, align 4
+ %arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
+ store float %extract_b, ptr %arrayidx4, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+; CHECK-COST-LABEL: sincos_f64
+; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { double, double } @llvm.sincos.f64(double %in_val)
+; CHECK-COST: Cost of 26 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+
+; CHECK-COST-ARMPL-LABEL: sincos_f64
+; CHECK-COST-ARMPL: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { double, double } @llvm.sincos.f64(double %in_val)
+; CHECK-COST-ARMPL: Cost of 12 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of 13 for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+
+define void @sincos_f64(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @sincos_f64(
+; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0]] {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[VECTOR_PH:.*:]]
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[TMP3:%.*]] = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; CHECK: [[TMP4:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 0
+; CHECK: [[TMP5:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 1
+; CHECK: store <2 x double> [[TMP4]], ptr [[TMP7:%.*]], align 8
+; CHECK: store <2 x double> [[TMP5]], ptr [[TMP9:%.*]], align 8
+; CHECK: [[MIDDLE_BLOCK:.*:]]
+; CHECK: [[SCALAR_PH:.*:]]
+; CHECK: [[FOR_BODY:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { double, double } @llvm.sincos.f64(double [[IN_VAL:%.*]])
+; CHECK: [[EXTRACT_A:%.*]] = extractvalue { double, double } [[CALL]], 0
+; CHECK: [[EXTRACT_B:%.*]] = extractvalue { double, double } [[CALL]], 1
+; CHECK: store double [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 8
+; CHECK: store double [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 8
+; CHECK: [[EXIT:.*:]]
+;
+; CHECK-ARMPL-LABEL: define void @sincos_f64(
+; CHECK-ARMPL-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0]] {
+; CHECK-ARMPL: [[ENTRY:.*:]]
+; CHECK-ARMPL: [[VECTOR_PH:.*:]]
+; CHECK-ARMPL: [[VECTOR_BODY:.*:]]
+; CHECK-ARMPL: [[TMP12:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.sincos.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD:%.*]])
+; CHECK-ARMPL: [[TMP13:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.sincos.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD1:%.*]])
+; CHECK-ARMPL: [[TMP14:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP12]], 0
+; CHECK-ARMPL: [[TMP15:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP13]], 0
+; CHECK-ARMPL: [[TMP16:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP12]], 1
+; CHECK-ARMPL: [[TMP17:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[TMP13]], 1
+; CHECK-ARMPL: store <vscale x 2 x double> [[TMP14]], ptr [[TMP19:%.*]], align 8
+; CHECK-ARMPL: store <vscale x 2 x double> [[TMP15]], ptr [[TMP22:%.*]], align 8
+; CHECK-ARMPL: store <vscale x 2 x double> [[TMP16]], ptr [[TMP24:%.*]], align 8
+; CHECK-ARMPL: store <vscale x 2 x double> [[TMP17]], ptr [[TMP27:%.*]], align 8
+; CHECK-ARMPL: [[MIDDLE_BLOCK:.*:]]
+; CHECK-ARMPL: [[SCALAR_PH:.*:]]
+; CHECK-ARMPL: [[FOR_BODY:.*:]]
+; CHECK-ARMPL: [[CALL:%.*]] = tail call { double, double } @llvm.sincos.f64(double [[IN_VAL:%.*]])
+; CHECK-ARMPL: [[EXTRACT_A:%.*]] = extractvalue { double, double } [[CALL]], 0
+; CHECK-ARMPL: [[EXTRACT_B:%.*]] = extractvalue { double, double } [[CALL]], 1
+; CHECK-ARMPL: store double [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 8
+; CHECK-ARMPL: store double [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 8
+; CHECK-ARMPL: [[EXIT:.*:]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds double, ptr %in, i64 %iv
+ %in_val = load double, ptr %arrayidx, align 8
+ %call = tail call { double, double } @llvm.sincos.f64(double %in_val)
+ %extract_a = extractvalue { double, double } %call, 0
+ %extract_b = extractvalue { double, double } %call, 1
+ %arrayidx2 = getelementptr inbounds double, ptr %out_a, i64 %iv
+ store double %extract_a, ptr %arrayidx2, align 8
+ %arrayidx4 = getelementptr inbounds double, ptr %out_b, i64 %iv
+ store double %extract_b, ptr %arrayidx4, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/Scalarizer/deinterleave2.ll b/llvm/test/Transforms/Scalarizer/deinterleave2.ll
new file mode 100644
index 0000000000000..bbc7c726857df
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/deinterleave2.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt %s -passes="function(scalarizer)" -S | FileCheck %s
+
+; Test to make sure that struct return intrinsics that are not `isTriviallyScalarizable` do not get scalarized.
+
+define <2 x float> @test_(<4 x float> %Val) {
+; CHECK-LABEL: define <2 x float> @test_(
+; CHECK-SAME: <4 x float> [[VAL:%.*]]) {
+; CHECK-NEXT: [[R:%.*]] = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> [[VAL]])
+; CHECK-NEXT: [[EL:%.*]] = extractvalue { <2 x float>, <2 x float> } [[R]], 0
+; CHECK-NEXT: ret <2 x float> [[EL]]
+;
+ %r = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %Val)
+ %el = extractvalue { <2 x float>, <2 x float> } %r, 0
+ ret <2 x float> %el
+}
+
diff --git a/llvm/test/Transforms/Scalarizer/sincos.ll b/llvm/test/Transforms/Scalarizer/sincos.ll
deleted file mode 100644
index 8db4ba3183290..0000000000000
--- a/llvm/test/Transforms/Scalarizer/sincos.ll
+++ /dev/null
@@ -1,17 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt %s -passes="function(scalarizer)" -S | FileCheck %s
-
-; Test to make sure that struct return intrinsics that are not `isTriviallyScalarizable` do not get scalarized.
-
-define <4 x float> @test_(<4 x float> %Val) {
-; CHECK-LABEL: define <4 x float> @test_(
-; CHECK-SAME: <4 x float> [[VAL:%.*]]) {
-; CHECK-NEXT: [[R:%.*]] = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> [[VAL]])
-; CHECK-NEXT: [[EL:%.*]] = extractvalue { <4 x float>, <4 x float> } [[R]], 0
-; CHECK-NEXT: ret <4 x float> [[EL]]
-;
- %r = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %Val)
- %el = extractvalue { <4 x float>, <4 x float> } %r, 0
- ret <4 x float> %el
-}
-
>From 25717a1dd532353a9134f1f467d28b7e46f05d7a Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 26 Feb 2025 15:43:08 +0000
Subject: [PATCH 2/4] Add simple predicated test case
---
.../LoopVectorize/AArch64/llvm.sincos.ll | 81 +++++++++++++++++++
1 file changed, 81 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll b/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
index 9c2b3a2f57f9c..5001ae921eeeb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
@@ -168,3 +168,84 @@ for.body:
exit:
ret void
}
+
+; CHECK-COST-LABEL: predicated_sincos
+; CHECK-COST: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
+; CHECK-COST: Cost of 26 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of 58 for VF 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST: Cost of Invalid for VF vscale x 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+
+; CHECK-COST-ARMPL-LABEL: predicated_sincos
+; CHECK-COST-ARMPL: LV: Found an estimated cost of 10 for VF 1 For instruction: %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
+; CHECK-COST-ARMPL: Cost of 26 for VF 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of 12 for VF 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of Invalid for VF vscale x 1: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of Invalid for VF vscale x 2: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+; CHECK-COST-ARMPL: Cost of 13 for VF vscale x 4: WIDEN-INTRINSIC ir<%call> = call llvm.sincos(ir<%in_val>)
+
+define void @predicated_sincos(float %x, ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @predicated_sincos(
+; CHECK-SAME: float [[X:%.*]], ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0]] {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[FOR_BODY:.*:]]
+; CHECK: [[IF_THEN:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
+; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
+; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
+; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
+; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
+; CHECK: [[IF_MERGE:.*:]]
+; CHECK: [[FOR_END:.*:]]
+;
+; CHECK-ARMPL-LABEL: define void @predicated_sincos(
+; CHECK-ARMPL-SAME: float [[X:%.*]], ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0]] {
+; CHECK-ARMPL: [[ENTRY:.*:]]
+; CHECK-ARMPL: [[VECTOR_PH:.*:]]
+; CHECK-ARMPL: [[VECTOR_BODY:.*:]]
+; CHECK-ARMPL: [[TMP15:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.sincos.nxv4f32(<vscale x 4 x float> [[WIDE_MASKED_LOAD:%.*]])
+; CHECK-ARMPL: [[TMP16:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP15]], 0
+; CHECK-ARMPL: [[TMP17:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[TMP15]], 1
+; CHECK-ARMPL: call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[TMP16]], ptr [[TMP19:%.*]], i32 4, <vscale x 4 x i1> [[TMP14:%.*]])
+; CHECK-ARMPL: call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[TMP17]], ptr [[TMP21:%.*]], i32 4, <vscale x 4 x i1> [[TMP14]])
+; CHECK-ARMPL: [[MIDDLE_BLOCK:.*:]]
+; CHECK-ARMPL: [[SCALAR_PH:.*:]]
+; CHECK-ARMPL: [[FOR_BODY:.*:]]
+; CHECK-ARMPL: [[IF_THEN:.*:]]
+; CHECK-ARMPL: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
+; CHECK-ARMPL: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
+; CHECK-ARMPL: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
+; CHECK-ARMPL: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
+; CHECK-ARMPL: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
+; CHECK-ARMPL: [[IF_MERGE:.*:]]
+; CHECK-ARMPL: [[FOR_END:.*:]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ %iv.next, %if.merge ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
+ %in_val = load float, ptr %arrayidx, align 4
+ %if_cond = fcmp olt float %in_val, %x
+ br i1 %if_cond, label %if.then, label %if.merge
+
+if.then:
+ %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
+ %extract_a = extractvalue { float, float } %call, 0
+ %extract_b = extractvalue { float, float } %call, 1
+ %arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
+ store float %extract_a, ptr %arrayidx2, align 4
+ %arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
+ store float %extract_b, ptr %arrayidx4, align 4
+ br label %if.merge
+
+if.merge:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, 1024
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
>From afeb08fb8c24c02ca271ec54e41b61f92a3b93fa Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Wed, 26 Feb 2025 16:44:11 +0000
Subject: [PATCH 3/4] Add target-independent tests + fixups
---
.../LoopVectorize/AArch64/llvm.sincos.ll | 4 +-
.../Transforms/LoopVectorize/llvm.sincos.ll | 157 ++++++++++++++++++
2 files changed, 159 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/llvm.sincos.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll b/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
index 5001ae921eeeb..a7e949838f762 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "(:|sincos|extractvalue|store)" --version 5
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-gnu-linux -mcpu=neoverse-v1 -mattr=+sve < %s -S -o - -debug-only=loop-vectorize 2>%t.1 | FileCheck %s --check-prefix=CHECK
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-gnu-linux -mcpu=neoverse-v1 -mattr=+sve -vector-library=ArmPL < %s -S -o - -debug-only=loop-vectorize 2>%t.2 | FileCheck %s --check-prefix=CHECK-ARMPL
-; RUN: cat %t.1 | FileCheck --check-prefix=CHECK-COST %s
-; RUN: cat %t.2 | FileCheck --check-prefix=CHECK-COST-ARMPL %s
+; RUN: FileCheck --input-file=%t.1 --check-prefix=CHECK-COST %s
+; RUN: FileCheck --input-file=%t.2 --check-prefix=CHECK-COST-ARMPL %s
; REQUIRES: asserts
; CHECK-COST-LABEL: sincos_f32
diff --git a/llvm/test/Transforms/LoopVectorize/llvm.sincos.ll b/llvm/test/Transforms/LoopVectorize/llvm.sincos.ll
new file mode 100644
index 0000000000000..c2936eb8bb8b2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/llvm.sincos.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "(:|sincos|extract|store)" --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s -S -o - | FileCheck %s
+
+define void @sincos_f32(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @sincos_f32(
+; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[VECTOR_PH:.*:]]
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[TMP3:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> [[WIDE_LOAD:%.*]])
+; CHECK: [[TMP4:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 0
+; CHECK: [[TMP5:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 1
+; CHECK: store <2 x float> [[TMP4]], ptr [[TMP7:%.*]], align 4
+; CHECK: store <2 x float> [[TMP5]], ptr [[TMP9:%.*]], align 4
+; CHECK: [[MIDDLE_BLOCK:.*:]]
+; CHECK: [[SCALAR_PH:.*:]]
+; CHECK: [[FOR_BODY:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
+; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
+; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
+; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
+; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
+; CHECK: [[EXIT:.*:]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
+ %in_val = load float, ptr %arrayidx, align 4
+ %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
+ %extract_a = extractvalue { float, float } %call, 0
+ %extract_b = extractvalue { float, float } %call, 1
+ %arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
+ store float %extract_a, ptr %arrayidx2, align 4
+ %arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
+ store float %extract_b, ptr %arrayidx4, align 4
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @sincos_f64(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @sincos_f64(
+; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[VECTOR_PH:.*:]]
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[TMP3:%.*]] = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
+; CHECK: [[TMP4:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 0
+; CHECK: [[TMP5:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 1
+; CHECK: store <2 x double> [[TMP4]], ptr [[TMP7:%.*]], align 8
+; CHECK: store <2 x double> [[TMP5]], ptr [[TMP9:%.*]], align 8
+; CHECK: [[MIDDLE_BLOCK:.*:]]
+; CHECK: [[SCALAR_PH:.*:]]
+; CHECK: [[FOR_BODY:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { double, double } @llvm.sincos.f64(double [[IN_VAL:%.*]])
+; CHECK: [[EXTRACT_A:%.*]] = extractvalue { double, double } [[CALL]], 0
+; CHECK: [[EXTRACT_B:%.*]] = extractvalue { double, double } [[CALL]], 1
+; CHECK: store double [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 8
+; CHECK: store double [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 8
+; CHECK: [[EXIT:.*:]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds double, ptr %in, i64 %iv
+ %in_val = load double, ptr %arrayidx, align 8
+ %call = tail call { double, double } @llvm.sincos.f64(double %in_val)
+ %extract_a = extractvalue { double, double } %call, 0
+ %extract_b = extractvalue { double, double } %call, 1
+ %arrayidx2 = getelementptr inbounds double, ptr %out_a, i64 %iv
+ store double %extract_a, ptr %arrayidx2, align 8
+ %arrayidx4 = getelementptr inbounds double, ptr %out_b, i64 %iv
+ store double %extract_b, ptr %arrayidx4, align 8
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, 1024
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @predicated_sincos(float %x, ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
+; CHECK-LABEL: define void @predicated_sincos(
+; CHECK-SAME: float [[X:%.*]], ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
+; CHECK: [[ENTRY:.*:]]
+; CHECK: [[VECTOR_BODY1:.*]]:
+; CHECK: [[VECTOR_BODY:.*:]]
+; CHECK: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY1]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1:.*]] ]
+; CHECK: [[TMP4:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> [[WIDE_LOAD:%.*]])
+; CHECK: [[TMP5:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP4]], 0
+; CHECK: [[TMP6:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP4]], 1
+; CHECK: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP3:%.*]], i32 0
+; CHECK: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
+; CHECK: [[TMP9:%.*]] = extractelement <2 x float> [[TMP5]], i32 0
+; CHECK: store float [[TMP9]], ptr [[TMP8:%.*]], align 4
+; CHECK: [[TMP11:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
+; CHECK: store float [[TMP11]], ptr [[TMP10:%.*]], align 4
+; CHECK: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
+; CHECK: br i1 [[TMP12]], label %[[PRED_STORE_IF1:.*]], label %[[FOR_BODY1]]
+; CHECK: [[PRED_STORE_IF1]]:
+; CHECK: [[TMP15:%.*]] = extractelement <2 x float> [[TMP5]], i32 1
+; CHECK: store float [[TMP15]], ptr [[TMP14:%.*]], align 4
+; CHECK: [[TMP17:%.*]] = extractelement <2 x float> [[TMP6]], i32 1
+; CHECK: store float [[TMP17]], ptr [[TMP16:%.*]], align 4
+; CHECK: br label %[[FOR_BODY1]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK: [[IF_THEN1:.*:]]
+; CHECK: [[IF_THEN2:.*:]]
+; CHECK: [[IF_THEN:.*:]]
+; CHECK: [[IF_THEN3:.*:]]
+; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
+; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
+; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
+; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
+; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
+; CHECK: [[IF_MERGE:.*:]]
+; CHECK: [[FOR_END:.*:]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ %iv.next, %if.merge ], [ 0, %entry ]
+ %arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
+ %in_val = load float, ptr %arrayidx, align 4
+ %if_cond = fcmp olt float %in_val, %x
+ br i1 %if_cond, label %if.then, label %if.merge
+
+if.then:
+ %call = tail call { float, float } @llvm.sincos.f32(float %in_val)
+ %extract_a = extractvalue { float, float } %call, 0
+ %extract_b = extractvalue { float, float } %call, 1
+ %arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
+ store float %extract_a, ptr %arrayidx2, align 4
+ %arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
+ store float %extract_b, ptr %arrayidx4, align 4
+ br label %if.merge
+
+if.merge:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cond = icmp slt i64 %iv.next, 1024
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
>From bb52c13d35a827ea8655048dabe66ed5d2d2f0dd Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Thu, 27 Feb 2025 08:24:25 +0000
Subject: [PATCH 4/4] mv tests
---
.../LoopVectorize/AArch64/{llvm.sincos.ll => sincos.ll} | 0
llvm/test/Transforms/LoopVectorize/{llvm.sincos.ll => sincos.ll} | 0
2 files changed, 0 insertions(+), 0 deletions(-)
rename llvm/test/Transforms/LoopVectorize/AArch64/{llvm.sincos.ll => sincos.ll} (100%)
rename llvm/test/Transforms/LoopVectorize/{llvm.sincos.ll => sincos.ll} (100%)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sincos.ll
similarity index 100%
rename from llvm/test/Transforms/LoopVectorize/AArch64/llvm.sincos.ll
rename to llvm/test/Transforms/LoopVectorize/AArch64/sincos.ll
diff --git a/llvm/test/Transforms/LoopVectorize/llvm.sincos.ll b/llvm/test/Transforms/LoopVectorize/sincos.ll
similarity index 100%
rename from llvm/test/Transforms/LoopVectorize/llvm.sincos.ll
rename to llvm/test/Transforms/LoopVectorize/sincos.ll
More information about the llvm-commits
mailing list