[llvm] 44601f4 - [AARCH64][NEON] Allow to sink operands for aarch64_neon_pmull
David Green via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 3 08:46:54 PST 2022
Author: Sunho Kim
Date: 2022-02-03T16:46:49Z
New Revision: 44601f4956062bee5fc2b481963ff06d24422767
URL: https://github.com/llvm/llvm-project/commit/44601f4956062bee5fc2b481963ff06d24422767
DIFF: https://github.com/llvm/llvm-project/commit/44601f4956062bee5fc2b481963ff06d24422767.diff
LOG: [AARCH64][NEON] Allow to sink operands for aarch64_neon_pmull
This teaches AArch64TargetLowering::shouldSinkOperands to sink the
operands of aarch64_neon_pmull intrinsic.
Differential Revision: https://reviews.llvm.org/D117944
Added:
llvm/test/CodeGen/AArch64/neon-vmull-high-p8.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c48f5c4b047f9..b7daae45ced70 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -12197,6 +12197,12 @@ bool AArch64TargetLowering::shouldSinkOperands(
Ops.push_back(&II->getOperandUse(1));
return !Ops.empty();
+ case Intrinsic::aarch64_neon_pmull:
+ if (!areExtractShuffleVectors(II->getOperand(0), II->getOperand(1)))
+ return false;
+ Ops.push_back(&II->getOperandUse(0));
+ Ops.push_back(&II->getOperandUse(1));
+ return true;
case Intrinsic::aarch64_neon_pmull64:
if (!areOperandsOfVmullHighP64(II->getArgOperand(0),
II->getArgOperand(1)))
diff --git a/llvm/test/CodeGen/AArch64/neon-vmull-high-p8.ll b/llvm/test/CodeGen/AArch64/neon-vmull-high-p8.ll
new file mode 100644
index 0000000000000..137a865ba86ad
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-vmull-high-p8.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -asm-verbose=0 -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s
+
+; Check that pmull2 instruction is used for vmull_high_p8 intrinsic
+; even if shufflevector instructions are located in
diff erent basic blocks,
+; which can happen when vmull_high_p8 is used inside a loop body.
+;
+
+define <8 x i16> @test_pmull2_sink(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, i1 %t) {
+; CHECK-LABEL: test_pmull2_sink:
+; CHECK: tbz w0, #0, .LBB0_2
+; CHECK-NEXT: pmull2 v0.8h, v0.16b, v1.16b
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB0_2:
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+entry:
+ %0 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ br i1 %t, label %if.then, label %cleanup
+
+if.then:
+ %1 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %0, <8 x i8> %1)
+ br label %cleanup
+
+cleanup:
+ %retval = phi <8 x i16> [ %res, %if.then ], [ %c, %entry ]
+ ret <8 x i16> %retval
+}
+
+define <8 x i16> @test_pmull2_sink2(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, i1 %t) {
+; CHECK-LABEL: test_pmull2_sink2:
+; CHECK: tbz w0, #0, .LBB1_2
+; CHECK-NEXT: pmull2 v0.8h, v0.16b, v0.16b
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB1_2:
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+entry:
+ %0 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ br i1 %t, label %if.then, label %cleanup
+
+if.then:
+ %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %0, <8 x i8> %1)
+ br label %cleanup
+
+cleanup:
+ %retval = phi <8 x i16> [ %res, %if.then ], [ %c, %entry ]
+ ret <8 x i16> %retval
+}
+
+define <8 x i16> @test_pmull2_sink3(<16 x i8> %a, <16 x i8> %b, <8 x i16> %c, i1 %t, i1 %t2) {
+; CHECK-LABEL: test_pmull2_sink3:
+; CHECK: tbz w0, #0, .LBB2_2
+; CHECK-NEXT: tbz w1, #0, .LBB2_3
+; CHECK-NEXT: .LBB2_2:
+; CHECK-NEXT: pmull2 v0.8h, v0.16b, v1.16b
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB2_3:
+; CHECK-NEXT: pmull2 v0.8h, v0.16b, v0.16b
+; CHECK-NEXT: ret
+entry:
+ %0 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ br i1 %t, label %if.then, label %if.then.2
+
+if.then:
+ %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %0, <8 x i8> %0)
+ br i1 %t2, label %if.then.2, label %cleanup
+
+if.then.2:
+ %1 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %res2 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %0, <8 x i8> %1)
+ br label %cleanup
+
+cleanup:
+ %retval = phi <8 x i16> [ %res2, %if.then.2 ], [ %res, %if.then ]
+ ret <8 x i16> %retval
+}
+
+declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>)
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
index a69d4652e277f..94164c08a4b38 100644
--- a/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
@@ -397,3 +397,100 @@ for.cond4.preheader.us.preheader: ; preds = %for.cond4.preheader
for.cond4.preheader.preheader: ; preds = %for.cond4.preheader.lr.ph
ret <4 x i32> zeroinitializer
}
+
+declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>)
+
+define <8 x i16> @sink_shufflevector_pmull(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @sink_shufflevector_pmull(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 poison, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[TMP0]], <8 x i8> [[S2]])
+; CHECK-NEXT: ret <8 x i16> [[VMULL0]]
+; CHECK: if.else:
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[S4]])
+; CHECK-NEXT: ret <8 x i16> [[VMULL1]]
+;
+entry:
+ %s1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %s3 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ br i1 poison, label %if.then, label %if.else
+
+if.then:
+ %s2 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %vmull0 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s1, <8 x i8> %s2)
+ ret <8 x i16> %vmull0
+
+if.else:
+ %s4 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %vmull1 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s3, <8 x i8> %s4)
+ ret <8 x i16> %vmull1
+}
+
+; Indexed pmull is not available on aarch64. Shuffle vector should not be sunk here.
+define <8 x i16> @no_sink_splatvector_pmull(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @no_sink_splatvector_pmull(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[S1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: br i1 poison, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[S1]], <8 x i8> [[S2]])
+; CHECK-NEXT: ret <8 x i16> [[VMULL0]]
+; CHECK: if.else:
+; CHECK-NEXT: [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[S3]], <8 x i8> [[S4]])
+; CHECK-NEXT: ret <8 x i16> [[VMULL1]]
+;
+entry:
+ %s1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+ %s3 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+ br i1 poison, label %if.then, label %if.else
+
+if.then:
+ %s2 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %vmull0 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s1, <8 x i8> %s2)
+ ret <8 x i16> %vmull0
+
+if.else:
+ %s4 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %vmull1 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s3, <8 x i8> %s4)
+ ret <8 x i16> %vmull1
+}
+
+; Mask used are not suitable for pmull. Shuffle vector should not be sunk here.
+define <8 x i16> @no_sink_shufflevector_pmull(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @no_sink_shufflevector_pmull(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[S1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: br i1 poison, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK: if.then:
+; CHECK-NEXT: [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[S1]], <8 x i8> [[S2]])
+; CHECK-NEXT: ret <8 x i16> [[VMULL0]]
+; CHECK: if.else:
+; CHECK-NEXT: [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> [[S3]], <8 x i8> [[S4]])
+; CHECK-NEXT: ret <8 x i16> [[VMULL1]]
+;
+entry:
+ %s1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %s3 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ br i1 poison, label %if.then, label %if.else
+
+if.then:
+ %s2 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %vmull0 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s1, <8 x i8> %s2)
+ ret <8 x i16> %vmull0
+
+if.else:
+ %s4 = shufflevector <16 x i8> %b, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %vmull1 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %s3, <8 x i8> %s4)
+ ret <8 x i16> %vmull1
+}
More information about the llvm-commits
mailing list