[llvm] [AArch64][ISel] Add clmul to pmullb/t lowering (PR #180568)
Matthew Devereau via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 10 06:36:27 PST 2026
https://github.com/MDevereau updated https://github.com/llvm/llvm-project/pull/180568
>From b3d1230edf159e726fef58d623bf6b0bc933e66c Mon Sep 17 00:00:00 2001
From: Matthew Devereau <matthew.devereau at arm.com>
Date: Mon, 9 Feb 2026 17:24:31 +0000
Subject: [PATCH 1/2] [AArch64][ISel] Add i64 clmul to pmullb/t lowering
---
.../Target/AArch64/AArch64ISelLowering.cpp | 5 +-
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 11 +
llvm/test/CodeGen/AArch64/clmul-scalable.ll | 740 +-----------------
3 files changed, 48 insertions(+), 708 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index add3a92343943..4d81ea1b7ddcf 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2002,7 +2002,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv8i16, Legal);
setPartialReduceMLAAction(MLAOps, MVT::nxv8i16, MVT::nxv16i8, Legal);
- setOperationAction(ISD::CLMUL, MVT::nxv16i8, Legal);
+ setOperationAction(ISD::CLMUL, {MVT::nxv16i8, MVT::nxv4i32}, Legal);
}
// Handle floating-point partial reduction
@@ -2015,6 +2015,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
}
}
+ if (Subtarget->hasSVE2AES())
+ setOperationAction(ISD::CLMUL, MVT::nxv2i64, Legal);
+
// Handle non-aliasing elements mask
if (Subtarget->hasSVE2() ||
(Subtarget->hasSME() && Subtarget->isStreaming())) {
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 61db07493260c..d7f402bf03a30 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4254,6 +4254,17 @@ let Predicates = [HasSVEAES, HasNonStreamingSVE_or_SSVE_AES] in {
defm PMULLT_ZZZ_Q : sve2_wide_int_arith_pmul<0b00, 0b11011, "pmullt", int_aarch64_sve_pmullt_pair>;
}
+let Predicates = [HasSVEAES, HasSVE2_or_SME] in {
+ def : Pat<(nxv2i64 (clmul nxv2i64:$Rn, nxv2i64:$Rm)),
+ (ZIP1_ZZZ_D (UZP1_ZZZ_D (PMULLB_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm)),
+ (UZP1_ZZZ_D (PMULLT_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm)))>;
+}
+let Predicates = [HasSVE2_or_SME] in {
+ def : Pat<(nxv4i32 (clmul nxv4i32:$Rn, nxv4i32:$Rm)),
+ (ZIP1_ZZZ_S (UZP1_ZZZ_S (PMULLB_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm)),
+ (UZP1_ZZZ_S (PMULLT_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm)))>;
+}
+
let Predicates = [HasSVESM4] in {
// SVE2 crypto constructive binary operations
defm SM4EKEY_ZZZ_S : sve2_crypto_cons_bin_op<0b0, "sm4ekey", ZPR32, int_aarch64_sve_sm4ekey, nxv4i32>;
diff --git a/llvm/test/CodeGen/AArch64/clmul-scalable.ll b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
index 58dbba833f67b..a6c1bfae6e1b9 100644
--- a/llvm/test/CodeGen/AArch64/clmul-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
@@ -377,234 +377,20 @@ define <vscale x 4 x i32> @clmul_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i3
;
; CHECK-SVE2-LABEL: clmul_nxv4i32:
; CHECK-SVE2: // %bb.0:
-; CHECK-SVE2-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z2.s, z2.s, #0x2
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x1
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x8
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x4
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x20
-; CHECK-SVE2-NEXT: mul z2.s, z0.s, z2.s
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x10
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x80
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x40
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x200
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x100
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x800
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x400
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x2000
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x1000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x8000
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x4000
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x20000
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x10000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x80000
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x40000
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x200000
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x100000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x800000
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x400000
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x2000000
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x1000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x8000000
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x4000000
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x20000000
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x10000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: and z1.s, z1.s, #0x40000000
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x80000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: mul z0.s, z0.s, z1.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-NEXT: pmullt z2.d, z0.s, z1.s
+; CHECK-SVE2-NEXT: pmullb z0.d, z0.s, z1.s
+; CHECK-SVE2-NEXT: uzp1 z1.s, z2.s, z2.s
+; CHECK-SVE2-NEXT: uzp1 z0.s, z0.s, z2.s
+; CHECK-SVE2-NEXT: zip1 z0.s, z0.s, z1.s
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-AES-LABEL: clmul_nxv4i32:
; CHECK-SVE2-AES: // %bb.0:
-; CHECK-SVE2-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z2.s, z2.s, #0x2
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x1
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x8
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x4
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x20
-; CHECK-SVE2-AES-NEXT: mul z2.s, z0.s, z2.s
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x10
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x80
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x40
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x200
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x100
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x800
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x400
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x2000
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x1000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x8000
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x4000
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x20000
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x10000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x80000
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x40000
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x200000
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x100000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x800000
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x400000
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x2000000
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x1000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x8000000
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x4000000
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x20000000
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x10000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z1.s, z1.s, #0x40000000
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x80000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: mul z0.s, z0.s, z1.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT: pmullt z2.d, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT: pmullb z0.d, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT: uzp1 z1.s, z2.s, z2.s
+; CHECK-SVE2-AES-NEXT: uzp1 z0.s, z0.s, z2.s
+; CHECK-SVE2-AES-NEXT: zip1 z0.s, z0.s, z1.s
; CHECK-SVE2-AES-NEXT: ret
%a = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y)
ret <vscale x 4 x i32> %a
@@ -1100,230 +886,11 @@ define <vscale x 2 x i64> @clmul_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i6
;
; CHECK-SVE2-AES-LABEL: clmul_nxv2i64:
; CHECK-SVE2-AES: // %bb.0:
-; CHECK-SVE2-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z2.d, z2.d, #0x2
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x1
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x8
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x4
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x20
-; CHECK-SVE2-AES-NEXT: mul z2.d, z0.d, z2.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x10
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x80
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x40
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x200
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x100
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x800
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x400
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x2000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x1000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x8000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x4000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x20000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x10000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x80000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x40000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x200000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x100000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x800000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x400000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x2000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x1000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x8000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x4000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x20000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x10000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x80000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x40000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x200000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x100000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x800000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x400000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x2000000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x1000000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x8000000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x4000000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x20000000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x10000000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x80000000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x40000000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x200000000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x100000000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x800000000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x400000000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x2000000000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x1000000000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x8000000000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x4000000000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x20000000000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x10000000000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x80000000000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x40000000000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x200000000000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x100000000000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x800000000000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x400000000000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x2000000000000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x1000000000000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z1.d, z1.d, #0x4000000000000000
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x8000000000000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: mul z0.d, z0.d, z1.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT: pmullt z2.q, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT: pmullb z0.q, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT: uzp1 z1.d, z2.d, z2.d
+; CHECK-SVE2-AES-NEXT: uzp1 z0.d, z0.d, z2.d
+; CHECK-SVE2-AES-NEXT: zip1 z0.d, z0.d, z1.d
; CHECK-SVE2-AES-NEXT: ret
%a = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y)
ret <vscale x 2 x i64> %a
@@ -1597,142 +1164,24 @@ define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4
;
; CHECK-SVE2-LABEL: clmul_nxv4i32_zext:
; CHECK-SVE2: // %bb.0:
-; CHECK-SVE2-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
; CHECK-SVE2-NEXT: and z0.s, z0.s, #0xffff
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z2.s, z2.s, #0x2
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x1
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x8
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x4
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x20
-; CHECK-SVE2-NEXT: mul z2.s, z0.s, z2.s
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x10
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x80
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x40
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x200
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x100
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: and z4.s, z4.s, #0x800
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x400
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.s, z3.s, #0x2000
-; CHECK-SVE2-NEXT: and z6.s, z6.s, #0x1000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: and z1.s, z1.s, #0x4000
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT: mul z4.s, z0.s, z6.s
-; CHECK-SVE2-NEXT: and z5.s, z5.s, #0x8000
-; CHECK-SVE2-NEXT: mul z1.s, z0.s, z1.s
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT: mul z3.s, z0.s, z5.s
-; CHECK-SVE2-NEXT: mul z0.s, z0.s, #0
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z1.d, z3.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-NEXT: and z1.s, z1.s, #0xffff
+; CHECK-SVE2-NEXT: pmullt z2.d, z0.s, z1.s
+; CHECK-SVE2-NEXT: pmullb z0.d, z0.s, z1.s
+; CHECK-SVE2-NEXT: uzp1 z1.s, z2.s, z2.s
+; CHECK-SVE2-NEXT: uzp1 z0.s, z0.s, z2.s
+; CHECK-SVE2-NEXT: zip1 z0.s, z0.s, z1.s
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-AES-LABEL: clmul_nxv4i32_zext:
; CHECK-SVE2-AES: // %bb.0:
-; CHECK-SVE2-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
; CHECK-SVE2-AES-NEXT: and z0.s, z0.s, #0xffff
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z2.s, z2.s, #0x2
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x1
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x8
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x4
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x20
-; CHECK-SVE2-AES-NEXT: mul z2.s, z0.s, z2.s
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x10
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x80
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x40
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x200
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x100
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: and z4.s, z4.s, #0x800
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x400
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT: mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.s, z3.s, #0x2000
-; CHECK-SVE2-AES-NEXT: and z6.s, z6.s, #0x1000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z1.s, z1.s, #0x4000
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT: mul z4.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT: and z5.s, z5.s, #0x8000
-; CHECK-SVE2-AES-NEXT: mul z1.s, z0.s, z1.s
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT: mul z0.s, z0.s, #0
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z1.d, z3.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT: and z1.s, z1.s, #0xffff
+; CHECK-SVE2-AES-NEXT: pmullt z2.d, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT: pmullb z0.d, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT: uzp1 z1.s, z2.s, z2.s
+; CHECK-SVE2-AES-NEXT: uzp1 z0.s, z0.s, z2.s
+; CHECK-SVE2-AES-NEXT: zip1 z0.s, z0.s, z1.s
; CHECK-SVE2-AES-NEXT: ret
%zextx = zext <vscale x 4 x i16> %x to <vscale x 4 x i32>
%zexty = zext <vscale x 4 x i16> %y to <vscale x 4 x i32>
@@ -2096,136 +1545,13 @@ define <vscale x 2 x i64> @clmul_nxv2i64_zext(<vscale x 2 x i32> %x, <vscale x 2
;
; CHECK-SVE2-AES-LABEL: clmul_nxv2i64_zext:
; CHECK-SVE2-AES: // %bb.0:
-; CHECK-SVE2-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
; CHECK-SVE2-AES-NEXT: and z0.d, z0.d, #0xffffffff
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z2.d, z2.d, #0x2
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x1
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x8
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x4
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x20
-; CHECK-SVE2-AES-NEXT: mul z2.d, z0.d, z2.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x10
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x80
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x40
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x200
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x100
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x800
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x400
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x2000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x1000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x8000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x4000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x20000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x10000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x80000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x40000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x200000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x100000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x800000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x400000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x2000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x1000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z4.d, z4.d, #0x8000000
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x4000000
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.d, z3.d, #0x20000000
-; CHECK-SVE2-AES-NEXT: and z6.d, z6.d, #0x10000000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z1.d, z1.d, #0x40000000
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z4.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT: and z5.d, z5.d, #0x80000000
-; CHECK-SVE2-AES-NEXT: mul z1.d, z0.d, z1.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT: mul z0.d, z0.d, #0
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z1.d, z3.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT: and z1.d, z1.d, #0xffffffff
+; CHECK-SVE2-AES-NEXT: pmullt z2.q, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT: pmullb z0.q, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT: uzp1 z1.d, z2.d, z2.d
+; CHECK-SVE2-AES-NEXT: uzp1 z0.d, z0.d, z2.d
+; CHECK-SVE2-AES-NEXT: zip1 z0.d, z0.d, z1.d
; CHECK-SVE2-AES-NEXT: ret
%zextx = zext <vscale x 2 x i32> %x to <vscale x 2 x i64>
%zexty = zext <vscale x 2 x i32> %y to <vscale x 2 x i64>
>From 4de7e03e1994ab89f611556cd450a8d6f8d5427f Mon Sep 17 00:00:00 2001
From: Matthew Devereau <matthew.devereau at arm.com>
Date: Tue, 10 Feb 2026 14:26:26 +0000
Subject: [PATCH 2/2] Use TRN1
---
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 6 ++---
llvm/test/CodeGen/AArch64/clmul-scalable.ll | 24 +++++--------------
2 files changed, 8 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index d7f402bf03a30..87e419b6d3667 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4256,13 +4256,11 @@ let Predicates = [HasSVEAES, HasNonStreamingSVE_or_SSVE_AES] in {
let Predicates = [HasSVEAES, HasSVE2_or_SME] in {
def : Pat<(nxv2i64 (clmul nxv2i64:$Rn, nxv2i64:$Rm)),
- (ZIP1_ZZZ_D (UZP1_ZZZ_D (PMULLB_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm)),
- (UZP1_ZZZ_D (PMULLT_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm)))>;
+ (TRN1_ZZZ_D (PMULLB_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm))>;
}
let Predicates = [HasSVE2_or_SME] in {
def : Pat<(nxv4i32 (clmul nxv4i32:$Rn, nxv4i32:$Rm)),
- (ZIP1_ZZZ_S (UZP1_ZZZ_S (PMULLB_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm)),
- (UZP1_ZZZ_S (PMULLT_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm)))>;
+ (TRN1_ZZZ_S (PMULLB_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm))>;
}
let Predicates = [HasSVESM4] in {
diff --git a/llvm/test/CodeGen/AArch64/clmul-scalable.ll b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
index a6c1bfae6e1b9..b974340ce6cd0 100644
--- a/llvm/test/CodeGen/AArch64/clmul-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
@@ -379,18 +379,14 @@ define <vscale x 4 x i32> @clmul_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i3
; CHECK-SVE2: // %bb.0:
; CHECK-SVE2-NEXT: pmullt z2.d, z0.s, z1.s
; CHECK-SVE2-NEXT: pmullb z0.d, z0.s, z1.s
-; CHECK-SVE2-NEXT: uzp1 z1.s, z2.s, z2.s
-; CHECK-SVE2-NEXT: uzp1 z0.s, z0.s, z2.s
-; CHECK-SVE2-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-SVE2-NEXT: trn1 z0.s, z0.s, z2.s
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-AES-LABEL: clmul_nxv4i32:
; CHECK-SVE2-AES: // %bb.0:
; CHECK-SVE2-AES-NEXT: pmullt z2.d, z0.s, z1.s
; CHECK-SVE2-AES-NEXT: pmullb z0.d, z0.s, z1.s
-; CHECK-SVE2-AES-NEXT: uzp1 z1.s, z2.s, z2.s
-; CHECK-SVE2-AES-NEXT: uzp1 z0.s, z0.s, z2.s
-; CHECK-SVE2-AES-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT: trn1 z0.s, z0.s, z2.s
; CHECK-SVE2-AES-NEXT: ret
%a = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y)
ret <vscale x 4 x i32> %a
@@ -888,9 +884,7 @@ define <vscale x 2 x i64> @clmul_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i6
; CHECK-SVE2-AES: // %bb.0:
; CHECK-SVE2-AES-NEXT: pmullt z2.q, z0.d, z1.d
; CHECK-SVE2-AES-NEXT: pmullb z0.q, z0.d, z1.d
-; CHECK-SVE2-AES-NEXT: uzp1 z1.d, z2.d, z2.d
-; CHECK-SVE2-AES-NEXT: uzp1 z0.d, z0.d, z2.d
-; CHECK-SVE2-AES-NEXT: zip1 z0.d, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT: trn1 z0.d, z0.d, z2.d
; CHECK-SVE2-AES-NEXT: ret
%a = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y)
ret <vscale x 2 x i64> %a
@@ -1168,9 +1162,7 @@ define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4
; CHECK-SVE2-NEXT: and z1.s, z1.s, #0xffff
; CHECK-SVE2-NEXT: pmullt z2.d, z0.s, z1.s
; CHECK-SVE2-NEXT: pmullb z0.d, z0.s, z1.s
-; CHECK-SVE2-NEXT: uzp1 z1.s, z2.s, z2.s
-; CHECK-SVE2-NEXT: uzp1 z0.s, z0.s, z2.s
-; CHECK-SVE2-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-SVE2-NEXT: trn1 z0.s, z0.s, z2.s
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-AES-LABEL: clmul_nxv4i32_zext:
@@ -1179,9 +1171,7 @@ define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4
; CHECK-SVE2-AES-NEXT: and z1.s, z1.s, #0xffff
; CHECK-SVE2-AES-NEXT: pmullt z2.d, z0.s, z1.s
; CHECK-SVE2-AES-NEXT: pmullb z0.d, z0.s, z1.s
-; CHECK-SVE2-AES-NEXT: uzp1 z1.s, z2.s, z2.s
-; CHECK-SVE2-AES-NEXT: uzp1 z0.s, z0.s, z2.s
-; CHECK-SVE2-AES-NEXT: zip1 z0.s, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT: trn1 z0.s, z0.s, z2.s
; CHECK-SVE2-AES-NEXT: ret
%zextx = zext <vscale x 4 x i16> %x to <vscale x 4 x i32>
%zexty = zext <vscale x 4 x i16> %y to <vscale x 4 x i32>
@@ -1549,9 +1539,7 @@ define <vscale x 2 x i64> @clmul_nxv2i64_zext(<vscale x 2 x i32> %x, <vscale x 2
; CHECK-SVE2-AES-NEXT: and z1.d, z1.d, #0xffffffff
; CHECK-SVE2-AES-NEXT: pmullt z2.q, z0.d, z1.d
; CHECK-SVE2-AES-NEXT: pmullb z0.q, z0.d, z1.d
-; CHECK-SVE2-AES-NEXT: uzp1 z1.d, z2.d, z2.d
-; CHECK-SVE2-AES-NEXT: uzp1 z0.d, z0.d, z2.d
-; CHECK-SVE2-AES-NEXT: zip1 z0.d, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT: trn1 z0.d, z0.d, z2.d
; CHECK-SVE2-AES-NEXT: ret
%zextx = zext <vscale x 2 x i32> %x to <vscale x 2 x i64>
%zexty = zext <vscale x 2 x i32> %y to <vscale x 2 x i64>
More information about the llvm-commits
mailing list