[llvm] [AArch64][ISel] Add clmul to pmullb/t lowering (PR #180568)

Matthew Devereau via llvm-commits llvm-commits at lists.llvm.org
Tue Feb 10 06:36:27 PST 2026


https://github.com/MDevereau updated https://github.com/llvm/llvm-project/pull/180568

>From b3d1230edf159e726fef58d623bf6b0bc933e66c Mon Sep 17 00:00:00 2001
From: Matthew Devereau <matthew.devereau at arm.com>
Date: Mon, 9 Feb 2026 17:24:31 +0000
Subject: [PATCH 1/2] [AArch64][ISel] Add i64 clmul to pmullb/t lowering

---
 .../Target/AArch64/AArch64ISelLowering.cpp    |   5 +-
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  11 +
 llvm/test/CodeGen/AArch64/clmul-scalable.ll   | 740 +-----------------
 3 files changed, 48 insertions(+), 708 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index add3a92343943..4d81ea1b7ddcf 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2002,7 +2002,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv8i16, Legal);
       setPartialReduceMLAAction(MLAOps, MVT::nxv8i16, MVT::nxv16i8, Legal);
 
-      setOperationAction(ISD::CLMUL, MVT::nxv16i8, Legal);
+      setOperationAction(ISD::CLMUL, {MVT::nxv16i8, MVT::nxv4i32}, Legal);
     }
 
     // Handle floating-point partial reduction
@@ -2015,6 +2015,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     }
   }
 
+  if (Subtarget->hasSVE2AES())
+    setOperationAction(ISD::CLMUL, MVT::nxv2i64, Legal);
+
   // Handle non-aliasing elements mask
   if (Subtarget->hasSVE2() ||
       (Subtarget->hasSME() && Subtarget->isStreaming())) {
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 61db07493260c..d7f402bf03a30 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4254,6 +4254,17 @@ let Predicates = [HasSVEAES, HasNonStreamingSVE_or_SSVE_AES] in {
   defm PMULLT_ZZZ_Q : sve2_wide_int_arith_pmul<0b00, 0b11011, "pmullt", int_aarch64_sve_pmullt_pair>;
 }
 
+let Predicates = [HasSVEAES, HasSVE2_or_SME] in {
+  def : Pat<(nxv2i64 (clmul nxv2i64:$Rn, nxv2i64:$Rm)),
+            (ZIP1_ZZZ_D (UZP1_ZZZ_D (PMULLB_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm)),
+                        (UZP1_ZZZ_D (PMULLT_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm)))>;
+}
+let Predicates = [HasSVE2_or_SME] in {
+  def : Pat<(nxv4i32 (clmul nxv4i32:$Rn, nxv4i32:$Rm)),
+            (ZIP1_ZZZ_S (UZP1_ZZZ_S (PMULLB_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm)),
+                        (UZP1_ZZZ_S (PMULLT_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm)))>;
+}
+
 let Predicates = [HasSVESM4] in {
   // SVE2 crypto constructive binary operations
   defm SM4EKEY_ZZZ_S : sve2_crypto_cons_bin_op<0b0, "sm4ekey", ZPR32, int_aarch64_sve_sm4ekey, nxv4i32>;
diff --git a/llvm/test/CodeGen/AArch64/clmul-scalable.ll b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
index 58dbba833f67b..a6c1bfae6e1b9 100644
--- a/llvm/test/CodeGen/AArch64/clmul-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
@@ -377,234 +377,20 @@ define <vscale x 4 x i32> @clmul_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i3
 ;
 ; CHECK-SVE2-LABEL: clmul_nxv4i32:
 ; CHECK-SVE2:       // %bb.0:
-; CHECK-SVE2-NEXT:    mov z2.d, z1.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z2.s, z2.s, #0x2
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x1
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x8
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x4
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x20
-; CHECK-SVE2-NEXT:    mul z2.s, z0.s, z2.s
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x10
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x80
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x40
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x200
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x100
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x800
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x400
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x2000
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x1000
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x8000
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x4000
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x20000
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x10000
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x80000
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x40000
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x200000
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x100000
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x800000
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x400000
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x2000000
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x1000000
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x8000000
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x4000000
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x20000000
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x10000000
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    and z1.s, z1.s, #0x40000000
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x80000000
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    mul z0.s, z0.s, z1.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-NEXT:    mov z0.d, z2.d
+; CHECK-SVE2-NEXT:    pmullt z2.d, z0.s, z1.s
+; CHECK-SVE2-NEXT:    pmullb z0.d, z0.s, z1.s
+; CHECK-SVE2-NEXT:    uzp1 z1.s, z2.s, z2.s
+; CHECK-SVE2-NEXT:    uzp1 z0.s, z0.s, z2.s
+; CHECK-SVE2-NEXT:    zip1 z0.s, z0.s, z1.s
 ; CHECK-SVE2-NEXT:    ret
 ;
 ; CHECK-SVE2-AES-LABEL: clmul_nxv4i32:
 ; CHECK-SVE2-AES:       // %bb.0:
-; CHECK-SVE2-AES-NEXT:    mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z2.s, z2.s, #0x2
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x1
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x8
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x4
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x20
-; CHECK-SVE2-AES-NEXT:    mul z2.s, z0.s, z2.s
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x10
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x80
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x40
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x200
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x100
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x800
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x400
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x2000
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x1000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x8000
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x4000
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x20000
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x10000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x80000
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x40000
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x200000
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x100000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x800000
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x400000
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x2000000
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x1000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x8000000
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x4000000
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x20000000
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x10000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z1.s, z1.s, #0x40000000
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x80000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    mul z0.s, z0.s, z1.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT:    pmullt z2.d, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT:    pmullb z0.d, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT:    uzp1 z1.s, z2.s, z2.s
+; CHECK-SVE2-AES-NEXT:    uzp1 z0.s, z0.s, z2.s
+; CHECK-SVE2-AES-NEXT:    zip1 z0.s, z0.s, z1.s
 ; CHECK-SVE2-AES-NEXT:    ret
   %a = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y)
   ret <vscale x 4 x i32> %a
@@ -1100,230 +886,11 @@ define <vscale x 2 x i64> @clmul_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i6
 ;
 ; CHECK-SVE2-AES-LABEL: clmul_nxv2i64:
 ; CHECK-SVE2-AES:       // %bb.0:
-; CHECK-SVE2-AES-NEXT:    mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z2.d, z2.d, #0x2
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x1
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x8
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x4
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x20
-; CHECK-SVE2-AES-NEXT:    mul z2.d, z0.d, z2.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x10
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x80
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x40
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x200
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x100
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x800
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x400
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x2000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x1000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x8000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x4000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x20000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x10000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x80000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x40000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x200000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x100000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x800000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x400000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x2000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x1000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x8000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x4000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x20000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x10000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x80000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x40000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x200000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x100000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x800000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x400000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x2000000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x1000000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x8000000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x4000000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x20000000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x10000000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x80000000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x40000000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x200000000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x100000000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x800000000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x400000000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x2000000000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x1000000000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x8000000000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x4000000000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x20000000000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x10000000000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x80000000000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x40000000000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x200000000000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x100000000000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x800000000000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x400000000000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x2000000000000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x1000000000000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z1.d, z1.d, #0x4000000000000000
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x8000000000000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    mul z0.d, z0.d, z1.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT:    pmullt z2.q, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT:    pmullb z0.q, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT:    uzp1 z1.d, z2.d, z2.d
+; CHECK-SVE2-AES-NEXT:    uzp1 z0.d, z0.d, z2.d
+; CHECK-SVE2-AES-NEXT:    zip1 z0.d, z0.d, z1.d
 ; CHECK-SVE2-AES-NEXT:    ret
   %a = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y)
   ret <vscale x 2 x i64> %a
@@ -1597,142 +1164,24 @@ define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4
 ;
 ; CHECK-SVE2-LABEL: clmul_nxv4i32_zext:
 ; CHECK-SVE2:       // %bb.0:
-; CHECK-SVE2-NEXT:    mov z2.d, z1.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
 ; CHECK-SVE2-NEXT:    and z0.s, z0.s, #0xffff
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z2.s, z2.s, #0x2
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x1
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x8
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x4
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x20
-; CHECK-SVE2-NEXT:    mul z2.s, z0.s, z2.s
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x10
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x80
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x40
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x200
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x100
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    and z4.s, z4.s, #0x800
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x400
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-NEXT:    and z3.s, z3.s, #0x2000
-; CHECK-SVE2-NEXT:    and z6.s, z6.s, #0x1000
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-NEXT:    and z1.s, z1.s, #0x4000
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-NEXT:    mul z4.s, z0.s, z6.s
-; CHECK-SVE2-NEXT:    and z5.s, z5.s, #0x8000
-; CHECK-SVE2-NEXT:    mul z1.s, z0.s, z1.s
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT:    mul z3.s, z0.s, z5.s
-; CHECK-SVE2-NEXT:    mul z0.s, z0.s, #0
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z1.d, z3.d
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-NEXT:    mov z0.d, z2.d
+; CHECK-SVE2-NEXT:    and z1.s, z1.s, #0xffff
+; CHECK-SVE2-NEXT:    pmullt z2.d, z0.s, z1.s
+; CHECK-SVE2-NEXT:    pmullb z0.d, z0.s, z1.s
+; CHECK-SVE2-NEXT:    uzp1 z1.s, z2.s, z2.s
+; CHECK-SVE2-NEXT:    uzp1 z0.s, z0.s, z2.s
+; CHECK-SVE2-NEXT:    zip1 z0.s, z0.s, z1.s
 ; CHECK-SVE2-NEXT:    ret
 ;
 ; CHECK-SVE2-AES-LABEL: clmul_nxv4i32_zext:
 ; CHECK-SVE2-AES:       // %bb.0:
-; CHECK-SVE2-AES-NEXT:    mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
 ; CHECK-SVE2-AES-NEXT:    and z0.s, z0.s, #0xffff
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z2.s, z2.s, #0x2
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x1
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x8
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x4
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x20
-; CHECK-SVE2-AES-NEXT:    mul z2.s, z0.s, z2.s
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x10
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x80
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x40
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x200
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x100
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z6.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    and z4.s, z4.s, #0x800
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x400
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z4.s
-; CHECK-SVE2-AES-NEXT:    mul z5.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.s, z3.s, #0x2000
-; CHECK-SVE2-AES-NEXT:    and z6.s, z6.s, #0x1000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z1.s, z1.s, #0x4000
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z3.s
-; CHECK-SVE2-AES-NEXT:    mul z4.s, z0.s, z6.s
-; CHECK-SVE2-AES-NEXT:    and z5.s, z5.s, #0x8000
-; CHECK-SVE2-AES-NEXT:    mul z1.s, z0.s, z1.s
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z3.s, z0.s, z5.s
-; CHECK-SVE2-AES-NEXT:    mul z0.s, z0.s, #0
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z1.d, z3.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT:    and z1.s, z1.s, #0xffff
+; CHECK-SVE2-AES-NEXT:    pmullt z2.d, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT:    pmullb z0.d, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT:    uzp1 z1.s, z2.s, z2.s
+; CHECK-SVE2-AES-NEXT:    uzp1 z0.s, z0.s, z2.s
+; CHECK-SVE2-AES-NEXT:    zip1 z0.s, z0.s, z1.s
 ; CHECK-SVE2-AES-NEXT:    ret
   %zextx = zext <vscale x 4 x i16> %x to <vscale x 4 x i32>
   %zexty = zext <vscale x 4 x i16> %y to <vscale x 4 x i32>
@@ -2096,136 +1545,13 @@ define <vscale x 2 x i64> @clmul_nxv2i64_zext(<vscale x 2 x i32> %x, <vscale x 2
 ;
 ; CHECK-SVE2-AES-LABEL: clmul_nxv2i64_zext:
 ; CHECK-SVE2-AES:       // %bb.0:
-; CHECK-SVE2-AES-NEXT:    mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
 ; CHECK-SVE2-AES-NEXT:    and z0.d, z0.d, #0xffffffff
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z2.d, z2.d, #0x2
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x1
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x8
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x4
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x20
-; CHECK-SVE2-AES-NEXT:    mul z2.d, z0.d, z2.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x10
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x80
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x40
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x200
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x100
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x800
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x400
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x2000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x1000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x8000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x4000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x20000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x10000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x80000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x40000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x200000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x100000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x800000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x400000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x2000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x1000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z6.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z4.d, z4.d, #0x8000000
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x4000000
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mul z5.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT:    mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z3.d, z3.d, #0x20000000
-; CHECK-SVE2-AES-NEXT:    and z6.d, z6.d, #0x10000000
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT:    mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT:    and z1.d, z1.d, #0x40000000
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z4.d, z0.d, z6.d
-; CHECK-SVE2-AES-NEXT:    and z5.d, z5.d, #0x80000000
-; CHECK-SVE2-AES-NEXT:    mul z1.d, z0.d, z1.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT:    mul z3.d, z0.d, z5.d
-; CHECK-SVE2-AES-NEXT:    mul z0.d, z0.d, #0
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z1.d, z3.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    eor3 z2.d, z2.d, z0.d, z0.d
-; CHECK-SVE2-AES-NEXT:    mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT:    and z1.d, z1.d, #0xffffffff
+; CHECK-SVE2-AES-NEXT:    pmullt z2.q, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT:    pmullb z0.q, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT:    uzp1 z1.d, z2.d, z2.d
+; CHECK-SVE2-AES-NEXT:    uzp1 z0.d, z0.d, z2.d
+; CHECK-SVE2-AES-NEXT:    zip1 z0.d, z0.d, z1.d
 ; CHECK-SVE2-AES-NEXT:    ret
   %zextx = zext <vscale x 2 x i32> %x to <vscale x 2 x i64>
   %zexty = zext <vscale x 2 x i32> %y to <vscale x 2 x i64>

>From 4de7e03e1994ab89f611556cd450a8d6f8d5427f Mon Sep 17 00:00:00 2001
From: Matthew Devereau <matthew.devereau at arm.com>
Date: Tue, 10 Feb 2026 14:26:26 +0000
Subject: [PATCH 2/2] Use TRN1

---
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  6 ++---
 llvm/test/CodeGen/AArch64/clmul-scalable.ll   | 24 +++++--------------
 2 files changed, 8 insertions(+), 22 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index d7f402bf03a30..87e419b6d3667 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4256,13 +4256,11 @@ let Predicates = [HasSVEAES, HasNonStreamingSVE_or_SSVE_AES] in {
 
 let Predicates = [HasSVEAES, HasSVE2_or_SME] in {
   def : Pat<(nxv2i64 (clmul nxv2i64:$Rn, nxv2i64:$Rm)),
-            (ZIP1_ZZZ_D (UZP1_ZZZ_D (PMULLB_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm)),
-                        (UZP1_ZZZ_D (PMULLT_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm)))>;
+            (TRN1_ZZZ_D (PMULLB_ZZZ_Q $Rn, $Rm), (PMULLT_ZZZ_Q $Rn, $Rm))>;
 }
 let Predicates = [HasSVE2_or_SME] in {
   def : Pat<(nxv4i32 (clmul nxv4i32:$Rn, nxv4i32:$Rm)),
-            (ZIP1_ZZZ_S (UZP1_ZZZ_S (PMULLB_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm)),
-                        (UZP1_ZZZ_S (PMULLT_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm)))>;
+            (TRN1_ZZZ_S (PMULLB_ZZZ_D $Rn, $Rm), (PMULLT_ZZZ_D $Rn, $Rm))>;
 }
 
 let Predicates = [HasSVESM4] in {
diff --git a/llvm/test/CodeGen/AArch64/clmul-scalable.ll b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
index a6c1bfae6e1b9..b974340ce6cd0 100644
--- a/llvm/test/CodeGen/AArch64/clmul-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
@@ -379,18 +379,14 @@ define <vscale x 4 x i32> @clmul_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i3
 ; CHECK-SVE2:       // %bb.0:
 ; CHECK-SVE2-NEXT:    pmullt z2.d, z0.s, z1.s
 ; CHECK-SVE2-NEXT:    pmullb z0.d, z0.s, z1.s
-; CHECK-SVE2-NEXT:    uzp1 z1.s, z2.s, z2.s
-; CHECK-SVE2-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-SVE2-NEXT:    zip1 z0.s, z0.s, z1.s
+; CHECK-SVE2-NEXT:    trn1 z0.s, z0.s, z2.s
 ; CHECK-SVE2-NEXT:    ret
 ;
 ; CHECK-SVE2-AES-LABEL: clmul_nxv4i32:
 ; CHECK-SVE2-AES:       // %bb.0:
 ; CHECK-SVE2-AES-NEXT:    pmullt z2.d, z0.s, z1.s
 ; CHECK-SVE2-AES-NEXT:    pmullb z0.d, z0.s, z1.s
-; CHECK-SVE2-AES-NEXT:    uzp1 z1.s, z2.s, z2.s
-; CHECK-SVE2-AES-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-SVE2-AES-NEXT:    zip1 z0.s, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT:    trn1 z0.s, z0.s, z2.s
 ; CHECK-SVE2-AES-NEXT:    ret
   %a = call <vscale x 4 x i32> @llvm.clmul.nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y)
   ret <vscale x 4 x i32> %a
@@ -888,9 +884,7 @@ define <vscale x 2 x i64> @clmul_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i6
 ; CHECK-SVE2-AES:       // %bb.0:
 ; CHECK-SVE2-AES-NEXT:    pmullt z2.q, z0.d, z1.d
 ; CHECK-SVE2-AES-NEXT:    pmullb z0.q, z0.d, z1.d
-; CHECK-SVE2-AES-NEXT:    uzp1 z1.d, z2.d, z2.d
-; CHECK-SVE2-AES-NEXT:    uzp1 z0.d, z0.d, z2.d
-; CHECK-SVE2-AES-NEXT:    zip1 z0.d, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT:    trn1 z0.d, z0.d, z2.d
 ; CHECK-SVE2-AES-NEXT:    ret
   %a = call <vscale x 2 x i64> @llvm.clmul.nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y)
   ret <vscale x 2 x i64> %a
@@ -1168,9 +1162,7 @@ define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4
 ; CHECK-SVE2-NEXT:    and z1.s, z1.s, #0xffff
 ; CHECK-SVE2-NEXT:    pmullt z2.d, z0.s, z1.s
 ; CHECK-SVE2-NEXT:    pmullb z0.d, z0.s, z1.s
-; CHECK-SVE2-NEXT:    uzp1 z1.s, z2.s, z2.s
-; CHECK-SVE2-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-SVE2-NEXT:    zip1 z0.s, z0.s, z1.s
+; CHECK-SVE2-NEXT:    trn1 z0.s, z0.s, z2.s
 ; CHECK-SVE2-NEXT:    ret
 ;
 ; CHECK-SVE2-AES-LABEL: clmul_nxv4i32_zext:
@@ -1179,9 +1171,7 @@ define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4
 ; CHECK-SVE2-AES-NEXT:    and z1.s, z1.s, #0xffff
 ; CHECK-SVE2-AES-NEXT:    pmullt z2.d, z0.s, z1.s
 ; CHECK-SVE2-AES-NEXT:    pmullb z0.d, z0.s, z1.s
-; CHECK-SVE2-AES-NEXT:    uzp1 z1.s, z2.s, z2.s
-; CHECK-SVE2-AES-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-SVE2-AES-NEXT:    zip1 z0.s, z0.s, z1.s
+; CHECK-SVE2-AES-NEXT:    trn1 z0.s, z0.s, z2.s
 ; CHECK-SVE2-AES-NEXT:    ret
   %zextx = zext <vscale x 4 x i16> %x to <vscale x 4 x i32>
   %zexty = zext <vscale x 4 x i16> %y to <vscale x 4 x i32>
@@ -1549,9 +1539,7 @@ define <vscale x 2 x i64> @clmul_nxv2i64_zext(<vscale x 2 x i32> %x, <vscale x 2
 ; CHECK-SVE2-AES-NEXT:    and z1.d, z1.d, #0xffffffff
 ; CHECK-SVE2-AES-NEXT:    pmullt z2.q, z0.d, z1.d
 ; CHECK-SVE2-AES-NEXT:    pmullb z0.q, z0.d, z1.d
-; CHECK-SVE2-AES-NEXT:    uzp1 z1.d, z2.d, z2.d
-; CHECK-SVE2-AES-NEXT:    uzp1 z0.d, z0.d, z2.d
-; CHECK-SVE2-AES-NEXT:    zip1 z0.d, z0.d, z1.d
+; CHECK-SVE2-AES-NEXT:    trn1 z0.d, z0.d, z2.d
 ; CHECK-SVE2-AES-NEXT:    ret
   %zextx = zext <vscale x 2 x i32> %x to <vscale x 2 x i64>
   %zexty = zext <vscale x 2 x i32> %y to <vscale x 2 x i64>



More information about the llvm-commits mailing list