[llvm] [AArch64][ISel] Add custom lowering for clmul nxv8i16 (PR #195893)
via llvm-commits
llvm-commits at lists.llvm.org
Tue May 5 10:54:25 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Matthew Devereau (MDevereau)
<details>
<summary>Changes</summary>
When sve2/sme are available, this sequence provides faster and smaller codegen than the current lowering:
```
clmul.i16(a, b) = xor(pmullb(a_lo, b_lo),
lsl(xor(pmul(a_hi, b_lo),
pmul(a_lo, b_hi)),
8))
```
Assisted-by: codex with gpt-5.5
---
Patch is 25.26 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/195893.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+44-3)
- (modified) llvm/test/CodeGen/AArch64/clmul-scalable.ll (+72-340)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e3c12b9ebd3ed..27fe5776a14fb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2051,6 +2051,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setPartialReduceMLAAction(MLAOps, MVT::nxv8i16, MVT::nxv16i8, Legal);
setOperationAction(ISD::CLMUL, {MVT::nxv16i8, MVT::nxv4i32}, Legal);
+ setOperationAction(ISD::CLMUL, {MVT::nxv8i16}, Custom);
setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::nxv4f32,
MVT::nxv8f16, Legal);
@@ -8089,10 +8090,50 @@ SDValue AArch64TargetLowering::LowerFMA(SDValue Op, SelectionDAG &DAG) const {
static SDValue LowerCLMUL(SDValue Op, SelectionDAG &DAG) {
EVT VT = Op.getValueType();
- assert(
- (VT == MVT::i64 || VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8) &&
- "Unexpected Type");
SDLoc DL(Op);
+ assert((VT == MVT::i64 || VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8 ||
+ VT == MVT::nxv8i16) &&
+ "Unexpected Type");
+
+ if (VT == MVT::nxv8i16) {
+ // clmul.i16(a, b) = xor(pmullb(a_lo, b_lo),
+ // lsl(xor(pmul(a_hi, b_lo),
+ // pmul(a_lo, b_hi)),
+ // 8))
+
+ SDValue OpA = Op.getOperand(0);
+ SDValue OpB = Op.getOperand(1);
+ SDValue SplatEight =
+ DAG.getSplatVector(VT, DL, DAG.getConstant(8, DL, MVT::i16));
+
+ // Shift >> 8 and bitcast to i8 for pmul
+ SDValue OpAHi = DAG.getNode(ISD::SRL, DL, VT, OpA, SplatEight);
+ OpAHi = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, OpAHi);
+ SDValue OpBHi = DAG.getNode(ISD::SRL, DL, VT, OpB, SplatEight);
+ OpBHi = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, OpBHi);
+
+ // Bitcast to i8 for pmullb
+ OpA = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, OpA);
+ OpB = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, OpB);
+
+ SDValue PMULLB =
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_pmullb_pair, DL, MVT::i64);
+ PMULLB = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::nxv16i8, PMULLB, OpA,
+ OpB);
+ PMULLB = DAG.getNode(ISD::BITCAST, DL, VT, PMULLB);
+
+ SDValue PMUL_OpALo_OpBHi =
+ DAG.getNode(ISD::CLMUL, DL, MVT::nxv16i8, OpA, OpBHi);
+ SDValue PMUL_OpAHi_OpBLo =
+ DAG.getNode(ISD::CLMUL, DL, MVT::nxv16i8, OpAHi, OpB);
+ SDValue Cross = DAG.getNode(ISD::XOR, DL, MVT::nxv16i8, PMUL_OpALo_OpBHi,
+ PMUL_OpAHi_OpBLo);
+ Cross = DAG.getNode(ISD::SHL, DL, VT,
+ DAG.getNode(ISD::BITCAST, DL, VT, Cross), SplatEight);
+
+ return DAG.getNode(ISD::XOR, DL, VT, PMULLB, Cross);
+ }
+
EVT VecVT = EVT::getVectorVT(*DAG.getContext(), VT, 64 / VT.getSizeInBits());
EVT CLMULTy = VT == MVT::i8 ? MVT::v8i8 : MVT::v1i64;
EVT ExtractTy = VT == MVT::i64 ? MVT::i64 : MVT::i32;
diff --git a/llvm/test/CodeGen/AArch64/clmul-scalable.ll b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
index ef903902b9e65..04114e1dc1807 100644
--- a/llvm/test/CodeGen/AArch64/clmul-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
@@ -237,242 +237,50 @@ define <vscale x 8 x i16> @clmul_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1
;
; CHECK-SME-STREAMING-LABEL: clmul_nxv8i16:
; CHECK-SME-STREAMING: // %bb.0:
-; CHECK-SME-STREAMING-NEXT: movprfx z2, z1
-; CHECK-SME-STREAMING-NEXT: and z2.h, z2.h, #0x2
-; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
-; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x1
-; CHECK-SME-STREAMING-NEXT: movprfx z4, z1
-; CHECK-SME-STREAMING-NEXT: and z4.h, z4.h, #0x8
-; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x10
-; CHECK-SME-STREAMING-NEXT: mul z2.h, z0.h, z2.h
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SME-STREAMING-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
-; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x20
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x40
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x200
-; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x100
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x800
-; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x400
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x2000
-; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x1000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x8000
-; CHECK-SME-STREAMING-NEXT: and z1.h, z1.h, #0x4000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: mul z0.h, z0.h, z1.h
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z0.d, z2.d
+; CHECK-SME-STREAMING-NEXT: lsr z2.h, z0.h, #8
+; CHECK-SME-STREAMING-NEXT: lsr z3.h, z1.h, #8
+; CHECK-SME-STREAMING-NEXT: pmul z2.b, z2.b, z1.b
+; CHECK-SME-STREAMING-NEXT: pmul z3.b, z0.b, z3.b
+; CHECK-SME-STREAMING-NEXT: pmullb z0.h, z0.b, z1.b
+; CHECK-SME-STREAMING-NEXT: eor z1.d, z3.d, z2.d
+; CHECK-SME-STREAMING-NEXT: lsl z1.h, z1.h, #8
+; CHECK-SME-STREAMING-NEXT: eor z0.d, z0.d, z1.d
; CHECK-SME-STREAMING-NEXT: ret
;
; CHECK-SME-STREAMING-SSVE-AES-LABEL: clmul_nxv8i16:
; CHECK-SME-STREAMING-SSVE-AES: // %bb.0:
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z2, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z2.h, z2.h, #0x2
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z3, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z4, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z4.h, z4.h, #0x8
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x10
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z2.h, z0.h, z2.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z3, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x20
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x40
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x200
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x100
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x800
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x400
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x2000
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x1000
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x8000
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z1.h, z1.h, #0x4000
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z0.h, z0.h, z1.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z0.d, z2.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: lsr z2.h, z0.h, #8
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: lsr z3.h, z1.h, #8
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: pmul z2.b, z2.b, z1.b
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: pmul z3.b, z0.b, z3.b
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: pmullb z0.h, z0.b, z1.b
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor z1.d, z3.d, z2.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: lsl z1.h, z1.h, #8
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor z0.d, z0.d, z1.d
; CHECK-SME-STREAMING-SSVE-AES-NEXT: ret
;
; CHECK-SVE2-LABEL: clmul_nxv8i16:
; CHECK-SVE2: // %bb.0:
-; CHECK-SVE2-NEXT: movprfx z2, z1
-; CHECK-SVE2-NEXT: and z2.h, z2.h, #0x2
-; CHECK-SVE2-NEXT: movprfx z3, z1
-; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x1
-; CHECK-SVE2-NEXT: movprfx z4, z1
-; CHECK-SVE2-NEXT: and z4.h, z4.h, #0x8
-; CHECK-SVE2-NEXT: movprfx z5, z1
-; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SVE2-NEXT: movprfx z6, z1
-; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x10
-; CHECK-SVE2-NEXT: mul z2.h, z0.h, z2.h
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SVE2-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT: movprfx z3, z1
-; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x20
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-NEXT: movprfx z5, z1
-; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SVE2-NEXT: movprfx z6, z1
-; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x40
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-NEXT: movprfx z5, z1
-; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x200
-; CHECK-SVE2-NEXT: movprfx z6, z1
-; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x100
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-NEXT: movprfx z5, z1
-; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x800
-; CHECK-SVE2-NEXT: movprfx z6, z1
-; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x400
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-NEXT: movprfx z5, z1
-; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x2000
-; CHECK-SVE2-NEXT: movprfx z6, z1
-; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x1000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-NEXT: movprfx z5, z1
-; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x8000
-; CHECK-SVE2-NEXT: and z1.h, z1.h, #0x4000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: mul z0.h, z0.h, z1.h
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-NEXT: lsr z2.h, z0.h, #8
+; CHECK-SVE2-NEXT: lsr z3.h, z1.h, #8
+; CHECK-SVE2-NEXT: pmul z2.b, z2.b, z1.b
+; CHECK-SVE2-NEXT: pmul z3.b, z0.b, z3.b
+; CHECK-SVE2-NEXT: pmullb z0.h, z0.b, z1.b
+; CHECK-SVE2-NEXT: eor z1.d, z3.d, z2.d
+; CHECK-SVE2-NEXT: lsl z1.h, z1.h, #8
+; CHECK-SVE2-NEXT: eor z0.d, z0.d, z1.d
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-AES-LABEL: clmul_nxv8i16:
; CHECK-SVE2-AES: // %bb.0:
-; CHECK-SVE2-AES-NEXT: movprfx z2, z1
-; CHECK-SVE2-AES-NEXT: and z2.h, z2.h, #0x2
-; CHECK-SVE2-AES-NEXT: movprfx z3, z1
-; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x1
-; CHECK-SVE2-AES-NEXT: movprfx z4, z1
-; CHECK-SVE2-AES-NEXT: and z4.h, z4.h, #0x8
-; CHECK-SVE2-AES-NEXT: movprfx z5, z1
-; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SVE2-AES-NEXT: movprfx z6, z1
-; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x10
-; CHECK-SVE2-AES-NEXT: mul z2.h, z0.h, z2.h
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SVE2-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT: movprfx z3, z1
-; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x20
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-AES-NEXT: movprfx z5, z1
-; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SVE2-AES-NEXT: movprfx z6, z1
-; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x40
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-AES-NEXT: movprfx z5, z1
-; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x200
-; CHECK-SVE2-AES-NEXT: movprfx z6, z1
-; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x100
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-AES-NEXT: movprfx z5, z1
-; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x800
-; CHECK-SVE2-AES-NEXT: movprfx z6, z1
-; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x400
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-AES-NEXT: movprfx z5, z1
-; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x2000
-; CHECK-SVE2-AES-NEXT: movprfx z6, z1
-; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x1000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-AES-NEXT: movprfx z5, z1
-; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x8000
-; CHECK-SVE2-AES-NEXT: and z1.h, z1.h, #0x4000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: mul z0.h, z0.h, z1.h
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT: lsr z2.h, z0.h, #8
+; CHECK-SVE2-AES-NEXT: lsr z3.h, z1.h, #8
+; CHECK-SVE2-AES-NEXT: pmul z2.b, z2.b, z1.b
+; CHECK-SVE2-AES-NEXT: pmul z3.b, z0.b, z3.b
+; CHECK-SVE2-AES-NEXT: pmullb z0.h, z0.b, z1.b
+; CHECK-SVE2-AES-NEXT: eor z1.d, z3.d, z2.d
+; CHECK-SVE2-AES-NEXT: lsl z1.h, z1.h, #8
+; CHECK-SVE2-AES-NEXT: eor z0.d, z0.d, z1.d
; CHECK-SVE2-AES-NEXT: ret
%a = call <vscale x 8 x i16> @llvm.clmul.nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y)
ret <vscale x 8 x i16> %a
@@ -1695,134 +1503,58 @@ define <vscale x 8 x i16> @clmul_nxv8i16_zext(<vscale x 8 x i8> %x, <vscale x 8
;
; CHECK-SME-STREAMING-LABEL: clmul_nxv8i16_zext:
; CHECK-SME-STREAMING: // %bb.0:
-; CHECK-SME-STREAMING-NEXT: movprfx z2, z0
-; CHECK-SME-STREAMING-NEXT: and z2.h, z2.h, #0xff
-; CHECK-SME-STREAMING-NEXT: movprfx z0, z1
-; CHECK-SME-STREAMING-NEXT: and z0.h, z0.h, #0x2
-; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
-; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x1
-; CHECK-SME-STREAMING-NEXT: movprfx z4, z1
-; CHECK-SME-STREAMING-NEXT: and z4.h, z4.h, #0x8
-; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x10
-; CHECK-SME-STREAMING-NEXT: mul z0.h, z2.h, z0.h
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z2.h, z3.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z2.h, z4.h
-; CHECK-SME-STREAMING-NEXT: mul z5.h, z2.h, z5.h
-; CHECK-SME-STREAMING-NEXT: eor z0.d, z3.d, z0.d
-; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
-; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x20
-; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z2.h, z3.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z2.h, z6.h
-; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SME-STREAMING-NEXT: and z1.h, z1.h, #0x40
-; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z2.h, z5.h
-; CHECK-SME-STREAMING-NEXT: mul z1.h, z2.h, z1.h
-; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z1.d, z3.d
+; CHECK-SME-STREAMING-NEXT: and z0.h, z0.h, #0xff
+; CHECK-SME-STREAMING-NEXT: and z1.h, z1.h, #0xff
+; CHECK-SME-STREAMING-NEXT: lsr z2.h, z0.h, #8
+; CHECK-SME-STREAMING-NEXT: lsr z3.h, z1.h, #8
+; CHECK-SME-STREAMING-NEXT: pmul z2.b, z2.b, z1.b
+; CHECK-SME-STREAMING-NEXT: pmul z3.b, z0.b, z3.b
+; CHECK-SME-STREAMING-NEXT: pmullb z0.h, z0.b, z1.b
+; CHECK-SME-STREAMING-NEXT: eor z1.d, z3.d, z2.d
+; CHECK-SME-STREAMING-NEXT: lsl z1.h, z1.h, #8
+; CHECK-SME-STREAMING-NEXT: eor z0.d, z0.d, z1.d
; CHECK-SME-STREAMING-NEXT: ret
;
; CHECK-SME-STREAMING-SSVE-AES-LABEL: clmul_nxv8i16_zext:
; CHECK-SME-STREAMING-SSVE-AES: // %bb.0:
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z2, z0
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z2.h, z2.h, #0xff
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z0, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z0.h, z0.h, #0x2
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z3, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z4, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z4.h, z4.h, #0x8
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x10
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z0.h, z2.h, z0.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z2.h, z3.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z2.h, z4.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/195893
More information about the llvm-commits
mailing list