[llvm] [AArch64][GlobalISel] Add legalization for G_VECREDUCE_MUL (PR #68398)

via llvm-commits llvm-commits at lists.llvm.org
Tue Oct 10 05:48:43 PDT 2023


https://github.com/chuongg3 updated https://github.com/llvm/llvm-project/pull/68398

>From 5d2a94fa717e30ef7faa900c82ccaab80a9de0ab Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh at arm.com>
Date: Thu, 5 Oct 2023 14:38:33 +0100
Subject: [PATCH 1/3] [AArch64][GlobalISel] Add legalization for
 G_VECREDUCE_MUL

---
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   8 +
 .../GlobalISel/legalizer-info-validation.mir  |   4 +-
 llvm/test/CodeGen/AArch64/aarch64-mulv.ll     | 668 ++++++++++++++++++
 3 files changed, 678 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/aarch64-mulv.ll

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index cb6f1c1b5fc5f90..e04897285498761 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -885,6 +885,14 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .clampMaxNumElements(1, s16, 8)
       .lower();
 
+  getActionDefinitionsBuilder(G_VECREDUCE_MUL)
+      .clampMaxNumElements(1, s64, 2)
+      .clampMaxNumElements(1, s32, 4)
+      .clampMaxNumElements(1, s16, 8)
+      .clampMaxNumElements(1, s8, 16)
+      .scalarize(1)
+      .lower();
+
   getActionDefinitionsBuilder(
       {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
       // Try to break down into smaller vectors as long as they're at least 64
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index bb915153c53a147..fce4794e7408e62 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -754,8 +754,8 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_VECREDUCE_MUL (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_VECREDUCE_AND (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
new file mode 100644
index 000000000000000..44f650c01f71225
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
@@ -0,0 +1,668 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK_GI:        warning: Instruction selection used fallback path for mulv_v3i62
+
+declare i8 @llvm.vector.reduce.mul.v2i8(<2 x i8>)
+declare i8 @llvm.vector.reduce.mul.v3i8(<3 x i8>)
+declare i8 @llvm.vector.reduce.mul.v4i8(<4 x i8>)
+declare i8 @llvm.vector.reduce.mul.v8i8(<8 x i8>)
+declare i8 @llvm.vector.reduce.mul.v16i8(<16 x i8>)
+declare i8 @llvm.vector.reduce.mul.v32i8(<32 x i8>)
+declare i16 @llvm.vector.reduce.mul.v2i16(<2 x i16>)
+declare i16 @llvm.vector.reduce.mul.v3i16(<3 x i16>)
+declare i16 @llvm.vector.reduce.mul.v4i16(<4 x i16>)
+declare i16 @llvm.vector.reduce.mul.v8i16(<8 x i16>)
+declare i16 @llvm.vector.reduce.mul.v16i16(<16 x i16>)
+declare i32 @llvm.vector.reduce.mul.v2i32(<2 x i32>)
+declare i32 @llvm.vector.reduce.mul.v3i32(<3 x i32>)
+declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.mul.v8i32(<8 x i32>)
+declare i64 @llvm.vector.reduce.mul.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.mul.v3i64(<3 x i64>)
+declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)
+declare i128 @llvm.vector.reduce.mul.v2i128(<2 x i128>)
+
+define i8 @mulv_v2i8(<2 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v2i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v3i8(<3 x i8> %a) {
+; CHECK-LABEL: mulv_v3i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mul w8, w0, w1
+; CHECK-NEXT:    mul w0, w8, w2
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v4i8(<4 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.h[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v8i8(<8 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v8i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w8, v0.b[1]
+; CHECK-SD-NEXT:    umov w9, v0.b[0]
+; CHECK-SD-NEXT:    umov w10, v0.b[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.b[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[4]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[5]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[6]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[7]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v8i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov b4, v0.b[4]
+; CHECK-GI-NEXT:    mov b5, v0.b[5]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov b6, v0.b[6]
+; CHECK-GI-NEXT:    mov b7, v0.b[7]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov w12, s5
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s6
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    fmov w12, s7
+; CHECK-GI-NEXT:    mul w8, w8, w10
+; CHECK-GI-NEXT:    mul w11, w11, w12
+; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v16i8(<16 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v16i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umov w8, v0.b[1]
+; CHECK-SD-NEXT:    umov w9, v0.b[0]
+; CHECK-SD-NEXT:    umov w10, v0.b[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.b[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[4]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[5]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[6]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[7]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v16i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov b4, v0.b[4]
+; CHECK-GI-NEXT:    mov b5, v0.b[5]
+; CHECK-GI-NEXT:    mov b6, v0.b[6]
+; CHECK-GI-NEXT:    mov b7, v0.b[7]
+; CHECK-GI-NEXT:    mov b16, v0.b[8]
+; CHECK-GI-NEXT:    mov b17, v0.b[9]
+; CHECK-GI-NEXT:    mov b18, v0.b[10]
+; CHECK-GI-NEXT:    mov b19, v0.b[11]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s3
+; CHECK-GI-NEXT:    mov b20, v0.b[12]
+; CHECK-GI-NEXT:    fmov w11, s5
+; CHECK-GI-NEXT:    mov b21, v0.b[13]
+; CHECK-GI-NEXT:    mov b22, v0.b[14]
+; CHECK-GI-NEXT:    fmov w12, s7
+; CHECK-GI-NEXT:    mov b23, v0.b[15]
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    fmov w13, s17
+; CHECK-GI-NEXT:    fmov w14, s19
+; CHECK-GI-NEXT:    fmov w15, s21
+; CHECK-GI-NEXT:    mul w9, w9, w10
+; CHECK-GI-NEXT:    fmov w10, s4
+; CHECK-GI-NEXT:    fmov w16, s23
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s6
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w11, w11, w12
+; CHECK-GI-NEXT:    fmov w12, s16
+; CHECK-GI-NEXT:    mul w12, w12, w13
+; CHECK-GI-NEXT:    fmov w13, s18
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w13, w13, w14
+; CHECK-GI-NEXT:    fmov w14, s20
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w14, w14, w15
+; CHECK-GI-NEXT:    fmov w15, s22
+; CHECK-GI-NEXT:    mul w10, w12, w13
+; CHECK-GI-NEXT:    mul w15, w15, w16
+; CHECK-GI-NEXT:    mul w11, w14, w15
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %a)
+  ret i8 %arg1
+}
+
+define i8 @mulv_v32i8(<32 x i8> %a) {
+; CHECK-SD-LABEL: mulv_v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mul v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umov w8, v0.b[1]
+; CHECK-SD-NEXT:    umov w9, v0.b[0]
+; CHECK-SD-NEXT:    umov w10, v0.b[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.b[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[4]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[5]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    umov w10, v0.b[6]
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    umov w9, v0.b[7]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mul v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov b3, v0.b[3]
+; CHECK-GI-NEXT:    mov b4, v0.b[4]
+; CHECK-GI-NEXT:    mov b5, v0.b[5]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov b6, v0.b[6]
+; CHECK-GI-NEXT:    mov b7, v0.b[7]
+; CHECK-GI-NEXT:    mov b16, v0.b[8]
+; CHECK-GI-NEXT:    mov b17, v0.b[9]
+; CHECK-GI-NEXT:    mov b18, v0.b[10]
+; CHECK-GI-NEXT:    mov b19, v0.b[11]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov w12, s5
+; CHECK-GI-NEXT:    mov b20, v0.b[12]
+; CHECK-GI-NEXT:    mov b21, v0.b[13]
+; CHECK-GI-NEXT:    fmov w13, s7
+; CHECK-GI-NEXT:    mov b22, v0.b[14]
+; CHECK-GI-NEXT:    mov b23, v0.b[15]
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    fmov w14, s17
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s6
+; CHECK-GI-NEXT:    fmov w15, s21
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    fmov w12, s16
+; CHECK-GI-NEXT:    fmov w16, s23
+; CHECK-GI-NEXT:    mul w11, w11, w13
+; CHECK-GI-NEXT:    fmov w13, s18
+; CHECK-GI-NEXT:    mul w8, w8, w10
+; CHECK-GI-NEXT:    mul w12, w12, w14
+; CHECK-GI-NEXT:    fmov w14, s19
+; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    mul w13, w13, w14
+; CHECK-GI-NEXT:    fmov w14, s20
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w14, w14, w15
+; CHECK-GI-NEXT:    fmov w15, s22
+; CHECK-GI-NEXT:    mul w10, w12, w13
+; CHECK-GI-NEXT:    mul w15, w15, w16
+; CHECK-GI-NEXT:    mul w11, w14, w15
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %a)
+  ret i8 %arg1
+}
+
+define i16 @mulv_v2i16(<2 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %a)
+  ret i16 %arg1
+}
+
+define i16 @mulv_v3i16(<3 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    mul w0, w8, w10
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v3i16(<3 x i16> %a)
+  ret i16 %arg1
+}
+
+define i16 @mulv_v4i16(<4 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v4i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.h[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v4i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %a)
+  ret i16 %arg1
+}
+
+define i16 @mulv_v8i16(<8 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.h[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    mov h4, v0.h[4]
+; CHECK-GI-NEXT:    mov h5, v0.h[5]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov h6, v0.h[6]
+; CHECK-GI-NEXT:    mov h7, v0.h[7]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov w12, s5
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s6
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    fmov w12, s7
+; CHECK-GI-NEXT:    mul w8, w8, w10
+; CHECK-GI-NEXT:    mul w11, w11, w12
+; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %a)
+  ret i16 %arg1
+}
+
+define i16 @mulv_v16i16(<16 x i16> %a) {
+; CHECK-SD-LABEL: mulv_v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mul v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    umov w8, v0.h[1]
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[2]
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    umov w9, v0.h[3]
+; CHECK-SD-NEXT:    mul w8, w8, w10
+; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mul v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-NEXT:    mov h4, v0.h[4]
+; CHECK-GI-NEXT:    mov h5, v0.h[5]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov h6, v0.h[6]
+; CHECK-GI-NEXT:    mov h7, v0.h[7]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov w12, s5
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    mul w10, w10, w11
+; CHECK-GI-NEXT:    fmov w11, s6
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    fmov w12, s7
+; CHECK-GI-NEXT:    mul w8, w8, w10
+; CHECK-GI-NEXT:    mul w11, w11, w12
+; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %a)
+  ret i16 %arg1
+}
+
+define i32 @mulv_v2i32(<2 x i32> %a) {
+; CHECK-SD-LABEL: mulv_v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %a)
+  ret i32 %arg1
+}
+
+define i32 @mulv_v3i32(<3 x i32> %a) {
+; CHECK-SD-LABEL: mulv_v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    mov v1.s[3], w8
+; CHECK-SD-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i32 @llvm.vector.reduce.mul.v3i32(<3 x i32> %a)
+  ret i32 %arg1
+}
+
+define i32 @mulv_v4i32(<4 x i32> %a) {
+; CHECK-SD-LABEL: mulv_v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov s3, v0.s[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a)
+  ret i32 %arg1
+}
+
+define i32 @mulv_v8i32(<8 x i32> %a) {
+; CHECK-SD-LABEL: mulv_v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    mul w0, w9, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-GI-NEXT:    mov s3, v0.s[3]
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w0, w8, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> %a)
+  ret i32 %arg1
+}
+
+define i64 @mulv_v2i64(<2 x i64> %a) {
+; CHECK-SD-LABEL: mulv_v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-SD-NEXT:    fmov x9, d0
+; CHECK-SD-NEXT:    mul x0, x9, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> %a)
+  ret i64 %arg1
+}
+
+define i64 @mulv_v3i64(<3 x i64> %a) {
+; CHECK-LABEL: mulv_v3i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov x8, d2
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    mul x8, x9, x8
+; CHECK-NEXT:    fmov x9, d1
+; CHECK-NEXT:    mul x0, x9, x8
+; CHECK-NEXT:    ret
+entry:
+  %arg1 = call i64 @llvm.vector.reduce.mul.v3i64(<3 x i64> %a)
+  ret i64 %arg1
+}
+
+define i64 @mulv_v4i64(<4 x i64> %a) {
+; CHECK-SD-LABEL: mulv_v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, v1.d[1]
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mul x8, x9, x8
+; CHECK-SD-NEXT:    fmov x9, d1
+; CHECK-SD-NEXT:    mul x9, x10, x9
+; CHECK-SD-NEXT:    mul x0, x9, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov x9, d2
+; CHECK-GI-NEXT:    fmov x10, d3
+; CHECK-GI-NEXT:    mul x9, x9, x10
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    mov v0.d[1], x9
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> %a)
+  ret i64 %arg1
+}
+
+define i128 @mulv_v2i128(<2 x i128> %a) {
+; CHECK-SD-LABEL: mulv_v2i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    umulh x8, x0, x2
+; CHECK-SD-NEXT:    madd x8, x0, x3, x8
+; CHECK-SD-NEXT:    mul x0, x0, x2
+; CHECK-SD-NEXT:    madd x1, x1, x2, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v2i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mul x9, x0, x3
+; CHECK-GI-NEXT:    umulh x8, x0, x2
+; CHECK-GI-NEXT:    madd x9, x1, x2, x9
+; CHECK-GI-NEXT:    mul x0, x0, x2
+; CHECK-GI-NEXT:    add x1, x9, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %arg1 = call i128 @llvm.vector.reduce.mul.v2i128(<2 x i128> %a)
+  ret i128 %arg1
+}

>From a7d0ca4c49ba35129fa0712c9d9ec5ec77ec6e71 Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh at arm.com>
Date: Mon, 9 Oct 2023 13:44:30 +0100
Subject: [PATCH 2/3] fixup! [AArch64][GlobalISel] Add legalization for
 G_VECREDUCE_MUL

---
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   8 +-
 llvm/test/CodeGen/AArch64/aarch64-mulv.ll     | 209 ++++++------------
 2 files changed, 72 insertions(+), 145 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index e04897285498761..f67efc39e98265b 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -886,10 +886,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .lower();
 
   getActionDefinitionsBuilder(G_VECREDUCE_MUL)
-      .clampMaxNumElements(1, s64, 2)
-      .clampMaxNumElements(1, s32, 4)
-      .clampMaxNumElements(1, s16, 8)
-      .clampMaxNumElements(1, s8, 16)
+      .clampMaxNumElements(1, s64, 1)
+      .clampMaxNumElements(1, s32, 2)
+      .clampMaxNumElements(1, s16, 4)
+      .clampMaxNumElements(1, s8, 8)
       .scalarize(1)
       .lower();
 
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
index 44f650c01f71225..a38ddf32c2216b2 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
@@ -165,51 +165,29 @@ define i8 @mulv_v16i8(<16 x i8> %a) {
 ;
 ; CHECK-GI-LABEL: mulv_v16i8:
 ; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v1.8b
 ; CHECK-GI-NEXT:    mov b1, v0.b[1]
 ; CHECK-GI-NEXT:    mov b2, v0.b[2]
 ; CHECK-GI-NEXT:    mov b3, v0.b[3]
-; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    mov b4, v0.b[4]
 ; CHECK-GI-NEXT:    mov b5, v0.b[5]
+; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    mov b6, v0.b[6]
 ; CHECK-GI-NEXT:    mov b7, v0.b[7]
-; CHECK-GI-NEXT:    mov b16, v0.b[8]
-; CHECK-GI-NEXT:    mov b17, v0.b[9]
-; CHECK-GI-NEXT:    mov b18, v0.b[10]
-; CHECK-GI-NEXT:    mov b19, v0.b[11]
 ; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    fmov w10, s3
-; CHECK-GI-NEXT:    mov b20, v0.b[12]
-; CHECK-GI-NEXT:    fmov w11, s5
-; CHECK-GI-NEXT:    mov b21, v0.b[13]
-; CHECK-GI-NEXT:    mov b22, v0.b[14]
-; CHECK-GI-NEXT:    fmov w12, s7
-; CHECK-GI-NEXT:    mov b23, v0.b[15]
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov w11, s3
+; CHECK-GI-NEXT:    fmov w12, s5
 ; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    fmov w9, s2
-; CHECK-GI-NEXT:    fmov w13, s17
-; CHECK-GI-NEXT:    fmov w14, s19
-; CHECK-GI-NEXT:    fmov w15, s21
-; CHECK-GI-NEXT:    mul w9, w9, w10
-; CHECK-GI-NEXT:    fmov w10, s4
-; CHECK-GI-NEXT:    fmov w16, s23
+; CHECK-GI-NEXT:    fmov w9, s4
 ; CHECK-GI-NEXT:    mul w10, w10, w11
 ; CHECK-GI-NEXT:    fmov w11, s6
-; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mul w9, w9, w12
+; CHECK-GI-NEXT:    fmov w12, s7
+; CHECK-GI-NEXT:    mul w8, w8, w10
 ; CHECK-GI-NEXT:    mul w11, w11, w12
-; CHECK-GI-NEXT:    fmov w12, s16
-; CHECK-GI-NEXT:    mul w12, w12, w13
-; CHECK-GI-NEXT:    fmov w13, s18
-; CHECK-GI-NEXT:    mul w9, w10, w11
-; CHECK-GI-NEXT:    mul w13, w13, w14
-; CHECK-GI-NEXT:    fmov w14, s20
-; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    mul w14, w14, w15
-; CHECK-GI-NEXT:    fmov w15, s22
-; CHECK-GI-NEXT:    mul w10, w12, w13
-; CHECK-GI-NEXT:    mul w15, w15, w16
-; CHECK-GI-NEXT:    mul w11, w14, w15
-; CHECK-GI-NEXT:    mul w9, w10, w11
+; CHECK-GI-NEXT:    mul w9, w9, w11
 ; CHECK-GI-NEXT:    mul w0, w8, w9
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -242,7 +220,11 @@ define i8 @mulv_v32i8(<32 x i8> %a) {
 ;
 ; CHECK-GI-LABEL: mulv_v32i8:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mul v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    mul v1.8b, v1.8b, v3.8b
+; CHECK-GI-NEXT:    mul v0.8b, v0.8b, v1.8b
 ; CHECK-GI-NEXT:    mov b1, v0.b[1]
 ; CHECK-GI-NEXT:    mov b2, v0.b[2]
 ; CHECK-GI-NEXT:    mov b3, v0.b[3]
@@ -251,43 +233,19 @@ define i8 @mulv_v32i8(<32 x i8> %a) {
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    mov b6, v0.b[6]
 ; CHECK-GI-NEXT:    mov b7, v0.b[7]
-; CHECK-GI-NEXT:    mov b16, v0.b[8]
-; CHECK-GI-NEXT:    mov b17, v0.b[9]
-; CHECK-GI-NEXT:    mov b18, v0.b[10]
-; CHECK-GI-NEXT:    mov b19, v0.b[11]
 ; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    fmov w10, s2
 ; CHECK-GI-NEXT:    fmov w11, s3
 ; CHECK-GI-NEXT:    fmov w12, s5
-; CHECK-GI-NEXT:    mov b20, v0.b[12]
-; CHECK-GI-NEXT:    mov b21, v0.b[13]
-; CHECK-GI-NEXT:    fmov w13, s7
-; CHECK-GI-NEXT:    mov b22, v0.b[14]
-; CHECK-GI-NEXT:    mov b23, v0.b[15]
 ; CHECK-GI-NEXT:    mul w8, w8, w9
 ; CHECK-GI-NEXT:    fmov w9, s4
-; CHECK-GI-NEXT:    fmov w14, s17
 ; CHECK-GI-NEXT:    mul w10, w10, w11
 ; CHECK-GI-NEXT:    fmov w11, s6
-; CHECK-GI-NEXT:    fmov w15, s21
 ; CHECK-GI-NEXT:    mul w9, w9, w12
-; CHECK-GI-NEXT:    fmov w12, s16
-; CHECK-GI-NEXT:    fmov w16, s23
-; CHECK-GI-NEXT:    mul w11, w11, w13
-; CHECK-GI-NEXT:    fmov w13, s18
+; CHECK-GI-NEXT:    fmov w12, s7
 ; CHECK-GI-NEXT:    mul w8, w8, w10
-; CHECK-GI-NEXT:    mul w12, w12, w14
-; CHECK-GI-NEXT:    fmov w14, s19
+; CHECK-GI-NEXT:    mul w11, w11, w12
 ; CHECK-GI-NEXT:    mul w9, w9, w11
-; CHECK-GI-NEXT:    mul w13, w13, w14
-; CHECK-GI-NEXT:    fmov w14, s20
-; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    mul w14, w14, w15
-; CHECK-GI-NEXT:    fmov w15, s22
-; CHECK-GI-NEXT:    mul w10, w12, w13
-; CHECK-GI-NEXT:    mul w15, w15, w16
-; CHECK-GI-NEXT:    mul w11, w14, w15
-; CHECK-GI-NEXT:    mul w9, w10, w11
 ; CHECK-GI-NEXT:    mul w0, w8, w9
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -392,27 +350,17 @@ define i16 @mulv_v8i16(<8 x i16> %a) {
 ;
 ; CHECK-GI-LABEL: mulv_v8i16:
 ; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
 ; CHECK-GI-NEXT:    mov h1, v0.h[1]
 ; CHECK-GI-NEXT:    mov h2, v0.h[2]
 ; CHECK-GI-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-NEXT:    mov h4, v0.h[4]
-; CHECK-GI-NEXT:    mov h5, v0.h[5]
 ; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov h6, v0.h[6]
-; CHECK-GI-NEXT:    mov h7, v0.h[7]
 ; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    fmov w10, s2
 ; CHECK-GI-NEXT:    fmov w11, s3
-; CHECK-GI-NEXT:    fmov w12, s5
 ; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    fmov w9, s4
-; CHECK-GI-NEXT:    mul w10, w10, w11
-; CHECK-GI-NEXT:    fmov w11, s6
-; CHECK-GI-NEXT:    mul w9, w9, w12
-; CHECK-GI-NEXT:    fmov w12, s7
-; CHECK-GI-NEXT:    mul w8, w8, w10
-; CHECK-GI-NEXT:    mul w11, w11, w12
-; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    mul w9, w10, w11
 ; CHECK-GI-NEXT:    mul w0, w8, w9
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -437,28 +385,20 @@ define i16 @mulv_v16i16(<16 x i16> %a) {
 ;
 ; CHECK-GI-LABEL: mulv_v16i16:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mul v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v2.4h
+; CHECK-GI-NEXT:    mul v1.4h, v1.4h, v3.4h
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
 ; CHECK-GI-NEXT:    mov h1, v0.h[1]
 ; CHECK-GI-NEXT:    mov h2, v0.h[2]
 ; CHECK-GI-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-NEXT:    mov h4, v0.h[4]
-; CHECK-GI-NEXT:    mov h5, v0.h[5]
 ; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    mov h6, v0.h[6]
-; CHECK-GI-NEXT:    mov h7, v0.h[7]
 ; CHECK-GI-NEXT:    fmov w9, s1
 ; CHECK-GI-NEXT:    fmov w10, s2
-; CHECK-GI-NEXT:    fmov w11, s3
-; CHECK-GI-NEXT:    fmov w12, s5
 ; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    fmov w9, s4
-; CHECK-GI-NEXT:    mul w10, w10, w11
-; CHECK-GI-NEXT:    fmov w11, s6
-; CHECK-GI-NEXT:    mul w9, w9, w12
-; CHECK-GI-NEXT:    fmov w12, s7
-; CHECK-GI-NEXT:    mul w8, w8, w10
-; CHECK-GI-NEXT:    mul w11, w11, w12
-; CHECK-GI-NEXT:    mul w9, w9, w11
+; CHECK-GI-NEXT:    fmov w9, s3
+; CHECK-GI-NEXT:    mul w9, w10, w9
 ; CHECK-GI-NEXT:    mul w0, w8, w9
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -489,28 +429,17 @@ entry:
 }
 
 define i32 @mulv_v3i32(<3 x i32> %a) {
-; CHECK-SD-LABEL: mulv_v3i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    mov w8, #1 // =0x1
-; CHECK-SD-NEXT:    mov v1.s[3], w8
-; CHECK-SD-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    mul w0, w9, w8
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: mulv_v3i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    mov s2, v0.s[2]
-; CHECK-GI-NEXT:    fmov w8, s0
-; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    fmov w9, s2
-; CHECK-GI-NEXT:    mul w0, w8, w9
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: mulv_v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:    mov v1.s[3], w8
+; CHECK-NEXT:    ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    mul w0, w9, w8
+; CHECK-NEXT:    ret
 entry:
   %arg1 = call i32 @llvm.vector.reduce.mul.v3i32(<3 x i32> %a)
   ret i32 %arg1
@@ -528,15 +457,11 @@ define i32 @mulv_v4i32(<4 x i32> %a) {
 ;
 ; CHECK-GI-LABEL: mulv_v4i32:
 ; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
 ; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    mov s2, v0.s[2]
-; CHECK-GI-NEXT:    mov s3, v0.s[3]
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    fmov w10, s2
-; CHECK-GI-NEXT:    fmov w11, s3
-; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    mul w9, w10, w11
 ; CHECK-GI-NEXT:    mul w0, w8, w9
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -557,16 +482,14 @@ define i32 @mulv_v8i32(<8 x i32> %a) {
 ;
 ; CHECK-GI-LABEL: mulv_v8i32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-GI-NEXT:    mov d3, v1.d[1]
+; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v2.2s
+; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v3.2s
+; CHECK-GI-NEXT:    mul v0.2s, v0.2s, v1.2s
 ; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    mov s2, v0.s[2]
-; CHECK-GI-NEXT:    mov s3, v0.s[3]
 ; CHECK-GI-NEXT:    fmov w8, s0
 ; CHECK-GI-NEXT:    fmov w9, s1
-; CHECK-GI-NEXT:    fmov w10, s2
-; CHECK-GI-NEXT:    fmov w11, s3
-; CHECK-GI-NEXT:    mul w8, w8, w9
-; CHECK-GI-NEXT:    mul w9, w10, w11
 ; CHECK-GI-NEXT:    mul w0, w8, w9
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -595,17 +518,26 @@ entry:
 }
 
 define i64 @mulv_v3i64(<3 x i64> %a) {
-; CHECK-LABEL: mulv_v3i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov x8, d2
-; CHECK-NEXT:    fmov x9, d0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    mul x8, x9, x8
-; CHECK-NEXT:    fmov x9, d1
-; CHECK-NEXT:    mul x0, x9, x8
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: mulv_v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    fmov x8, d2
+; CHECK-SD-NEXT:    fmov x9, d0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    mul x8, x9, x8
+; CHECK-SD-NEXT:    fmov x9, d1
+; CHECK-SD-NEXT:    mul x0, x9, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: mulv_v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov x9, d2
+; CHECK-GI-NEXT:    mul x0, x8, x9
+; CHECK-GI-NEXT:    ret
 entry:
   %arg1 = call i64 @llvm.vector.reduce.mul.v3i64(<3 x i64> %a)
   ret i64 %arg1
@@ -628,16 +560,11 @@ define i64 @mulv_v4i64(<4 x i64> %a) {
 ; CHECK-GI-NEXT:    mov d2, v0.d[1]
 ; CHECK-GI-NEXT:    mov d3, v1.d[1]
 ; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    mul x8, x8, x9
 ; CHECK-GI-NEXT:    fmov x9, d2
 ; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    mul x9, x9, x10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    mov v0.d[1], x9
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    mul x8, x8, x9
 ; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    mul x9, x9, x10
 ; CHECK-GI-NEXT:    mul x0, x8, x9
 ; CHECK-GI-NEXT:    ret
 entry:

>From 47d44c9c990f12f50e226acda853a99dbd1fae5d Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh at arm.com>
Date: Tue, 10 Oct 2023 11:50:12 +0100
Subject: [PATCH 3/3] fixup! [AArch64][GlobalISel] Add legalization for
 G_VECREDUCE_MUL

---
 llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp | 1 -
 llvm/test/CodeGen/AArch64/aarch64-mulv.ll              | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index f67efc39e98265b..f31a2ad8294d6ab 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -886,7 +886,6 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .lower();
 
   getActionDefinitionsBuilder(G_VECREDUCE_MUL)
-      .clampMaxNumElements(1, s64, 1)
       .clampMaxNumElements(1, s32, 2)
       .clampMaxNumElements(1, s16, 4)
       .clampMaxNumElements(1, s8, 8)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
index a38ddf32c2216b2..995023e80c44bef 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-; CHECK_GI:        warning: Instruction selection used fallback path for mulv_v3i62
+; CHECK_GI:        warning: Instruction selection used fallback path for mulv_v3i64
 
 declare i8 @llvm.vector.reduce.mul.v2i8(<2 x i8>)
 declare i8 @llvm.vector.reduce.mul.v3i8(<3 x i8>)



More information about the llvm-commits mailing list