[llvm] [AArch64][GlobalISel] Legalize more G_VECREDUCE_ADD operations. (PR #123392)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jan 17 12:01:21 PST 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: David Green (davemgreen)
<details>
<summary>Changes</summary>
Non-power-2 vectors will now be padded with zero elements, smaller vectors will be widened using anyext, which I believe will be better in many situations than padding with zeros, although some small types may prefer being scalarized depending on the code. Padding with zeros may not be best for all sizes (v5i8 being the worst), we can hopefully improve that in the future but they no longer fall back. We scalarize other types like i128.
---
Patch is 259.96 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/123392.diff
7 Files Affected:
- (modified) llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp (+9)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp (+3-1)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir (+11-6)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir (+2-2)
- (modified) llvm/test/CodeGen/AArch64/aarch64-addv.ll (+61-38)
- (modified) llvm/test/CodeGen/AArch64/neon-dotreduce.ll (+3905-1760)
- (modified) llvm/test/CodeGen/AArch64/vecreduce-add.ll (+150-204)
``````````diff
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index d0a62340a5f322..5a16d72594363d 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3342,6 +3342,15 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Observer.changedInstr(MI);
return Legalized;
}
+ case TargetOpcode::G_VECREDUCE_ADD: {
+ if (TypeIdx != 1)
+ return UnableToLegalize;
+ Observer.changingInstr(MI);
+ widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
+ widenScalarDst(MI, WideTy.getScalarType(), 0, TargetOpcode::G_TRUNC);
+ Observer.changedInstr(MI);
+ return Legalized;
+ }
case TargetOpcode::G_VECREDUCE_FADD:
case TargetOpcode::G_VECREDUCE_FMUL:
case TargetOpcode::G_VECREDUCE_FMIN:
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 93461e39f95597..fdedf44e0ba1bf 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -1215,11 +1215,13 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
{s32, v4s32},
{s32, v2s32},
{s64, v2s64}})
+ .moreElementsToNextPow2(1)
.clampMaxNumElements(1, s64, 2)
.clampMaxNumElements(1, s32, 4)
.clampMaxNumElements(1, s16, 8)
.clampMaxNumElements(1, s8, 16)
- .lower();
+ .widenVectorEltsToVectorMinSize(1, 64)
+ .scalarize(1);
getActionDefinitionsBuilder({G_VECREDUCE_FMIN, G_VECREDUCE_FMAX,
G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir
index 76fdfd0c301f6d..a18bd0f2f90eb7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-reduce-add.mir
@@ -157,12 +157,17 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
- ; CHECK-NEXT: [[VECREDUCE_ADD:%[0-9]+]]:_(s64) = G_VECREDUCE_ADD [[COPY]](<2 x s64>)
- ; CHECK-NEXT: [[VECREDUCE_ADD1:%[0-9]+]]:_(s64) = G_VECREDUCE_ADD [[COPY1]](<2 x s64>)
- ; CHECK-NEXT: [[VECREDUCE_ADD2:%[0-9]+]]:_(s64) = G_VECREDUCE_ADD [[COPY2]](<2 x s64>)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[VECREDUCE_ADD]], [[VECREDUCE_ADD1]]
- ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[ADD]], [[VECREDUCE_ADD2]]
- ; CHECK-NEXT: $x0 = COPY [[ADD1]](s64)
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[DEF]](s64), [[DEF]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[IVEC:%[0-9]+]]:_(<2 x s64>) = G_INSERT_VECTOR_ELT [[BUILD_VECTOR]], [[C]](s64), [[C]](s64)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+ ; CHECK-NEXT: [[IVEC1:%[0-9]+]]:_(<2 x s64>) = G_INSERT_VECTOR_ELT [[IVEC]], [[C]](s64), [[C1]](s64)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<2 x s64>) = G_ADD [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(<2 x s64>) = G_ADD [[COPY2]], [[IVEC1]]
+ ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(<2 x s64>) = G_ADD [[ADD]], [[ADD1]]
+ ; CHECK-NEXT: [[VECREDUCE_ADD:%[0-9]+]]:_(s64) = G_VECREDUCE_ADD [[ADD2]](<2 x s64>)
+ ; CHECK-NEXT: $x0 = COPY [[VECREDUCE_ADD]](s64)
; CHECK-NEXT: RET_ReallyLR implicit $x0
%0:_(<2 x s64>) = COPY $q0
%1:_(<2 x s64>) = COPY $q1
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index c2c77b9326cb64..0260e65520774e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -867,8 +867,8 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_VECREDUCE_ADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
# DEBUG-NEXT: G_VECREDUCE_MUL (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/aarch64-addv.ll b/llvm/test/CodeGen/AArch64/aarch64-addv.ll
index aba284b4e0d292..104b6d1236d715 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-addv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-addv.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64 -aarch64-neon-syntax=generic | FileCheck %s -check-prefixes=CHECK,CHECK-SD
-; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -global-isel-abort=2 -aarch64-neon-syntax=generic 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc < %s -mtriple=aarch64 -aarch64-neon-syntax=generic | FileCheck %s -check-prefixes=CHECK,SDAG
+; RUN: llc < %s -mtriple=aarch64 -global-isel=1 -aarch64-neon-syntax=generic | FileCheck %s --check-prefixes=CHECK,GISEL
declare i8 @llvm.vector.reduce.add.v2i8(<2 x i8>)
declare i8 @llvm.vector.reduce.add.v3i8(<3 x i8>)
@@ -22,15 +22,6 @@ declare i64 @llvm.vector.reduce.add.v3i64(<3 x i64>)
declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
declare i128 @llvm.vector.reduce.add.v2i128(<2 x i128>)
-; CHECK-GI: warning: Instruction selection used fallback path for addv_v2i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for addv_v3i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for addv_v4i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for addv_v2i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for addv_v3i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for addv_v3i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for addv_v3i64
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for addv_v2i128
-
define i8 @add_B(ptr %arr) {
; CHECK-LABEL: add_B:
@@ -256,15 +247,26 @@ entry:
}
define i8 @addv_v3i8(<3 x i8> %a) {
-; CHECK-LABEL: addv_v3i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov v0.h[0], w0
-; CHECK-NEXT: mov v0.h[1], w1
-; CHECK-NEXT: mov v0.h[2], w2
-; CHECK-NEXT: addv h0, v0.4h
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
+; SDAG-LABEL: addv_v3i8:
+; SDAG: // %bb.0: // %entry
+; SDAG-NEXT: movi v0.2d, #0000000000000000
+; SDAG-NEXT: mov v0.h[0], w0
+; SDAG-NEXT: mov v0.h[1], w1
+; SDAG-NEXT: mov v0.h[2], w2
+; SDAG-NEXT: addv h0, v0.4h
+; SDAG-NEXT: fmov w0, s0
+; SDAG-NEXT: ret
+;
+; GISEL-LABEL: addv_v3i8:
+; GISEL: // %bb.0: // %entry
+; GISEL-NEXT: fmov s0, w0
+; GISEL-NEXT: mov w8, #0 // =0x0
+; GISEL-NEXT: mov v0.h[1], w1
+; GISEL-NEXT: mov v0.h[2], w2
+; GISEL-NEXT: mov v0.h[3], w8
+; GISEL-NEXT: addv h0, v0.4h
+; GISEL-NEXT: fmov w0, s0
+; GISEL-NEXT: ret
entry:
%arg1 = call i8 @llvm.vector.reduce.add.v3i8(<3 x i8> %a)
ret i8 %arg1
@@ -327,13 +329,22 @@ entry:
}
define i16 @addv_v3i16(<3 x i16> %a) {
-; CHECK-LABEL: addv_v3i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v0.h[3], wzr
-; CHECK-NEXT: addv h0, v0.4h
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
+; SDAG-LABEL: addv_v3i16:
+; SDAG: // %bb.0: // %entry
+; SDAG-NEXT: // kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT: mov v0.h[3], wzr
+; SDAG-NEXT: addv h0, v0.4h
+; SDAG-NEXT: fmov w0, s0
+; SDAG-NEXT: ret
+;
+; GISEL-LABEL: addv_v3i16:
+; GISEL: // %bb.0: // %entry
+; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0
+; GISEL-NEXT: mov w8, #0 // =0x0
+; GISEL-NEXT: mov v0.h[3], w8
+; GISEL-NEXT: addv h0, v0.4h
+; GISEL-NEXT: fmov w0, s0
+; GISEL-NEXT: ret
entry:
%arg1 = call i16 @llvm.vector.reduce.add.v3i16(<3 x i16> %a)
ret i16 %arg1
@@ -431,17 +442,29 @@ entry:
}
define i64 @addv_v3i64(<3 x i64> %a) {
-; CHECK-LABEL: addv_v3i64:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-NEXT: mov v2.d[1], xzr
-; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
-; CHECK-NEXT: addp d0, v0.2d
-; CHECK-NEXT: fmov x0, d0
-; CHECK-NEXT: ret
+; SDAG-LABEL: addv_v3i64:
+; SDAG: // %bb.0: // %entry
+; SDAG-NEXT: // kill: def $d2 killed $d2 def $q2
+; SDAG-NEXT: // kill: def $d0 killed $d0 def $q0
+; SDAG-NEXT: // kill: def $d1 killed $d1 def $q1
+; SDAG-NEXT: mov v0.d[1], v1.d[0]
+; SDAG-NEXT: mov v2.d[1], xzr
+; SDAG-NEXT: add v0.2d, v0.2d, v2.2d
+; SDAG-NEXT: addp d0, v0.2d
+; SDAG-NEXT: fmov x0, d0
+; SDAG-NEXT: ret
+;
+; GISEL-LABEL: addv_v3i64:
+; GISEL: // %bb.0: // %entry
+; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0
+; GISEL-NEXT: // kill: def $d2 killed $d2 def $q2
+; GISEL-NEXT: // kill: def $d1 killed $d1 def $q1
+; GISEL-NEXT: mov v0.d[1], v1.d[0]
+; GISEL-NEXT: mov v2.d[1], xzr
+; GISEL-NEXT: add v0.2d, v0.2d, v2.2d
+; GISEL-NEXT: addp d0, v0.2d
+; GISEL-NEXT: fmov x0, d0
+; GISEL-NEXT: ret
entry:
%arg1 = call i64 @llvm.vector.reduce.add.v3i64(<3 x i64> %a)
ret i64 %arg1
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 748555d7bdfa15..8e12446164e89e 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -1,22 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-
-; CHECK-GI: warning: Instruction selection used fallback path for test_udot_v5i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v5i8_nomla
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double_nomla
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8_nomla
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double_nomla
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8_nomla
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double_nomla
+; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm -global-isel < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>)
@@ -413,19 +397,50 @@ entry:
}
define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_udot_v5i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: umull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
-; CHECK-NEXT: mov v1.s[0], v2.s[0]
-; CHECK-NEXT: uaddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test_udot_v5i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: ldr d0, [x0]
+; CHECK-SD-NEXT: ldr d1, [x1]
+; CHECK-SD-NEXT: umull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT: mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT: uaddw v0.4s, v1.4s, v0.4h
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_udot_v5i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldrb w8, [x0, #4]
+; CHECK-GI-NEXT: ldrb w9, [x1, #4]
+; CHECK-GI-NEXT: ldrb w10, [x1]
+; CHECK-GI-NEXT: mul w8, w9, w8
+; CHECK-GI-NEXT: ldrb w9, [x0]
+; CHECK-GI-NEXT: mov v0.s[0], w10
+; CHECK-GI-NEXT: mov v1.s[0], w9
+; CHECK-GI-NEXT: ldrb w9, [x1, #1]
+; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: ldrb w8, [x0, #1]
+; CHECK-GI-NEXT: mov v0.s[1], w9
+; CHECK-GI-NEXT: ldrb w9, [x1, #2]
+; CHECK-GI-NEXT: mov v1.s[1], w8
+; CHECK-GI-NEXT: ldrb w8, [x0, #2]
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: mov v0.s[2], w9
+; CHECK-GI-NEXT: ldrb w9, [x1, #3]
+; CHECK-GI-NEXT: mov v1.s[2], w8
+; CHECK-GI-NEXT: ldrb w8, [x0, #3]
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v0.s[3], w9
+; CHECK-GI-NEXT: mov v1.s[3], w8
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: mla v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: addv s0, v2.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ret
entry:
%0 = load <5 x i8>, ptr %a
%1 = zext <5 x i8> %0 to <5 x i32>
@@ -438,17 +453,37 @@ entry:
}
define i32 @test_udot_v5i8_nomla(ptr nocapture readonly %a1) {
-; CHECK-LABEL: test_udot_v5i8_nomla:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
-; CHECK-NEXT: mov v1.s[0], v2.s[0]
-; CHECK-NEXT: uaddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test_udot_v5i8_nomla:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: ldr d0, [x0]
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT: mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT: uaddw v0.4s, v1.4s, v0.4h
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_udot_v5i8_nomla:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldrb w8, [x0]
+; CHECK-GI-NEXT: ldrb w9, [x0, #4]
+; CHECK-GI-NEXT: mov v0.s[0], w8
+; CHECK-GI-NEXT: mov v1.s[0], w9
+; CHECK-GI-NEXT: ldrb w8, [x0, #1]
+; CHECK-GI-NEXT: mov v0.s[1], w8
+; CHECK-GI-NEXT: mov v1.s[1], wzr
+; CHECK-GI-NEXT: ldrb w8, [x0, #2]
+; CHECK-GI-NEXT: mov v0.s[2], w8
+; CHECK-GI-NEXT: mov v1.s[2], wzr
+; CHECK-GI-NEXT: ldrb w8, [x0, #3]
+; CHECK-GI-NEXT: mov v0.s[3], w8
+; CHECK-GI-NEXT: mov v1.s[3], wzr
+; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: addv s0, v0.4s
+; CHECK-GI-NEXT: fmov w0, s0
+; CHECK-GI-NEXT: ret
entry:
%0 = load <5 x i8>, ptr %a1
%1 = zext <5 x i8> %0 to <5 x i32>
@@ -456,19 +491,50 @@ entry:
ret i32 %2
}
define i32 @test_sdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
-; CHECK-LABEL: test_sdot_v5i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: ldr d1, [x1]
-; CHECK-NEXT: smull v0.8h, v1.8b, v0.8b
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0
-; CHECK-NEXT: mov v1.s[0], v2.s[0]
-; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: add w0, w8, w2
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test_sdot_v5i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: ldr d0, [x0]
+; CHECK-SD-NEXT: ldr d1, [x1]
+; CHECK-SD-NEXT: smull v0.8h, v1.8b, v0.8b
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-SD-NEXT: mov v1.s[0], v2.s[0]
+; CHECK-SD-NEXT: saddw v0.4s, v1.4s, v0.4h
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w8, s0
+; CHECK-SD-NEXT: add w0, w8, w2
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_sdot_v5i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldrsb w8, [x0, #4]
+; CHECK-GI-NEXT: ldrsb w9, [x1, #4]
+; CHECK-GI-NEXT: ldrsb w10, [x1]
+; CHECK-GI-NEXT: mul w8, w9, w8
+; CHECK-GI-NEXT: ldrsb w9, [x0]
+; CHECK-GI-NEXT: mov v0.s[0], w10
+; CHECK-GI-NEXT: mov v1.s[0], w9
+; CHECK-GI-NEXT: ldrsb w9, [x1, #1]
+; CHECK-GI-NEXT: mov v2.s[0], w8
+; CHECK-GI-NEXT: ldrsb w8, [x0, #1]
+; CHECK-GI-NEXT: mov v0.s[1], w9
+; CHECK-GI-NEXT: ldrsb w9, [x1, #2]
+; CHECK-GI-NEXT: mov v1.s[1], w8
+; CHECK-GI-NEXT: ldrsb w8, [x0, #2]
+; CHECK-GI-NEXT: mov v2.s[1], wzr
+; CHECK-GI-NEXT: mov v0.s[2], w9
+; CHECK-GI-NEXT: ldrsb w9, [x1, #3]
+; CHECK-GI-NEXT: mov v1.s[2], w8
+; CHECK-GI-NEXT: ldrsb w8, [x0, #3]
+; CHECK-GI-NEXT: mov v2.s[2], wzr
+; CHECK-GI-NEXT: mov v0.s[3], w9
+; CHECK-GI-NEXT: mov v1.s[3], w8
+; CHECK-GI-NEXT: mov v2.s[3], wzr
+; CHECK-GI-NEXT: mla v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: addv s0, v2.4s
+; CHECK-GI-NEXT: fmov w8, s0
+; CHECK-GI-NEXT: add w0, w8, w2
+; CHECK-GI-NEXT: ret
entry:
%0 = load <5 x i8>, ptr %a
%1 = sext <5 x i8> %0 to <5 x i32>
@@ -481,22 +547,83 @@ entry:
}
define i32 @test_sdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) {
-; CHECK-LABEL: test_sdot_v5i8_double:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: smull v2.8h, v2.8b, v3.8b
-; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: movi v3.2d, #0000000000000000
-; CHECK-NEXT: sshll2 v4.4s, v0.8h, #0
-; CHECK-NEXT: sshll2 v5.4s, v2.8h, #0
-; CHECK-NEXT: mov v3.s[0], v4.s[0]
-; CHECK-NEXT: mov v1.s[0], v5.s[0]
-; CHECK-NEXT: saddw v0.4s, v3.4s, v0.4h
-; CHECK-NEXT: saddw v1.4s, v1.4s, v2.4h
-; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: addv s0, v0.4s
-; CHECK-NEXT: fmov w0, s0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: test_sdot_v5i8_double:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: smull v2.8h, v2.8b, v3.8b
+; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SD-NEXT: movi v3.2d, #0000000000000000
+; CHECK-SD-NEXT: sshll2 v4.4s, v0.8h, #0
+; CHECK-SD-NEXT: sshll2 v5.4s, v2.8h, #0
+; CHECK-SD-NEXT: mov v3.s[0], v4.s[0]
+; CHECK-SD-NEXT: mov v1.s[0], v5.s[0]
+; CHECK-SD-NEXT: saddw v0.4s, v3.4s, v0.4h
+; CHECK-SD-NEXT: saddw v1.4s, v1.4s, v2.4h
+; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: addv s0, v0.4s
+; CHECK-SD-NEXT: fmov w0, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: test_sdot_v5i8_double:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-GI-NEXT: // kill...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/123392
More information about the llvm-commits
mailing list