[llvm] [AArch64][SDAG] Enable +sve2p2/+sme2p2 lowerings for VECTOR_COMPRESS (PR #215219)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 01:49:50 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Benjamin Maxwell (MacDue)
<details>
<summary>Changes</summary>
Enables the +sve2p2/+sme2p2 lowerings for `VECTOR_COMPRESS`, which allows lowering the full range of legal types to `compact`.
---
Patch is 30.68 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/215219.diff
3 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+30-20)
- (modified) llvm/test/CodeGen/AArch64/sve-vector-compress.ll (+295-114)
- (added) llvm/test/CodeGen/AArch64/sve2p2-vector-compress.ll (+154)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b01dc35531f4b..f62405655eacc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2166,6 +2166,36 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::CLMUL, MVT::nxv2i64, Custom);
}
+ if (Subtarget->isSVEAvailable() ||
+ (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) {
+ // We can lower types that have <vscale x {2|4}> elements to compact.
+ for (auto VT :
+ {MVT::nxv4i32, MVT::nxv2i64, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv2f64})
+ setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
+
+ // If we have SVE, we can use SVE logic for legal NEON vectors in the lowest
+ // bits of the SVE register.
+ for (auto VT : {MVT::v2i32, MVT::v4i32, MVT::v2i64, MVT::v2f32, MVT::v4f32,
+ MVT::v2f64})
+ setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
+
+ if (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()) {
+ // With +sve2p2/+sme2p2 the full range of vector types are supported.
+ for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16})
+ setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
+
+ for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v4f16,
+ MVT::v8f16, MVT::v4bf16, MVT::v8bf16})
+ setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
+ } else {
+ // Promote v4i16/f16 to v4i32/f32 as the SVE container for v4i16 is nxv8,
+ // which is not supported with for compact (with only +sve).
+ setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4bf16, MVT::v4i16);
+ setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4f16, MVT::v4i16);
+ setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4i16, MVT::v4i32);
+ }
+ }
+
// Handle non-aliasing elements mask
if (Subtarget->hasSVE2() ||
(Subtarget->hasSME() && Subtarget->isStreaming())) {
@@ -2199,23 +2229,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
MVT::v2f32, MVT::v4f32, MVT::v2f64})
setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
- // We can lower types that have <vscale x {2|4}> elements to compact.
- for (auto VT :
- {MVT::nxv4i32, MVT::nxv2i64, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv2f64})
- setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
-
- // If we have SVE, we can use SVE logic for legal NEON vectors in the lowest
- // bits of the SVE register.
- for (auto VT : {MVT::v2i32, MVT::v4i32, MVT::v2i64, MVT::v2f32, MVT::v4f32,
- MVT::v2f64})
- setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
-
- // Promote v4i16/f16 to v4i32/f32 as the SVE container for v4i16 is nxv8,
- // which is not supported with for compact (with only +sve).
- setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4bf16, MVT::v4i16);
- setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4f16, MVT::v4i16);
- setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4i16, MVT::v4i32);
-
for (auto VT : {MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64,
MVT::nxv2f32, MVT::nxv2f64, MVT::nxv4i8, MVT::nxv4i16,
MVT::nxv4i32, MVT::nxv4f32}) {
@@ -8064,9 +8077,6 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorCompressToSVE(
SDValue AArch64TargetLowering::LowerVECTOR_COMPRESS(SDValue Op,
SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
- if (!Subtarget->isSVEAvailable())
- return SDValue();
-
if (VT.isFixedLengthVector())
return LowerFixedLengthVectorCompressToSVE(Op, DAG);
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-compress.ll b/llvm/test/CodeGen/AArch64/sve-vector-compress.ll
index 017d27c9a5bc8..e66b632ad7fad 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-compress.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-compress.ll
@@ -1,5 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE,CHECK-SVE1
+; RUN: llc -mtriple=aarch64 -mattr=+sve,+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE,CHECK-SVE2p2
+; RUN: llc -mtriple=aarch64 -mattr=+sme,+sme2p2 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SME2p2
define <vscale x 2 x i8> @test_compress_nxv2i8(<vscale x 2 x i8> %vec, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: test_compress_nxv2i8:
@@ -137,30 +139,84 @@ define <vscale x 4 x i32> @test_compress_numsignbits_sext(<vscale x 4 x i16> %ve
}
define <vscale x 8 x i32> @test_compress_large(<vscale x 8 x i32> %vec, <vscale x 8 x i1> %mask) {
-; CHECK-LABEL: test_compress_large:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-2
-; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: punpklo p1.h, p0.b
-; CHECK-NEXT: cnth x9
-; CHECK-NEXT: punpkhi p0.h, p0.b
-; CHECK-NEXT: sub x9, x9, #1
-; CHECK-NEXT: cntp x8, p1, p1.s
-; CHECK-NEXT: compact z0.s, p1, z0.s
-; CHECK-NEXT: compact z1.s, p0, z1.s
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: cmp x8, x9
-; CHECK-NEXT: str z0, [sp]
-; CHECK-NEXT: csel x8, x8, x9, lo
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: st1w { z1.s }, p0, [x9, x8, lsl #2]
-; CHECK-NEXT: ldr z0, [sp]
-; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
-; CHECK-NEXT: addvl sp, sp, #2
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; CHECK-SVE1-LABEL: test_compress_large:
+; CHECK-SVE1: // %bb.0:
+; CHECK-SVE1-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SVE1-NEXT: addvl sp, sp, #-2
+; CHECK-SVE1-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-SVE1-NEXT: .cfi_offset w29, -16
+; CHECK-SVE1-NEXT: punpklo p1.h, p0.b
+; CHECK-SVE1-NEXT: cnth x9
+; CHECK-SVE1-NEXT: punpkhi p0.h, p0.b
+; CHECK-SVE1-NEXT: sub x9, x9, #1
+; CHECK-SVE1-NEXT: cntp x8, p1, p1.s
+; CHECK-SVE1-NEXT: compact z0.s, p1, z0.s
+; CHECK-SVE1-NEXT: compact z1.s, p0, z1.s
+; CHECK-SVE1-NEXT: ptrue p0.s
+; CHECK-SVE1-NEXT: cmp x8, x9
+; CHECK-SVE1-NEXT: str z0, [sp]
+; CHECK-SVE1-NEXT: csel x8, x8, x9, lo
+; CHECK-SVE1-NEXT: mov x9, sp
+; CHECK-SVE1-NEXT: st1w { z1.s }, p0, [x9, x8, lsl #2]
+; CHECK-SVE1-NEXT: ldr z0, [sp]
+; CHECK-SVE1-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-SVE1-NEXT: addvl sp, sp, #2
+; CHECK-SVE1-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SVE1-NEXT: ret
+;
+; CHECK-SVE2p2-LABEL: test_compress_large:
+; CHECK-SVE2p2: // %bb.0:
+; CHECK-SVE2p2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SVE2p2-NEXT: addvl sp, sp, #-2
+; CHECK-SVE2p2-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-SVE2p2-NEXT: .cfi_offset w29, -16
+; CHECK-SVE2p2-NEXT: punpklo p1.h, p0.b
+; CHECK-SVE2p2-NEXT: mov x8, #-1 // =0xffffffffffffffff
+; CHECK-SVE2p2-NEXT: punpkhi p0.h, p0.b
+; CHECK-SVE2p2-NEXT: inch x8
+; CHECK-SVE2p2-NEXT: cntp x9, p1, p1.s
+; CHECK-SVE2p2-NEXT: compact z0.s, p1, z0.s
+; CHECK-SVE2p2-NEXT: compact z1.s, p0, z1.s
+; CHECK-SVE2p2-NEXT: ptrue p0.s
+; CHECK-SVE2p2-NEXT: cmp x9, x8
+; CHECK-SVE2p2-NEXT: str z0, [sp]
+; CHECK-SVE2p2-NEXT: csel x8, x9, x8, lo
+; CHECK-SVE2p2-NEXT: mov x9, sp
+; CHECK-SVE2p2-NEXT: st1w { z1.s }, p0, [x9, x8, lsl #2]
+; CHECK-SVE2p2-NEXT: ldr z0, [sp]
+; CHECK-SVE2p2-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-SVE2p2-NEXT: addvl sp, sp, #2
+; CHECK-SVE2p2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SVE2p2-NEXT: ret
+;
+; CHECK-SME2p2-LABEL: test_compress_large:
+; CHECK-SME2p2: // %bb.0:
+; CHECK-SME2p2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SME2p2-NEXT: addvl sp, sp, #-1
+; CHECK-SME2p2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; CHECK-SME2p2-NEXT: addvl sp, sp, #-2
+; CHECK-SME2p2-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-SME2p2-NEXT: .cfi_offset w29, -16
+; CHECK-SME2p2-NEXT: punpklo p1.h, p0.b
+; CHECK-SME2p2-NEXT: mov x8, #-1 // =0xffffffffffffffff
+; CHECK-SME2p2-NEXT: ptrue pn8.s
+; CHECK-SME2p2-NEXT: punpkhi p0.h, p0.b
+; CHECK-SME2p2-NEXT: inch x8
+; CHECK-SME2p2-NEXT: cntp x9, p1, p1.s
+; CHECK-SME2p2-NEXT: compact z0.s, p1, z0.s
+; CHECK-SME2p2-NEXT: compact z1.s, p0, z1.s
+; CHECK-SME2p2-NEXT: ptrue p0.s
+; CHECK-SME2p2-NEXT: cmp x9, x8
+; CHECK-SME2p2-NEXT: str z0, [sp]
+; CHECK-SME2p2-NEXT: csel x8, x9, x8, lo
+; CHECK-SME2p2-NEXT: mov x9, sp
+; CHECK-SME2p2-NEXT: st1w { z1.s }, p0, [x9, x8, lsl #2]
+; CHECK-SME2p2-NEXT: ld1w { z0.s, z1.s }, pn8/z, [sp]
+; CHECK-SME2p2-NEXT: addvl sp, sp, #2
+; CHECK-SME2p2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; CHECK-SME2p2-NEXT: addvl sp, sp, #1
+; CHECK-SME2p2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SME2p2-NEXT: ret
%out = call <vscale x 8 x i32> @llvm.experimental.vector.compress(<vscale x 8 x i32> %vec, <vscale x 8 x i1> %mask, <vscale x 8 x i32> poison)
ret <vscale x 8 x i32> %out
}
@@ -191,125 +247,250 @@ define <vscale x 4 x i32> @test_compress_poison_mask(<vscale x 4 x i32> %ignore,
}
define <4 x i32> @test_compress_v4i32_with_sve(<4 x i32> %vec, <4 x i1> %mask) {
-; CHECK-LABEL: test_compress_v4i32_with_sve:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT: shl v1.4s, v1.4s, #31
-; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0
-; CHECK-NEXT: compact z0.s, p1, z0.s
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: test_compress_v4i32_with_sve:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-SVE-NEXT: ptrue p0.s, vl4
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SVE-NEXT: shl v1.4s, v1.4s, #31
+; CHECK-SVE-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-SVE-NEXT: compact z0.s, p1, z0.s
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME2p2-LABEL: test_compress_v4i32_with_sve:
+; CHECK-SME2p2: // %bb.0:
+; CHECK-SME2p2-NEXT: uunpklo z1.s, z1.h
+; CHECK-SME2p2-NEXT: ptrue p0.s, vl4
+; CHECK-SME2p2-NEXT: lsl z1.s, z1.s, #31
+; CHECK-SME2p2-NEXT: asr z1.s, z1.s, #31
+; CHECK-SME2p2-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-SME2p2-NEXT: compact z0.s, p1, z0.s
+; CHECK-SME2p2-NEXT: ret
%out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> %mask, <4 x i32> poison)
ret <4 x i32> %out
}
define <4 x i16> @test_compress_v4i16_with_sve(<4 x i16> %vec, <4 x i1> %mask) {
-; CHECK-LABEL: test_compress_v4i16_with_sve:
-; CHECK: // %bb.0:
-; CHECK-NEXT: shl v1.4h, v1.4h, #15
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
-; CHECK-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0
-; CHECK-NEXT: compact z0.s, p1, z0.s
-; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: ret
+; CHECK-SVE1-LABEL: test_compress_v4i16_with_sve:
+; CHECK-SVE1: // %bb.0:
+; CHECK-SVE1-NEXT: shl v1.4h, v1.4h, #15
+; CHECK-SVE1-NEXT: ptrue p0.s, vl4
+; CHECK-SVE1-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SVE1-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-SVE1-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-SVE1-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-SVE1-NEXT: compact z0.s, p1, z0.s
+; CHECK-SVE1-NEXT: xtn v0.4h, v0.4s
+; CHECK-SVE1-NEXT: ret
+;
+; CHECK-SVE2p2-LABEL: test_compress_v4i16_with_sve:
+; CHECK-SVE2p2: // %bb.0:
+; CHECK-SVE2p2-NEXT: shl v1.4h, v1.4h, #15
+; CHECK-SVE2p2-NEXT: ptrue p0.h, vl4
+; CHECK-SVE2p2-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-SVE2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SVE2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SVE2p2-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-SVE2p2-NEXT: ret
+;
+; CHECK-SME2p2-LABEL: test_compress_v4i16_with_sve:
+; CHECK-SME2p2: // %bb.0:
+; CHECK-SME2p2-NEXT: lsl z1.h, z1.h, #15
+; CHECK-SME2p2-NEXT: ptrue p0.h, vl4
+; CHECK-SME2p2-NEXT: asr z1.h, z1.h, #15
+; CHECK-SME2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SME2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SME2p2-NEXT: ret
%out = call <4 x i16> @llvm.experimental.vector.compress(<4 x i16> %vec, <4 x i1> %mask, <4 x i16> poison)
ret <4 x i16> %out
}
define <4 x half> @test_compress_v4f16_with_sve(<4 x half> %vec, <4 x i1> %mask) {
-; CHECK-LABEL: test_compress_v4f16_with_sve:
-; CHECK: // %bb.0:
-; CHECK-NEXT: shl v1.4h, v1.4h, #15
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
-; CHECK-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0
-; CHECK-NEXT: compact z0.s, p1, z0.s
-; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: ret
+; CHECK-SVE1-LABEL: test_compress_v4f16_with_sve:
+; CHECK-SVE1: // %bb.0:
+; CHECK-SVE1-NEXT: shl v1.4h, v1.4h, #15
+; CHECK-SVE1-NEXT: ptrue p0.s, vl4
+; CHECK-SVE1-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SVE1-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-SVE1-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-SVE1-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-SVE1-NEXT: compact z0.s, p1, z0.s
+; CHECK-SVE1-NEXT: xtn v0.4h, v0.4s
+; CHECK-SVE1-NEXT: ret
+;
+; CHECK-SVE2p2-LABEL: test_compress_v4f16_with_sve:
+; CHECK-SVE2p2: // %bb.0:
+; CHECK-SVE2p2-NEXT: shl v1.4h, v1.4h, #15
+; CHECK-SVE2p2-NEXT: ptrue p0.h, vl4
+; CHECK-SVE2p2-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-SVE2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SVE2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SVE2p2-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-SVE2p2-NEXT: ret
+;
+; CHECK-SME2p2-LABEL: test_compress_v4f16_with_sve:
+; CHECK-SME2p2: // %bb.0:
+; CHECK-SME2p2-NEXT: lsl z1.h, z1.h, #15
+; CHECK-SME2p2-NEXT: ptrue p0.h, vl4
+; CHECK-SME2p2-NEXT: asr z1.h, z1.h, #15
+; CHECK-SME2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SME2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SME2p2-NEXT: ret
%out = call <4 x half> @llvm.experimental.vector.compress(<4 x half> %vec, <4 x i1> %mask, <4 x half> poison)
ret <4 x half> %out
}
define <4 x bfloat> @test_compress_v4bf16_with_sve(<4 x bfloat> %vec, <4 x i1> %mask) {
-; CHECK-LABEL: test_compress_v4bf16_with_sve:
-; CHECK: // %bb.0:
-; CHECK-NEXT: shl v1.4h, v1.4h, #15
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-NEXT: cmlt v1.4h, v1.4h, #0
-; CHECK-NEXT: sshll v1.4s, v1.4h, #0
-; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0
-; CHECK-NEXT: compact z0.s, p1, z0.s
-; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: ret
+; CHECK-SVE1-LABEL: test_compress_v4bf16_with_sve:
+; CHECK-SVE1: // %bb.0:
+; CHECK-SVE1-NEXT: shl v1.4h, v1.4h, #15
+; CHECK-SVE1-NEXT: ptrue p0.s, vl4
+; CHECK-SVE1-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-SVE1-NEXT: cmlt v1.4h, v1.4h, #0
+; CHECK-SVE1-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-SVE1-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-SVE1-NEXT: compact z0.s, p1, z0.s
+; CHECK-SVE1-NEXT: xtn v0.4h, v0.4s
+; CHECK-SVE1-NEXT: ret
+;
+; CHECK-SVE2p2-LABEL: test_compress_v4bf16_with_sve:
+; CHECK-SVE2p2: // %bb.0:
+; CHECK-SVE2p2-NEXT: shl v1.4h, v1.4h, #15
+; CHECK-SVE2p2-NEXT: ptrue p0.h, vl4
+; CHECK-SVE2p2-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-SVE2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SVE2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SVE2p2-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-SVE2p2-NEXT: ret
+;
+; CHECK-SME2p2-LABEL: test_compress_v4bf16_with_sve:
+; CHECK-SME2p2: // %bb.0:
+; CHECK-SME2p2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SME2p2-NEXT: addvl sp, sp, #-1
+; CHECK-SME2p2-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-SME2p2-NEXT: .cfi_offset w29, -16
+; CHECK-SME2p2-NEXT: lsl z1.h, z1.h, #15
+; CHECK-SME2p2-NEXT: ptrue p0.h, vl4
+; CHECK-SME2p2-NEXT: asr z1.h, z1.h, #15
+; CHECK-SME2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SME2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SME2p2-NEXT: str z0, [sp]
+; CHECK-SME2p2-NEXT: ldr d0, [sp]
+; CHECK-SME2p2-NEXT: addvl sp, sp, #1
+; CHECK-SME2p2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-SME2p2-NEXT: ret
%out = call <4 x bfloat> @llvm.experimental.vector.compress(<4 x bfloat> %vec, <4 x i1> %mask, <4 x bfloat> poison)
ret <4 x bfloat> %out
}
define <1 x i32> @test_compress_v1i32_with_sve(<1 x i32> %vec, <1 x i1> %mask) {
-; CHECK-LABEL: test_compress_v1i32_with_sve:
-; CHECK: // %bb.0:
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: sbfx w8, w0, #0, #1
-; CHECK-NEXT: ptrue p0.s, vl2
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT: mov v1.s[0], w8
-; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0
-; CHECK-NEXT: compact z0.s, p1, z0.s
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: test_compress_v1i32_with_sve:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: movi v1.2d, #0000000000000000
+; CHECK-SVE-NEXT: sbfx w8, w0, #0, #1
+; CHECK-SVE-NEXT: ptrue p0.s, vl2
+; CHECK-SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-SVE-NEXT: mov v1.s[0], w8
+; CHECK-SVE-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-SVE-NEXT: compact z0.s, p1, z0.s
+; CHECK-SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME2p2-LABEL: test_compress_v1i32_with_sve:
+; CHECK-SME2p2: // %bb.0:
+; CHECK-SME2p2-NEXT: sbfx w8, w0, #0, #1
+; CHECK-SME2p2-NEXT: fmov s1, wzr
+; CHECK-SME2p2-NEXT: ptrue p0.s, vl2
+; CHECK-SME2p2-NEXT: fmov s2, w8
+; CHECK-SME2p2-NEXT: zip1 z1.s, z2.s, z1.s
+; CHECK-SME2p2-NEXT: cmpne p1.s, p0/z, z1.s, #0
+; CHECK-SME2p2-NEXT: compact z0.s, p1, z0.s
+; CHECK-SME2p2-NEXT: ret
%out = call <1 x i32> @llvm.experimental.vector.compress(<1 x i32> %vec, <1 x i1> %mask, <1 x i32> poison)
ret <1 x i32> %out
}
define <4 x double> @test_compress_v4f64_with_sve(<4 x double> %vec, <4 x i1> %mask) {
-; CHECK-LABEL: test_compress_v4f64_with_sve:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #32
-; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: ushll v2.4s, v2.4h, #0
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
-; CHECK-NEXT: mov x9, sp
-; CHECK-NEXT: ushll v3.2d, v2.2s, #0
-; CHECK-NEXT: ushll2 v4.2d, v2.4s, #0
-; CHECK-NEXT: and z2.s, z2.s, #0x1
-; CHECK-NEXT: shl v3.2d, v3.2d, #63
-; CHECK-NEXT: shl v4.2d, v4.2d, #63
-; CHECK-NEXT: addp v2.2s, v2.2s, v2.2s
-; CHECK-NEXT: cmpne p1.d, p0/z, z3.d, #0
-; CHECK-NEXT: cmpne p2.d, p0/z, z4.d, #0
-; CHECK-NEXT: fmov w8, s2
-; CHECK-NEXT: and x8, x8, #0x3
-; CHECK-NEXT: compact z0.d, p1, z0.d
-; CHECK-NEXT: compact z1.d, p2, z1.d
-; CHECK-NEXT: lsl x8, x8, #3
-; CHECK-NEXT: str q0, [sp]
-; CHECK-NEXT: str q1, [x9, x8]
-; CHECK-NEXT: ldp q0, q1, [sp], #32
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: test_compress_v4f64_with_sve:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: sub sp, sp, #32
+; CHECK-SVE-NEXT: .cfi_def_cfa_offset 32
+; CHECK-SVE-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SVE-NEXT: // ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/215219
More information about the llvm-commits
mailing list