[llvm] 5fee83d - [AArch64] Promote unpacked SVE f16/bf16 compressstores 32/64-bit (#222321)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 15 01:29:03 PDT 2026
Author: Benjamin Maxwell
Date: 2026-09-15T09:28:58+01:00
New Revision: 5fee83dfe895db41930566a9146fdade0e30dd8c
URL: https://github.com/llvm/llvm-project/commit/5fee83dfe895db41930566a9146fdade0e30dd8c
DIFF: https://github.com/llvm/llvm-project/commit/5fee83dfe895db41930566a9146fdade0e30dd8c.diff
LOG: [AArch64] Promote unpacked SVE f16/bf16 compressstores 32/64-bit (#222321)
To support compressstore for unpacked 16-bit SVE floats (e.g., nxv2f16),
we need to expand the compressstore before type legalization. If we
expand after type legalization the `VECTOR_COMPRESS` node inserted won't
be promoted to a 32/64-bit element size (which means we'd have to
duplicate some type legalization in the lowering).
Added:
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 26ca67ed132e5..e324b1e385a7f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -28349,15 +28349,53 @@ static SDValue performLegalizedInterleavedStoreCombine(
return NewStore;
}
+/// Expand a scalable compressing store to a VECTOR_COMPRESS + a masked store.
+static SDValue expandScalableCompressingStore(MaskedStoreSDNode *Store,
+ SelectionDAG &DAG) {
+ SDLoc DL(Store);
+ EVT VT = Store->getValue().getValueType();
+ assert(VT.isScalableVector() && Store->isCompressingStore() &&
+ "Expected a scalable compressing store");
+
+ EVT MaskVT = Store->getMask().getValueType();
+ SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
+ SDValue CntActive = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64),
+ Store->getMask(), Store->getMask());
+
+ SDValue CompressedValue =
+ DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, Store->getValue(),
+ Store->getMask(), DAG.getPOISON(VT));
+ SDValue CompressedMask =
+ DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, MaskVT, Zero, CntActive);
+
+ return DAG.getMaskedStore(Store->getChain(), DL, CompressedValue,
+ Store->getBasePtr(), Store->getOffset(),
+ CompressedMask, Store->getMemoryVT(),
+ Store->getMemOperand(), Store->getAddressingMode(),
+ Store->isTruncatingStore(),
+ /*isCompressing=*/false);
+}
+
static SDValue performMSTORECombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
- const AArch64Subtarget *Subtarget) {
+ const AArch64Subtarget *Subtarget,
+ const AArch64TargetLowering &TLI) {
MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
SDValue Value = MST->getValue();
SDValue Mask = MST->getMask();
SDLoc DL(N);
+ // If MST is a compressing store and VECTOR_COMPRESS can be lowered for the VT
+ // expand the store early. This allows type promotion to apply to unpacked
+ // SVE float types.
+ EVT VT = MST->getValue().getValueType();
+ if (MST->isCompressingStore() && VT.isScalableVector() &&
+ TLI.isOperationLegalOrCustomOrPromote(ISD::VECTOR_COMPRESS, VT))
+ return expandScalableCompressingStore(MST, DAG);
+
if (SDValue Res = performInterleavedStoreCombine(N, DCI, DAG))
return Res;
@@ -31769,7 +31807,7 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
case ISD::STORE:
return performSTORECombine(N, DCI, DAG, Subtarget);
case ISD::MSTORE:
- return performMSTORECombine(N, DCI, DAG, Subtarget);
+ return performMSTORECombine(N, DCI, DAG, Subtarget, *this);
case ISD::MGATHER:
case ISD::MSCATTER:
case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
@@ -34212,25 +34250,7 @@ SDValue AArch64TargetLowering::LowerMSTORE(SDValue Op,
if (!Store->isCompressingStore())
return SDValue();
- EVT MaskVT = Store->getMask().getValueType();
- SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
- SDValue CntActive = DAG.getNode(
- ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
- DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64),
- Store->getMask(), Store->getMask());
-
- SDValue CompressedValue =
- DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, Store->getValue(),
- Store->getMask(), DAG.getPOISON(VT));
- SDValue CompressedMask =
- DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, MaskVT, Zero, CntActive);
-
- return DAG.getMaskedStore(Store->getChain(), DL, CompressedValue,
- Store->getBasePtr(), Store->getOffset(),
- CompressedMask, Store->getMemoryVT(),
- Store->getMemOperand(), Store->getAddressingMode(),
- Store->isTruncatingStore(),
- /*isCompressing=*/false);
+ return expandScalableCompressingStore(Store, DAG);
}
SDValue AArch64TargetLowering::LowerFixedLengthVectorMStoreToSVE(
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll b/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
index b6fa9ea943c75..01530547b6acb 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-compressstore.ll
@@ -102,6 +102,30 @@ define void @test_compressstore_nxv2i8(ptr %p, <vscale x 2 x i8> %vec, <vscale x
ret void
}
+define void @test_compressstore_nxv2bf16(ptr %p, <vscale x 2 x bfloat> %vec, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: test_compressstore_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cntp x8, p0, p0.d
+; CHECK-NEXT: compact z0.d, p0, z0.d
+; CHECK-NEXT: whilelo p0.d, xzr, x8
+; CHECK-NEXT: st1h { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ tail call void @llvm.masked.compressstore.nxv2bf16(<vscale x 2 x bfloat> %vec, ptr align 2 %p, <vscale x 2 x i1> %mask)
+ ret void
+}
+
+define void @test_compressstore_nxv2f16(ptr %p, <vscale x 2 x half> %vec, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: test_compressstore_nxv2f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cntp x8, p0, p0.d
+; CHECK-NEXT: compact z0.d, p0, z0.d
+; CHECK-NEXT: whilelo p0.d, xzr, x8
+; CHECK-NEXT: st1h { z0.d }, p0, [x0]
+; CHECK-NEXT: ret
+ tail call void @llvm.masked.compressstore.nxv2f16(<vscale x 2 x half> %vec, ptr align 2 %p, <vscale x 2 x i1> %mask)
+ ret void
+}
+
define void @test_compressstore_nxv4i16(ptr %p, <vscale x 4 x i16> %vec, <vscale x 4 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv4i16:
; CHECK: // %bb.0:
@@ -114,6 +138,30 @@ define void @test_compressstore_nxv4i16(ptr %p, <vscale x 4 x i16> %vec, <vscale
ret void
}
+define void @test_compressstore_nxv4bf16(ptr %p, <vscale x 4 x bfloat> %vec, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: test_compressstore_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cntp x8, p0, p0.s
+; CHECK-NEXT: compact z0.s, p0, z0.s
+; CHECK-NEXT: whilelo p0.s, xzr, x8
+; CHECK-NEXT: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ tail call void @llvm.masked.compressstore.nxv4bf16(<vscale x 4 x bfloat> %vec, ptr align 2 %p, <vscale x 4 x i1> %mask)
+ ret void
+}
+
+define void @test_compressstore_nxv4f16(ptr %p, <vscale x 4 x half> %vec, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: test_compressstore_nxv4f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cntp x8, p0, p0.s
+; CHECK-NEXT: compact z0.s, p0, z0.s
+; CHECK-NEXT: whilelo p0.s, xzr, x8
+; CHECK-NEXT: st1h { z0.s }, p0, [x0]
+; CHECK-NEXT: ret
+ tail call void @llvm.masked.compressstore.nxv4f16(<vscale x 4 x half> %vec, ptr align 2 %p, <vscale x 4 x i1> %mask)
+ ret void
+}
+
;; NEON vector types (promoted to SVE)
define void @test_compressstore_v2f64(ptr %p, <2 x double> %vec, <2 x i1> %mask) {
@@ -334,9 +382,9 @@ define void @test_compressstore_v8i32(ptr %p, <8 x i32> %vec, <8 x i1> %mask) {
; CHECK-BASE: // %bb.0:
; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-BASE-NEXT: zip1 v3.8b, v2.8b, v0.8b
-; CHECK-BASE-NEXT: adrp x8, .LCPI11_0
+; CHECK-BASE-NEXT: adrp x8, .LCPI15_0
; CHECK-BASE-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-BASE-NEXT: ldr d5, [x8, :lo12:.LCPI11_0]
+; CHECK-BASE-NEXT: ldr d5, [x8, :lo12:.LCPI15_0]
; CHECK-BASE-NEXT: ptrue p0.s
; CHECK-BASE-NEXT: // kill: def $q1 killed $q1 def $z1
; CHECK-BASE-NEXT: ptrue p1.s, vl4
@@ -389,8 +437,8 @@ define void @test_compressstore_v8i32(ptr %p, <8 x i32> %vec, <8 x i1> %mask) {
; CHECK-SME2p2-NEXT: // kill: def $q0 killed $q0 def $z0
; CHECK-SME2p2-NEXT: zip1 v3.8b, v2.8b, v0.8b
; CHECK-SME2p2-NEXT: zip2 v2.8b, v2.8b, v0.8b
-; CHECK-SME2p2-NEXT: adrp x8, .LCPI11_0
-; CHECK-SME2p2-NEXT: ldr d5, [x8, :lo12:.LCPI11_0]
+; CHECK-SME2p2-NEXT: adrp x8, .LCPI15_0
+; CHECK-SME2p2-NEXT: ldr d5, [x8, :lo12:.LCPI15_0]
; CHECK-SME2p2-NEXT: ptrue p0.s, vl4
; CHECK-SME2p2-NEXT: // kill: def $q1 killed $q1 def $z1
; CHECK-SME2p2-NEXT: shl v4.4h, v3.4h, #15
More information about the llvm-commits
mailing list