[llvm-branch-commits] [llvm] release/23.x: [AArch64][SVE] Support lowering masked loads/stores of <4 x bf16> and <8 x bf16> (#208744) (PR #210497)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Jul 18 01:03:42 PDT 2026
https://github.com/llvmbot created https://github.com/llvm/llvm-project/pull/210497
Backport b9869c8c920a7dfa983e215bc30729b13d8b353b
Requested by: @davemgreen
>From 8cd1a9a50252c566b80d5c665e37cf619e666912 Mon Sep 17 00:00:00 2001
From: Shanzhi Chen <chenshanzhi at huawei.com>
Date: Fri, 17 Jul 2026 09:21:32 +0800
Subject: [PATCH] [AArch64][SVE] Support lowering masked loads/stores of <4 x
bf16> and <8 x bf16> (#208744)
Add support for lowering masked loads/stores of <4 x bf16> and <8 x
bf16> when target features contain "+sve" and "+bf16".
Fixes: #201149
(cherry picked from commit b9869c8c920a7dfa983e215bc30729b13d8b353b)
---
.../Target/AArch64/AArch64ISelLowering.cpp | 20 ++-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 5 +-
...sve-fixed-length-masked-64-128bit-loads.ll | 148 ++++++++++++++++++
...ve-fixed-length-masked-64-128bit-stores.ll | 143 +++++++++++++++++
4 files changed, 308 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d6f2633297e51..b206869732d5e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1792,9 +1792,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// NEON doesn't support masked loads/stores, but SME and SVE do.
for (auto VT :
- {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v1f64,
- MVT::v2f64, MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
- MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
+ {MVT::v4f16, MVT::v8f16, MVT::v4bf16, MVT::v8bf16, MVT::v2f32,
+ MVT::v4f32, MVT::v1f64, MVT::v2f64, MVT::v8i8, MVT::v16i8, MVT::v4i16,
+ MVT::v8i16, MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
setOperationAction(ISD::MLOAD, VT, Custom);
setOperationAction(ISD::MSTORE, VT, Custom);
}
@@ -2522,7 +2522,7 @@ void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
// Mark floating-point truncating stores/extending loads as having custom
// lowering
- if (VT.isFloatingPoint()) {
+ if (VT.getScalarType() == MVT::f32 || VT.getScalarType() == MVT::f64) {
MVT InnerVT = VT.changeVectorElementType(MVT::f16);
while (InnerVT != VT) {
setTruncStoreAction(VT, InnerVT, Custom);
@@ -7537,7 +7537,8 @@ SDValue AArch64TargetLowering::LowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
assert(LoadNode && "Expected custom lowering of a masked load node");
EVT VT = Op->getValueType(0);
- if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
+ if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true,
+ /*AllowBF16=*/true))
return LowerFixedLengthVectorMLoadToSVE(Op, DAG);
SDValue PassThru = LoadNode->getPassThru();
@@ -8851,8 +8852,9 @@ bool AArch64TargetLowering::mergeStoresAfterLegalization(EVT VT) const {
return !Subtarget->useSVEForFixedLengthVectors();
}
-bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
- EVT VT, bool OverrideNEON) const {
+bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(EVT VT,
+ bool OverrideNEON,
+ bool AllowBF16) const {
if (!VT.isFixedLengthVector() || !VT.isSimple())
return false;
@@ -8863,6 +8865,10 @@ bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
case MVT::i1:
default:
return false;
+ case MVT::bf16:
+ if (!AllowBF16)
+ return false;
+ break;
case MVT::i8:
case MVT::i16:
case MVT::i32:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 6e395e004f519..5cc4e9fa1b063 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -588,7 +588,10 @@ class AArch64TargetLowering : public TargetLowering {
// Normally SVE is only used for byte size vectors that do not fit within a
// NEON vector. This changes when OverrideNEON is true, allowing SVE to be
// used for 64bit and 128bit vectors as well.
- bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false) const;
+ // FIXME: AllowBF16 is used to incrementally enable SVE code generation for
+ // all the fixed-length vectors of bf16 and will be removed in the future.
+ bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false,
+ bool AllowBF16 = false) const;
// Follow NEON ABI rules even when using SVE for fixed length vectors.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
index 91af81c617c41..14431f826a75b 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
@@ -35,6 +35,83 @@ define <8 x half> @masked_load_v8f16(ptr %src, <8 x i1> %mask) {
ret <8 x half> %load
}
+define <8 x bfloat> @masked_load_v8bf16_without_bf16_attr(ptr %src, <8 x i1> %mask) {
+; CHECK-LABEL: masked_load_v8bf16_without_bf16_attr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv b1, v0.8b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: tbnz w8, #0, .LBB2_9
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: tbnz w8, #1, .LBB2_10
+; CHECK-NEXT: .LBB2_2: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB2_11
+; CHECK-NEXT: .LBB2_3: // %else5
+; CHECK-NEXT: tbnz w8, #3, .LBB2_12
+; CHECK-NEXT: .LBB2_4: // %else8
+; CHECK-NEXT: tbnz w8, #4, .LBB2_13
+; CHECK-NEXT: .LBB2_5: // %else11
+; CHECK-NEXT: tbnz w8, #5, .LBB2_14
+; CHECK-NEXT: .LBB2_6: // %else14
+; CHECK-NEXT: tbnz w8, #6, .LBB2_15
+; CHECK-NEXT: .LBB2_7: // %else17
+; CHECK-NEXT: tbnz w8, #7, .LBB2_16
+; CHECK-NEXT: .LBB2_8: // %else20
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB2_9: // %cond.load
+; CHECK-NEXT: ldr h0, [x0]
+; CHECK-NEXT: tbz w8, #1, .LBB2_2
+; CHECK-NEXT: .LBB2_10: // %cond.load1
+; CHECK-NEXT: add x9, x0, #2
+; CHECK-NEXT: ld1 { v0.h }[1], [x9]
+; CHECK-NEXT: tbz w8, #2, .LBB2_3
+; CHECK-NEXT: .LBB2_11: // %cond.load4
+; CHECK-NEXT: add x9, x0, #4
+; CHECK-NEXT: ld1 { v0.h }[2], [x9]
+; CHECK-NEXT: tbz w8, #3, .LBB2_4
+; CHECK-NEXT: .LBB2_12: // %cond.load7
+; CHECK-NEXT: add x9, x0, #6
+; CHECK-NEXT: ld1 { v0.h }[3], [x9]
+; CHECK-NEXT: tbz w8, #4, .LBB2_5
+; CHECK-NEXT: .LBB2_13: // %cond.load10
+; CHECK-NEXT: add x9, x0, #8
+; CHECK-NEXT: ld1 { v0.h }[4], [x9]
+; CHECK-NEXT: tbz w8, #5, .LBB2_6
+; CHECK-NEXT: .LBB2_14: // %cond.load13
+; CHECK-NEXT: add x9, x0, #10
+; CHECK-NEXT: ld1 { v0.h }[5], [x9]
+; CHECK-NEXT: tbz w8, #6, .LBB2_7
+; CHECK-NEXT: .LBB2_15: // %cond.load16
+; CHECK-NEXT: add x9, x0, #12
+; CHECK-NEXT: ld1 { v0.h }[6], [x9]
+; CHECK-NEXT: tbz w8, #7, .LBB2_8
+; CHECK-NEXT: .LBB2_16: // %cond.load19
+; CHECK-NEXT: add x8, x0, #14
+; CHECK-NEXT: ld1 { v0.h }[7], [x8]
+; CHECK-NEXT: ret
+ %load = call <8 x bfloat> @llvm.masked.load.v8bf16(ptr %src, i32 8, <8 x i1> %mask, <8 x bfloat> zeroinitializer)
+ ret <8 x bfloat> %load
+}
+
+define <8 x bfloat> @masked_load_v8bf16_with_bf16_attr(ptr %src, <8 x i1> %mask) #0 {
+; CHECK-LABEL: masked_load_v8bf16_with_bf16_attr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: shl v0.8h, v0.8h, #15
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT: ret
+ %load = call <8 x bfloat> @llvm.masked.load.v8bf16(ptr %src, i32 8, <8 x i1> %mask, <8 x bfloat> zeroinitializer)
+ ret <8 x bfloat> %load
+}
+
define <4 x float> @masked_load_v4f32(ptr %src, <4 x i1> %mask) {
; CHECK-LABEL: masked_load_v4f32:
; CHECK: // %bb.0:
@@ -134,3 +211,74 @@ define <4 x half> @masked_load_v4f16(ptr %ap, ptr %bp) {
%load = call <4 x half> @llvm.masked.load.v4f16(ptr %ap, i32 2, <4 x i1> %mask, <4 x half> zeroinitializer)
ret <4 x half> %load
}
+
+define <4 x bfloat> @masked_load_v4bf16_without_bf16_attr(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_load_v4bf16_without_bf16_attr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: adrp x8, .LCPI10_0
+; CHECK-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI10_0]
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv h1, v0.4h
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: tbnz w8, #0, .LBB10_5
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: tbnz w8, #1, .LBB10_6
+; CHECK-NEXT: .LBB10_2: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB10_7
+; CHECK-NEXT: .LBB10_3: // %else5
+; CHECK-NEXT: tbnz w8, #3, .LBB10_8
+; CHECK-NEXT: .LBB10_4: // %else8
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB10_5: // %cond.load
+; CHECK-NEXT: ldr h0, [x0]
+; CHECK-NEXT: tbz w8, #1, .LBB10_2
+; CHECK-NEXT: .LBB10_6: // %cond.load1
+; CHECK-NEXT: add x9, x0, #2
+; CHECK-NEXT: ld1 { v0.h }[1], [x9]
+; CHECK-NEXT: tbz w8, #2, .LBB10_3
+; CHECK-NEXT: .LBB10_7: // %cond.load4
+; CHECK-NEXT: add x9, x0, #4
+; CHECK-NEXT: ld1 { v0.h }[2], [x9]
+; CHECK-NEXT: tbz w8, #3, .LBB10_4
+; CHECK-NEXT: .LBB10_8: // %cond.load7
+; CHECK-NEXT: add x8, x0, #6
+; CHECK-NEXT: ld1 { v0.h }[3], [x8]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: ret
+ %a = load <4 x bfloat>, ptr %ap
+ %b = load <4 x bfloat>, ptr %bp
+ %mask = fcmp oeq <4 x bfloat> %a, %b
+ %load = call <4 x bfloat> @llvm.masked.load.v4bf16(ptr %ap, i32 2, <4 x i1> %mask, <4 x bfloat> zeroinitializer)
+ ret <4 x bfloat> %load
+}
+
+define <4 x bfloat> @masked_load_v4bf16_with_bf16_attr(ptr %ap, ptr %bp) #0 {
+; CHECK-LABEL: masked_load_v4bf16_with_bf16_attr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %a = load <4 x bfloat>, ptr %ap
+ %b = load <4 x bfloat>, ptr %bp
+ %mask = fcmp oeq <4 x bfloat> %a, %b
+ %load = call <4 x bfloat> @llvm.masked.load.v4bf16(ptr %ap, i32 2, <4 x i1> %mask, <4 x bfloat> zeroinitializer)
+ ret <4 x bfloat> %load
+}
+
+attributes #0 = { "target-features"="+bf16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
index 439c61339a631..f1b0a0fafb9cd 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
@@ -35,6 +35,83 @@ define void @masked_store_v8f16(ptr %dst, <8 x i1> %mask) {
ret void
}
+define void @masked_store_v8bf16_without_bf16_attr(ptr %dst, <8 x i1> %mask) {
+; CHECK-LABEL: masked_store_v8bf16_without_bf16_attr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: tbnz w8, #0, .LBB2_9
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: tbnz w8, #1, .LBB2_10
+; CHECK-NEXT: .LBB2_2: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB2_11
+; CHECK-NEXT: .LBB2_3: // %else4
+; CHECK-NEXT: tbnz w8, #3, .LBB2_12
+; CHECK-NEXT: .LBB2_4: // %else6
+; CHECK-NEXT: tbnz w8, #4, .LBB2_13
+; CHECK-NEXT: .LBB2_5: // %else8
+; CHECK-NEXT: tbnz w8, #5, .LBB2_14
+; CHECK-NEXT: .LBB2_6: // %else10
+; CHECK-NEXT: tbnz w8, #6, .LBB2_15
+; CHECK-NEXT: .LBB2_7: // %else12
+; CHECK-NEXT: tbnz w8, #7, .LBB2_16
+; CHECK-NEXT: .LBB2_8: // %else14
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB2_9: // %cond.store
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: str h0, [x0]
+; CHECK-NEXT: tbz w8, #1, .LBB2_2
+; CHECK-NEXT: .LBB2_10: // %cond.store1
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: str h0, [x0, #2]
+; CHECK-NEXT: tbz w8, #2, .LBB2_3
+; CHECK-NEXT: .LBB2_11: // %cond.store3
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: str h0, [x0, #4]
+; CHECK-NEXT: tbz w8, #3, .LBB2_4
+; CHECK-NEXT: .LBB2_12: // %cond.store5
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: str h0, [x0, #6]
+; CHECK-NEXT: tbz w8, #4, .LBB2_5
+; CHECK-NEXT: .LBB2_13: // %cond.store7
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: str h0, [x0, #8]
+; CHECK-NEXT: tbz w8, #5, .LBB2_6
+; CHECK-NEXT: .LBB2_14: // %cond.store9
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: str h0, [x0, #10]
+; CHECK-NEXT: tbz w8, #6, .LBB2_7
+; CHECK-NEXT: .LBB2_15: // %cond.store11
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: str h0, [x0, #12]
+; CHECK-NEXT: tbz w8, #7, .LBB2_8
+; CHECK-NEXT: .LBB2_16: // %cond.store13
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: str h0, [x0, #14]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.v8bf16(<8 x bfloat> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v8bf16_with_bf16_attr(ptr %dst, <8 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_v8bf16_with_bf16_attr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: shl v0.8h, v0.8h, #15
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: st1h { z0.h }, p1, [x0]
+; CHECK-NEXT: ret
+ call void @llvm.masked.store.v8bf16(<8 x bfloat> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
+ ret void
+}
+
define void @masked_store_v4f32(ptr %dst, <4 x i1> %mask) {
; CHECK-LABEL: masked_store_v4f32:
; CHECK: // %bb.0:
@@ -116,3 +193,69 @@ define void @masked_store_v4f16(ptr %ap, ptr %bp) {
ret void
}
+define void @masked_store_v4bf16_without_bf16_attr(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_store_v4bf16_without_bf16_attr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: adrp x8, .LCPI9_0
+; CHECK-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-NEXT: fcmeq v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI9_0]
+; CHECK-NEXT: xtn v1.4h, v1.4s
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: addv h1, v1.4h
+; CHECK-NEXT: fmov w8, s1
+; CHECK-NEXT: tbnz w8, #0, .LBB9_5
+; CHECK-NEXT: // %bb.1: // %else
+; CHECK-NEXT: tbnz w8, #1, .LBB9_6
+; CHECK-NEXT: .LBB9_2: // %else2
+; CHECK-NEXT: tbnz w8, #2, .LBB9_7
+; CHECK-NEXT: .LBB9_3: // %else4
+; CHECK-NEXT: tbnz w8, #3, .LBB9_8
+; CHECK-NEXT: .LBB9_4: // %else6
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB9_5: // %cond.store
+; CHECK-NEXT: st1 { v0.h }[0], [x1]
+; CHECK-NEXT: tbz w8, #1, .LBB9_2
+; CHECK-NEXT: .LBB9_6: // %cond.store1
+; CHECK-NEXT: add x9, x1, #2
+; CHECK-NEXT: st1 { v0.h }[1], [x9]
+; CHECK-NEXT: tbz w8, #2, .LBB9_3
+; CHECK-NEXT: .LBB9_7: // %cond.store3
+; CHECK-NEXT: add x9, x1, #4
+; CHECK-NEXT: st1 { v0.h }[2], [x9]
+; CHECK-NEXT: tbz w8, #3, .LBB9_4
+; CHECK-NEXT: .LBB9_8: // %cond.store5
+; CHECK-NEXT: add x8, x1, #6
+; CHECK-NEXT: st1 { v0.h }[3], [x8]
+; CHECK-NEXT: ret
+ %a = load <4 x bfloat>, ptr %ap
+ %b = load <4 x bfloat>, ptr %bp
+ %mask = fcmp oeq <4 x bfloat> %a, %b
+ call void @llvm.masked.store.v4bf16(<4 x bfloat> %a, ptr %bp, i32 2, <4 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v4bf16_with_bf16_attr(ptr %ap, ptr %bp) #0 {
+; CHECK-LABEL: masked_store_v4bf16_with_bf16_attr:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-NEXT: fcmeq v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: xtn v1.4h, v1.4s
+; CHECK-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-NEXT: st1h { z0.h }, p1, [x1]
+; CHECK-NEXT: ret
+ %a = load <4 x bfloat>, ptr %ap
+ %b = load <4 x bfloat>, ptr %bp
+ %mask = fcmp oeq <4 x bfloat> %a, %b
+ call void @llvm.masked.store.v4bf16(<4 x bfloat> %a, ptr %bp, i32 2, <4 x i1> %mask)
+ ret void
+}
+
+attributes #0 = { "target-features"="+bf16" }
More information about the llvm-branch-commits
mailing list