[llvm] [AArch64] Use sve for 64 wide masked load/store (PR #203480)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 01:15:14 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Yashwant Singh (yashssh)
<details>
<summary>Changes</summary>
---
Full diff: https://github.com/llvm/llvm-project/pull/203480.diff
3 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h (+5-4)
- (renamed) llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll (+55-4)
- (renamed) llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll (+53-4)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 200b4c9fba196..68af38aed185f 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -315,10 +315,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
if (!ST->isSVEorStreamingSVEAvailable())
return false;
- // For fixed vectors, avoid scalarization if using SVE for them.
- if (isa<FixedVectorType>(DataType) && !ST->useSVEForFixedLengthVectors() &&
- DataType->getPrimitiveSizeInBits() != 128)
- return false; // Fall back to scalarization of masked operations.
+ if (isa<FixedVectorType>(DataType) && !ST->useSVEForFixedLengthVectors()) {
+ unsigned Bits = DataType->getPrimitiveSizeInBits();
+ if (Bits != 64 && Bits != 128)
+ return false; // Fall back to scalarization of masked operations.
+ }
return isElementTypeLegalForScalableVector(DataType->getScalarType());
}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
similarity index 61%
rename from llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll
rename to llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
index b2af99bd96ab6..91af81c617c41 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-loads.ll
@@ -79,7 +79,58 @@ define <2 x double> @masked_load_passthru_v2f64(ptr %src, <2 x i1> %mask, <2 x d
ret <2 x double> %load
}
-declare <16 x i8> @llvm.masked.load.v16i8(ptr, i32, <16 x i1>, <16 x i8>)
-declare <8 x half> @llvm.masked.load.v8f16(ptr, i32, <8 x i1>, <8 x half>)
-declare <4 x float> @llvm.masked.load.v4f32(ptr, i32, <4 x i1>, <4 x float>)
-declare <2 x double> @llvm.masked.load.v2f64(ptr, i32, <2 x i1>, <2 x double>)
+;
+; 64-bit Masked Loads
+;
+
+define <8 x i8> @masked_load_v8i8(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_load_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl8
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %a = load <8 x i8>, ptr %ap
+ %b = load <8 x i8>, ptr %bp
+ %mask = icmp eq <8 x i8> %a, %b
+ %load = call <8 x i8> @llvm.masked.load.v8i8(ptr %ap, i32 1, <8 x i1> %mask, <8 x i8> zeroinitializer)
+ ret <8 x i8> %load
+}
+
+define <4 x i16> @masked_load_v4i16(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_load_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %a = load <4 x i16>, ptr %ap
+ %b = load <4 x i16>, ptr %bp
+ %mask = icmp eq <4 x i16> %a, %b
+ %load = call <4 x i16> @llvm.masked.load.v4i16(ptr %ap, i32 2, <4 x i1> %mask, <4 x i16> zeroinitializer)
+ ret <4 x i16> %load
+}
+
+define <4 x half> @masked_load_v4f16(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_load_v4f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: fcmeq v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0]
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT: ret
+ %a = load <4 x half>, ptr %ap
+ %b = load <4 x half>, ptr %bp
+ %mask = fcmp oeq <4 x half> %a, %b
+ %load = call <4 x half> @llvm.masked.load.v4f16(ptr %ap, i32 2, <4 x i1> %mask, <4 x half> zeroinitializer)
+ ret <4 x half> %load
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
similarity index 57%
rename from llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll
rename to llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
index 660213d1c7f33..439c61339a631 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-64-128bit-stores.ll
@@ -63,7 +63,56 @@ define void @masked_store_v2f64(ptr %dst, <2 x i1> %mask) {
ret void
}
-declare void @llvm.masked.store.v16i8(<16 x i8>, ptr, i32, <16 x i1>)
-declare void @llvm.masked.store.v8f16(<8 x half>, ptr, i32, <8 x i1>)
-declare void @llvm.masked.store.v4f32(<4 x float>, ptr, i32, <4 x i1>)
-declare void @llvm.masked.store.v2f64(<2 x double>, ptr, i32, <2 x i1>)
+;
+; 64-bit Masked Stores
+;
+
+define void @masked_store_v8i8(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_store_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b, vl8
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: st1b { z0.b }, p1, [x1]
+; CHECK-NEXT: ret
+ %a = load <8 x i8>, ptr %ap
+ %b = load <8 x i8>, ptr %bp
+ %mask = icmp eq <8 x i8> %a, %b
+ call void @llvm.masked.store.v8i8(<8 x i8> %a, ptr %bp, i32 1, <8 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v4i16(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_store_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT: st1h { z0.h }, p1, [x1]
+; CHECK-NEXT: ret
+ %a = load <4 x i16>, ptr %ap
+ %b = load <4 x i16>, ptr %bp
+ %mask = icmp eq <4 x i16> %a, %b
+ call void @llvm.masked.store.v4i16(<4 x i16> %a, ptr %bp, i32 2, <4 x i1> %mask)
+ ret void
+}
+
+define void @masked_store_v4f16(ptr %ap, ptr %bp) {
+; CHECK-LABEL: masked_store_v4f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldr d0, [x0]
+; CHECK-NEXT: ldr d1, [x1]
+; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: fcmeq v1.4h, v0.4h, v1.4h
+; CHECK-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-NEXT: st1h { z0.h }, p1, [x1]
+; CHECK-NEXT: ret
+ %a = load <4 x half>, ptr %ap
+ %b = load <4 x half>, ptr %bp
+ %mask = fcmp oeq <4 x half> %a, %b
+ call void @llvm.masked.store.v4f16(<4 x half> %a, ptr %bp, i32 2, <4 x i1> %mask)
+ ret void
+}
+
``````````
</details>
https://github.com/llvm/llvm-project/pull/203480
More information about the llvm-commits
mailing list