[Mlir-commits] [mlir] [mlir][llvm] Add `align` attribute to `llvm.intr.masked.{expandload, compressstore}` (PR #153063)
Erick Ochoa Lopez
llvmlistbot at llvm.org
Mon Aug 11 11:14:27 PDT 2025
https://github.com/amd-eochoalo created https://github.com/llvm/llvm-project/pull/153063
The LLVM intrinsics [`llvm.intr.masked.expandload`](https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics) and [`llvm.intr.masked.compressstore`](https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics) both allow an optional align parameter attribute to be set which defaults to one.
Inlining the documentation below for [`llvm.intr.masked.expandload` 's
](https://llvm.org/docs/LangRef.html#id1522) and [`llvm.intr.masked.compressstore`'s arguments:](https://llvm.org/docs/LangRef.html#id1522) respectively
> The `align` parameter attribute can be provided for the first argument. The pointer alignment defaults to 1.
> The `align` parameter attribute can be provided for the second argument. The pointer alignment defaults to 1.
>From bddaec92a3eb4bd1b55dd70cb0ae3c9e5ff75729 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 7 Aug 2025 20:57:15 -0700
Subject: [PATCH 1/4] Adds names to parameters of llvm intrinsics:
* masked.compressstore
* masked.expandload
---
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 76b08e664ee76..1c871ea922962 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1070,13 +1070,17 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
/// Create a call to Masked Expand Load intrinsic.
def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
- let arguments = (ins LLVM_AnyPointer, LLVM_VectorOf<I1>, LLVM_AnyVector);
+ let arguments = (ins LLVM_AnyPointer:$base,
+ LLVM_VectorOf<I1>:$mask,
+ LLVM_AnyVector:$passthru);
}
/// Create a call to Masked Compress Store intrinsic.
def LLVM_masked_compressstore
: LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
- let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf<I1>);
+ let arguments = (ins LLVM_AnyVector:$vector,
+ LLVM_AnyPointer:$base,
+ LLVM_VectorOf<I1>:$mask);
}
//
>From 7467a429aac0bbf08ddfac5c7d2a5ee04753889d Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Fri, 8 Aug 2025 16:27:54 -0700
Subject: [PATCH 2/4] Add alignment to llvm.intr.masked.compressstore
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 19 ++++++++++++++++++-
.../vector-to-llvm-interface.mlir | 4 ++--
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 11 ++++++++++-
3 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 1c871ea922962..90a18b8f00ab3 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1080,7 +1080,24 @@ def LLVM_masked_compressstore
: LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
let arguments = (ins LLVM_AnyVector:$vector,
LLVM_AnyPointer:$base,
- LLVM_VectorOf<I1>:$mask);
+ LLVM_VectorOf<I1>:$mask,
+ DefaultValuedAttr<I32Attr, "1">:$align);
+
+ let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
+
+ let assemblyFormat = "`(` $vector `,` $base `,` $mask `)` attr-dict `:` functional-type(operands, results)";
+
+ string llvmBuilder = [{
+ builder.CreateMaskedCompressStore(
+ $vector, $base, $align != 1 ? llvm::Align($align) : llvm::MaybeAlign(), $mask);
+ }];
+ string mlirBuilder = [{
+ auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
+ llvm::MaybeAlign alignment = intrinInst->getParamAlign(1);
+ IntegerAttr attr = $_builder.getI32IntegerAttr(alignment.has_value() ? alignment->value() : 1);
+ $_op = LLVM::masked_compressstore::create($_builder, $_location,
+ $vector, $base, $mask, attr);
+ }];
}
//
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 5a424a8ac0d5f..ddec6a49ebbbd 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -2163,7 +2163,7 @@ func.func @compress_store_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2:
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %[[P]], %{{.*}}) : (vector<11xf32>, !llvm.ptr, vector<11xi1>) -> ()
+// CHECK: llvm.intr.masked.compressstore(%{{.*}}, %[[P]], %{{.*}}) : (vector<11xf32>, !llvm.ptr, vector<11xi1>) -> ()
// -----
@@ -2173,7 +2173,7 @@ func.func @compress_store_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>,
return
}
// CHECK-LABEL: func @compress_store_op_index
-// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<11xi64>, !llvm.ptr, vector<11xi1>) -> ()
+// CHECK: llvm.intr.masked.compressstore(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<11xi64>, !llvm.ptr, vector<11xi1>) -> ()
// -----
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 9f882ad6f22e8..c607f9e453c3d 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -540,11 +540,20 @@ define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) {
define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float> %2) {
; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
%4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
- ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ ; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr %0, <7 x i1> %1)
ret void
}
+; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics_with_alignment
+define void @masked_expand_compress_intrinsics_with_alignment(ptr %0, <7 x i1> %1, <7 x float> %2) {
+ ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
+ ; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) {align = 8 : i32} : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr align 8 %0, <7 x i1> %1)
+ ret void
+}
+
; CHECK-LABEL: llvm.func @annotate_intrinsics
define void @annotate_intrinsics(ptr %var, ptr %ptr, i16 %int, ptr %annotation, ptr %fileName, i32 %line, ptr %args) {
; CHECK: "llvm.intr.var.annotation"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, !llvm.ptr, !llvm.ptr, i32, !llvm.ptr) -> ()
>From 6e4966cff7b747ea788173cf8e182e405c18b1ba Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Mon, 11 Aug 2025 14:58:54 -0700
Subject: [PATCH 3/4] Add align attribute to llvm.intr.masked.expandload
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 17 ++++++++++++++++-
.../VectorToLLVM/vector-to-llvm-interface.mlir | 4 ++--
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 6 +++---
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 90a18b8f00ab3..e2ad3a112ada2 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1072,7 +1072,22 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
let arguments = (ins LLVM_AnyPointer:$base,
LLVM_VectorOf<I1>:$mask,
- LLVM_AnyVector:$passthru);
+ LLVM_AnyVector:$passthru,
+ DefaultValuedAttr<I32Attr, "1">:$align);
+
+ let assemblyFormat = "`(` $base`,` $mask`,` $passthru `)` attr-dict `:` functional-type(operands, results)";
+
+ string llvmBuilder = [{
+ $res = builder.CreateMaskedExpandLoad(
+ $_resultType, $base, $align != 1 ? llvm::Align($align) : llvm::MaybeAlign(), $mask, $passthru);
+ }];
+ string mlirBuilder = [{
+ auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
+ llvm::MaybeAlign alignment = intrinInst->getParamAlign(0);
+ IntegerAttr attr = $_builder.getI32IntegerAttr(alignment.has_value() ? alignment->value() : 1);
+ $res = LLVM::masked_expandload::create($_builder, $_location,
+ $_resultType, $base, $mask, $passthru, attr);
+ }];
}
/// Create a call to Masked Compress Store intrinsic.
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index ddec6a49ebbbd..60dce615cfc05 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -2134,7 +2134,7 @@ func.func @expand_load_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vec
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[E:.*]] = "llvm.intr.masked.expandload"(%[[P]], %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xf32>) -> vector<11xf32>
+// CHECK: %[[E:.*]] = llvm.intr.masked.expandload(%[[P]], %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xf32>) -> vector<11xf32>
// CHECK: return %[[E]] : vector<11xf32>
// -----
@@ -2145,7 +2145,7 @@ func.func @expand_load_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>, %a
return %0 : vector<11xindex>
}
// CHECK-LABEL: func @expand_load_op_index
-// CHECK: %{{.*}} = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xi64>) -> vector<11xi64>
+// CHECK: %{{.*}} = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xi64>) -> vector<11xi64>
// -----
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index c607f9e453c3d..63bfdfce59521 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -538,7 +538,7 @@ define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) {
; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics
define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float> %2) {
- ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
%4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr %0, <7 x i1> %1)
@@ -547,8 +547,8 @@ define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float>
; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics_with_alignment
define void @masked_expand_compress_intrinsics_with_alignment(ptr %0, <7 x i1> %1, <7 x float> %2) {
- ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
- %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
+ ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) {align = 8 : i32} : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %0, <7 x i1> %1, <7 x float> %2)
; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) {align = 8 : i32} : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr align 8 %0, <7 x i1> %1)
ret void
>From 420351bd26f8619b52e264c07c14461dc0d18717 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Mon, 11 Aug 2025 18:45:01 -0700
Subject: [PATCH 4/4] Add target test
---
mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 2b420ed246fb2..11590391493ce 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -577,6 +577,17 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1
llvm.return
}
+// CHECK-LABEL: @masked_expand_compress_intrinsics_with_alignment
+llvm.func @masked_expand_compress_intrinsics_with_alignment(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthru: vector<7xf32>) {
+ // CHECK: call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
+ %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru) { align = 8 : i32 }
+ : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> (vector<7xf32>)
+ // CHECK: call void @llvm.masked.compressstore.v7f32(<7 x float> %{{.*}}, ptr align 8 %{{.*}}, <7 x i1> %{{.*}})
+ "llvm.intr.masked.compressstore"(%0, %ptr, %mask) { align = 8 : i32 }
+ : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ llvm.return
+}
+
// CHECK-LABEL: @annotate_intrinsics
llvm.func @annotate_intrinsics(%var: !llvm.ptr, %int: i16, %ptr: !llvm.ptr, %annotation: !llvm.ptr, %fileName: !llvm.ptr, %line: i32, %attr: !llvm.ptr) {
// CHECK: call void @llvm.var.annotation.p0.p0(ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, i32 %{{.*}}, ptr %{{.*}})
More information about the Mlir-commits
mailing list