[Mlir-commits] [mlir] [mlir][llvm] Add `align` attribute to `llvm.intr.masked.{expandload, compressstore}` (PR #153063)
Erick Ochoa Lopez
llvmlistbot at llvm.org
Fri Aug 15 05:20:12 PDT 2025
https://github.com/amd-eochoalo updated https://github.com/llvm/llvm-project/pull/153063
>From bddaec92a3eb4bd1b55dd70cb0ae3c9e5ff75729 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 7 Aug 2025 20:57:15 -0700
Subject: [PATCH 01/18] Adds names to parameters of llvm intrinsics:
* masked.compressstore
* masked.expandload
---
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 76b08e664ee76..1c871ea922962 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1070,13 +1070,17 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
/// Create a call to Masked Expand Load intrinsic.
def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
- let arguments = (ins LLVM_AnyPointer, LLVM_VectorOf<I1>, LLVM_AnyVector);
+ let arguments = (ins LLVM_AnyPointer:$base,
+ LLVM_VectorOf<I1>:$mask,
+ LLVM_AnyVector:$passthru);
}
/// Create a call to Masked Compress Store intrinsic.
def LLVM_masked_compressstore
: LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
- let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf<I1>);
+ let arguments = (ins LLVM_AnyVector:$vector,
+ LLVM_AnyPointer:$base,
+ LLVM_VectorOf<I1>:$mask);
}
//
>From 7467a429aac0bbf08ddfac5c7d2a5ee04753889d Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Fri, 8 Aug 2025 16:27:54 -0700
Subject: [PATCH 02/18] Add alignment to llvm.intr.masked.compressstore
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 19 ++++++++++++++++++-
.../vector-to-llvm-interface.mlir | 4 ++--
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 11 ++++++++++-
3 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 1c871ea922962..90a18b8f00ab3 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1080,7 +1080,24 @@ def LLVM_masked_compressstore
: LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
let arguments = (ins LLVM_AnyVector:$vector,
LLVM_AnyPointer:$base,
- LLVM_VectorOf<I1>:$mask);
+ LLVM_VectorOf<I1>:$mask,
+ DefaultValuedAttr<I32Attr, "1">:$align);
+
+ let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
+
+ let assemblyFormat = "`(` $vector `,` $base `,` $mask `)` attr-dict `:` functional-type(operands, results)";
+
+ string llvmBuilder = [{
+ builder.CreateMaskedCompressStore(
+ $vector, $base, $align != 1 ? llvm::Align($align) : llvm::MaybeAlign(), $mask);
+ }];
+ string mlirBuilder = [{
+ auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
+ llvm::MaybeAlign alignment = intrinInst->getParamAlign(1);
+ IntegerAttr attr = $_builder.getI32IntegerAttr(alignment.has_value() ? alignment->value() : 1);
+ $_op = LLVM::masked_compressstore::create($_builder, $_location,
+ $vector, $base, $mask, attr);
+ }];
}
//
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 5a424a8ac0d5f..ddec6a49ebbbd 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -2163,7 +2163,7 @@ func.func @compress_store_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2:
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %[[P]], %{{.*}}) : (vector<11xf32>, !llvm.ptr, vector<11xi1>) -> ()
+// CHECK: llvm.intr.masked.compressstore(%{{.*}}, %[[P]], %{{.*}}) : (vector<11xf32>, !llvm.ptr, vector<11xi1>) -> ()
// -----
@@ -2173,7 +2173,7 @@ func.func @compress_store_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>,
return
}
// CHECK-LABEL: func @compress_store_op_index
-// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<11xi64>, !llvm.ptr, vector<11xi1>) -> ()
+// CHECK: llvm.intr.masked.compressstore(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<11xi64>, !llvm.ptr, vector<11xi1>) -> ()
// -----
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 9f882ad6f22e8..c607f9e453c3d 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -540,11 +540,20 @@ define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) {
define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float> %2) {
; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
%4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
- ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ ; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr %0, <7 x i1> %1)
ret void
}
+; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics_with_alignment
+define void @masked_expand_compress_intrinsics_with_alignment(ptr %0, <7 x i1> %1, <7 x float> %2) {
+ ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
+ ; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) {align = 8 : i32} : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr align 8 %0, <7 x i1> %1)
+ ret void
+}
+
; CHECK-LABEL: llvm.func @annotate_intrinsics
define void @annotate_intrinsics(ptr %var, ptr %ptr, i16 %int, ptr %annotation, ptr %fileName, i32 %line, ptr %args) {
; CHECK: "llvm.intr.var.annotation"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, !llvm.ptr, !llvm.ptr, i32, !llvm.ptr) -> ()
>From 6e4966cff7b747ea788173cf8e182e405c18b1ba Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Mon, 11 Aug 2025 14:58:54 -0700
Subject: [PATCH 03/18] Add align attribute to llvm.intr.masked.expandload
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 17 ++++++++++++++++-
.../VectorToLLVM/vector-to-llvm-interface.mlir | 4 ++--
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 6 +++---
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 90a18b8f00ab3..e2ad3a112ada2 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1072,7 +1072,22 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
let arguments = (ins LLVM_AnyPointer:$base,
LLVM_VectorOf<I1>:$mask,
- LLVM_AnyVector:$passthru);
+ LLVM_AnyVector:$passthru,
+ DefaultValuedAttr<I32Attr, "1">:$align);
+
+ let assemblyFormat = "`(` $base`,` $mask`,` $passthru `)` attr-dict `:` functional-type(operands, results)";
+
+ string llvmBuilder = [{
+ $res = builder.CreateMaskedExpandLoad(
+ $_resultType, $base, $align != 1 ? llvm::Align($align) : llvm::MaybeAlign(), $mask, $passthru);
+ }];
+ string mlirBuilder = [{
+ auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
+ llvm::MaybeAlign alignment = intrinInst->getParamAlign(0);
+ IntegerAttr attr = $_builder.getI32IntegerAttr(alignment.has_value() ? alignment->value() : 1);
+ $res = LLVM::masked_expandload::create($_builder, $_location,
+ $_resultType, $base, $mask, $passthru, attr);
+ }];
}
/// Create a call to Masked Compress Store intrinsic.
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index ddec6a49ebbbd..60dce615cfc05 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -2134,7 +2134,7 @@ func.func @expand_load_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vec
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[E:.*]] = "llvm.intr.masked.expandload"(%[[P]], %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xf32>) -> vector<11xf32>
+// CHECK: %[[E:.*]] = llvm.intr.masked.expandload(%[[P]], %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xf32>) -> vector<11xf32>
// CHECK: return %[[E]] : vector<11xf32>
// -----
@@ -2145,7 +2145,7 @@ func.func @expand_load_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>, %a
return %0 : vector<11xindex>
}
// CHECK-LABEL: func @expand_load_op_index
-// CHECK: %{{.*}} = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xi64>) -> vector<11xi64>
+// CHECK: %{{.*}} = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xi64>) -> vector<11xi64>
// -----
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index c607f9e453c3d..63bfdfce59521 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -538,7 +538,7 @@ define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) {
; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics
define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float> %2) {
- ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
%4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr %0, <7 x i1> %1)
@@ -547,8 +547,8 @@ define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float>
; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics_with_alignment
define void @masked_expand_compress_intrinsics_with_alignment(ptr %0, <7 x i1> %1, <7 x float> %2) {
- ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
- %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
+ ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) {align = 8 : i32} : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %0, <7 x i1> %1, <7 x float> %2)
; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) {align = 8 : i32} : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr align 8 %0, <7 x i1> %1)
ret void
>From 420351bd26f8619b52e264c07c14461dc0d18717 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Mon, 11 Aug 2025 18:45:01 -0700
Subject: [PATCH 04/18] Add target test
---
mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 2b420ed246fb2..11590391493ce 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -577,6 +577,17 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1
llvm.return
}
+// CHECK-LABEL: @masked_expand_compress_intrinsics_with_alignment
+llvm.func @masked_expand_compress_intrinsics_with_alignment(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthru: vector<7xf32>) {
+ // CHECK: call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
+ %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru) { align = 8 : i32 }
+ : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> (vector<7xf32>)
+ // CHECK: call void @llvm.masked.compressstore.v7f32(<7 x float> %{{.*}}, ptr align 8 %{{.*}}, <7 x i1> %{{.*}})
+ "llvm.intr.masked.compressstore"(%0, %ptr, %mask) { align = 8 : i32 }
+ : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ llvm.return
+}
+
// CHECK-LABEL: @annotate_intrinsics
llvm.func @annotate_intrinsics(%var: !llvm.ptr, %int: i16, %ptr: !llvm.ptr, %annotation: !llvm.ptr, %fileName: !llvm.ptr, %line: i32, %attr: !llvm.ptr) {
// CHECK: call void @llvm.var.annotation.p0.p0(ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, i32 %{{.*}}, ptr %{{.*}})
>From a007d04827763ad2f03b541044e12e5b390d2141 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 14:08:47 -0700
Subject: [PATCH 05/18] Revert previous changes
This commit reverts and fixed up the following commits:
* 420351bd26f8619b52e264c07c14461dc0d18717
* 6e4966cff7b747ea788173cf8e182e405c18b1ba
* 7467a429aac0bbf08ddfac5c7d2a5ee04753889d
* bddaec92a3eb4bd1b55dd70cb0ae3c9e5ff75729
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 40 +------------------
.../vector-to-llvm-interface.mlir | 8 ++--
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 13 +-----
.../test/Target/LLVMIR/llvmir-intrinsics.mlir | 11 -----
4 files changed, 8 insertions(+), 64 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index e2ad3a112ada2..76b08e664ee76 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1070,49 +1070,13 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
/// Create a call to Masked Expand Load intrinsic.
def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
- let arguments = (ins LLVM_AnyPointer:$base,
- LLVM_VectorOf<I1>:$mask,
- LLVM_AnyVector:$passthru,
- DefaultValuedAttr<I32Attr, "1">:$align);
-
- let assemblyFormat = "`(` $base`,` $mask`,` $passthru `)` attr-dict `:` functional-type(operands, results)";
-
- string llvmBuilder = [{
- $res = builder.CreateMaskedExpandLoad(
- $_resultType, $base, $align != 1 ? llvm::Align($align) : llvm::MaybeAlign(), $mask, $passthru);
- }];
- string mlirBuilder = [{
- auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
- llvm::MaybeAlign alignment = intrinInst->getParamAlign(0);
- IntegerAttr attr = $_builder.getI32IntegerAttr(alignment.has_value() ? alignment->value() : 1);
- $res = LLVM::masked_expandload::create($_builder, $_location,
- $_resultType, $base, $mask, $passthru, attr);
- }];
+ let arguments = (ins LLVM_AnyPointer, LLVM_VectorOf<I1>, LLVM_AnyVector);
}
/// Create a call to Masked Compress Store intrinsic.
def LLVM_masked_compressstore
: LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
- let arguments = (ins LLVM_AnyVector:$vector,
- LLVM_AnyPointer:$base,
- LLVM_VectorOf<I1>:$mask,
- DefaultValuedAttr<I32Attr, "1">:$align);
-
- let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
-
- let assemblyFormat = "`(` $vector `,` $base `,` $mask `)` attr-dict `:` functional-type(operands, results)";
-
- string llvmBuilder = [{
- builder.CreateMaskedCompressStore(
- $vector, $base, $align != 1 ? llvm::Align($align) : llvm::MaybeAlign(), $mask);
- }];
- string mlirBuilder = [{
- auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
- llvm::MaybeAlign alignment = intrinInst->getParamAlign(1);
- IntegerAttr attr = $_builder.getI32IntegerAttr(alignment.has_value() ? alignment->value() : 1);
- $_op = LLVM::masked_compressstore::create($_builder, $_location,
- $vector, $base, $mask, attr);
- }];
+ let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf<I1>);
}
//
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 60dce615cfc05..5a424a8ac0d5f 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -2134,7 +2134,7 @@ func.func @expand_load_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2: vec
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[E:.*]] = llvm.intr.masked.expandload(%[[P]], %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xf32>) -> vector<11xf32>
+// CHECK: %[[E:.*]] = "llvm.intr.masked.expandload"(%[[P]], %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xf32>) -> vector<11xf32>
// CHECK: return %[[E]] : vector<11xf32>
// -----
@@ -2145,7 +2145,7 @@ func.func @expand_load_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>, %a
return %0 : vector<11xindex>
}
// CHECK-LABEL: func @expand_load_op_index
-// CHECK: %{{.*}} = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xi64>) -> vector<11xi64>
+// CHECK: %{{.*}} = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<11xi1>, vector<11xi64>) -> vector<11xi64>
// -----
@@ -2163,7 +2163,7 @@ func.func @compress_store_op(%arg0: memref<?xf32>, %arg1: vector<11xi1>, %arg2:
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.intr.masked.compressstore(%{{.*}}, %[[P]], %{{.*}}) : (vector<11xf32>, !llvm.ptr, vector<11xi1>) -> ()
+// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %[[P]], %{{.*}}) : (vector<11xf32>, !llvm.ptr, vector<11xi1>) -> ()
// -----
@@ -2173,7 +2173,7 @@ func.func @compress_store_op_index(%arg0: memref<?xindex>, %arg1: vector<11xi1>,
return
}
// CHECK-LABEL: func @compress_store_op_index
-// CHECK: llvm.intr.masked.compressstore(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<11xi64>, !llvm.ptr, vector<11xi1>) -> ()
+// CHECK: "llvm.intr.masked.compressstore"(%{{.*}}, %{{.*}}, %{{.*}}) : (vector<11xi64>, !llvm.ptr, vector<11xi1>) -> ()
// -----
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 63bfdfce59521..9f882ad6f22e8 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -538,22 +538,13 @@ define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) {
; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics
define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float> %2) {
- ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
%4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
- ; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr %0, <7 x i1> %1)
ret void
}
-; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics_with_alignment
-define void @masked_expand_compress_intrinsics_with_alignment(ptr %0, <7 x i1> %1, <7 x float> %2) {
- ; CHECK: %[[val1:.+]] = llvm.intr.masked.expandload(%{{.*}}, %{{.*}}, %{{.*}}) {align = 8 : i32} : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
- %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %0, <7 x i1> %1, <7 x float> %2)
- ; CHECK: llvm.intr.masked.compressstore(%[[val1]], %{{.*}}, %{{.*}}) {align = 8 : i32} : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
- call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr align 8 %0, <7 x i1> %1)
- ret void
-}
-
; CHECK-LABEL: llvm.func @annotate_intrinsics
define void @annotate_intrinsics(ptr %var, ptr %ptr, i16 %int, ptr %annotation, ptr %fileName, i32 %line, ptr %args) {
; CHECK: "llvm.intr.var.annotation"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, !llvm.ptr, !llvm.ptr, i32, !llvm.ptr) -> ()
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 11590391493ce..2b420ed246fb2 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -577,17 +577,6 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1
llvm.return
}
-// CHECK-LABEL: @masked_expand_compress_intrinsics_with_alignment
-llvm.func @masked_expand_compress_intrinsics_with_alignment(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthru: vector<7xf32>) {
- // CHECK: call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
- %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru) { align = 8 : i32 }
- : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> (vector<7xf32>)
- // CHECK: call void @llvm.masked.compressstore.v7f32(<7 x float> %{{.*}}, ptr align 8 %{{.*}}, <7 x i1> %{{.*}})
- "llvm.intr.masked.compressstore"(%0, %ptr, %mask) { align = 8 : i32 }
- : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
- llvm.return
-}
-
// CHECK-LABEL: @annotate_intrinsics
llvm.func @annotate_intrinsics(%var: !llvm.ptr, %int: i16, %ptr: !llvm.ptr, %annotation: !llvm.ptr, %fileName: !llvm.ptr, %line: i32, %attr: !llvm.ptr) {
// CHECK: call void @llvm.var.annotation.p0.p0(ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, i32 %{{.*}}, ptr %{{.*}})
>From ef4b27302a1c1f6c76c53aad39a59c3d6edb3b8b Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Tue, 12 Aug 2025 17:33:45 -0700
Subject: [PATCH 06/18] [mlir][llvm] Add align to masked.compressstore.
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 40 ++++++++++++++++++-
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 9 +++++
.../test/Target/LLVMIR/llvmir-intrinsics.mlir | 11 +++++
3 files changed, 58 insertions(+), 2 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 76b08e664ee76..a76890532d650 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1075,8 +1075,44 @@ def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
/// Create a call to Masked Compress Store intrinsic.
def LLVM_masked_compressstore
- : LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
- let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf<I1>);
+ : LLVM_ZeroResultIntrOp<"masked.compressstore", [0],
+ /*traits=*/[], /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
+ /*requiresArgAndResultAttrs=*/1, /*requiresOpBundles=*/0,
+ /*immArgPositions=*/[], /*immArgAttrNames=*/[]> {
+ dag args = (ins LLVM_AnyVector:$value,
+ LLVM_AnyPointer:$ptr,
+ LLVM_VectorOf<I1>:$mask);
+
+ let arguments = !con(args, baseArgs);
+
+ string llvmBuilder = [{
+ auto argAttrs = op.getArgAttrs();
+ uint64_t alignment = 1;
+ if (argAttrs.has_value()) {
+ NamedAttrList argAttr = cast<DictionaryAttr>((*argAttrs)[1]);
+ auto maybeAlignmentAttr = argAttr.get("llvm.align");
+ IntegerAttr alignmentAttr = maybeAlignmentAttr ? dyn_cast<IntegerAttr>(maybeAlignmentAttr) : nullptr;
+ alignment = alignmentAttr ? alignmentAttr.getValue().getZExtValue() : 1;
+ }
+ builder.CreateMaskedCompressStore(
+ $value, $ptr, alignment != 1 ? llvm::Align(alignment) : llvm::MaybeAlign(), $mask);
+ }];
+
+ let builders = [
+ OpBuilder<(ins "Value":$value, "Value":$ptr, "Value":$mask, CArg<"uint64_t", "1">:$align), [{
+ ArrayAttr callArgs = nullptr;
+ if (align != 1) {
+ auto emptyDictAttr = $_builder.getDictionaryAttr({});
+ auto alignmentAttr = $_builder.getI64IntegerAttr(align);
+ auto namedAttr = $_builder.getNamedAttr("llvm.align", alignmentAttr);
+ SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
+ auto alignDictAttr = $_builder.getDictionaryAttr(attrs);
+ callArgs = $_builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
+ }
+ build($_builder, $_state, value, ptr, mask, /*arg_attrs=*/callArgs, /*res_attrs=*/nullptr);
+ }]>
+ ];
+
}
//
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 9f882ad6f22e8..7830fc97629ab 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -545,6 +545,15 @@ define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float>
ret void
}
+; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics_with_alignment
+define void @masked_expand_compress_intrinsics_with_alignment(ptr %0, <7 x i1> %1, <7 x float> %2) {
+ ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
+ ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) <{arg_attrs = [{}, {llvm.align = 8 : i64}, {}]}> : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr align 8 %0, <7 x i1> %1)
+ ret void
+}
+
; CHECK-LABEL: llvm.func @annotate_intrinsics
define void @annotate_intrinsics(ptr %var, ptr %ptr, i16 %int, ptr %annotation, ptr %fileName, i32 %line, ptr %args) {
; CHECK: "llvm.intr.var.annotation"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, !llvm.ptr, !llvm.ptr, i32, !llvm.ptr) -> ()
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 2b420ed246fb2..9078c5b63e56a 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -577,6 +577,17 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1
llvm.return
}
+// CHECK-LABEL: @masked_expand_compress_intrinsics_with_alignment
+llvm.func @masked_expand_compress_intrinsics_with_alignment(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthru: vector<7xf32>) {
+ // CHECK: call <7 x float> @llvm.masked.expandload.v7f32(ptr %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
+ %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru)
+ : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> (vector<7xf32>)
+ // CHECK: call void @llvm.masked.compressstore.v7f32(<7 x float> %{{.*}}, ptr align 8 %{{.*}}, <7 x i1> %{{.*}})
+ "llvm.intr.masked.compressstore"(%0, %ptr, %mask) {arg_attrs = [{}, {llvm.align = 8 : i32}, {}]}
+ : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+ llvm.return
+}
+
// CHECK-LABEL: @annotate_intrinsics
llvm.func @annotate_intrinsics(%var: !llvm.ptr, %int: i16, %ptr: !llvm.ptr, %annotation: !llvm.ptr, %fileName: !llvm.ptr, %line: i32, %attr: !llvm.ptr) {
// CHECK: call void @llvm.var.annotation.p0.p0(ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, i32 %{{.*}}, ptr %{{.*}})
>From fc9ab380278314d2f7387f0b4e989ab77e93dc4a Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 03:02:19 -0700
Subject: [PATCH 07/18] Add requiresArgAndResultAttrs to LLVM_OneResultIntrOp
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 19 ++++++++++---------
.../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 3 ++-
2 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index a76890532d650..2cb06f7a8c359 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -87,21 +87,21 @@ class LLVM_TernarySameArgsIntrOpF<string func, list<Trait> traits = []> :
class LLVM_CountZerosIntrOp<string func, list<Trait> traits = []> :
LLVM_OneResultIntrOp<func, [], [0],
!listconcat([Pure, SameOperandsAndResultType], traits),
- /*requiresFastmath=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["is_zero_poison"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
I1Attr:$is_zero_poison);
}
def LLVM_AbsOp : LLVM_OneResultIntrOp<"abs", [], [0], [Pure],
- /*requiresFastmath=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["is_int_min_poison"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
I1Attr:$is_int_min_poison);
}
def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure],
- /*requiresFastmath=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["bit"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$in, I32Attr:$bit);
}
@@ -360,8 +360,8 @@ def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">;
def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1],
[DeclareOpInterfaceMethods<PromotableOpInterface>],
- /*requiresFastmath=*/0, /*immArgPositions=*/[0],
- /*immArgAttrNames=*/["size"]> {
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
+ /*immArgPositions=*/[0], /*immArgAttrNames=*/["size"]> {
let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
let results = (outs LLVM_DefaultPointer:$res);
let assemblyFormat = "$size `,` $ptr attr-dict `:` qualified(type($ptr))";
@@ -412,6 +412,7 @@ class LLVM_ConstrainedIntr<string mnem, int numArgs,
!gt(hasRoundingMode, 0) : [DeclareOpInterfaceMethods<RoundingModeOpInterface>],
true : []),
/*requiresFastmath=*/0,
+ /*requiresArgAndResultAttrs*/0,
/*immArgPositions=*/[],
/*immArgAttrNames=*/[]> {
dag regularArgs = !dag(ins, !listsplat(LLVM_Type, numArgs), !foreach(i, !range(numArgs), "arg_" #i));
@@ -589,7 +590,7 @@ def LLVM_ExpectOp
def LLVM_ExpectWithProbabilityOp
: LLVM_OneResultIntrOp<"expect.with.probability", [], [0],
[Pure, AllTypesMatch<["val", "expected", "res"]>],
- /*requiresFastmath=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
/*immArgPositions=*/[2], /*immArgAttrNames=*/["prob"]> {
let arguments = (ins AnySignlessInteger:$val,
AnySignlessInteger:$expected,
@@ -825,7 +826,7 @@ class LLVM_VecReductionAccBase<string mnem, Type element>
/*overloadedResults=*/[],
/*overloadedOperands=*/[1],
/*traits=*/[Pure, SameOperandsAndResultElementType],
- /*equiresFastmath=*/1>,
+ /*requiresFastmath=*/1>,
Arguments<(ins element:$start_value,
LLVM_VectorOf<element>:$input,
DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags)>;
@@ -1191,7 +1192,7 @@ def LLVM_vector_insert
PredOpTrait<"it is not inserting scalable into fixed-length vectors.",
CPred<"!isScalableVectorType($srcvec.getType()) || "
"isScalableVectorType($dstvec.getType())">>],
- /*requiresFastmath=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultattrs=*/0,
/*immArgPositions=*/[2], /*immArgAttrNames=*/["pos"]> {
let arguments = (ins LLVM_AnyVector:$dstvec, LLVM_AnyVector:$srcvec,
I64Attr:$pos);
@@ -1225,7 +1226,7 @@ def LLVM_vector_extract
PredOpTrait<"it is not extracting scalable from fixed-length vectors.",
CPred<"!isScalableVectorType($res.getType()) || "
"isScalableVectorType($srcvec.getType())">>],
- /*requiresFastmath=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultattrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["pos"]> {
let arguments = (ins LLVM_AnyVector:$srcvec, I64Attr:$pos);
let results = (outs LLVM_AnyVector:$res);
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index a8d7cf2069547..d6aa9580870a8 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -475,11 +475,12 @@ class LLVM_OneResultIntrOp<string mnem, list<int> overloadedResults = [],
list<int> overloadedOperands = [],
list<Trait> traits = [],
bit requiresFastmath = 0,
+ bit requiresArgAndResultAttrs = 0,
list<int> immArgPositions = [],
list<string> immArgAttrNames = []>
: LLVM_IntrOp<mnem, overloadedResults, overloadedOperands, traits, 1,
/*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
- requiresFastmath, /*requiresArgAndResultAttrs=*/0,
+ requiresFastmath, requiresArgAndResultAttrs,
/*requiresOpBundles=*/0, immArgPositions,
immArgAttrNames>;
>From dd50b3d46dbf6c7bcf44fa283a8e30d585ebf5ce Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 03:18:40 -0700
Subject: [PATCH 08/18] Add alignment to intr.masked.expandload.
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 41 ++++++++++++++++++-
mlir/test/Target/LLVMIR/Import/intrinsic.ll | 4 +-
.../test/Target/LLVMIR/llvmir-intrinsics.mlir | 4 +-
3 files changed, 43 insertions(+), 6 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 2cb06f7a8c359..21758b5fc9114 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1070,8 +1070,45 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
}
/// Create a call to Masked Expand Load intrinsic.
-def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
- let arguments = (ins LLVM_AnyPointer, LLVM_VectorOf<I1>, LLVM_AnyVector);
+def LLVM_masked_expandload
+ : LLVM_OneResultIntrOp<"masked.expandload", [0], [],
+ /*traits=*/[], /*requiresFastMath=*/0, /*requiresArgAndResultAttrs=*/1,
+ /*immArgPositions=*/[], /*immArgAttrNames=*/[]> {
+ dag args = (ins LLVM_AnyPointer:$ptr,
+ LLVM_VectorOf<I1>:$mask,
+ LLVM_AnyVector:$passthru);
+
+ let arguments = !con(args, baseArgs);
+
+ let builders = [LLVM_OneResultOpBuilder];
+
+ string llvmBuilder = [{
+ auto argAttrs = op.getArgAttrs();
+ uint64_t alignment = 1;
+ if (argAttrs.has_value()) {
+ NamedAttrList argAttr = cast<DictionaryAttr>((*argAttrs)[0]);
+ auto maybeAlignmentAttr = argAttr.get("llvm.align");
+ IntegerAttr alignmentAttr = maybeAlignmentAttr ? dyn_cast<IntegerAttr>(maybeAlignmentAttr) : nullptr;
+ alignment = alignmentAttr ? alignmentAttr.getValue().getZExtValue() : 1;
+ }
+ $res = builder.CreateMaskedExpandLoad(
+ $_resultType, $ptr, alignment != 1 ? llvm::Align(alignment) : llvm::MaybeAlign(), $mask, $passthru);
+ }];
+
+ let builders = [
+ OpBuilder<(ins "TypeRange":$resTy, "Value":$ptr, "Value":$mask, "Value":$passthru, CArg<"uint64_t", "1">:$align), [{
+ ArrayAttr callArgs = nullptr;
+ if (align != 1) {
+ auto emptyDictAttr = $_builder.getDictionaryAttr({});
+ auto alignmentAttr = $_builder.getI64IntegerAttr(align);
+ auto namedAttr = $_builder.getNamedAttr("llvm.align", alignmentAttr);
+ SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
+ auto alignDictAttr = $_builder.getDictionaryAttr(attrs);
+ callArgs = $_builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
+ }
+ build($_builder, $_state, resTy, ptr, mask, passthru, /*arg_attrs=*/callArgs, /*res_attrs=*/nullptr);
+ }]>
+ ];
}
/// Create a call to Masked Compress Store intrinsic.
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 7830fc97629ab..07d22120153fe 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -547,8 +547,8 @@ define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float>
; CHECK-LABEL: llvm.func @masked_expand_compress_intrinsics_with_alignment
define void @masked_expand_compress_intrinsics_with_alignment(ptr %0, <7 x i1> %1, <7 x float> %2) {
- ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
- %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr %0, <7 x i1> %1, <7 x float> %2)
+ ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) <{arg_attrs = [{llvm.align = 8 : i64}, {}, {}]}> : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %0, <7 x i1> %1, <7 x float> %2)
; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) <{arg_attrs = [{}, {llvm.align = 8 : i64}, {}]}> : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr align 8 %0, <7 x i1> %1)
ret void
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 9078c5b63e56a..c99dde36f5ccb 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -579,8 +579,8 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1
// CHECK-LABEL: @masked_expand_compress_intrinsics_with_alignment
llvm.func @masked_expand_compress_intrinsics_with_alignment(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthru: vector<7xf32>) {
- // CHECK: call <7 x float> @llvm.masked.expandload.v7f32(ptr %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
- %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru)
+ // CHECK: call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
+ %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru) {arg_attrs = [{llvm.align = 8 : i32}, {}, {}]}
: (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> (vector<7xf32>)
// CHECK: call void @llvm.masked.compressstore.v7f32(<7 x float> %{{.*}}, ptr align 8 %{{.*}}, <7 x i1> %{{.*}})
"llvm.intr.masked.compressstore"(%0, %ptr, %mask) {arg_attrs = [{}, {llvm.align = 8 : i32}, {}]}
>From 876962b2ff60381d0fda6cef142be7a224d51526 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 17:39:21 -0700
Subject: [PATCH 09/18] Fix placement of attribute alignment.
---
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 21758b5fc9114..cae2a93b12b68 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1104,7 +1104,7 @@ def LLVM_masked_expandload
auto namedAttr = $_builder.getNamedAttr("llvm.align", alignmentAttr);
SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
auto alignDictAttr = $_builder.getDictionaryAttr(attrs);
- callArgs = $_builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
+ callArgs = $_builder.getArrayAttr({alignDictAttr, emptyDictAttr, emptyDictAttr});
}
build($_builder, $_state, resTy, ptr, mask, passthru, /*arg_attrs=*/callArgs, /*res_attrs=*/nullptr);
}]>
>From 990dea1810feb6eb903c19cf6504a354559eeb27 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 19:12:56 -0700
Subject: [PATCH 10/18] Fix typos
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index cae2a93b12b68..43a949f6ce668 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -87,21 +87,21 @@ class LLVM_TernarySameArgsIntrOpF<string func, list<Trait> traits = []> :
class LLVM_CountZerosIntrOp<string func, list<Trait> traits = []> :
LLVM_OneResultIntrOp<func, [], [0],
!listconcat([Pure, SameOperandsAndResultType], traits),
- /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["is_zero_poison"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
I1Attr:$is_zero_poison);
}
def LLVM_AbsOp : LLVM_OneResultIntrOp<"abs", [], [0], [Pure],
- /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["is_int_min_poison"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
I1Attr:$is_int_min_poison);
}
def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure],
- /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["bit"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$in, I32Attr:$bit);
}
@@ -360,7 +360,7 @@ def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">;
def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1],
[DeclareOpInterfaceMethods<PromotableOpInterface>],
- /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[0], /*immArgAttrNames=*/["size"]> {
let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
let results = (outs LLVM_DefaultPointer:$res);
@@ -412,7 +412,7 @@ class LLVM_ConstrainedIntr<string mnem, int numArgs,
!gt(hasRoundingMode, 0) : [DeclareOpInterfaceMethods<RoundingModeOpInterface>],
true : []),
/*requiresFastmath=*/0,
- /*requiresArgAndResultAttrs*/0,
+ /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[],
/*immArgAttrNames=*/[]> {
dag regularArgs = !dag(ins, !listsplat(LLVM_Type, numArgs), !foreach(i, !range(numArgs), "arg_" #i));
@@ -590,7 +590,7 @@ def LLVM_ExpectOp
def LLVM_ExpectWithProbabilityOp
: LLVM_OneResultIntrOp<"expect.with.probability", [], [0],
[Pure, AllTypesMatch<["val", "expected", "res"]>],
- /*requiresFastmath=*/0, /*requiresArgAndResultAttrs*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[2], /*immArgAttrNames=*/["prob"]> {
let arguments = (ins AnySignlessInteger:$val,
AnySignlessInteger:$expected,
@@ -1229,7 +1229,7 @@ def LLVM_vector_insert
PredOpTrait<"it is not inserting scalable into fixed-length vectors.",
CPred<"!isScalableVectorType($srcvec.getType()) || "
"isScalableVectorType($dstvec.getType())">>],
- /*requiresFastmath=*/0, /*requiresArgAndResultattrs=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[2], /*immArgAttrNames=*/["pos"]> {
let arguments = (ins LLVM_AnyVector:$dstvec, LLVM_AnyVector:$srcvec,
I64Attr:$pos);
@@ -1263,7 +1263,7 @@ def LLVM_vector_extract
PredOpTrait<"it is not extracting scalable from fixed-length vectors.",
CPred<"!isScalableVectorType($res.getType()) || "
"isScalableVectorType($srcvec.getType())">>],
- /*requiresFastmath=*/0, /*requiresArgAndResultattrs=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["pos"]> {
let arguments = (ins LLVM_AnyVector:$srcvec, I64Attr:$pos);
let results = (outs LLVM_AnyVector:$res);
>From 37ce068655d4b79fe76b9ddaa62fe78a68fbf9c4 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 19:22:41 -0700
Subject: [PATCH 11/18] Use base class's builder
---
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 2 --
1 file changed, 2 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 43a949f6ce668..b32ecb72351f4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1080,8 +1080,6 @@ def LLVM_masked_expandload
let arguments = !con(args, baseArgs);
- let builders = [LLVM_OneResultOpBuilder];
-
string llvmBuilder = [{
auto argAttrs = op.getArgAttrs();
uint64_t alignment = 1;
>From 06d938f3597a8ae39ae41924853fda319cfc5852 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 19:26:58 -0700
Subject: [PATCH 12/18] Use function to get attribute name.
---
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index b32ecb72351f4..c8708ad48cfad 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1085,7 +1085,7 @@ def LLVM_masked_expandload
uint64_t alignment = 1;
if (argAttrs.has_value()) {
NamedAttrList argAttr = cast<DictionaryAttr>((*argAttrs)[0]);
- auto maybeAlignmentAttr = argAttr.get("llvm.align");
+ auto maybeAlignmentAttr = argAttr.get(LLVMDialect::getAlignAttrName());
IntegerAttr alignmentAttr = maybeAlignmentAttr ? dyn_cast<IntegerAttr>(maybeAlignmentAttr) : nullptr;
alignment = alignmentAttr ? alignmentAttr.getValue().getZExtValue() : 1;
}
@@ -1099,7 +1099,7 @@ def LLVM_masked_expandload
if (align != 1) {
auto emptyDictAttr = $_builder.getDictionaryAttr({});
auto alignmentAttr = $_builder.getI64IntegerAttr(align);
- auto namedAttr = $_builder.getNamedAttr("llvm.align", alignmentAttr);
+ auto namedAttr = $_builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
auto alignDictAttr = $_builder.getDictionaryAttr(attrs);
callArgs = $_builder.getArrayAttr({alignDictAttr, emptyDictAttr, emptyDictAttr});
@@ -1126,7 +1126,7 @@ def LLVM_masked_compressstore
uint64_t alignment = 1;
if (argAttrs.has_value()) {
NamedAttrList argAttr = cast<DictionaryAttr>((*argAttrs)[1]);
- auto maybeAlignmentAttr = argAttr.get("llvm.align");
+ auto maybeAlignmentAttr = argAttr.get(LLVMDialect::getAlignAttrName());
IntegerAttr alignmentAttr = maybeAlignmentAttr ? dyn_cast<IntegerAttr>(maybeAlignmentAttr) : nullptr;
alignment = alignmentAttr ? alignmentAttr.getValue().getZExtValue() : 1;
}
@@ -1140,7 +1140,7 @@ def LLVM_masked_compressstore
if (align != 1) {
auto emptyDictAttr = $_builder.getDictionaryAttr({});
auto alignmentAttr = $_builder.getI64IntegerAttr(align);
- auto namedAttr = $_builder.getNamedAttr("llvm.align", alignmentAttr);
+ auto namedAttr = $_builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
auto alignDictAttr = $_builder.getDictionaryAttr(attrs);
callArgs = $_builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
>From aa6c08df6b3a14a552d0534dbdcb29ef55d102cf Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 19:40:15 -0700
Subject: [PATCH 13/18] Move masked_expandload::build out of tblgen
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 13 +----------
mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 23 +++++++++++++++++++
2 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index c8708ad48cfad..85880609215e8 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1094,18 +1094,7 @@ def LLVM_masked_expandload
}];
let builders = [
- OpBuilder<(ins "TypeRange":$resTy, "Value":$ptr, "Value":$mask, "Value":$passthru, CArg<"uint64_t", "1">:$align), [{
- ArrayAttr callArgs = nullptr;
- if (align != 1) {
- auto emptyDictAttr = $_builder.getDictionaryAttr({});
- auto alignmentAttr = $_builder.getI64IntegerAttr(align);
- auto namedAttr = $_builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
- SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
- auto alignDictAttr = $_builder.getDictionaryAttr(attrs);
- callArgs = $_builder.getArrayAttr({alignDictAttr, emptyDictAttr, emptyDictAttr});
- }
- build($_builder, $_state, resTy, ptr, mask, passthru, /*arg_attrs=*/callArgs, /*res_attrs=*/nullptr);
- }]>
+ OpBuilder<(ins "TypeRange":$resTy, "Value":$ptr, "Value":$mask, "Value":$passthru, CArg<"uint64_t", "1">:$align)>
];
}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 422039f81855a..509151f93f96b 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -4116,6 +4116,29 @@ LogicalResult LLVM::masked_scatter::verify() {
return success();
}
+//===----------------------------------------------------------------------===//
+// masked_expandload (intrinsic)
+//===----------------------------------------------------------------------===//
+
+void LLVM::masked_expandload::build(OpBuilder &builder, OperationState &state,
+ mlir::TypeRange resTys, Value ptr,
+ Value mask, Value passthru,
+ uint64_t align) {
+ ArrayAttr callArgs = nullptr;
+ if (align != 1) {
+ auto emptyDictAttr = builder.getDictionaryAttr({});
+ auto alignmentAttr = builder.getI64IntegerAttr(align);
+ auto namedAttr =
+ builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
+ SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
+ auto alignDictAttr = builder.getDictionaryAttr(attrs);
+ callArgs =
+ builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
+ }
+ build(builder, state, resTys, ptr, mask, passthru, /*arg_attrs=*/callArgs,
+ /*res_attrs=*/nullptr);
+}
+
//===----------------------------------------------------------------------===//
// InlineAsmOp
//===----------------------------------------------------------------------===//
>From 483c1c71648ede20e6d2489fceb6c3eda56e52ab Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 20:01:47 -0700
Subject: [PATCH 14/18] Move masked_compressstore::build out of tblgen
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 13 +----------
mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 23 +++++++++++++++++++
2 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 85880609215e8..09ce369c75d35 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1124,18 +1124,7 @@ def LLVM_masked_compressstore
}];
let builders = [
- OpBuilder<(ins "Value":$value, "Value":$ptr, "Value":$mask, CArg<"uint64_t", "1">:$align), [{
- ArrayAttr callArgs = nullptr;
- if (align != 1) {
- auto emptyDictAttr = $_builder.getDictionaryAttr({});
- auto alignmentAttr = $_builder.getI64IntegerAttr(align);
- auto namedAttr = $_builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
- SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
- auto alignDictAttr = $_builder.getDictionaryAttr(attrs);
- callArgs = $_builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
- }
- build($_builder, $_state, value, ptr, mask, /*arg_attrs=*/callArgs, /*res_attrs=*/nullptr);
- }]>
+ OpBuilder<(ins "Value":$value, "Value":$ptr, "Value":$mask, CArg<"uint64_t", "1">:$align)>
];
}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 509151f93f96b..11dc30175727e 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -4139,6 +4139,29 @@ void LLVM::masked_expandload::build(OpBuilder &builder, OperationState &state,
/*res_attrs=*/nullptr);
}
+//===----------------------------------------------------------------------===//
+// masked_compressstore (intrinsic)
+//===----------------------------------------------------------------------===//
+
+void LLVM::masked_compressstore::build(OpBuilder &builder,
+ OperationState &state, Value value,
+ Value ptr, Value mask, uint64_t align) {
+
+ ArrayAttr callArgs = nullptr;
+ if (align != 1) {
+ auto emptyDictAttr = builder.getDictionaryAttr({});
+ auto alignmentAttr = builder.getI64IntegerAttr(align);
+ auto namedAttr =
+ builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
+ SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
+ auto alignDictAttr = builder.getDictionaryAttr(attrs);
+ callArgs =
+ builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
+ }
+ build(builder, state, value, ptr, mask, /*arg_attrs=*/callArgs,
+ /*res_attrs=*/nullptr);
+}
+
//===----------------------------------------------------------------------===//
// InlineAsmOp
//===----------------------------------------------------------------------===//
>From be27a32d6da29395089e8ee8a2a5eccc301cc0ab Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 20:32:50 -0700
Subject: [PATCH 15/18] Outline the generation of callArgs into a static helper
function
---
mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 58 ++++++++++++++--------
1 file changed, 36 insertions(+), 22 deletions(-)
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 11dc30175727e..1268b94cafada 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -141,6 +141,39 @@ static ParseResult parseLLVMLinkage(OpAsmParser &p, LinkageAttr &val) {
return success();
}
+static ArrayAttr getLLVMAlignParamForCompressExpand(OpBuilder &builder,
+ bool isExpandLoad,
+ uint64_t alignment = 1) {
+ // From
+ // https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics
+ // https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics
+ //
+ // The pointer alignment defaults to 1.
+ if (alignment == 1) {
+ return nullptr;
+ }
+
+ auto emptyDictAttr = builder.getDictionaryAttr({});
+ auto alignmentAttr = builder.getI64IntegerAttr(alignment);
+ auto namedAttr =
+ builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
+ SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
+ auto alignDictAttr = builder.getDictionaryAttr(attrs);
+ // From
+ // https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics
+ // https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics
+ //
+ // The align parameter attribute can be provided for [expandload]'s first
+ // argument. The align parameter attribute can be provided for
+ // [compressstore]'s second argument.
+ int pos = isExpandLoad ? 0 : 1;
+ auto arrayAttr =
+ pos == 0
+ ? builder.getArrayAttr({alignDictAttr, emptyDictAttr, emptyDictAttr})
+ : builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
+ return builder.getArrayAttr(arrayAttr);
+}
+
//===----------------------------------------------------------------------===//
// Operand bundle helpers.
//===----------------------------------------------------------------------===//
@@ -4124,17 +4157,7 @@ void LLVM::masked_expandload::build(OpBuilder &builder, OperationState &state,
mlir::TypeRange resTys, Value ptr,
Value mask, Value passthru,
uint64_t align) {
- ArrayAttr callArgs = nullptr;
- if (align != 1) {
- auto emptyDictAttr = builder.getDictionaryAttr({});
- auto alignmentAttr = builder.getI64IntegerAttr(align);
- auto namedAttr =
- builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
- SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
- auto alignDictAttr = builder.getDictionaryAttr(attrs);
- callArgs =
- builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
- }
+ ArrayAttr callArgs = getLLVMAlignParamForCompressExpand(builder, true, align);
build(builder, state, resTys, ptr, mask, passthru, /*arg_attrs=*/callArgs,
/*res_attrs=*/nullptr);
}
@@ -4147,17 +4170,8 @@ void LLVM::masked_compressstore::build(OpBuilder &builder,
OperationState &state, Value value,
Value ptr, Value mask, uint64_t align) {
- ArrayAttr callArgs = nullptr;
- if (align != 1) {
- auto emptyDictAttr = builder.getDictionaryAttr({});
- auto alignmentAttr = builder.getI64IntegerAttr(align);
- auto namedAttr =
- builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
- SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
- auto alignDictAttr = builder.getDictionaryAttr(attrs);
- callArgs =
- builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
- }
+ ArrayAttr callArgs =
+ getLLVMAlignParamForCompressExpand(builder, false, align);
build(builder, state, value, ptr, mask, /*arg_attrs=*/callArgs,
/*res_attrs=*/nullptr);
}
>From e2afeb6eb84dcf44657cc64817a1bfd9bfec427a Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 20:36:31 -0700
Subject: [PATCH 16/18] Use the base class's llvmBuilder.
---
.../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 27 -------------------
1 file changed, 27 deletions(-)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 09ce369c75d35..5e2461b4508e4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -1080,19 +1080,6 @@ def LLVM_masked_expandload
let arguments = !con(args, baseArgs);
- string llvmBuilder = [{
- auto argAttrs = op.getArgAttrs();
- uint64_t alignment = 1;
- if (argAttrs.has_value()) {
- NamedAttrList argAttr = cast<DictionaryAttr>((*argAttrs)[0]);
- auto maybeAlignmentAttr = argAttr.get(LLVMDialect::getAlignAttrName());
- IntegerAttr alignmentAttr = maybeAlignmentAttr ? dyn_cast<IntegerAttr>(maybeAlignmentAttr) : nullptr;
- alignment = alignmentAttr ? alignmentAttr.getValue().getZExtValue() : 1;
- }
- $res = builder.CreateMaskedExpandLoad(
- $_resultType, $ptr, alignment != 1 ? llvm::Align(alignment) : llvm::MaybeAlign(), $mask, $passthru);
- }];
-
let builders = [
OpBuilder<(ins "TypeRange":$resTy, "Value":$ptr, "Value":$mask, "Value":$passthru, CArg<"uint64_t", "1">:$align)>
];
@@ -1110,23 +1097,9 @@ def LLVM_masked_compressstore
let arguments = !con(args, baseArgs);
- string llvmBuilder = [{
- auto argAttrs = op.getArgAttrs();
- uint64_t alignment = 1;
- if (argAttrs.has_value()) {
- NamedAttrList argAttr = cast<DictionaryAttr>((*argAttrs)[1]);
- auto maybeAlignmentAttr = argAttr.get(LLVMDialect::getAlignAttrName());
- IntegerAttr alignmentAttr = maybeAlignmentAttr ? dyn_cast<IntegerAttr>(maybeAlignmentAttr) : nullptr;
- alignment = alignmentAttr ? alignmentAttr.getValue().getZExtValue() : 1;
- }
- builder.CreateMaskedCompressStore(
- $value, $ptr, alignment != 1 ? llvm::Align(alignment) : llvm::MaybeAlign(), $mask);
- }];
-
let builders = [
OpBuilder<(ins "Value":$value, "Value":$ptr, "Value":$mask, CArg<"uint64_t", "1">:$align)>
];
-
}
//
>From c78134ad05654c5de124f16bc1b73fbb4bc15060 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 13 Aug 2025 21:26:38 -0700
Subject: [PATCH 17/18] Fix return type.
---
mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 1268b94cafada..c7a193c31a59f 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -167,11 +167,10 @@ static ArrayAttr getLLVMAlignParamForCompressExpand(OpBuilder &builder,
// argument. The align parameter attribute can be provided for
// [compressstore]'s second argument.
int pos = isExpandLoad ? 0 : 1;
- auto arrayAttr =
- pos == 0
- ? builder.getArrayAttr({alignDictAttr, emptyDictAttr, emptyDictAttr})
- : builder.getArrayAttr({emptyDictAttr, alignDictAttr, emptyDictAttr});
- return builder.getArrayAttr(arrayAttr);
+ return pos == 0 ? builder.getArrayAttr(
+ {alignDictAttr, emptyDictAttr, emptyDictAttr})
+ : builder.getArrayAttr(
+ {emptyDictAttr, alignDictAttr, emptyDictAttr});
}
//===----------------------------------------------------------------------===//
>From 954a26b1ce4d7bdcd84577080476398aad612328 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 14 Aug 2025 13:51:23 -0700
Subject: [PATCH 18/18] Style: address review comments.
---
mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index c7a193c31a59f..a6e89f63c822b 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -4156,8 +4156,8 @@ void LLVM::masked_expandload::build(OpBuilder &builder, OperationState &state,
mlir::TypeRange resTys, Value ptr,
Value mask, Value passthru,
uint64_t align) {
- ArrayAttr callArgs = getLLVMAlignParamForCompressExpand(builder, true, align);
- build(builder, state, resTys, ptr, mask, passthru, /*arg_attrs=*/callArgs,
+ ArrayAttr argAttrs = getLLVMAlignParamForCompressExpand(builder, true, align);
+ build(builder, state, resTys, ptr, mask, passthru, /*arg_attrs=*/argAttrs,
/*res_attrs=*/nullptr);
}
@@ -4168,10 +4168,9 @@ void LLVM::masked_expandload::build(OpBuilder &builder, OperationState &state,
void LLVM::masked_compressstore::build(OpBuilder &builder,
OperationState &state, Value value,
Value ptr, Value mask, uint64_t align) {
-
- ArrayAttr callArgs =
+ ArrayAttr argAttrs =
getLLVMAlignParamForCompressExpand(builder, false, align);
- build(builder, state, value, ptr, mask, /*arg_attrs=*/callArgs,
+ build(builder, state, value, ptr, mask, /*arg_attrs=*/argAttrs,
/*res_attrs=*/nullptr);
}
More information about the Mlir-commits
mailing list