[Mlir-commits] [mlir] [mlir][amdgpu] Adds make_dma_gather_base (PR #171857)
Erick Ochoa Lopez
llvmlistbot at llvm.org
Fri Dec 12 05:56:56 PST 2025
https://github.com/amd-eochoalo updated https://github.com/llvm/llvm-project/pull/171857
>From cf7c396632c7d991fa4d7b613f5e68542629e904 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 10 Dec 2025 11:58:52 -0500
Subject: [PATCH 01/12] [mlir][amdgpu] Add make_gather_dma_base.
---
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td | 60 ++++++++--
.../AMDGPUToROCDL/AMDGPUToROCDL.cpp | 104 ++++++++++--------
.../Conversion/AMDGPUToROCDL/gfx1250.mlir | 13 ++-
3 files changed, 120 insertions(+), 57 deletions(-)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index 56160d3e8fe85..82398c2c82200 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -94,6 +94,9 @@ def AMDGPU_TDMBaseType : AMDGPU_Type<"TDMBase", "tdm_base"> {
let description = [{
This type is opaque and it is used to represent a struct of two addresses.
One address is in LDS while the other is in global memory.
+
+ The value defined by this operation is only intended to be used by
+ amdgpu.tdm_make_descriptor.
}];
let parameters = (ins "Type":$elementType);
let builders = [
@@ -104,6 +107,27 @@ def AMDGPU_TDMBaseType : AMDGPU_Type<"TDMBase", "tdm_base"> {
let assemblyFormat = "`<` $elementType `>`";
}
+def AMDGPU_TDMGatherBaseType : AMDGPU_Type<"TDMGatherBase", "tdm_gather_base"> {
+ let summary = "Pair of base addresses that move data between LDS and global storage.";
+ let description = [{
+ This type is opaque and it is used to represent a struct of two addresses.
+ One address is in LDS while the other is in global memory.
+
+ This operation is similar to amdgpu.tdm_make_base but intended to be
+ used in gather mode.
+
+ The value defined by this operation is only intended to be used by
+ amdgpu.tdm_make_gather_descriptor.
+ }];
+ let parameters = (ins "Type":$elementType, "unsigned":$indexSize);
+ let builders = [
+ TypeBuilderWithInferredContext<(ins "Type":$elementType, "unsigned": $indexSize), [{
+ return $_get(elementType.getContext(), elementType, indexSize);
+ }]>
+ ];
+ let assemblyFormat = "`<` $elementType `,` $indexSize `>`";
+}
+
def AMDGPU_TDMDescriptorType : AMDGPU_Type<"TDMDescriptor", "tdm_descriptor"> {
let summary = "Descriptors used in tensor store/load operations.";
let description = [{
@@ -1229,17 +1253,37 @@ def AMDGPU_ScaledMFMAOp :
let hasCanonicalizer = 1;
}
-def AMDGPU_MakeDmaBaseOp :
- AMDGPU_Op<"make_dma_base", [Pure, AttrSizedOperandSegments, AllElementTypesMatch<["global", "lds"]>]>,
+
+class AMDGPU_DmaBaseOp<string mnemonic, Type outType> :
+ AMDGPU_Op<mnemonic, [Pure, AttrSizedOperandSegments, AllElementTypesMatch<["global", "lds"]>]>,
Arguments<(ins Arg<AnyMemRef>:$global,
Variadic<Index>:$global_indices,
Arg<AnyMemRef>:$lds,
Variadic<Index>:$lds_indices)>,
- Results<(outs AMDGPU_TDMBaseType: $base)> {
+ Results<(outs outType: $base)> {
// TODO:
// * Add verifiers to make sure that the number of indices do not exceed the number of dimensions.
+ let assemblyFormat = [{
+ $global `[` $global_indices `]` `,` $lds `[` $lds_indices `]` attr-dict `:` type($global) `,` type($lds) `->` type(results)
+ }];
+}
+
+def AMDGPU_MakeGatherDmaBaseOp : AMDGPU_DmaBaseOp<"make_gather_dma_base", AMDGPU_TDMGatherBaseType> {
+ let summary = "Pair of based addresses used when moving tiles between LDS and global memory.";
+ let description = "TODO";
+
+ let extraClassDeclaration = [{
+ constexpr int64_t isGather() {
+ return true;
+ }
+ }];
+}
+
+
+def AMDGPU_MakeDmaBaseOp : AMDGPU_DmaBaseOp<"make_dma_base", AMDGPU_TDMBaseType> {
+
let summary = "Pair of based addresses used when moving tiles between LDS and global memory.";
let description = [{
This operation creates a pair of addresses that will be used by tensor_load_to_lds
@@ -1279,11 +1323,13 @@ def AMDGPU_MakeDmaBaseOp :
These tensor DMA operations were introduced in gfx1250.
}];
- let assemblyFormat = [{
- $global `[` $global_indices `]` `,` $lds `[` $lds_indices `]` attr-dict `:` type($global) `,` type($lds) `->` type(results)
- }];
-
let hasVerifier = 1;
+
+ let extraClassDeclaration = [{
+ constexpr int64_t isGather() {
+ return false;
+ }
+ }];
}
def AMDGPU_MakeDmaDescriptorOp :
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 91154b846f567..55c68ba73c19b 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -2276,45 +2276,84 @@ struct AMDGPUPermlaneLowering : public ConvertOpToLLVMPattern<PermlaneSwapOp> {
}
};
-struct AMDGPUMakeDmaBaseLowering
- : public ConvertOpToLLVMPattern<MakeDmaBaseOp> {
- using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+static Value setValueAtOffset(ConversionPatternRewriter &rewriter, Location loc,
+ Value accumulator, Value value, int64_t shift) {
+ shift = shift % 32;
+ Value shiftAmount;
+ if (shift != 0) {
+ shiftAmount = createI32Constant(rewriter, loc, shift % 32);
+ value = LLVM::ShlOp::create(rewriter, loc, value, shiftAmount);
+ }
+
+ if (matchPattern(accumulator, mlir::m_Zero()))
+ return value;
+
+ return LLVM::OrOp::create(rewriter, loc, accumulator, value);
+}
+
+template <typename BaseOp>
+struct AMDGPUMakeDmaBaseLowering : public ConvertOpToLLVMPattern<BaseOp> {
+ using ConvertOpToLLVMPattern<BaseOp>::ConvertOpToLLVMPattern;
+ using Adaptor = typename ConvertOpToLLVMPattern<BaseOp>::OpAdaptor;
AMDGPUMakeDmaBaseLowering(const LLVMTypeConverter &converter, Chipset chipset)
- : ConvertOpToLLVMPattern<MakeDmaBaseOp>(converter), chipset(chipset) {}
+ : ConvertOpToLLVMPattern<BaseOp>(converter), chipset(chipset) {}
Chipset chipset;
LogicalResult
- matchAndRewrite(MakeDmaBaseOp op, OpAdaptor adaptor,
+ matchAndRewrite(BaseOp op, Adaptor adaptor,
ConversionPatternRewriter &rewriter) const override {
if (chipset < kGfx1250)
return op->emitOpError("make_dma_base is only supported on gfx1250");
Location loc = op.getLoc();
+ constexpr int32_t constlen = 4;
+ Value consts[constlen];
+ for (int64_t i = 0; i < constlen; i++)
+ consts[i] = createI32Constant(rewriter, loc, i);
+
+ constexpr int32_t sgprslen = constlen;
+ Value sgprs[sgprslen];
+ for (int64_t i = 0; i < sgprslen; i++) {
+ sgprs[i] = consts[0];
+ }
+
+ sgprs[0] = consts[1];
+
+ if (op.isGather()) {
+ sgprs[0] = setValueAtOffset(rewriter, loc, sgprs[0], consts[1], 30);
+
+ auto type = cast<TDMGatherBaseType>(op.getResult().getType());
+ unsigned indexSize = type.getIndexSize();
+ assert(llvm::is_contained<unsigned>({16, 32}, indexSize) &&
+ "expected index_size to be 16 or 32");
+ unsigned idx = (indexSize / 16) - 1;
+ sgprs[0] = setValueAtOffset(rewriter, loc, sgprs[0], consts[idx], 31);
+ }
+
ValueRange ldsIndices = adaptor.getLdsIndices();
Value lds = adaptor.getLds();
auto ldsMemRefType = cast<MemRefType>(op.getLds().getType());
- Value ldsPtr =
- getStridedElementPtr(rewriter, loc, ldsMemRefType, lds, ldsIndices);
+ Value ldsPtr = ConvertToLLVMPattern::getStridedElementPtr(
+ rewriter, loc, ldsMemRefType, lds, ldsIndices);
ValueRange globalIndices = adaptor.getGlobalIndices();
Value global = adaptor.getGlobal();
auto globalMemRefType = cast<MemRefType>(op.getGlobal().getType());
- Value globalPtr = getStridedElementPtr(rewriter, loc, globalMemRefType,
- global, globalIndices);
+ Value globalPtr = ConvertToLLVMPattern::getStridedElementPtr(
+ rewriter, loc, globalMemRefType, global, globalIndices);
Type i32 = rewriter.getI32Type();
Type i64 = rewriter.getI64Type();
- Value castForLdsAddr = LLVM::PtrToIntOp::create(rewriter, loc, i32, ldsPtr);
+ sgprs[1] = LLVM::PtrToIntOp::create(rewriter, loc, i32, ldsPtr);
Value castForGlobalAddr =
LLVM::PtrToIntOp::create(rewriter, loc, i64, globalPtr);
- Value lowHalf =
- LLVM::TruncOp::create(rewriter, loc, i32, castForGlobalAddr);
+ sgprs[2] = LLVM::TruncOp::create(rewriter, loc, i32, castForGlobalAddr);
Value shift = LLVM::LShrOp::create(rewriter, loc, castForGlobalAddr,
createI64Constant(rewriter, loc, 32));
@@ -2322,26 +2361,17 @@ struct AMDGPUMakeDmaBaseLowering
Value highHalf = LLVM::TruncOp::create(rewriter, loc, i32, shift);
Value mask = createI32Constant(rewriter, loc, (1ull << 25) - 1);
- Value validHighHalf = LLVM::AndOp::create(rewriter, loc, highHalf, mask);
-
- Value typeField = createI32Constant(rewriter, loc, 2 << 30);
- Value highHalfPlusType =
- LLVM::OrOp::create(rewriter, loc, validHighHalf, typeField);
+ highHalf = LLVM::AndOp::create(rewriter, loc, highHalf, mask);
- Value c0 = createI32Constant(rewriter, loc, 0);
- Value c1 = createI32Constant(rewriter, loc, 1);
- Value c2 = createI32Constant(rewriter, loc, 2);
- Value c3 = createI32Constant(rewriter, loc, 3);
+ sgprs[3] = setValueAtOffset(rewriter, loc, highHalf, consts[2], 30);
Type v4i32 = this->typeConverter->convertType(VectorType::get(4, i32));
assert(v4i32 && "expected type conversion to succeed");
Value result = LLVM::PoisonOp::create(rewriter, loc, v4i32);
- result = LLVM::InsertElementOp::create(rewriter, loc, result, c1, c0);
- result = LLVM::InsertElementOp::create(rewriter, loc, result,
- castForLdsAddr, c1);
- result = LLVM::InsertElementOp::create(rewriter, loc, result, lowHalf, c2);
- result = LLVM::InsertElementOp::create(rewriter, loc, result,
- highHalfPlusType, c3);
+
+ for (auto [sgpr, constant] : llvm::zip_equal(sgprs, consts))
+ result =
+ LLVM::InsertElementOp::create(rewriter, loc, result, sgpr, constant);
rewriter.replaceOp(op, result);
return success();
@@ -2360,21 +2390,6 @@ struct AMDGPUMakeDmaDescriptorLowering
Value getDGroup0(OpAdaptor adaptor) const { return adaptor.getBase(); }
- Value setValueAtOffset(ConversionPatternRewriter &rewriter, Location loc,
- Value accumulator, Value value, int64_t shift) const {
- shift = shift % 32;
- Value shiftAmount;
- if (shift != 0) {
- shiftAmount = createI32Constant(rewriter, loc, shift % 32);
- value = LLVM::ShlOp::create(rewriter, loc, value, shiftAmount);
- }
-
- if (matchPattern(accumulator, mlir::m_Zero()))
- return value;
-
- return LLVM::OrOp::create(rewriter, loc, accumulator, value);
- }
-
Value setWorkgroupMask(MakeDmaDescriptorOp op, OpAdaptor adaptor,
ConversionPatternRewriter &rewriter, Location loc,
Value sgpr0) const {
@@ -2797,7 +2812,8 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
ScaledExtPackedOpLowering, PackedScaledTruncOpLowering,
PackedTrunc2xFp8OpLowering, PackedStochRoundFp8OpLowering,
GatherToLDSOpLowering, TransposeLoadOpLowering, AMDGPUPermlaneLowering,
- AMDGPUMakeDmaBaseLowering, AMDGPUMakeDmaDescriptorLowering>(converter,
- chipset);
+ AMDGPUMakeDmaBaseLowering<MakeDmaBaseOp>,
+ AMDGPUMakeDmaBaseLowering<MakeGatherDmaBaseOp>,
+ AMDGPUMakeDmaDescriptorLowering>(converter, chipset);
patterns.add<AMDGPUSwizzleBitModeLowering>(converter);
}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
index a94e17ab5b9a5..5c433b5c42bb9 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
@@ -200,6 +200,11 @@ func.func @make_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>
// CHECK-DAG: %[[MEMREF_DESC_MEM:.+]] = builtin.unrealized_conversion_cast %[[MEM]] : memref<8xi32, 1>
// CHECK-DAG: %[[MEMREF_DESC_SMEM:.+]] = builtin.unrealized_conversion_cast %[[SMEM]] : memref<8xi32, 3>
+ // CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(0 : i32) : i32
+ // CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32
+ // CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(2 : i32) : i32
+ // CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(3 : i32) : i32
+
// CHECK-DAG: %[[MEM_BASE_PTR:.+]] = llvm.extractvalue %[[MEMREF_DESC_MEM]][1] : !llvm.struct<(ptr<1>
// CHECK-DAG: %[[SMEM_BASE_PTR:.+]] = llvm.extractvalue %[[MEMREF_DESC_SMEM]][1] : !llvm.struct<(ptr<3>
@@ -216,14 +221,10 @@ func.func @make_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>
// CHECK-DAG: %[[MASK:.+]] = llvm.mlir.constant(33554431 : i32)
// CHECK: %[[VALID_MEM_INT_HIGH:.+]] = llvm.and %[[MEM_INT_HIGH]], %[[MASK]]
- // CHECK-DAG: %[[TYPE_FIELD:.+]] = llvm.mlir.constant(-2147483648 : i32)
+ // CHECK: %[[SHIFT:.+]] = llvm.mlir.constant(30 : i32)
+ // CHECK: %[[TYPE_FIELD:.+]] = llvm.shl %[[C2]], %[[SHIFT]]
// CHECK: %[[MEM_INT_HIGH_TYPE:.+]] = llvm.or %[[VALID_MEM_INT_HIGH]], %[[TYPE_FIELD]]
- // CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(0 : i32) : i32
- // CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32
- // CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(2 : i32) : i32
- // CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(3 : i32) : i32
-
// CHECK: %[[V4I32_0_0:.+]] = llvm.mlir.poison : vector<4xi32>
// CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[C1]], %[[V4I32_0_0]][%[[C0]] : i32]
// CHECK: %[[V4I32_0_2:.+]] = llvm.insertelement %[[SMEM_INT]], %[[V4I32_0_1]][%[[C1]] : i32]
>From f2ebb8142657a0d2aac1fd377df56f52a5ba1da8 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Wed, 10 Dec 2025 16:14:22 -0500
Subject: [PATCH 02/12] Add test
---
.../AMDGPUToROCDL/AMDGPUToROCDL.cpp | 4 +-
.../Conversion/AMDGPUToROCDL/gfx1250.mlir | 45 +++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 55c68ba73c19b..ff4798961ef73 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -2329,7 +2329,9 @@ struct AMDGPUMakeDmaBaseLowering : public ConvertOpToLLVMPattern<BaseOp> {
assert(llvm::is_contained<unsigned>({16, 32}, indexSize) &&
"expected index_size to be 16 or 32");
unsigned idx = (indexSize / 16) - 1;
- sgprs[0] = setValueAtOffset(rewriter, loc, sgprs[0], consts[idx], 31);
+
+ if (idx)
+ sgprs[0] = setValueAtOffset(rewriter, loc, sgprs[0], consts[idx], 31);
}
ValueRange ldsIndices = adaptor.getLdsIndices();
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
index 5c433b5c42bb9..4cbfec52cf0a7 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
@@ -238,6 +238,51 @@ func.func @make_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>
// -----
+#gpu_global_addrspace = 1
+#gpu_lds_addrspace = 3
+
+// CHECK-LABEL: func @make_gather_dma_base
+// CHECK-SAME: (%[[IDX:.+]]: index, %[[MEM:.+]]: memref<8xi32, 1>, %[[SMEM:.+]]: memref<8xi32, 3>)
+func.func @make_gather_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>, %smem: memref<8xi32,#gpu_lds_addrspace>) -> (!amdgpu.tdm_gather_base<i32,16>, !amdgpu.tdm_gather_base<i32, 32>) {
+
+ // CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(0 : i32) : i32
+ // CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32
+ // CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(2 : i32) : i32
+ // CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(3 : i32) : i32
+
+ // CHECK-DAG: %[[GATHER_MODE_OFFSET:.+]] = llvm.mlir.constant(30 : i32) : i32
+ // CHECK-DAG: %[[GATHER_MODE_BIT:.+]] = llvm.shl %[[C1]], %[[GATHER_MODE_OFFSET]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or %[[C1]], %[[GATHER_MODE_BIT]]
+
+ // CHECK: %[[V4I32_0_0:.+]] = llvm.mlir.poison : vector<4xi32>
+ // CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[SGPR0]], %[[V4I32_0_0]][%[[C0]] : i32]
+
+ %0 = amdgpu.make_gather_dma_base %mem[%idx], %smem[%idx] : memref<8xi32, #gpu_global_addrspace>, memref<8xi32, #gpu_lds_addrspace> -> !amdgpu.tdm_gather_base<i32,16>
+
+ // CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(0 : i32) : i32
+ // CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32
+ // CHECK-DAG: %[[C2:.+]] = llvm.mlir.constant(2 : i32) : i32
+ // CHECK-DAG: %[[C3:.+]] = llvm.mlir.constant(3 : i32) : i32
+
+ // CHECK-DAG: %[[GATHER_MODE_OFFSET:.+]] = llvm.mlir.constant(30 : i32) : i32
+ // CHECK-DAG: %[[GATHER_MODE_BIT:.+]] = llvm.shl %[[C1]], %[[GATHER_MODE_OFFSET]]
+ // CHECK: %[[SGPR0_0:.+]] = llvm.or %[[C1]], %[[GATHER_MODE_BIT]]
+
+ // CHECK-DAG: %[[INDEX_SIZE_OFFSET:.+]] = llvm.mlir.constant(31 : i32) : i32
+ // CHECK-DAG: %[[INDEX_SIZE_BIT:.+]] = llvm.shl %[[C1]], %[[INDEX_SIZE_OFFSET]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or %[[SGPR0_0]], %[[INDEX_SIZE_BIT]]
+
+ // CHECK: %[[V4I32_0_0:.+]] = llvm.mlir.poison : vector<4xi32>
+ // CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[SGPR0]], %[[V4I32_0_0]][%[[C0]] : i32]
+
+
+ %1 = amdgpu.make_gather_dma_base %mem[%idx], %smem[%idx] : memref<8xi32, #gpu_global_addrspace>, memref<8xi32, #gpu_lds_addrspace> -> !amdgpu.tdm_gather_base<i32,32>
+
+ func.return %0, %1 : !amdgpu.tdm_gather_base<i32,16>, !amdgpu.tdm_gather_base<i32, 32>
+}
+
+// -----
+
// CHECK-LABEL: func @make_dma_descriptor
// CHECK-SAME: (%[[BASE:.+]]: !amdgpu.tdm_base<i32>)
func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_descriptor {
>From e86509f75799b7da730563db3cdfb65a5c5bb7a6 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 09:54:32 -0500
Subject: [PATCH 03/12] Add description
---
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td | 19 ++++++++++++++++++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index 82398c2c82200..67d7f053a177b 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -1272,7 +1272,24 @@ class AMDGPU_DmaBaseOp<string mnemonic, Type outType> :
def AMDGPU_MakeGatherDmaBaseOp : AMDGPU_DmaBaseOp<"make_gather_dma_base", AMDGPU_TDMGatherBaseType> {
let summary = "Pair of based addresses used when moving tiles between LDS and global memory.";
- let description = "TODO";
+
+ let description = [{
+ This operation creates a pair of addresses that will be used by `tensor_load_to_lds`
+ and `tensor_store_from_lds`.
+
+ This operation creates a value corresponding to the tensor descriptor (D#) group 0
+ found in TensorLoadToLDSOp and TensorStoreFromLDSOp in the rocdl dialect.
+
+ Unlike `make_dma_base`, this operation returns `!amdgpu.tdm_gather_base<$element_type, $index_size>`
+ which is only compatible with `make_gather_dma_descriptor`. Using the descriptor returned
+ by `make_gather_dma_descriptor` will set the `tensor_load_to_lds` and `tensor_store_from_lds` to gather mode.
+
+ ```mlir
+ %base = amdgpu.make_gather_dma_base %global[%idx0, %idx1], %lds[%idx2, %idx3] : memref<64x64xi32>, memref<64x64xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_gather_base<i32, 16>
+ %descriptor = amdgpu.make_gather_dma_descriptor %base globalSize [2, 2] globalStride [2, 1] sharedSize [2, 2] : !amdgpu.tdm_gather_base<i32, 16> -> !amdgpu.tdm_descriptor
+ amdgpu.tensor_load_to_lds %descriptor : !amdgpu.tdm_descriptor
+ ```
+ }];
let extraClassDeclaration = [{
constexpr int64_t isGather() {
>From 5828b4d2e9c707501e69dd2790fc773fb322ea87 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 10:22:40 -0500
Subject: [PATCH 04/12] Add verifier to make_gather_dma_base
---
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td | 2 ++
mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp | 23 ++++++++++++-------
mlir/test/Dialect/AMDGPU/invalid.mlir | 14 +++++++++++
3 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index 67d7f053a177b..fd4cc603f7d5b 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -1291,6 +1291,8 @@ def AMDGPU_MakeGatherDmaBaseOp : AMDGPU_DmaBaseOp<"make_gather_dma_base", AMDGPU
```
}];
+ let hasVerifier = 1;
+
let extraClassDeclaration = [{
constexpr int64_t isGather() {
return true;
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index b7a665b0f5367..db3b224da8cd5 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -755,28 +755,35 @@ LogicalResult TransposeLoadOp::verify() {
// MakeDmaBaseOp
//===----------------------------------------------------------------------===//
-LogicalResult MakeDmaBaseOp::verify() {
-
- auto ldsType = cast<MemRefType>(getLds().getType());
- auto globalType = cast<MemRefType>(getGlobal().getType());
+template <typename BaseOp>
+static LogicalResult verifyBase(BaseOp op) {
+ auto ldsType = cast<MemRefType>(op.getLds().getType());
+ auto globalType = cast<MemRefType>(op.getGlobal().getType());
if (!hasWorkgroupMemorySpace(ldsType.getMemorySpace()))
- return emitOpError(
+ return op.emitOpError(
"lds memref must have workgroup address space attribute.");
if (!hasGlobalMemorySpace(globalType.getMemorySpace()))
- return emitOpError(
+ return op.emitOpError(
"global memref must have global address space attribute.");
Type elementType = ldsType.getElementType();
unsigned width = elementType.getIntOrFloatBitWidth();
if (!llvm::is_contained<unsigned>({8, 16, 32, 64}, width))
- return emitOpError(
+ return op.emitOpError(
"element type must be 1, 2, 4, or 8 bytes long but type was ")
<< width << " bits long.";
-
return success();
}
+LogicalResult MakeDmaBaseOp::verify() { return verifyBase(*this); }
+
+//===----------------------------------------------------------------------===//
+// MakeGatherDmaBaseOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult MakeGatherDmaBaseOp::verify() { return verifyBase(*this); }
+
//===----------------------------------------------------------------------===//
// MakeDmaDescriptorOp
//===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index 6308ea9a6a096..4f2ec8e7be8a5 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -371,6 +371,20 @@ func.func @make_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32,
// -----
+func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %mem: memref<8xi32>) {
+ // expected-error at +1 {{'amdgpu.make_gather_dma_base' op lds memref must have workgroup address space attribute.}}
+ amdgpu.make_gather_dma_base %mem[%idx], %mem[%idx] : memref<8xi32>, memref<8xi32> -> !amdgpu.tdm_gather_base<i32, 16>
+}
+
+// -----
+
+func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32, #gpu.address_space<workgroup>>) {
+ // expected-error at +1 {{'amdgpu.make_gather_dma_base' op global memref must have global address space attribute.}}
+ amdgpu.make_gather_dma_base %smem[%idx], %smem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_gather_base<i32, 16>
+}
+
+// -----
+
func.func @make_dma_base_invalid_barrier(%base: !amdgpu.tdm_base<i32>, %barrier: memref<8xi32>, %idx: index) {
// expected-error at +1 {{'amdgpu.make_dma_descriptor' op atomic barrier address must be in LDS.}}
amdgpu.make_dma_descriptor %base globalSize [64, 64] globalStride [64, 1] sharedSize [64, 64] atomicBarrier(%barrier[%idx] : memref<8xi32>) : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor
>From 59c12622fe62e424ebfe2427fc230f26e65f880d Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 10:31:45 -0500
Subject: [PATCH 05/12] Change indexSize to indexType
---
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td | 10 +++++-----
mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp | 3 ++-
mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir | 8 ++++----
mlir/test/Dialect/AMDGPU/invalid.mlir | 4 ++--
4 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index fd4cc603f7d5b..ad7f0cf158d8b 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -119,13 +119,13 @@ def AMDGPU_TDMGatherBaseType : AMDGPU_Type<"TDMGatherBase", "tdm_gather_base"> {
The value defined by this operation is only intended to be used by
amdgpu.tdm_make_gather_descriptor.
}];
- let parameters = (ins "Type":$elementType, "unsigned":$indexSize);
+ let parameters = (ins "Type":$elementType, "Type":$indexType);
let builders = [
- TypeBuilderWithInferredContext<(ins "Type":$elementType, "unsigned": $indexSize), [{
- return $_get(elementType.getContext(), elementType, indexSize);
+ TypeBuilderWithInferredContext<(ins "Type":$elementType, "Type": $indexType), [{
+ return $_get(elementType.getContext(), elementType, indexType);
}]>
];
- let assemblyFormat = "`<` $elementType `,` $indexSize `>`";
+ let assemblyFormat = "`<` $elementType `,` $indexType`>`";
}
def AMDGPU_TDMDescriptorType : AMDGPU_Type<"TDMDescriptor", "tdm_descriptor"> {
@@ -1280,7 +1280,7 @@ def AMDGPU_MakeGatherDmaBaseOp : AMDGPU_DmaBaseOp<"make_gather_dma_base", AMDGPU
This operation creates a value corresponding to the tensor descriptor (D#) group 0
found in TensorLoadToLDSOp and TensorStoreFromLDSOp in the rocdl dialect.
- Unlike `make_dma_base`, this operation returns `!amdgpu.tdm_gather_base<$element_type, $index_size>`
+ Unlike `make_dma_base`, this operation returns `!amdgpu.tdm_gather_base<$element_type, $index_type>`
which is only compatible with `make_gather_dma_descriptor`. Using the descriptor returned
by `make_gather_dma_descriptor` will set the `tensor_load_to_lds` and `tensor_store_from_lds` to gather mode.
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index ff4798961ef73..6497b970285f3 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -2325,7 +2325,8 @@ struct AMDGPUMakeDmaBaseLowering : public ConvertOpToLLVMPattern<BaseOp> {
sgprs[0] = setValueAtOffset(rewriter, loc, sgprs[0], consts[1], 30);
auto type = cast<TDMGatherBaseType>(op.getResult().getType());
- unsigned indexSize = type.getIndexSize();
+ Type indexType = type.getIndexType();
+ unsigned indexSize = indexType.getIntOrFloatBitWidth();
assert(llvm::is_contained<unsigned>({16, 32}, indexSize) &&
"expected index_size to be 16 or 32");
unsigned idx = (indexSize / 16) - 1;
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
index 4cbfec52cf0a7..5dbd38571f630 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
@@ -243,7 +243,7 @@ func.func @make_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>
// CHECK-LABEL: func @make_gather_dma_base
// CHECK-SAME: (%[[IDX:.+]]: index, %[[MEM:.+]]: memref<8xi32, 1>, %[[SMEM:.+]]: memref<8xi32, 3>)
-func.func @make_gather_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>, %smem: memref<8xi32,#gpu_lds_addrspace>) -> (!amdgpu.tdm_gather_base<i32,16>, !amdgpu.tdm_gather_base<i32, 32>) {
+func.func @make_gather_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>, %smem: memref<8xi32,#gpu_lds_addrspace>) -> (!amdgpu.tdm_gather_base<i32, i16>, !amdgpu.tdm_gather_base<i32, i32>) {
// CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(0 : i32) : i32
// CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32
@@ -257,7 +257,7 @@ func.func @make_gather_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_add
// CHECK: %[[V4I32_0_0:.+]] = llvm.mlir.poison : vector<4xi32>
// CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[SGPR0]], %[[V4I32_0_0]][%[[C0]] : i32]
- %0 = amdgpu.make_gather_dma_base %mem[%idx], %smem[%idx] : memref<8xi32, #gpu_global_addrspace>, memref<8xi32, #gpu_lds_addrspace> -> !amdgpu.tdm_gather_base<i32,16>
+ %0 = amdgpu.make_gather_dma_base %mem[%idx], %smem[%idx] : memref<8xi32, #gpu_global_addrspace>, memref<8xi32, #gpu_lds_addrspace> -> !amdgpu.tdm_gather_base<i32, i16>
// CHECK-DAG: %[[C0:.+]] = llvm.mlir.constant(0 : i32) : i32
// CHECK-DAG: %[[C1:.+]] = llvm.mlir.constant(1 : i32) : i32
@@ -276,9 +276,9 @@ func.func @make_gather_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_add
// CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[SGPR0]], %[[V4I32_0_0]][%[[C0]] : i32]
- %1 = amdgpu.make_gather_dma_base %mem[%idx], %smem[%idx] : memref<8xi32, #gpu_global_addrspace>, memref<8xi32, #gpu_lds_addrspace> -> !amdgpu.tdm_gather_base<i32,32>
+ %1 = amdgpu.make_gather_dma_base %mem[%idx], %smem[%idx] : memref<8xi32, #gpu_global_addrspace>, memref<8xi32, #gpu_lds_addrspace> -> !amdgpu.tdm_gather_base<i32, i32>
- func.return %0, %1 : !amdgpu.tdm_gather_base<i32,16>, !amdgpu.tdm_gather_base<i32, 32>
+ func.return %0, %1 : !amdgpu.tdm_gather_base<i32,i16>, !amdgpu.tdm_gather_base<i32,i32>
}
// -----
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index 4f2ec8e7be8a5..2f6906cc64ee7 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -373,14 +373,14 @@ func.func @make_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32,
func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %mem: memref<8xi32>) {
// expected-error at +1 {{'amdgpu.make_gather_dma_base' op lds memref must have workgroup address space attribute.}}
- amdgpu.make_gather_dma_base %mem[%idx], %mem[%idx] : memref<8xi32>, memref<8xi32> -> !amdgpu.tdm_gather_base<i32, 16>
+ amdgpu.make_gather_dma_base %mem[%idx], %mem[%idx] : memref<8xi32>, memref<8xi32> -> !amdgpu.tdm_gather_base<i32, i16>
}
// -----
func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32, #gpu.address_space<workgroup>>) {
// expected-error at +1 {{'amdgpu.make_gather_dma_base' op global memref must have global address space attribute.}}
- amdgpu.make_gather_dma_base %smem[%idx], %smem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_gather_base<i32, 16>
+ amdgpu.make_gather_dma_base %smem[%idx], %smem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_gather_base<i32, i16>
}
// -----
>From d80d2dfbc8c083055e3512c201ddb0ea8c365383 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 10:47:37 -0500
Subject: [PATCH 06/12] Add verification to TDMGatherBaseType
---
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td | 1 +
mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp | 17 +++++++++++++++++
mlir/test/Dialect/AMDGPU/invalid.mlir | 7 +++++++
3 files changed, 25 insertions(+)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index ad7f0cf158d8b..d479976d16ad5 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -126,6 +126,7 @@ def AMDGPU_TDMGatherBaseType : AMDGPU_Type<"TDMGatherBase", "tdm_gather_base"> {
}]>
];
let assemblyFormat = "`<` $elementType `,` $indexType`>`";
+ let genVerifyDecl = 1;
}
def AMDGPU_TDMDescriptorType : AMDGPU_Type<"TDMDescriptor", "tdm_descriptor"> {
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index db3b224da8cd5..00482ec9d01dd 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -782,6 +782,23 @@ LogicalResult MakeDmaBaseOp::verify() { return verifyBase(*this); }
// MakeGatherDmaBaseOp
//===----------------------------------------------------------------------===//
+LogicalResult
+TDMGatherBaseType::verify(function_ref<InFlightDiagnostic()> emitError,
+ Type elementType, Type indexType) {
+ unsigned width = elementType.getIntOrFloatBitWidth();
+ if (!llvm::is_contained<unsigned>({8, 16, 32, 64}, width))
+ return emitError()
+ << "element type must be 1, 2, 4, or 8 bytes wide but type "
+ << elementType << " is " << width / 8 << " bytes wide.";
+ MLIRContext *ctx = elementType.getContext();
+ Type i16 = IntegerType::get(ctx, 32);
+ Type i32 = IntegerType::get(ctx, 16);
+ if (!llvm::is_contained<Type>({i16, i32}, indexType))
+ return emitError() << "index type must be i16 or i32 but index type is "
+ << indexType << ".";
+ return success();
+}
+
LogicalResult MakeGatherDmaBaseOp::verify() { return verifyBase(*this); }
//===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index 2f6906cc64ee7..a2d60c9cedea1 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -378,6 +378,13 @@ func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %mem: memref<8
// -----
+func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %mem: memref<8xi32>) {
+ // expected-error at +1 {{index type must be i16 or i32 but index type is 'i64'.}}
+ amdgpu.make_gather_dma_base %mem[%idx], %mem[%idx] : memref<8xi32>, memref<8xi32> -> !amdgpu.tdm_gather_base<i32, i64>
+}
+
+// -----
+
func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32, #gpu.address_space<workgroup>>) {
// expected-error at +1 {{'amdgpu.make_gather_dma_base' op global memref must have global address space attribute.}}
amdgpu.make_gather_dma_base %smem[%idx], %smem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_gather_base<i32, i16>
>From f810a416284b364c92be1d9b805a410659a00f33 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 10:49:33 -0500
Subject: [PATCH 07/12] documentation
---
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index d479976d16ad5..e48614beb542b 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -1286,8 +1286,9 @@ def AMDGPU_MakeGatherDmaBaseOp : AMDGPU_DmaBaseOp<"make_gather_dma_base", AMDGPU
by `make_gather_dma_descriptor` will set the `tensor_load_to_lds` and `tensor_store_from_lds` to gather mode.
```mlir
- %base = amdgpu.make_gather_dma_base %global[%idx0, %idx1], %lds[%idx2, %idx3] : memref<64x64xi32>, memref<64x64xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_gather_base<i32, 16>
- %descriptor = amdgpu.make_gather_dma_descriptor %base globalSize [2, 2] globalStride [2, 1] sharedSize [2, 2] : !amdgpu.tdm_gather_base<i32, 16> -> !amdgpu.tdm_descriptor
+ %base = amdgpu.make_gather_dma_base %global[%idx0, %idx1], %lds[%idx2, %idx3] : memref<64x64xi32>, memref<64x64xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_gather_base<i32, i16>
+ // %indices : i16
+ %descriptor = amdgpu.make_gather_dma_descriptor %base[%indices] globalSize [2, 2] globalStride [2, 1] sharedSize [2, 2] : !amdgpu.tdm_gather_base<i32, i16>, i16 -> !amdgpu.tdm_descriptor
amdgpu.tensor_load_to_lds %descriptor : !amdgpu.tdm_descriptor
```
}];
>From c34761b548754a76e2e51dae17998be4b327a6d0 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 10:53:50 -0500
Subject: [PATCH 08/12] Change test name
---
mlir/test/Dialect/AMDGPU/invalid.mlir | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index a2d60c9cedea1..a816553fbf951 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -378,7 +378,7 @@ func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %mem: memref<8
// -----
-func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %mem: memref<8xi32>) {
+func.func @make_gather_dma_base_invalid_index_type(%idx: index, %mem: memref<8xi32>) {
// expected-error at +1 {{index type must be i16 or i32 but index type is 'i64'.}}
amdgpu.make_gather_dma_base %mem[%idx], %mem[%idx] : memref<8xi32>, memref<8xi32> -> !amdgpu.tdm_gather_base<i32, i64>
}
>From b3292986df53a1e6915a00e82f9c3c6bc953d01d Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 10:57:49 -0500
Subject: [PATCH 09/12] Correct tests
---
mlir/test/Dialect/AMDGPU/invalid.mlir | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index a816553fbf951..d3f0f43d039ae 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -367,6 +367,7 @@ func.func @make_dma_base_invalid_addressspace(%idx: index, %mem: memref<8xi32>)
func.func @make_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32, #gpu.address_space<workgroup>>) {
// expected-error at +1 {{'amdgpu.make_dma_base' op global memref must have global address space attribute.}}
amdgpu.make_dma_base %smem[%idx], %smem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>
+ return
}
// -----
@@ -374,13 +375,15 @@ func.func @make_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32,
func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %mem: memref<8xi32>) {
// expected-error at +1 {{'amdgpu.make_gather_dma_base' op lds memref must have workgroup address space attribute.}}
amdgpu.make_gather_dma_base %mem[%idx], %mem[%idx] : memref<8xi32>, memref<8xi32> -> !amdgpu.tdm_gather_base<i32, i16>
+ return
}
// -----
-func.func @make_gather_dma_base_invalid_index_type(%idx: index, %mem: memref<8xi32>) {
+func.func @make_gather_dma_base_invalid_index_type(%idx: index, %smem: memref<8xi32, #gpu.address_space<workgroup>>, %mem: memref<8xi32>) {
// expected-error at +1 {{index type must be i16 or i32 but index type is 'i64'.}}
- amdgpu.make_gather_dma_base %mem[%idx], %mem[%idx] : memref<8xi32>, memref<8xi32> -> !amdgpu.tdm_gather_base<i32, i64>
+ amdgpu.make_gather_dma_base %smem[%idx], %mem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32> -> !amdgpu.tdm_gather_base<i32, i64>
+ return
}
// -----
@@ -388,6 +391,7 @@ func.func @make_gather_dma_base_invalid_index_type(%idx: index, %mem: memref<8xi
func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %smem : memref<8xi32, #gpu.address_space<workgroup>>) {
// expected-error at +1 {{'amdgpu.make_gather_dma_base' op global memref must have global address space attribute.}}
amdgpu.make_gather_dma_base %smem[%idx], %smem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_gather_base<i32, i16>
+ return
}
// -----
@@ -395,6 +399,7 @@ func.func @make_gather_dma_base_invalid_addressspace(%idx: index, %smem : memref
func.func @make_dma_base_invalid_barrier(%base: !amdgpu.tdm_base<i32>, %barrier: memref<8xi32>, %idx: index) {
// expected-error at +1 {{'amdgpu.make_dma_descriptor' op atomic barrier address must be in LDS.}}
amdgpu.make_dma_descriptor %base globalSize [64, 64] globalStride [64, 1] sharedSize [64, 64] atomicBarrier(%barrier[%idx] : memref<8xi32>) : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor
+ return
}
// -----
>From 23a3b45fbf7e9e0d8d6b2458d8a286224184ee68 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 12:05:59 -0500
Subject: [PATCH 10/12] no type parameter when using llvm::is_contained
---
mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp | 11 +++++------
mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp | 8 ++++----
2 files changed, 9 insertions(+), 10 deletions(-)
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 6497b970285f3..c8ad5b3eae18a 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -1644,7 +1644,7 @@ int32_t getScaleSel(int32_t blockSize, unsigned bitWidth, int32_t scaleWaveHalf,
// those values are merged together. (Note: scaleWaveHalf isn't a high-level
// attribute but is derifed from firstScaleLane).
assert(llvm::is_contained({16, 32}, blockSize));
- assert(llvm::is_contained(llvm::ArrayRef<unsigned>{4, 6, 8}, bitWidth));
+ assert(llvm::is_contained({4u, 6u, 8u}, bitWidth));
const bool isFp8 = bitWidth == 8;
const bool isBlock16 = blockSize == 16;
@@ -2327,12 +2327,12 @@ struct AMDGPUMakeDmaBaseLowering : public ConvertOpToLLVMPattern<BaseOp> {
auto type = cast<TDMGatherBaseType>(op.getResult().getType());
Type indexType = type.getIndexType();
unsigned indexSize = indexType.getIntOrFloatBitWidth();
- assert(llvm::is_contained<unsigned>({16, 32}, indexSize) &&
+ assert(llvm::is_contained({16u, 32u}, indexSize) &&
"expected index_size to be 16 or 32");
unsigned idx = (indexSize / 16) - 1;
if (idx)
- sgprs[0] = setValueAtOffset(rewriter, loc, sgprs[0], consts[idx], 31);
+ sgprs[0] = setValueAtOffset(rewriter, loc, sgprs[0], consts[1], 31);
}
ValueRange ldsIndices = adaptor.getLdsIndices();
@@ -2410,9 +2410,8 @@ struct AMDGPUMakeDmaDescriptorLowering
Value sgpr0, ArrayRef<Value> consts) const {
// Compute data_size.
unsigned elementTypeWidthInBits = op.getElementTypeWidth();
- assert(
- llvm::is_contained<unsigned>({8, 16, 32, 64}, elementTypeWidthInBits) &&
- "expected type width to be 8, 16, 32, or 64.");
+ assert(llvm::is_contained({8u, 16u, 32u, 64u}, elementTypeWidthInBits) &&
+ "expected type width to be 8, 16, 32, or 64.");
int64_t dataSize = llvm::Log2_32(elementTypeWidthInBits / 8);
Value size = createI32Constant(rewriter, loc, dataSize);
return setValueAtOffset(rewriter, loc, sgpr0, size, 16);
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index 00482ec9d01dd..241c8841d65a4 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -769,7 +769,7 @@ static LogicalResult verifyBase(BaseOp op) {
Type elementType = ldsType.getElementType();
unsigned width = elementType.getIntOrFloatBitWidth();
- if (!llvm::is_contained<unsigned>({8, 16, 32, 64}, width))
+ if (!llvm::is_contained({8u, 16u, 32u, 64u}, width))
return op.emitOpError(
"element type must be 1, 2, 4, or 8 bytes long but type was ")
<< width << " bits long.";
@@ -786,14 +786,14 @@ LogicalResult
TDMGatherBaseType::verify(function_ref<InFlightDiagnostic()> emitError,
Type elementType, Type indexType) {
unsigned width = elementType.getIntOrFloatBitWidth();
- if (!llvm::is_contained<unsigned>({8, 16, 32, 64}, width))
+ if (!llvm::is_contained({8u, 16u, 32u, 64u}, width))
return emitError()
<< "element type must be 1, 2, 4, or 8 bytes wide but type "
<< elementType << " is " << width / 8 << " bytes wide.";
MLIRContext *ctx = elementType.getContext();
Type i16 = IntegerType::get(ctx, 32);
Type i32 = IntegerType::get(ctx, 16);
- if (!llvm::is_contained<Type>({i16, i32}, indexType))
+ if (!llvm::is_contained({i16, i32}, indexType))
return emitError() << "index type must be i16 or i32 but index type is "
<< indexType << ".";
return success();
@@ -825,7 +825,7 @@ LogicalResult MakeDmaDescriptorOp::verify() {
return emitOpError("tensor must have same rank as tile.");
unsigned elementTypeWidth = getElementTypeWidth();
- if (!llvm::is_contained<unsigned>({8, 16, 32, 64}, elementTypeWidth))
+ if (!llvm::is_contained({8u, 16u, 32u, 64u}, elementTypeWidth))
return emitOpError(
"element type width must be 1, 2, 4 or 8 bytes, but was ")
<< elementTypeWidth << " bits long";
>From 9cbf8d8883232bc571b6e987eca6b235c6f48854 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 15:58:04 -0500
Subject: [PATCH 11/12] change int64_t to bool
---
mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index a3a52958f5a30..2a6ced8711cd4 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -1301,7 +1301,7 @@ def AMDGPU_MakeGatherDmaBaseOp : AMDGPU_DmaBaseOp<"make_gather_dma_base", AMDGPU
let hasVerifier = 1;
let extraClassDeclaration = [{
- constexpr int64_t isGather() {
+ constexpr bool isGather() {
return true;
}
}];
@@ -1352,7 +1352,7 @@ def AMDGPU_MakeDmaBaseOp : AMDGPU_DmaBaseOp<"make_dma_base", AMDGPU_TDMBaseType>
let hasVerifier = 1;
let extraClassDeclaration = [{
- constexpr int64_t isGather() {
+ constexpr bool isGather() {
return false;
}
}];
>From f6cbb710a0ed06a8c955722f723ee1112d1b3606 Mon Sep 17 00:00:00 2001
From: Erick Ochoa <erick.ochoalopez at amd.com>
Date: Thu, 11 Dec 2025 16:15:08 -0500
Subject: [PATCH 12/12] Add disjoint to llvm.or
---
.../AMDGPUToROCDL/AMDGPUToROCDL.cpp | 3 +-
.../Conversion/AMDGPUToROCDL/gfx1250.mlir | 52 +++++++++----------
2 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 21108b269248f..455e59c4a272a 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -2288,7 +2288,8 @@ static Value setValueAtOffset(ConversionPatternRewriter &rewriter, Location loc,
if (matchPattern(accumulator, mlir::m_Zero()))
return value;
- return LLVM::OrOp::create(rewriter, loc, accumulator, value);
+ constexpr bool isDisjoint = true;
+ return LLVM::OrOp::create(rewriter, loc, accumulator, value, isDisjoint);
}
template <typename BaseOp>
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
index e3e07a4dd5ef9..58014e52fa191 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/gfx1250.mlir
@@ -223,7 +223,7 @@ func.func @make_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_addrspace>
// CHECK: %[[SHIFT:.+]] = llvm.mlir.constant(30 : i32)
// CHECK: %[[TYPE_FIELD:.+]] = llvm.shl %[[C2]], %[[SHIFT]]
- // CHECK: %[[MEM_INT_HIGH_TYPE:.+]] = llvm.or %[[VALID_MEM_INT_HIGH]], %[[TYPE_FIELD]]
+ // CHECK: %[[MEM_INT_HIGH_TYPE:.+]] = llvm.or disjoint %[[VALID_MEM_INT_HIGH]], %[[TYPE_FIELD]]
// CHECK: %[[V4I32_0_0:.+]] = llvm.mlir.poison : vector<4xi32>
// CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[C1]], %[[V4I32_0_0]][%[[C0]] : i32]
@@ -252,7 +252,7 @@ func.func @make_gather_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_add
// CHECK-DAG: %[[GATHER_MODE_OFFSET:.+]] = llvm.mlir.constant(30 : i32) : i32
// CHECK-DAG: %[[GATHER_MODE_BIT:.+]] = llvm.shl %[[C1]], %[[GATHER_MODE_OFFSET]]
- // CHECK: %[[SGPR0:.+]] = llvm.or %[[C1]], %[[GATHER_MODE_BIT]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or disjoint %[[C1]], %[[GATHER_MODE_BIT]]
// CHECK: %[[V4I32_0_0:.+]] = llvm.mlir.poison : vector<4xi32>
// CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[SGPR0]], %[[V4I32_0_0]][%[[C0]] : i32]
@@ -266,11 +266,11 @@ func.func @make_gather_dma_base(%idx: index, %mem: memref<8xi32, #gpu_global_add
// CHECK-DAG: %[[GATHER_MODE_OFFSET:.+]] = llvm.mlir.constant(30 : i32) : i32
// CHECK-DAG: %[[GATHER_MODE_BIT:.+]] = llvm.shl %[[C1]], %[[GATHER_MODE_OFFSET]]
- // CHECK: %[[SGPR0_0:.+]] = llvm.or %[[C1]], %[[GATHER_MODE_BIT]]
+ // CHECK: %[[SGPR0_0:.+]] = llvm.or disjoint %[[C1]], %[[GATHER_MODE_BIT]]
// CHECK-DAG: %[[INDEX_SIZE_OFFSET:.+]] = llvm.mlir.constant(31 : i32) : i32
// CHECK-DAG: %[[INDEX_SIZE_BIT:.+]] = llvm.shl %[[C1]], %[[INDEX_SIZE_OFFSET]]
- // CHECK: %[[SGPR0:.+]] = llvm.or %[[SGPR0_0]], %[[INDEX_SIZE_BIT]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or disjoint %[[SGPR0_0]], %[[INDEX_SIZE_BIT]]
// CHECK: %[[V4I32_0_0:.+]] = llvm.mlir.poison : vector<4xi32>
// CHECK: %[[V4I32_0_1:.+]] = llvm.insertelement %[[SGPR0]], %[[V4I32_0_0]][%[[C0]] : i32]
@@ -313,7 +313,7 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TENSOR_DIM_1:.+]] = llvm.mlir.constant(128 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[TENSOR_DIM_1_SHIFTED:.+]] = llvm.shl %[[TENSOR_DIM_1]], %[[C16]]
- // CHECK: %[[SGPR2:.+]] = llvm.or %[[SGPR2_0]], %[[TENSOR_DIM_1_SHIFTED]]
+ // CHECK: %[[SGPR2:.+]] = llvm.or disjoint %[[SGPR2_0]], %[[TENSOR_DIM_1_SHIFTED]]
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[SGPR3_0:.+]] = llvm.lshr %[[TENSOR_DIM_1]], %[[C16]]
@@ -321,7 +321,7 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TILE_DIM_0:.+]] = llvm.mlir.constant(64 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[TILE_DIM_0_SHIFTED:.+]] = llvm.shl %[[TILE_DIM_0:.+]], %[[C16]]
- // CHECK: %[[SGPR3:.+]] = llvm.or %[[SGPR3_0]], %[[TILE_DIM_0_SHIFTED]]
+ // CHECK: %[[SGPR3:.+]] = llvm.or disjoint %[[SGPR3_0]], %[[TILE_DIM_0_SHIFTED]]
// CHECK-DAG: %[[SGPR4:.+]] = llvm.mlir.constant(128 : i32)
@@ -340,7 +340,7 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TENSOR_DIM_1_STRIDE_LOW:.+]] = llvm.trunc %[[TENSOR_DIM_1_STRIDE_MASKED]]
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i32) : i32
// CHECK: %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED:.+]] = llvm.shl %[[TENSOR_DIM_1_STRIDE_LOW]], %[[SHIFT]]
- // CHECK-DAG: %[[SGPR6:.+]] = llvm.or %[[SGPR6_0]], %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED]]
+ // CHECK-DAG: %[[SGPR6:.+]] = llvm.or disjoint %[[SGPR6_0]], %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED]]
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i64) : i64
// CHECK: %[[TENSOR_DIM_1_STRIDE_SHIFTED:.+]] = llvm.lshr %[[TENSOR_DIM_1_STRIDE_MASKED]], %[[SHIFT]]
@@ -391,7 +391,7 @@ func.func @make_dma_descriptor_atomic_barrier(%base: !amdgpu.tdm_base<i32>, %bar
// CHECK-DAG: %[[ATOMIC_BARRIER_ENABLE_OFFSET:.+]] = llvm.mlir.constant(18 : i32)
// CHECK: %[[ATOMIC_BARRIER_ENABLE_FIELD:.+]] = llvm.shl %[[C1]], %[[ATOMIC_BARRIER_ENABLE_OFFSET]]
- // CHECK: %[[SGPR0:.+]] = llvm.or %[[SGPR0_0]], %[[ATOMIC_BARRIER_ENABLE_FIELD]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or disjoint %[[SGPR0_0]], %[[ATOMIC_BARRIER_ENABLE_FIELD]]
// CHECK: %[[ATOMIC_BARRIER_ALIGNED_PTR:.+]] = llvm.extractvalue %[[BARRIER_MEMREF_DESC]][1]
// CHECK: %[[ATOMIC_BARRIER_ADDR:.+]] = llvm.getelementptr %[[ATOMIC_BARRIER_ALIGNED_PTR]][%[[INDEX]]
@@ -403,7 +403,7 @@ func.func @make_dma_descriptor_atomic_barrier(%base: !amdgpu.tdm_base<i32>, %bar
// CHECK-DAG: %[[TENSOR_DIM_0:.+]] = llvm.mlir.constant(64 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[SGPR1_0:.+]] = llvm.shl %[[TENSOR_DIM_0]], %[[C16]]
- // CHECK: %[[SGPR1:.+]] = llvm.or %[[ATOMIC_BARRIER]], %[[SGPR1_0]]
+ // CHECK: %[[SGPR1:.+]] = llvm.or disjoint %[[ATOMIC_BARRIER]], %[[SGPR1_0]]
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[SGPR2_0:.+]] = llvm.lshr %[[TENSOR_DIM_0]], %[[C16]]
@@ -439,7 +439,7 @@ func.func @make_dma_descriptor_iterate(%base: !amdgpu.tdm_base<i32>, %idx : inde
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(19 : i32)
// CHECK: %[[ITERATE_ENABLE:.+]] = llvm.shl %[[C1]], %[[SHIFT]]
- // CHECK: %[[SGPR0:.+]] = llvm.or %[[SGPR0_0]], %[[ITERATE_ENABLE]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or disjoint %[[SGPR0_0]], %[[ITERATE_ENABLE]]
// CHECK: %[[V8I32:.+]] = llvm.mlir.poison : vector<8xi32>
// CHECK: %[[DGROUP1_0:.+]] = llvm.insertelement %[[SGPR0]], %[[V8I32]][%[[C0]] : i32]
@@ -456,7 +456,7 @@ func.func @make_dma_descriptor_iterate(%base: !amdgpu.tdm_base<i32>, %idx : inde
// CHECK: %[[ITERATE_COUNT_M1:.+]] = llvm.sub %[[ITERATE_COUNT]], %[[C1]]
// CHECK: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[ITERATE_COUNT_SHIFTED:.+]] = llvm.shl %[[ITERATE_COUNT_M1]], %[[SHIFT]]
- // CHECK: %[[SGPR3:.+]] = llvm.or %[[SGPR3_LOW]], %[[ITERATE_COUNT_SHIFTED]]
+ // CHECK: %[[SGPR3:.+]] = llvm.or disjoint %[[SGPR3_LOW]], %[[ITERATE_COUNT_SHIFTED]]
// CHECK: %[[V4I32:.+]] = llvm.mlir.poison : vector<4xi32>
// CHECK: %[[DGROUP2_0:.+]] = llvm.insertelement %[[C0]], %[[V4I32]][%[[C0]]
@@ -487,18 +487,18 @@ func.func @make_dma_descriptor_pad_enable(%base: !amdgpu.tdm_base<i32>, %pad_amo
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(20 : i32)
// CHECK: %[[PAD_ENABLE:.+]] = llvm.shl %[[C1]], %[[SHIFT]]
- // CHECK: %[[SGPR0:.+]] = llvm.or %[[SGPR0_BASE:.+]], %[[PAD_ENABLE]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or disjoint %[[SGPR0_BASE:.+]], %[[PAD_ENABLE]]
// CHECK: %[[PAD_INTERVAL_CTTZ:.+]] = "llvm.intr.cttz"(%[[PAD_INTERVAL]]) <{is_zero_poison = false}> : (i32) -> i32
// CHECK: %[[PAD_INTERVAL_M1:.+]] = llvm.sub %[[PAD_INTERVAL_CTTZ]], %[[C1]]
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(22 : i32)
// CHECK: %[[PAD_INTERVAL:.+]] = llvm.shl %[[PAD_INTERVAL_M1]], %[[SHIFT]]
- // CHECK: %[[SGPR0:.+]] = llvm.or %[[SGPR0_BASE:.+]], %[[PAD_INTERVAL]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or disjoint %[[SGPR0_BASE:.+]], %[[PAD_INTERVAL]]
// CHECK: %[[PAD_AMOUNT_M1:.+]] = llvm.sub %[[PAD_AMOUNT]], %[[C1]]
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(25 : i32)
// CHECK: %[[PAD_AMOUNT_SHIFTED:.+]] = llvm.shl %[[PAD_AMOUNT_M1]], %[[SHIFT]]
- // CHECK: llvm.or %[[SGPR0:.+]], %[[PAD_AMOUNT_SHIFTED]]
+ // CHECK: llvm.or disjoint %[[SGPR0:.+]], %[[PAD_AMOUNT_SHIFTED]]
%descriptor = amdgpu.make_dma_descriptor %base globalSize [128, 64] globalStride [64, 1] sharedSize [128, 64] padShared(%pad_amount every %pad_interval) : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor
func.return %descriptor : !amdgpu.tdm_descriptor
@@ -583,7 +583,7 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TENSOR_DIM_1:.+]] = llvm.mlir.constant(128 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[TENSOR_DIM_1_SHIFTED:.+]] = llvm.shl %[[TENSOR_DIM_1]], %[[C16]]
- // CHECK: %[[SGPR2:.+]] = llvm.or %[[SGPR2_0]], %[[TENSOR_DIM_1_SHIFTED]]
+ // CHECK: %[[SGPR2:.+]] = llvm.or disjoint %[[SGPR2_0]], %[[TENSOR_DIM_1_SHIFTED]]
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[SGPR3_0:.+]] = llvm.lshr %[[TENSOR_DIM_1]], %[[C16]]
@@ -591,13 +591,13 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TILE_DIM_0:.+]] = llvm.mlir.constant(64 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[TILE_DIM_0_SHIFTED:.+]] = llvm.shl %[[TILE_DIM_0]], %[[C16]]
- // CHECK: %[[SGPR3:.+]] = llvm.or %[[SGPR3_0]], %[[TILE_DIM_0_SHIFTED]]
+ // CHECK: %[[SGPR3:.+]] = llvm.or disjoint %[[SGPR3_0]], %[[TILE_DIM_0_SHIFTED]]
// CHECK-DAG: %[[TILE_DIM_1:.+]] = llvm.mlir.constant(128 : i32)
// CHECK-DAG: %[[TILE_DIM_2:.+]] = llvm.mlir.constant(64 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[TILE_DIM_2_SHIFTED:.+]] = llvm.shl %[[TILE_DIM_2]], %[[C16]]
- // CHECK: %[[SGPR4:.+]] = llvm.or %[[TILE_DIM_1]], %[[TILE_DIM_2_SHIFTED]]
+ // CHECK: %[[SGPR4:.+]] = llvm.or disjoint %[[TILE_DIM_1]], %[[TILE_DIM_2_SHIFTED]]
// CHECK-DAG: %[[TENSOR_DIM_0_STRIDE:.+]] = llvm.mlir.constant(1 : i64) : i64
// CHECK-DAG: %[[MASK:.+]] = llvm.mlir.constant(281474976710655 : i64) : i64
@@ -613,7 +613,7 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TENSOR_DIM_1_STRIDE_LOW:.+]] = llvm.trunc %[[TENSOR_DIM_1_STRIDE_MASKED]]
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i32) : i32
// CHECK: %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED:.+]] = llvm.shl %[[TENSOR_DIM_1_STRIDE_LOW]], %[[SHIFT]]
- // CHECK-DAG: %[[SGPR6:.+]] = llvm.or %[[SGPR6_0]], %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED]]
+ // CHECK-DAG: %[[SGPR6:.+]] = llvm.or disjoint %[[SGPR6_0]], %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED]]
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i64) : i64
// CHECK: %[[TENSOR_DIM_1_STRIDE_SHIFTED:.+]] = llvm.lshr %[[TENSOR_DIM_1_STRIDE_MASKED]], %[[SHIFT]]
@@ -645,7 +645,7 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TILE_DIM_3:.+]] = llvm.mlir.constant(64 : i32) : i32
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i32) : i32
// CHECK: %[[TILE_DIM_3_SHIFTED:.+]] = llvm.shl %[[TILE_DIM_3]], %[[SHIFT]]
- // CHECK: %[[SGPR3:.+]] = llvm.or %[[SGPR3_0]], %[[TILE_DIM_3_SHIFTED]]
+ // CHECK: %[[SGPR3:.+]] = llvm.or disjoint %[[SGPR3_0]], %[[TILE_DIM_3_SHIFTED]]
// CHECK-DAG: %[[V4I32:.+]] = llvm.mlir.poison : vector<4xi32>
// CHECK: %[[DGROUP2_0:.+]] = llvm.insertelement %[[SGPR0]], %[[V4I32]][%[[C0]] : i32]
@@ -664,7 +664,7 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TENSOR_DIM_4:.+]] = llvm.mlir.constant(64 : i32)
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i32)
// CHECK-DAG: %[[TENSOR_DIM_4_LOW:.+]] = llvm.shl %[[TENSOR_DIM_4]], %[[SHIFT]]
- // CHECK: %[[SGPR1:.+]] = llvm.or %[[TENSOR_DIM3_STRIDE_HIGH]], %[[TENSOR_DIM_4_LOW]]
+ // CHECK: %[[SGPR1:.+]] = llvm.or disjoint %[[TENSOR_DIM3_STRIDE_HIGH]], %[[TENSOR_DIM_4_LOW]]
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[SGPR2_0:.+]] = llvm.lshr %[[TENSOR_DIM_4]], %[[SHIFT]]
@@ -672,7 +672,7 @@ func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>) -> !amdgpu.tdm_desc
// CHECK-DAG: %[[TILE_DIM_4:.+]] = llvm.mlir.constant(64 : i32) : i32
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i32) : i32
// CHECK: %[[TILE_DIM_4_SHIFTED:.+]] = llvm.shl %[[TILE_DIM_4]], %[[SHIFT]]
- // CHECK: %[[SGPR2:.+]] = llvm.or %[[SGPR2_0]], %[[TILE_DIM_4_SHIFTED]]
+ // CHECK: %[[SGPR2:.+]] = llvm.or disjoint %[[SGPR2_0]], %[[TILE_DIM_4_SHIFTED]]
// CHECK: %[[V4I32:.+]] = llvm.mlir.poison : vector<4xi32>
// CHECK: %[[DGROUP3_0:.+]] = llvm.insertelement %[[TENSOR_DIM3_STRIDE_LOW]], %[[V4I32]][%[[C0]] : i32]
@@ -705,11 +705,11 @@ func.func @make_dma_descriptor_workgroup_mask(%base: !amdgpu.tdm_base<i32>, %wg_
// CHECK-DAG: %[[WG_MASK_EXT:.+]] = llvm.zext %[[WG_MASK_CAST]]
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[DATA_SIZE_SHIFTED:.+]] = llvm.shl %[[C2]], %[[C16]]
- // CHECK: %[[SGPR0_BASE:.+]] = llvm.or %[[WG_MASK_EXT]], %[[DATA_SIZE_SHIFTED]]
+ // CHECK: %[[SGPR0_BASE:.+]] = llvm.or disjoint %[[WG_MASK_EXT]], %[[DATA_SIZE_SHIFTED]]
// CHECK-DAG: %[[C21:.+]] = llvm.mlir.constant(21 : i32)
// CHECK: %[[TIMEOUT_SHIFTED:.+]] = llvm.shl %[[C1]], %[[C21]]
- // CHECK: %[[SGPR0:.+]] = llvm.or %[[SGPR0_BASE]], %[[TIMEOUT_SHIFTED]]
+ // CHECK: %[[SGPR0:.+]] = llvm.or disjoint %[[SGPR0_BASE]], %[[TIMEOUT_SHIFTED]]
// CHECK-DAG: %[[TENSOR_DIM_0:.+]] = llvm.mlir.constant(64 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
@@ -721,7 +721,7 @@ func.func @make_dma_descriptor_workgroup_mask(%base: !amdgpu.tdm_base<i32>, %wg_
// CHECK-DAG: %[[TENSOR_DIM_1:.+]] = llvm.mlir.constant(128 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[TENSOR_DIM_1_SHIFTED:.+]] = llvm.shl %[[TENSOR_DIM_1]], %[[C16]]
- // CHECK: %[[SGPR2:.+]] = llvm.or %[[SGPR2_0]], %[[TENSOR_DIM_1_SHIFTED]]
+ // CHECK: %[[SGPR2:.+]] = llvm.or disjoint %[[SGPR2_0]], %[[TENSOR_DIM_1_SHIFTED]]
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[SGPR3_0:.+]] = llvm.lshr %[[TENSOR_DIM_1]], %[[C16]]
@@ -729,7 +729,7 @@ func.func @make_dma_descriptor_workgroup_mask(%base: !amdgpu.tdm_base<i32>, %wg_
// CHECK-DAG: %[[TILE_DIM_0:.+]] = llvm.mlir.constant(64 : i32)
// CHECK-DAG: %[[C16:.+]] = llvm.mlir.constant(16 : i32)
// CHECK: %[[TILE_DIM_0_SHIFTED:.+]] = llvm.shl %[[TILE_DIM_0:.+]], %[[C16]]
- // CHECK: %[[SGPR3:.+]] = llvm.or %[[SGPR3_0]], %[[TILE_DIM_0_SHIFTED]]
+ // CHECK: %[[SGPR3:.+]] = llvm.or disjoint %[[SGPR3_0]], %[[TILE_DIM_0_SHIFTED]]
// CHECK-DAG: %[[SGPR4:.+]] = llvm.mlir.constant(128 : i32)
@@ -749,7 +749,7 @@ func.func @make_dma_descriptor_workgroup_mask(%base: !amdgpu.tdm_base<i32>, %wg_
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i32) : i32
// CHECK: %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED:.+]] = llvm.shl %[[TENSOR_DIM_1_STRIDE_LOW]], %[[SHIFT]]
- // CHECK-DAG: %[[SGPR6:.+]] = llvm.or %[[SGPR6_0]], %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED]]
+ // CHECK-DAG: %[[SGPR6:.+]] = llvm.or disjoint %[[SGPR6_0]], %[[TENSOR_DIM_1_STRIDE_LOW_SHIFTED]]
// CHECK-DAG: %[[SHIFT:.+]] = llvm.mlir.constant(16 : i64) : i64
// CHECK: %[[TENSOR_DIM_1_STRIDE_SHIFTED:.+]] = llvm.lshr %[[TENSOR_DIM_1_STRIDE_MASKED]], %[[SHIFT]]
More information about the Mlir-commits
mailing list