[Mlir-commits] [mlir] [MLIR][NVVM] Support sp{de}compress MLIR Ops (PR #232731)
Kirill Vedernikov
llvmlistbot at llvm.org
Fri Oct 16 06:03:34 PDT 2026
https://github.com/kvederni created https://github.com/llvm/llvm-project/pull/232731
This change adds support for `spcompress` and `spdecompress` MLIR Ops.
>From 2df362221330105c3c08ee1bc34647bde2d52ff1 Mon Sep 17 00:00:00 2001
From: Kirill Vedernikov <kvedernikov at nvidia.com>
Date: Fri, 16 Oct 2026 12:05:14 +0200
Subject: [PATCH] [MLIR][NVVM] Support sp{de}compress MLIR Ops
---
mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 276 +++++++++++
mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp | 145 ++++++
.../LLVMIR/nvvm/spcompress-invalid.mlir | 80 ++++
.../Target/LLVMIR/nvvm/spcompress-sp2to4.mlir | 449 ++++++++++++++++++
.../LLVMIR/nvvm/spdecompress-invalid.mlir | 110 +++++
.../LLVMIR/nvvm/spdecompress-sp1to16.mlir | 158 ++++++
.../LLVMIR/nvvm/spdecompress-sp1to2.mlir | 369 ++++++++++++++
.../LLVMIR/nvvm/spdecompress-sp1to4.mlir | 399 ++++++++++++++++
.../LLVMIR/nvvm/spdecompress-sp1to8.mlir | 186 ++++++++
.../LLVMIR/nvvm/spdecompress-sp2to16.mlir | 156 ++++++
.../LLVMIR/nvvm/spdecompress-sp2to4.mlir | 395 +++++++++++++++
.../LLVMIR/nvvm/spdecompress-sp2to8.mlir | 184 +++++++
.../LLVMIR/nvvm/spdecompress-sp4to16.mlir | 85 ++++
.../LLVMIR/nvvm/spdecompress-sp4to8.mlir | 99 ++++
14 files changed, 3091 insertions(+)
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spcompress-invalid.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spcompress-sp2to4.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-invalid.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to16.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to2.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to4.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to8.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to16.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to4.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to8.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp4to16.mlir
create mode 100644 mlir/test/Target/LLVMIR/nvvm/spdecompress-sp4to8.mlir
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 79eda3fee335cc..230cdd18fc8eb9 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -7200,6 +7200,282 @@ def NVVM_Tcgen05MMABlockScaleDecompressBOp :
}];
}
+//===----------------------------------------------------------------------===//
+// NVVM spcompress and spdecompress Ops
+//===----------------------------------------------------------------------===//
+
+// Structured sparsity factor: spXtoY keeps X out of every Y elements.
+def SPFactor1To2 : I32EnumAttrCase<"SP1TO2", 0, "sp1to2">;
+def SPFactor1To4 : I32EnumAttrCase<"SP1TO4", 1, "sp1to4">;
+def SPFactor1To8 : I32EnumAttrCase<"SP1TO8", 2, "sp1to8">;
+def SPFactor1To16 : I32EnumAttrCase<"SP1TO16", 3, "sp1to16">;
+def SPFactor2To4 : I32EnumAttrCase<"SP2TO4", 4, "sp2to4">;
+def SPFactor2To8 : I32EnumAttrCase<"SP2TO8", 5, "sp2to8">;
+def SPFactor2To16 : I32EnumAttrCase<"SP2TO16", 6, "sp2to16">;
+def SPFactor4To8 : I32EnumAttrCase<"SP4TO8", 7, "sp4to8">;
+def SPFactor4To16 : I32EnumAttrCase<"SP4TO16", 8, "sp4to16">;
+def SPFactor : I32EnumAttr<"SPFactor", "Structured sparsity factor", [
+ SPFactor1To2,
+ SPFactor1To4,
+ SPFactor1To8,
+ SPFactor1To16,
+ SPFactor2To4,
+ SPFactor2To8,
+ SPFactor2To16,
+ SPFactor4To8,
+ SPFactor4To16,
+ ]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::NVVM";
+}
+def SPFactorAttr : EnumAttr<NVVM_Dialect, SPFactor, "sp_factor">;
+
+// Structured sparsity metadata index size
+def SPIndexSizeB2 : I32EnumAttrCase<"B2", 2, "b2">;
+def SPIndexSizeB4 : I32EnumAttrCase<"B4", 4, "b4">;
+def SPIndexSize : I32EnumAttr<"SPIndexSize",
+ "Structured sparsity metadata index size", [
+ SPIndexSizeB2,
+ SPIndexSizeB4,
+ ]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::NVVM";
+}
+def SPIndexSizeAttr : EnumAttr<NVVM_Dialect, SPIndexSize, "sp_index_size">;
+
+// Structured sparsity element size
+def SPElemSizeB8 : I32EnumAttrCase<"B8", 8, "b8">;
+def SPElemSizeB16 : I32EnumAttrCase<"B16", 16, "b16">;
+def SPElemSize : I32EnumAttr<"SPElemSize", "Structured sparsity element size", [
+ SPElemSizeB8,
+ SPElemSizeB16,
+ ]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::NVVM";
+}
+def SPElemSizeAttr : EnumAttr<NVVM_Dialect, SPElemSize, "sp_elem_size">;
+
+// Structured sparsity repetition factor
+def SPRepFactorX1 : I32EnumAttrCase<"X1", 1, "x1">;
+def SPRepFactorX2 : I32EnumAttrCase<"X2", 2, "x2">;
+def SPRepFactorX4 : I32EnumAttrCase<"X4", 4, "x4">;
+def SPRepFactorX8 : I32EnumAttrCase<"X8", 8, "x8">;
+def SPRepFactorX16 : I32EnumAttrCase<"X16", 16, "x16">;
+def SPRepFactorX32 : I32EnumAttrCase<"X32", 32, "x32">;
+def SPRepFactorX64 : I32EnumAttrCase<"X64", 64, "x64">;
+def SPRepFactor : I32EnumAttr<"SPRepFactor",
+ "Structured sparsity repetition factor",
+ [SPRepFactorX1, SPRepFactorX2, SPRepFactorX4, SPRepFactorX8,
+ SPRepFactorX16, SPRepFactorX32, SPRepFactorX64]> {
+ let genSpecializedAttr = 0;
+ let cppNamespace = "::mlir::NVVM";
+}
+def SPRepFactorAttr : EnumAttr<NVVM_Dialect, SPRepFactor, "sp_rep_factor">;
+
+// spcompress only supports the 2:4 sparsity factor.
+def SPCompressFactorAttr : ConfinedAttr<SPFactorAttr, [
+ EnumAttrIsOneOf<SPFactorAttr, [
+ SPFactor2To4,
+ ]>
+ ]>;
+
+// Register bundles are i32 or fixed-length, rank-1 vectors of i32.
+defvar SPDataType = FixedVectorOfRankAndLengthAndType<[1],
+ [2, 4, 8, 16, 32, 64, 128], [I32]>;
+defvar SPRegType = AnyTypeOf<[I32,
+ FixedVectorOfRankAndLengthAndType<[1], [2, 4, 8, 16, 32, 64], [I32]>]>;
+
+def NVVM_SPCompressOp : NVVM_Op<"spcompress",
+ [Pure, NVVMRequiresSMa<[107]>]> {
+ let summary = "Compress dense vector to structured sparse vector";
+ let description = [{
+ Compresses elements from a dense vector `data` into a structured sparse
+ vector `compressed_data`. The indices of the selected elements are
+ returned in `metadata`.
+
+ All vectors are bundles of 32-bit registers. The elements are packed into
+ these registers according to `elemSize`.
+
+ Arguments:
+ - `spDesc`: Sparsity descriptor specifying the selection operation and the
+ element data type.
+ - `data`: Input dense vector of `2 * repFactor` registers containing the
+ elements to be compressed.
+ - `compFactor`: Compression factor. Only `sp2to4` is supported.
+ - `indexSize`: Size in bits of each metadata index (`b2` or `b4`).
+ - `elemSize`: Size in bits of each element in `data` (`b8` or `b16`).
+ - `repFactor`: Repeat factor of the operation (`x1` to `x64`).
+
+ Results:
+ - `metadata`: Indices of the selected elements, stored in
+ `ceil(repFactor * indexSize / elemSize)` registers.
+ - `compressed_data`: Selected elements, stored in `repFactor` registers.
+
+ Examples:
+ ```mlir
+ // 8-bit elements, 2-bit indices, repeat factor of 1.
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, vector<2xi32> -> i32, i32
+
+ // 8-bit elements, 4-bit indices, repeat factor of 2.
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, vector<4xi32> -> i32, vector<2xi32>
+
+ // 16-bit elements, 4-bit indices, repeat factor of 8.
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x8
+ : i32, vector<16xi32> -> vector<2xi32>, vector<8xi32>
+ ```
+
+ [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-spcompress)
+ }];
+
+ let arguments = (ins
+ I32:$spDesc,
+ SPDataType:$data,
+ SPCompressFactorAttr:$compFactor,
+ SPIndexSizeAttr:$indexSize,
+ SPElemSizeAttr:$elemSize,
+ SPRepFactorAttr:$repFactor);
+
+ let results = (outs
+ SPRegType:$metadata,
+ SPRegType:$compressed_data);
+
+ let assemblyFormat = [{
+ $spDesc `,` $data
+ `comp_factor` `=` enum($compFactor)
+ `index_size` `=` enum($indexSize)
+ `elem_size` `=` enum($elemSize)
+ `rep_factor` `=` enum($repFactor)
+ attr-dict `:` type(operands) `->` type($metadata) `,` type($compressed_data)
+ }];
+
+ let extraClassDeclaration = [{
+ static NVVM::IDArgPair getIntrinsicIDAndArgs(Operation &op,
+ LLVM::ModuleTranslation &mt,
+ llvm::IRBuilderBase &builder);
+ }];
+
+ let hasVerifier = 1;
+
+ string llvmBuilder = [{
+ auto [id, args] = NVVM::SPCompressOp::getIntrinsicIDAndArgs(
+ *op, moduleTranslation, builder);
+ // The intrinsic returns the compressed data with one lane per element.
+ auto *dataTy = llvm::cast<llvm::FixedVectorType>(args[0]->getType());
+ llvm::Type *compressedTy = llvm::FixedVectorType::get(
+ dataTy->getElementType(), dataTy->getNumElements() / 2);
+ llvm::Type *metadataTy = moduleTranslation.convertType(
+ op.getMetadata().getType());
+ llvm::Type *resultTy = llvm::StructType::get(
+ builder.getContext(), {metadataTy, compressedTy});
+ llvm::CallInst *result = createIntrinsicCall(builder, id, resultTy, args);
+ $metadata = builder.CreateExtractValue(result, 0);
+ $compressed_data = builder.CreateBitCast(
+ builder.CreateExtractValue(result, 1),
+ moduleTranslation.convertType(op.getCompressedData().getType()));
+ }];
+}
+
+def NVVM_SPDecompressOp : NVVM_Op<"spdecompress",
+ [Pure, NVVMRequiresSMa<[107]>]> {
+ let summary = "Decompress structured sparse vector to dense vector";
+ let description = [{
+ Decompresses elements from a structured sparse vector `compressed_data`
+ into a dense vector `data`, using the indices stored in `metadata`. Dense
+ positions not selected by `metadata` are set to zero.
+
+ All vectors are bundles of 32-bit registers. The elements are packed into
+ these registers according to `elemSize`. For a decompression factor
+ `spXtoY`, the number of registers in each bundle is:
+ - `metadata`: `ceil(X * indexSize * repFactor / 32)`
+ - `compressed_data`: `ceil(X * elemSize * repFactor / 32)`
+ - `data`: `Y * elemSize * repFactor / 32`
+
+ The following conditions must hold:
+ - `Y * elemSize * repFactor` is between 32 and 4096.
+ - `X * elemSize` is at most 32.
+ - `indexSize` is `b2` only if `Y` is at most 4.
+
+ Arguments:
+ - `metadata`: Indices of the selected elements.
+ - `compressed_data`: Compressed vector containing the selected elements.
+ - `compFactor`: Decompression factor (`sp1to2`, `sp1to4`, `sp1to8`,
+ `sp1to16`, `sp2to4`, `sp2to8`, `sp2to16`, `sp4to8`, or `sp4to16`).
+ - `indexSize`: Size in bits of each metadata index (`b2` or `b4`).
+ - `elemSize`: Size in bits of each element (`b8` or `b16`).
+ - `repFactor`: Repeat factor of the operation (`x1` to `x64`).
+
+ Results:
+ - `data`: Output dense vector containing the decompressed elements.
+
+ Examples:
+ ```mlir
+ // 8-bit elements, 2-bit indices, repeat factor of 1.
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, i32 -> i32
+
+ // 8-bit elements, 4-bit indices, repeat factor of 2.
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<2xi32>
+
+ // 16-bit elements, 4-bit indices, repeat factor of 64.
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x64
+ : vector<16xi32>, vector<64xi32> -> vector<128xi32>
+ ```
+
+ [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-spdecompress)
+ }];
+
+ let arguments = (ins
+ SPRegType:$metadata,
+ SPRegType:$compressed_data,
+ SPFactorAttr:$compFactor,
+ SPIndexSizeAttr:$indexSize,
+ SPElemSizeAttr:$elemSize,
+ SPRepFactorAttr:$repFactor);
+
+ let results = (outs
+ AnyTypeOf<[I32, SPDataType]>:$data);
+
+ let assemblyFormat = [{
+ $metadata `,` $compressed_data
+ `comp_factor` `=` enum($compFactor)
+ `index_size` `=` enum($indexSize)
+ `elem_size` `=` enum($elemSize)
+ `rep_factor` `=` enum($repFactor)
+ attr-dict `:` type($metadata) `,` type($compressed_data) `->` type($data)
+ }];
+
+ let extraClassDeclaration = [{
+ static NVVM::IDArgPair getIntrinsicIDAndArgs(Operation &op,
+ LLVM::ModuleTranslation &mt,
+ llvm::IRBuilderBase &builder);
+ }];
+
+ let hasVerifier = 1;
+
+ string llvmBuilder = [{
+ auto [id, args] = NVVM::SPDecompressOp::getIntrinsicIDAndArgs(
+ *op, moduleTranslation, builder);
+ // The intrinsic returns the dense data with one lane per element.
+ auto *compressedTy = llvm::cast<llvm::FixedVectorType>(args[1]->getType());
+ unsigned numTgt = llvm::cast<llvm::ConstantInt>(args[3])->getZExtValue();
+ llvm::Type *dataTy = llvm::FixedVectorType::get(
+ compressedTy->getElementType(),
+ numTgt * static_cast<unsigned>($repFactor));
+ llvm::Value *logicalData = createIntrinsicCall(builder, id, dataTy, args);
+ $data = builder.CreateBitCast(
+ logicalData, moduleTranslation.convertType(op.getData().getType()));
+ }];
+}
+
def SIMTFloatType : AnyTypeOf<[F16, BF16, F32, F64,
VectorOfLengthAndType<[2], [F16, BF16, F32, F64]>]>;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index 1a300b1e7f41c7..f59957cbf08cf7 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -37,6 +37,7 @@
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Support/NVPTXAddrSpace.h"
#include "llvm/Support/raw_ostream.h"
#include <array>
@@ -7068,6 +7069,150 @@ LogicalResult Tcgen05LdRedOp::verify() {
return success();
}
+//===----------------------------------------------------------------------===//
+// spcompress and spdecompress Ops
+//===----------------------------------------------------------------------===//
+
+// Returns the number of 32-bit registers in a register bundle, which is either
+// an i32 or a vector of i32.
+static unsigned getNumSPRegisters(Type type) {
+ if (auto vecTy = dyn_cast<VectorType>(type))
+ return vecTy.getNumElements();
+ return 1;
+}
+
+// Returns the {source, target} group sizes of the sparsity factor.
+static std::pair<unsigned, unsigned> getSPFactor(NVVM::SPFactor compFactor) {
+ switch (compFactor) {
+ case NVVM::SPFactor::SP1TO2:
+ return {1, 2};
+ case NVVM::SPFactor::SP1TO4:
+ return {1, 4};
+ case NVVM::SPFactor::SP1TO8:
+ return {1, 8};
+ case NVVM::SPFactor::SP1TO16:
+ return {1, 16};
+ case NVVM::SPFactor::SP2TO4:
+ return {2, 4};
+ case NVVM::SPFactor::SP2TO8:
+ return {2, 8};
+ case NVVM::SPFactor::SP2TO16:
+ return {2, 16};
+ case NVVM::SPFactor::SP4TO8:
+ return {4, 8};
+ case NVVM::SPFactor::SP4TO16:
+ return {4, 16};
+ }
+ llvm_unreachable("Invalid SPFactor");
+}
+
+LogicalResult SPCompressOp::verify() {
+ unsigned repFactor = static_cast<unsigned>(getRepFactor());
+ unsigned indexSize = static_cast<unsigned>(getIndexSize());
+ unsigned elemSize = static_cast<unsigned>(getElemSize());
+
+ if (getNumSPRegisters(getData().getType()) != 2 * repFactor)
+ return emitError(
+ "Operation attributes and provided input data size doesn't match");
+
+ if (getNumSPRegisters(getCompressedData().getType()) != repFactor)
+ return emitError("Operation attributes and provided output compressed data "
+ "size doesn't match");
+
+ if (getNumSPRegisters(getMetadata().getType()) !=
+ llvm::divideCeil(repFactor * indexSize, elemSize))
+ return emitError(
+ "Operation attributes and provided output metadata size doesn't match");
+
+ return success();
+}
+
+LogicalResult SPDecompressOp::verify() {
+ unsigned repFactor = static_cast<unsigned>(getRepFactor());
+ unsigned indexSize = static_cast<unsigned>(getIndexSize());
+ unsigned elemSize = static_cast<unsigned>(getElemSize());
+ auto [numSrc, numTgt] = getSPFactor(getCompFactor());
+
+ unsigned dataBits = numTgt * elemSize * repFactor;
+ if (dataBits < 32 || dataBits > 4096)
+ return emitError(
+ "Invalid compFactor, element size, and repeat factor combination");
+
+ if (numSrc * elemSize > 32)
+ return emitError("Invalid compFactor and element size combination");
+
+ if (indexSize == 2 && numTgt > 4)
+ return emitError("Invalid index size and target combination, target "
+ "should be at most 4 if index size is 2");
+
+ if (getNumSPRegisters(getData().getType()) != dataBits / 32)
+ return emitError(
+ "Operation attributes and provided output data size doesn't match");
+
+ if (getNumSPRegisters(getCompressedData().getType()) !=
+ llvm::divideCeil(numSrc * elemSize * repFactor, 32))
+ return emitError("Operation attributes and provided input compressed data "
+ "size doesn't match");
+
+ if (getNumSPRegisters(getMetadata().getType()) !=
+ llvm::divideCeil(numSrc * indexSize * repFactor, 32))
+ return emitError(
+ "Operation attributes and provided input metadata size doesn't match");
+
+ return success();
+}
+
+NVVM::IDArgPair
+SPCompressOp::getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
+ llvm::IRBuilderBase &builder) {
+ auto thisOp = cast<NVVM::SPCompressOp>(op);
+ unsigned elemSize = static_cast<unsigned>(thisOp.getElemSize());
+ unsigned repFactor = static_cast<unsigned>(thisOp.getRepFactor());
+
+ // The intrinsic takes the dense data with one lane per element. The data
+ // occupies 2 * repFactor 32-bit registers.
+ llvm::Type *dataTy = llvm::FixedVectorType::get(builder.getIntNTy(elemSize),
+ 64 * repFactor / elemSize);
+ llvm::Value *data =
+ builder.CreateBitCast(mt.lookupValue(thisOp.getData()), dataTy);
+
+ llvm::SmallVector<llvm::Value *> args{
+ data,
+ mt.lookupValue(thisOp.getSpDesc()),
+ builder.getInt32(static_cast<unsigned>(thisOp.getIndexSize())),
+ // spcompress only supports the sp2to4 compression factor.
+ builder.getInt32(4),
+ };
+ return {llvm::Intrinsic::nvvm_spcompress, std::move(args)};
+}
+
+NVVM::IDArgPair SPDecompressOp::getIntrinsicIDAndArgs(
+ Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
+ auto thisOp = cast<NVVM::SPDecompressOp>(op);
+ auto [numSrc, numTgt] = getSPFactor(thisOp.getCompFactor());
+ unsigned elemSize = static_cast<unsigned>(thisOp.getElemSize());
+ unsigned repFactor = static_cast<unsigned>(thisOp.getRepFactor());
+
+ // The intrinsic takes the compressed data with one lane per element.
+ // Compressed data narrower than 32 bits is carried in the low bits of an i32.
+ llvm::Value *compressedData = mt.lookupValue(thisOp.getCompressedData());
+ unsigned compressedBits = numSrc * elemSize * repFactor;
+ if (compressedBits < 32)
+ compressedData =
+ builder.CreateTrunc(compressedData, builder.getIntNTy(compressedBits));
+ llvm::Type *compressedTy = llvm::FixedVectorType::get(
+ builder.getIntNTy(elemSize), numSrc * repFactor);
+ compressedData = builder.CreateBitCast(compressedData, compressedTy);
+
+ llvm::SmallVector<llvm::Value *> args{
+ mt.lookupValue(thisOp.getMetadata()),
+ compressedData,
+ builder.getInt32(static_cast<unsigned>(thisOp.getIndexSize())),
+ builder.getInt32(numTgt),
+ };
+ return {llvm::Intrinsic::nvvm_spdecompress, std::move(args)};
+}
+
//===----------------------------------------------------------------------===//
// NVVMDialect initialization, type parsing, and registration.
//===----------------------------------------------------------------------===//
diff --git a/mlir/test/Target/LLVMIR/nvvm/spcompress-invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/spcompress-invalid.mlir
new file mode 100644
index 00000000000000..3eb906346a33cc
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spcompress-invalid.mlir
@@ -0,0 +1,80 @@
+// RUN: mlir-translate --mlir-to-llvmir -verify-diagnostics -split-input-file %s
+
+llvm.func @spcompress_invalid_data_size(%spdesc: i32, %data: vector<2xi32>) {
+ // expected-error @below {{Operation attributes and provided input data size doesn't match}}
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x2
+ : i32, vector<2xi32> -> i32, vector<2xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spcompress_invalid_compressed_data_size(%spdesc: i32, %data: vector<4xi32>) {
+ // expected-error @below {{Operation attributes and provided output compressed data size doesn't match}}
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x2
+ : i32, vector<4xi32> -> i32, vector<4xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spcompress_invalid_metadata_size(%spdesc: i32, %data: vector<4xi32>) {
+ // expected-error @below {{Operation attributes and provided output metadata size doesn't match}}
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x2
+ : i32, vector<4xi32> -> vector<2xi32>, vector<2xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spcompress_invalid_elem_size(%spdesc: i32, %data: vector<2xi32>) {
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ // expected-error @below {{expected string or keyword containing one of the following enum values for attribute 'elemSize' [b8, b16]}}
+ comp_factor = sp2to4 index_size = b2 elem_size = b32 rep_factor = x1
+ : i32, vector<2xi32> -> i32, i32
+ llvm.return
+}
+
+// -----
+
+llvm.func @spcompress_invalid_comp_factor(%spdesc: i32, %data: vector<2xi32>) {
+ // expected-error @below {{attribute 'compFactor' failed to satisfy constraint: Structured sparsity factor whose value is one of {sp2to4}}}
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, vector<2xi32> -> i32, i32
+ llvm.return
+}
+
+// -----
+
+llvm.func @spcompress_scalable_data(%spdesc: i32, %data: vector<[2]xi32>) {
+ // expected-error @below {{operand #1 must be fixed-length vector of 32-bit signless integer values of length 2/4/8/16/32/64/128 of ranks 1, but got 'vector<[2]xi32>'}}
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, vector<[2]xi32> -> i32, i32
+ llvm.return
+}
+
+// -----
+
+llvm.func @spcompress_rank2_data(%spdesc: i32, %arg: vector<2xi32>) {
+ %data = builtin.unrealized_conversion_cast %arg : vector<2xi32> to vector<1x2xi32>
+ // expected-error @below {{operand #1 must be fixed-length vector of 32-bit signless integer values of length 2/4/8/16/32/64/128 of ranks 1, but got 'vector<1x2xi32>'}}
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, vector<1x2xi32> -> i32, i32
+ llvm.return
+}
+
+// -----
+
+llvm.func @spcompress_scalable_compressed_data(%spdesc: i32, %data: vector<4xi32>) {
+ // expected-error @below {{result #1 must be 32-bit signless integer or fixed-length vector of 32-bit signless integer values of length 2/4/8/16/32/64 of ranks 1, but got 'vector<[2]xi32>'}}
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x2
+ : i32, vector<4xi32> -> i32, vector<[2]xi32>
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spcompress-sp2to4.mlir b/mlir/test/Target/LLVMIR/nvvm/spcompress-sp2to4.mlir
new file mode 100644
index 00000000000000..ab35ec98c9c66f
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spcompress-sp2to4.mlir
@@ -0,0 +1,449 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: data=2, compressed_data=1, metadata=1
+llvm.func @spcompress_elmsz_b8_idxsz_b2_rpf_1(%spdesc: i32, %data: vector<2xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b8_idxsz_b2_rpf_1(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <4 x i8> } @llvm.nvvm.spcompress.i32.v4i8.v8i8(<8 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <4 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <4 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <4 x i8> %[[CDATA]] to i32
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, vector<2xi32> -> i32, i32
+ llvm.return %metadata : i32
+}
+
+// rpf=2: data=4, compressed_data=2, metadata=1
+llvm.func @spcompress_elmsz_b8_idxsz_b2_rpf_2(%spdesc: i32, %data: vector<4xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b8_idxsz_b2_rpf_2(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <8 x i8> } @llvm.nvvm.spcompress.i32.v8i8.v16i8(<16 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <8 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <8 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <8 x i8> %[[CDATA]] to <2 x i32>
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x2
+ : i32, vector<4xi32> -> i32, vector<2xi32>
+ llvm.return %metadata : i32
+}
+
+// rpf=4: data=8, compressed_data=4, metadata=1
+llvm.func @spcompress_elmsz_b8_idxsz_b2_rpf_4(%spdesc: i32, %data: vector<8xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b8_idxsz_b2_rpf_4(i32 %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <16 x i8> } @llvm.nvvm.spcompress.i32.v16i8.v32i8(<32 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <16 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <16 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <16 x i8> %[[CDATA]] to <4 x i32>
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x4
+ : i32, vector<8xi32> -> i32, vector<4xi32>
+ llvm.return %metadata : i32
+}
+
+// rpf=8: data=16, compressed_data=8, metadata=2
+llvm.func @spcompress_elmsz_b8_idxsz_b2_rpf_8(%spdesc: i32, %data: vector<16xi32>) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spcompress_elmsz_b8_idxsz_b2_rpf_8(i32 %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <2 x i32>, <32 x i8> } @llvm.nvvm.spcompress.v2i32.v32i8.v64i8(<64 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <2 x i32>, <32 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <2 x i32>, <32 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <32 x i8> %[[CDATA]] to <8 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x8
+ : i32, vector<16xi32> -> vector<2xi32>, vector<8xi32>
+ llvm.return %metadata : vector<2xi32>
+}
+
+// rpf=16: data=32, compressed_data=16, metadata=4
+llvm.func @spcompress_elmsz_b8_idxsz_b2_rpf_16(%spdesc: i32, %data: vector<32xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spcompress_elmsz_b8_idxsz_b2_rpf_16(i32 %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i32> %1 to <128 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <4 x i32>, <64 x i8> } @llvm.nvvm.spcompress.v4i32.v64i8.v128i8(<128 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <4 x i32>, <64 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <4 x i32>, <64 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <64 x i8> %[[CDATA]] to <16 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x16
+ : i32, vector<32xi32> -> vector<4xi32>, vector<16xi32>
+ llvm.return %metadata : vector<4xi32>
+}
+
+// rpf=32: data=64, compressed_data=32, metadata=8
+llvm.func @spcompress_elmsz_b8_idxsz_b2_rpf_32(%spdesc: i32, %data: vector<64xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spcompress_elmsz_b8_idxsz_b2_rpf_32(i32 %0, <64 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i32> %1 to <256 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <8 x i32>, <128 x i8> } @llvm.nvvm.spcompress.v8i32.v128i8.v256i8(<256 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <8 x i32>, <128 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <8 x i32>, <128 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <128 x i8> %[[CDATA]] to <32 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x32
+ : i32, vector<64xi32> -> vector<8xi32>, vector<32xi32>
+ llvm.return %metadata : vector<8xi32>
+}
+
+// rpf=64: data=128, compressed_data=64, metadata=16
+llvm.func @spcompress_elmsz_b8_idxsz_b2_rpf_64(%spdesc: i32, %data: vector<128xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spcompress_elmsz_b8_idxsz_b2_rpf_64(i32 %0, <128 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i32> %1 to <512 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <16 x i32>, <256 x i8> } @llvm.nvvm.spcompress.v16i32.v256i8.v512i8(<512 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <16 x i32>, <256 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <16 x i32>, <256 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <256 x i8> %[[CDATA]] to <64 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x64
+ : i32, vector<128xi32> -> vector<16xi32>, vector<64xi32>
+ llvm.return %metadata : vector<16xi32>
+}
+
+// rpf=1: data=2, compressed_data=1, metadata=1
+llvm.func @spcompress_elmsz_b8_idxsz_b4_rpf_1(%spdesc: i32, %data: vector<2xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b8_idxsz_b4_rpf_1(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <4 x i8> } @llvm.nvvm.spcompress.i32.v4i8.v8i8(<8 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <4 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <4 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <4 x i8> %[[CDATA]] to i32
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, vector<2xi32> -> i32, i32
+ llvm.return %metadata : i32
+}
+
+// rpf=2: data=4, compressed_data=2, metadata=1
+llvm.func @spcompress_elmsz_b8_idxsz_b4_rpf_2(%spdesc: i32, %data: vector<4xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b8_idxsz_b4_rpf_2(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <8 x i8> } @llvm.nvvm.spcompress.i32.v8i8.v16i8(<16 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <8 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <8 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <8 x i8> %[[CDATA]] to <2 x i32>
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, vector<4xi32> -> i32, vector<2xi32>
+ llvm.return %metadata : i32
+}
+
+// rpf=4: data=8, compressed_data=4, metadata=2
+llvm.func @spcompress_elmsz_b8_idxsz_b4_rpf_4(%spdesc: i32, %data: vector<8xi32>) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spcompress_elmsz_b8_idxsz_b4_rpf_4(i32 %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <2 x i32>, <16 x i8> } @llvm.nvvm.spcompress.v2i32.v16i8.v32i8(<32 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <2 x i32>, <16 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <2 x i32>, <16 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <16 x i8> %[[CDATA]] to <4 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x4
+ : i32, vector<8xi32> -> vector<2xi32>, vector<4xi32>
+ llvm.return %metadata : vector<2xi32>
+}
+
+// rpf=8: data=16, compressed_data=8, metadata=4
+llvm.func @spcompress_elmsz_b8_idxsz_b4_rpf_8(%spdesc: i32, %data: vector<16xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spcompress_elmsz_b8_idxsz_b4_rpf_8(i32 %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <4 x i32>, <32 x i8> } @llvm.nvvm.spcompress.v4i32.v32i8.v64i8(<64 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <4 x i32>, <32 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <4 x i32>, <32 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <32 x i8> %[[CDATA]] to <8 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x8
+ : i32, vector<16xi32> -> vector<4xi32>, vector<8xi32>
+ llvm.return %metadata : vector<4xi32>
+}
+
+// rpf=16: data=32, compressed_data=16, metadata=8
+llvm.func @spcompress_elmsz_b8_idxsz_b4_rpf_16(%spdesc: i32, %data: vector<32xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spcompress_elmsz_b8_idxsz_b4_rpf_16(i32 %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i32> %1 to <128 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <8 x i32>, <64 x i8> } @llvm.nvvm.spcompress.v8i32.v64i8.v128i8(<128 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <8 x i32>, <64 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <8 x i32>, <64 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <64 x i8> %[[CDATA]] to <16 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x16
+ : i32, vector<32xi32> -> vector<8xi32>, vector<16xi32>
+ llvm.return %metadata : vector<8xi32>
+}
+
+// rpf=32: data=64, compressed_data=32, metadata=16
+llvm.func @spcompress_elmsz_b8_idxsz_b4_rpf_32(%spdesc: i32, %data: vector<64xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spcompress_elmsz_b8_idxsz_b4_rpf_32(i32 %0, <64 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i32> %1 to <256 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <16 x i32>, <128 x i8> } @llvm.nvvm.spcompress.v16i32.v128i8.v256i8(<256 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <16 x i32>, <128 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <16 x i32>, <128 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <128 x i8> %[[CDATA]] to <32 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x32
+ : i32, vector<64xi32> -> vector<16xi32>, vector<32xi32>
+ llvm.return %metadata : vector<16xi32>
+}
+
+// rpf=64: data=128, compressed_data=64, metadata=32
+llvm.func @spcompress_elmsz_b8_idxsz_b4_rpf_64(%spdesc: i32, %data: vector<128xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spcompress_elmsz_b8_idxsz_b4_rpf_64(i32 %0, <128 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i32> %1 to <512 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <32 x i32>, <256 x i8> } @llvm.nvvm.spcompress.v32i32.v256i8.v512i8(<512 x i8> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <32 x i32>, <256 x i8> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <32 x i32>, <256 x i8> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <256 x i8> %[[CDATA]] to <64 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x64
+ : i32, vector<128xi32> -> vector<32xi32>, vector<64xi32>
+ llvm.return %metadata : vector<32xi32>
+}
+
+// rpf=1: data=2, compressed_data=1, metadata=1
+llvm.func @spcompress_elmsz_b16_idxsz_b2_rpf_1(%spdesc: i32, %data: vector<2xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b16_idxsz_b2_rpf_1(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <2 x i16> } @llvm.nvvm.spcompress.i32.v2i16.v4i16(<4 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <2 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <2 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <2 x i16> %[[CDATA]] to i32
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x1
+ : i32, vector<2xi32> -> i32, i32
+ llvm.return %metadata : i32
+}
+
+// rpf=2: data=4, compressed_data=2, metadata=1
+llvm.func @spcompress_elmsz_b16_idxsz_b2_rpf_2(%spdesc: i32, %data: vector<4xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b16_idxsz_b2_rpf_2(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <4 x i16> } @llvm.nvvm.spcompress.i32.v4i16.v8i16(<8 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <4 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <4 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <4 x i16> %[[CDATA]] to <2 x i32>
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x2
+ : i32, vector<4xi32> -> i32, vector<2xi32>
+ llvm.return %metadata : i32
+}
+
+// rpf=4: data=8, compressed_data=4, metadata=1
+llvm.func @spcompress_elmsz_b16_idxsz_b2_rpf_4(%spdesc: i32, %data: vector<8xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b16_idxsz_b2_rpf_4(i32 %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <8 x i16> } @llvm.nvvm.spcompress.i32.v8i16.v16i16(<16 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <8 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <8 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <8 x i16> %[[CDATA]] to <4 x i32>
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x4
+ : i32, vector<8xi32> -> i32, vector<4xi32>
+ llvm.return %metadata : i32
+}
+
+// rpf=8: data=16, compressed_data=8, metadata=1
+llvm.func @spcompress_elmsz_b16_idxsz_b2_rpf_8(%spdesc: i32, %data: vector<16xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b16_idxsz_b2_rpf_8(i32 %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <16 x i16> } @llvm.nvvm.spcompress.i32.v16i16.v32i16(<32 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <16 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <16 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <16 x i16> %[[CDATA]] to <8 x i32>
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x8
+ : i32, vector<16xi32> -> i32, vector<8xi32>
+ llvm.return %metadata : i32
+}
+
+// rpf=16: data=32, compressed_data=16, metadata=2
+llvm.func @spcompress_elmsz_b16_idxsz_b2_rpf_16(%spdesc: i32, %data: vector<32xi32>) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spcompress_elmsz_b16_idxsz_b2_rpf_16(i32 %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <2 x i32>, <32 x i16> } @llvm.nvvm.spcompress.v2i32.v32i16.v64i16(<64 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <2 x i32>, <32 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <2 x i32>, <32 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <32 x i16> %[[CDATA]] to <16 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x16
+ : i32, vector<32xi32> -> vector<2xi32>, vector<16xi32>
+ llvm.return %metadata : vector<2xi32>
+}
+
+// rpf=32: data=64, compressed_data=32, metadata=4
+llvm.func @spcompress_elmsz_b16_idxsz_b2_rpf_32(%spdesc: i32, %data: vector<64xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spcompress_elmsz_b16_idxsz_b2_rpf_32(i32 %0, <64 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i32> %1 to <128 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <4 x i32>, <64 x i16> } @llvm.nvvm.spcompress.v4i32.v64i16.v128i16(<128 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <4 x i32>, <64 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <4 x i32>, <64 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <64 x i16> %[[CDATA]] to <32 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x32
+ : i32, vector<64xi32> -> vector<4xi32>, vector<32xi32>
+ llvm.return %metadata : vector<4xi32>
+}
+
+// rpf=64: data=128, compressed_data=64, metadata=8
+llvm.func @spcompress_elmsz_b16_idxsz_b2_rpf_64(%spdesc: i32, %data: vector<128xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spcompress_elmsz_b16_idxsz_b2_rpf_64(i32 %0, <128 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i32> %1 to <256 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <8 x i32>, <128 x i16> } @llvm.nvvm.spcompress.v8i32.v128i16.v256i16(<256 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <8 x i32>, <128 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <8 x i32>, <128 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <128 x i16> %[[CDATA]] to <64 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x64
+ : i32, vector<128xi32> -> vector<8xi32>, vector<64xi32>
+ llvm.return %metadata : vector<8xi32>
+}
+
+// rpf=1: data=2, compressed_data=1, metadata=1
+llvm.func @spcompress_elmsz_b16_idxsz_b4_rpf_1(%spdesc: i32, %data: vector<2xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b16_idxsz_b4_rpf_1(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <2 x i16> } @llvm.nvvm.spcompress.i32.v2i16.v4i16(<4 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <2 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <2 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <2 x i16> %[[CDATA]] to i32
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, vector<2xi32> -> i32, i32
+ llvm.return %metadata : i32
+}
+
+// rpf=2: data=4, compressed_data=2, metadata=1
+llvm.func @spcompress_elmsz_b16_idxsz_b4_rpf_2(%spdesc: i32, %data: vector<4xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b16_idxsz_b4_rpf_2(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <4 x i16> } @llvm.nvvm.spcompress.i32.v4i16.v8i16(<8 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <4 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <4 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <4 x i16> %[[CDATA]] to <2 x i32>
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x2
+ : i32, vector<4xi32> -> i32, vector<2xi32>
+ llvm.return %metadata : i32
+}
+
+// rpf=4: data=8, compressed_data=4, metadata=1
+llvm.func @spcompress_elmsz_b16_idxsz_b4_rpf_4(%spdesc: i32, %data: vector<8xi32>) -> i32 {
+ // CHECK-LABEL: define i32 @spcompress_elmsz_b16_idxsz_b4_rpf_4(i32 %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { i32, <8 x i16> } @llvm.nvvm.spcompress.i32.v8i16.v16i16(<16 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { i32, <8 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { i32, <8 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <8 x i16> %[[CDATA]] to <4 x i32>
+ // CHECK-NEXT: ret i32 %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x4
+ : i32, vector<8xi32> -> i32, vector<4xi32>
+ llvm.return %metadata : i32
+}
+
+// rpf=8: data=16, compressed_data=8, metadata=2
+llvm.func @spcompress_elmsz_b16_idxsz_b4_rpf_8(%spdesc: i32, %data: vector<16xi32>) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spcompress_elmsz_b16_idxsz_b4_rpf_8(i32 %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <2 x i32>, <16 x i16> } @llvm.nvvm.spcompress.v2i32.v16i16.v32i16(<32 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <2 x i32>, <16 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <2 x i32>, <16 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <16 x i16> %[[CDATA]] to <8 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x8
+ : i32, vector<16xi32> -> vector<2xi32>, vector<8xi32>
+ llvm.return %metadata : vector<2xi32>
+}
+
+// rpf=16: data=32, compressed_data=16, metadata=4
+llvm.func @spcompress_elmsz_b16_idxsz_b4_rpf_16(%spdesc: i32, %data: vector<32xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spcompress_elmsz_b16_idxsz_b4_rpf_16(i32 %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <4 x i32>, <32 x i16> } @llvm.nvvm.spcompress.v4i32.v32i16.v64i16(<64 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <4 x i32>, <32 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <4 x i32>, <32 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <32 x i16> %[[CDATA]] to <16 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x16
+ : i32, vector<32xi32> -> vector<4xi32>, vector<16xi32>
+ llvm.return %metadata : vector<4xi32>
+}
+
+// rpf=32: data=64, compressed_data=32, metadata=8
+llvm.func @spcompress_elmsz_b16_idxsz_b4_rpf_32(%spdesc: i32, %data: vector<64xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spcompress_elmsz_b16_idxsz_b4_rpf_32(i32 %0, <64 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i32> %1 to <128 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <8 x i32>, <64 x i16> } @llvm.nvvm.spcompress.v8i32.v64i16.v128i16(<128 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <8 x i32>, <64 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <8 x i32>, <64 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <64 x i16> %[[CDATA]] to <32 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x32
+ : i32, vector<64xi32> -> vector<8xi32>, vector<32xi32>
+ llvm.return %metadata : vector<8xi32>
+}
+
+// rpf=64: data=128, compressed_data=64, metadata=16
+llvm.func @spcompress_elmsz_b16_idxsz_b4_rpf_64(%spdesc: i32, %data: vector<128xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spcompress_elmsz_b16_idxsz_b4_rpf_64(i32 %0, <128 x i32> %1) {
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i32> %1 to <256 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call { <16 x i32>, <128 x i16> } @llvm.nvvm.spcompress.v16i32.v128i16.v256i16(<256 x i16> %[[DATA]], i32 %0, /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[MDATA:[0-9]+]] = extractvalue { <16 x i32>, <128 x i16> } %[[RES]], 0
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = extractvalue { <16 x i32>, <128 x i16> } %[[RES]], 1
+ // CHECK-NEXT: %{{[0-9]+}} = bitcast <128 x i16> %[[CDATA]] to <64 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[MDATA]]
+ // CHECK-NEXT: }
+ %metadata, %compressed_data = nvvm.spcompress %spdesc, %data
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x64
+ : i32, vector<128xi32> -> vector<16xi32>, vector<64xi32>
+ llvm.return %metadata : vector<16xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-invalid.mlir
new file mode 100644
index 00000000000000..63589511d9e281
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-invalid.mlir
@@ -0,0 +1,110 @@
+// RUN: mlir-translate --mlir-to-llvmir -verify-diagnostics -split-input-file %s
+
+llvm.func @spdecompress_data_too_small(%metadata: i32, %compdata: i32) {
+ // expected-error @below {{Invalid compFactor, element size, and repeat factor combination}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> i32
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_data_too_large(%metadata: i32, %compdata: vector<16xi32>) {
+ // expected-error @below {{Invalid compFactor, element size, and repeat factor combination}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b16 rep_factor = x32
+ : i32, vector<16xi32> -> vector<128xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_invalid_comp_factor_elem_size(%metadata: i32, %compdata: vector<2xi32>) {
+ // expected-error @below {{Invalid compFactor and element size combination}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to8 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_invalid_index_size(%metadata: i32, %compdata: i32) {
+ // expected-error @below {{Invalid index size and target combination, target should be at most 4 if index size is 2}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_invalid_data_size(%metadata: i32, %compdata: i32) {
+ // expected-error @below {{Operation attributes and provided output data size doesn't match}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> i32
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_invalid_compressed_data_size(%metadata: i32, %compdata: vector<2xi32>) {
+ // expected-error @below {{Operation attributes and provided input compressed data size doesn't match}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, vector<2xi32> -> vector<2xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_invalid_metadata_size(%metadata: vector<2xi32>, %compdata: i32) {
+ // expected-error @below {{Operation attributes and provided input metadata size doesn't match}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x2
+ : vector<2xi32>, i32 -> vector<2xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_invalid_comp_factor(%metadata: i32, %compdata: i32) {
+ %res = nvvm.spdecompress %metadata, %compdata
+ // expected-error @below {{expected string or keyword containing one of the following enum values for attribute 'compFactor' [sp1to2, sp1to4, sp1to8, sp1to16, sp2to4, sp2to8, sp2to16, sp4to8, sp4to16]}}
+ comp_factor = sp3to4 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_scalable_compressed_data(%metadata: i32, %compdata: vector<[2]xi32>) {
+ // expected-error @below {{operand #1 must be 32-bit signless integer or fixed-length vector of 32-bit signless integer values of length 2/4/8/16/32/64 of ranks 1, but got 'vector<[2]xi32>'}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x8
+ : i32, vector<[2]xi32> -> vector<4xi32>
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_rank2_metadata(%arg: vector<2xi32>, %compdata: i32) {
+ %metadata = builtin.unrealized_conversion_cast %arg : vector<2xi32> to vector<1x2xi32>
+ // expected-error @below {{operand #0 must be 32-bit signless integer or fixed-length vector of 32-bit signless integer values of length 2/4/8/16/32/64 of ranks 1, but got 'vector<1x2xi32>'}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x2
+ : vector<1x2xi32>, i32 -> i32
+ llvm.return
+}
+
+// -----
+
+llvm.func @spdecompress_scalable_data(%metadata: i32, %compdata: i32) {
+ // expected-error @below {{result #0 must be 32-bit signless integer or fixed-length vector of 32-bit signless integer values of length 2/4/8/16/32/64/128 of ranks 1, but got 'vector<[2]xi32>'}}
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x4
+ : i32, i32 -> vector<[2]xi32>
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to16.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to16.mlir
new file mode 100644
index 00000000000000..f1cc5665b9b70d
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to16.mlir
@@ -0,0 +1,158 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i8
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i8 %[[CDATA_TRUNC]] to <1 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v1i8(i32 %0, <1 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=8
+llvm.func @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=1, data=16
+llvm.func @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_4(%metadata: i32, %compdata: i32) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_4(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b8 rep_factor = x4
+ : i32, i32 -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=2, data=32
+llvm.func @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_8(%metadata: i32, %compdata: vector<2xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_8(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b8 rep_factor = x8
+ : i32, vector<2xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=4, data=64
+llvm.func @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_16(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<2xi32>, vector<4xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=8, data=128
+llvm.func @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp1to16_elmsz_b8_idxsz_b4_rpf_32(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <512 x i8> @llvm.nvvm.spdecompress.v512i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <512 x i8> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<4xi32>, vector<8xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=8
+llvm.func @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <1 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v1i16(i32 %0, <1 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, i32 -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=16
+llvm.func @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b16 rep_factor = x2
+ : i32, i32 -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=32
+llvm.func @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b16 rep_factor = x4
+ : i32, vector<2xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=4, data=64
+llvm.func @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_8(%metadata: i32, %compdata: vector<4xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_8(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b16 rep_factor = x8
+ : i32, vector<4xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=8, data=128
+llvm.func @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_16(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp1to16_elmsz_b16_idxsz_b4_rpf_16(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i16> @llvm.nvvm.spdecompress.v256i16.v2i32.v16i16(<2 x i32> %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i16> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to16 index_size = b4 elem_size = b16 rep_factor = x16
+ : vector<2xi32>, vector<8xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to2.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to2.mlir
new file mode 100644
index 00000000000000..38d2bddfdb80b5
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to2.mlir
@@ -0,0 +1,369 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=2: metadata=1, compressed_data=1, data=1
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_2(%metadata: i32, %compdata: i32) -> i32 {
+ // CHECK-LABEL: define i32 @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i8> @llvm.nvvm.spdecompress.v4i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i8> %[[RES]] to i32
+ // CHECK-NEXT: ret i32 %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x2
+ : i32, i32 -> i32
+ llvm.return %res : i32
+}
+
+// rpf=4: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_4(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_4(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x4
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_8(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_8(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x8
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=16: metadata=1, compressed_data=4, data=8
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_16(%metadata: i32, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_16(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v16i8(i32 %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x16
+ : i32, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=32: metadata=2, compressed_data=8, data=16
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_32(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_32(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.v2i32.v32i8(<2 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x32
+ : vector<2xi32>, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=64: metadata=4, compressed_data=16, data=32
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_64(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b2_rpf_64(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v4i32.v64i8(<4 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b8 rep_factor = x64
+ : vector<4xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=1
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> i32 {
+ // CHECK-LABEL: define i32 @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i8> @llvm.nvvm.spdecompress.v4i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i8> %[[RES]] to i32
+ // CHECK-NEXT: ret i32 %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> i32
+ llvm.return %res : i32
+}
+
+// rpf=4: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_4(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_4(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b8 rep_factor = x4
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_8(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_8(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b8 rep_factor = x8
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=4, data=8
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_16(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<2xi32>, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=8, data=16
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_32(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<4xi32>, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=64: metadata=8, compressed_data=16, data=32
+llvm.func @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_64(%metadata: vector<8xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to2_elmsz_b8_idxsz_b4_rpf_64(<8 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v8i32.v64i8(<8 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b8 rep_factor = x64
+ : vector<8xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=1
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_1(%metadata: i32, %compdata: i32) -> i32 {
+ // CHECK-LABEL: define i32 @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <1 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <2 x i16> @llvm.nvvm.spdecompress.v2i16.i32.v1i16(i32 %0, <1 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <2 x i16> %[[RES]] to i32
+ // CHECK-NEXT: ret i32 %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b16 rep_factor = x1
+ : i32, i32 -> i32
+ llvm.return %res : i32
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_2(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i16> @llvm.nvvm.spdecompress.v4i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i16> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b16 rep_factor = x2
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i16> @llvm.nvvm.spdecompress.v8i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i16> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b16 rep_factor = x4
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=4, data=8
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_8(%metadata: i32, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_8(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b16 rep_factor = x8
+ : i32, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=16: metadata=1, compressed_data=8, data=16
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_16(%metadata: i32, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_16(i32 %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.i32.v16i16(i32 %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b16 rep_factor = x16
+ : i32, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=32: metadata=2, compressed_data=16, data=32
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_32(%metadata: vector<2xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_32(<2 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.v2i32.v32i16(<2 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b16 rep_factor = x32
+ : vector<2xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=64: metadata=4, compressed_data=32, data=64
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_64(%metadata: vector<4xi32>, %compdata: vector<32xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b2_rpf_64(<4 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v4i32.v64i16(<4 x i32> %0, <64 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b2 elem_size = b16 rep_factor = x64
+ : vector<4xi32>, vector<32xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=1
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> i32 {
+ // CHECK-LABEL: define i32 @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <1 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <2 x i16> @llvm.nvvm.spdecompress.v2i16.i32.v1i16(i32 %0, <1 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <2 x i16> %[[RES]] to i32
+ // CHECK-NEXT: ret i32 %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, i32 -> i32
+ llvm.return %res : i32
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i16> @llvm.nvvm.spdecompress.v4i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i16> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b16 rep_factor = x2
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i16> @llvm.nvvm.spdecompress.v8i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i16> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b16 rep_factor = x4
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=4, data=8
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_8(%metadata: i32, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_8(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b16 rep_factor = x8
+ : i32, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=8, data=16
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_16(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_16(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.v2i32.v16i16(<2 x i32> %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b16 rep_factor = x16
+ : vector<2xi32>, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=16, data=32
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_32(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_32(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.v4i32.v32i16(<4 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b16 rep_factor = x32
+ : vector<4xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=64: metadata=8, compressed_data=32, data=64
+llvm.func @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_64(%metadata: vector<8xi32>, %compdata: vector<32xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to2_elmsz_b16_idxsz_b4_rpf_64(<8 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v8i32.v64i16(<8 x i32> %0, <64 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 2)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to2 index_size = b4 elem_size = b16 rep_factor = x64
+ : vector<8xi32>, vector<32xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to4.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to4.mlir
new file mode 100644
index 00000000000000..5e1d0e76fa10d9
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to4.mlir
@@ -0,0 +1,399 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: metadata=1, compressed_data=1, data=1
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_1(%metadata: i32, %compdata: i32) -> i32 {
+ // CHECK-LABEL: define i32 @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i8
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i8 %[[CDATA_TRUNC]] to <1 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i8> @llvm.nvvm.spdecompress.v4i8.i32.v1i8(i32 %0, <1 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i8> %[[RES]] to i32
+ // CHECK-NEXT: ret i32 %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, i32 -> i32
+ llvm.return %res : i32
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_2(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_4(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_4(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b8 rep_factor = x4
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=2, data=8
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_8(%metadata: i32, %compdata: vector<2xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_8(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b8 rep_factor = x8
+ : i32, vector<2xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=16: metadata=1, compressed_data=4, data=16
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_16(%metadata: i32, %compdata: vector<4xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_16(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.i32.v16i8(i32 %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b8 rep_factor = x16
+ : i32, vector<4xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=32: metadata=2, compressed_data=8, data=32
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_32(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_32(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v2i32.v32i8(<2 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b8 rep_factor = x32
+ : vector<2xi32>, vector<8xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=64: metadata=4, compressed_data=16, data=64
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_64(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b2_rpf_64(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v4i32.v64i8(<4 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b8 rep_factor = x64
+ : vector<4xi32>, vector<16xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=1
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> i32 {
+ // CHECK-LABEL: define i32 @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i8
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i8 %[[CDATA_TRUNC]] to <1 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i8> @llvm.nvvm.spdecompress.v4i8.i32.v1i8(i32 %0, <1 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i8> %[[RES]] to i32
+ // CHECK-NEXT: ret i32 %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> i32
+ llvm.return %res : i32
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_4(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_4(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b8 rep_factor = x4
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=2, data=8
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_8(%metadata: i32, %compdata: vector<2xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_8(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b8 rep_factor = x8
+ : i32, vector<2xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=4, data=16
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_16(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<2xi32>, vector<4xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=8, data=32
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_32(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<4xi32>, vector<8xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=64: metadata=8, compressed_data=16, data=64
+llvm.func @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_64(%metadata: vector<8xi32>, %compdata: vector<16xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to4_elmsz_b8_idxsz_b4_rpf_64(<8 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v8i32.v64i8(<8 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b8 rep_factor = x64
+ : vector<8xi32>, vector<16xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_1(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <1 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i16> @llvm.nvvm.spdecompress.v4i16.i32.v1i16(i32 %0, <1 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i16> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b16 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_2(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i16> @llvm.nvvm.spdecompress.v8i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i16> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b16 rep_factor = x2
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=8
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b16 rep_factor = x4
+ : i32, vector<2xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=4, data=16
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_8(%metadata: i32, %compdata: vector<4xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_8(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b16 rep_factor = x8
+ : i32, vector<4xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=16: metadata=1, compressed_data=8, data=32
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_16(%metadata: i32, %compdata: vector<8xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_16(i32 %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.i32.v16i16(i32 %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b16 rep_factor = x16
+ : i32, vector<8xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=32: metadata=2, compressed_data=16, data=64
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_32(%metadata: vector<2xi32>, %compdata: vector<16xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_32(<2 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v2i32.v32i16(<2 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b16 rep_factor = x32
+ : vector<2xi32>, vector<16xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=64: metadata=4, compressed_data=32, data=128
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_64(%metadata: vector<4xi32>, %compdata: vector<32xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b2_rpf_64(<4 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i16> @llvm.nvvm.spdecompress.v256i16.v4i32.v64i16(<4 x i32> %0, <64 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i16> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b2 elem_size = b16 rep_factor = x64
+ : vector<4xi32>, vector<32xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <1 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i16> @llvm.nvvm.spdecompress.v4i16.i32.v1i16(i32 %0, <1 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i16> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i16> @llvm.nvvm.spdecompress.v8i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i16> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b16 rep_factor = x2
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=8
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b16 rep_factor = x4
+ : i32, vector<2xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=4, data=16
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_8(%metadata: i32, %compdata: vector<4xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_8(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b16 rep_factor = x8
+ : i32, vector<4xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=8, data=32
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_16(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_16(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.v2i32.v16i16(<2 x i32> %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b16 rep_factor = x16
+ : vector<2xi32>, vector<8xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=16, data=64
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_32(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_32(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v4i32.v32i16(<4 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b16 rep_factor = x32
+ : vector<4xi32>, vector<16xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=64: metadata=8, compressed_data=32, data=128
+llvm.func @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_64(%metadata: vector<8xi32>, %compdata: vector<32xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp1to4_elmsz_b16_idxsz_b4_rpf_64(<8 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i16> @llvm.nvvm.spdecompress.v256i16.v8i32.v64i16(<8 x i32> %0, <64 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i16> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to4 index_size = b4 elem_size = b16 rep_factor = x64
+ : vector<8xi32>, vector<32xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to8.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to8.mlir
new file mode 100644
index 00000000000000..9be00dbd908606
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp1to8.mlir
@@ -0,0 +1,186 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i8
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i8 %[[CDATA_TRUNC]] to <1 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v1i8(i32 %0, <1 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=1, data=8
+llvm.func @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_4(%metadata: i32, %compdata: i32) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_4(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b8 rep_factor = x4
+ : i32, i32 -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=2, data=16
+llvm.func @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_8(%metadata: i32, %compdata: vector<2xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_8(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b8 rep_factor = x8
+ : i32, vector<2xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=4, data=32
+llvm.func @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_16(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<2xi32>, vector<4xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=8, data=64
+llvm.func @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_32(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<4xi32>, vector<8xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=64: metadata=8, compressed_data=16, data=128
+llvm.func @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_64(%metadata: vector<8xi32>, %compdata: vector<16xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp1to8_elmsz_b8_idxsz_b4_rpf_64(<8 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <512 x i8> @llvm.nvvm.spdecompress.v512i8.v8i32.v64i8(<8 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <512 x i8> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b8 rep_factor = x64
+ : vector<8xi32>, vector<16xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <1 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i16> @llvm.nvvm.spdecompress.v8i16.i32.v1i16(i32 %0, <1 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i16> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=8
+llvm.func @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b16 rep_factor = x2
+ : i32, i32 -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=16
+llvm.func @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b16 rep_factor = x4
+ : i32, vector<2xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=4, data=32
+llvm.func @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_8(%metadata: i32, %compdata: vector<4xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_8(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b16 rep_factor = x8
+ : i32, vector<4xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=8, data=64
+llvm.func @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_16(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_16(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v2i32.v16i16(<2 x i32> %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b16 rep_factor = x16
+ : vector<2xi32>, vector<8xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=16, data=128
+llvm.func @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_32(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp1to8_elmsz_b16_idxsz_b4_rpf_32(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i16> @llvm.nvvm.spdecompress.v256i16.v4i32.v32i16(<4 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i16> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp1to8 index_size = b4 elem_size = b16 rep_factor = x32
+ : vector<4xi32>, vector<16xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to16.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to16.mlir
new file mode 100644
index 00000000000000..3ffd0f43a760b6
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to16.mlir
@@ -0,0 +1,156 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=8
+llvm.func @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=16
+llvm.func @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b8 rep_factor = x4
+ : i32, vector<2xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=8: metadata=2, compressed_data=4, data=32
+llvm.func @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_8(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_8(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b8 rep_factor = x8
+ : vector<2xi32>, vector<4xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=16: metadata=4, compressed_data=8, data=64
+llvm.func @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_16(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<4xi32>, vector<8xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=32: metadata=8, compressed_data=16, data=128
+llvm.func @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<8xi32>, %compdata: vector<16xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp2to16_elmsz_b8_idxsz_b4_rpf_32(<8 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <512 x i8> @llvm.nvvm.spdecompress.v512i8.v8i32.v64i8(<8 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <512 x i8> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<8xi32>, vector<16xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=8
+llvm.func @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, i32 -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=2, data=16
+llvm.func @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_2(%metadata: i32, %compdata: vector<2xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_2(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b16 rep_factor = x2
+ : i32, vector<2xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=4, data=32
+llvm.func @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<4xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_4(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b16 rep_factor = x4
+ : i32, vector<4xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=8: metadata=2, compressed_data=8, data=64
+llvm.func @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_8(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_8(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v2i32.v16i16(<2 x i32> %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b16 rep_factor = x8
+ : vector<2xi32>, vector<8xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=16: metadata=4, compressed_data=16, data=128
+llvm.func @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_16(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp2to16_elmsz_b16_idxsz_b4_rpf_16(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i16> @llvm.nvvm.spdecompress.v256i16.v4i32.v32i16(<4 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i16> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to16 index_size = b4 elem_size = b16 rep_factor = x16
+ : vector<4xi32>, vector<16xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to4.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to4.mlir
new file mode 100644
index 00000000000000..c418ec19391395
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to4.mlir
@@ -0,0 +1,395 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: metadata=1, compressed_data=1, data=1
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_1(%metadata: i32, %compdata: i32) -> i32 {
+ // CHECK-LABEL: define i32 @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i8> @llvm.nvvm.spdecompress.v4i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i8> %[[RES]] to i32
+ // CHECK-NEXT: ret i32 %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x1
+ : i32, i32 -> i32
+ llvm.return %res : i32
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_2(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x4
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=4, data=8
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_8(%metadata: i32, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_8(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v16i8(i32 %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x8
+ : i32, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=8, data=16
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_16(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_16(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.v2i32.v32i8(<2 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x16
+ : vector<2xi32>, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=16, data=32
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_32(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_32(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v4i32.v64i8(<4 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x32
+ : vector<4xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=64: metadata=8, compressed_data=32, data=64
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_64(%metadata: vector<8xi32>, %compdata: vector<32xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b2_rpf_64(<8 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <128 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v8i32.v128i8(<8 x i32> %0, <128 x i8> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b8 rep_factor = x64
+ : vector<8xi32>, vector<32xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=1
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> i32 {
+ // CHECK-LABEL: define i32 @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i8> @llvm.nvvm.spdecompress.v4i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i8> %[[RES]] to i32
+ // CHECK-NEXT: ret i32 %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> i32
+ llvm.return %res : i32
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x4
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=8: metadata=2, compressed_data=4, data=8
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_8(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_8(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x8
+ : vector<2xi32>, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=16: metadata=4, compressed_data=8, data=16
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_16(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<4xi32>, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=32: metadata=8, compressed_data=16, data=32
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<8xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_32(<8 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v8i32.v64i8(<8 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<8xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=64: metadata=16, compressed_data=32, data=64
+llvm.func @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_64(%metadata: vector<16xi32>, %compdata: vector<32xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp2to4_elmsz_b8_idxsz_b4_rpf_64(<16 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <128 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v16i32.v128i8(<16 x i32> %0, <128 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b8 rep_factor = x64
+ : vector<16xi32>, vector<32xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_1(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i16> @llvm.nvvm.spdecompress.v4i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i16> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_2(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_2(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i16> @llvm.nvvm.spdecompress.v8i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i16> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x2
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=4, data=8
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_4(%metadata: i32, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_4(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x4
+ : i32, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=8: metadata=1, compressed_data=8, data=16
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_8(%metadata: i32, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_8(i32 %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.i32.v16i16(i32 %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x8
+ : i32, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=16: metadata=2, compressed_data=16, data=32
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_16(%metadata: vector<2xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_16(<2 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.v2i32.v32i16(<2 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x16
+ : vector<2xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=32: metadata=4, compressed_data=32, data=64
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_32(%metadata: vector<4xi32>, %compdata: vector<32xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_32(<4 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v4i32.v64i16(<4 x i32> %0, <64 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x32
+ : vector<4xi32>, vector<32xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=64: metadata=8, compressed_data=64, data=128
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_64(%metadata: vector<8xi32>, %compdata: vector<64xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b2_rpf_64(<8 x i32> %0, <64 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <64 x i32> %1 to <128 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i16> @llvm.nvvm.spdecompress.v256i16.v8i32.v128i16(<8 x i32> %0, <128 x i16> %[[CDATA]], /* idx_size= */ i32 2, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i16> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b2 elem_size = b16 rep_factor = x64
+ : vector<8xi32>, vector<64xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <4 x i16> @llvm.nvvm.spdecompress.v4i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <4 x i16> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_2(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_2(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i16> @llvm.nvvm.spdecompress.v8i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i16> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x2
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=4, data=8
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_4(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x4
+ : i32, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=8: metadata=2, compressed_data=8, data=16
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_8(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_8(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.v2i32.v16i16(<2 x i32> %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x8
+ : vector<2xi32>, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=16: metadata=4, compressed_data=16, data=32
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_16(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_16(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.v4i32.v32i16(<4 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x16
+ : vector<4xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=32: metadata=8, compressed_data=32, data=64
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_32(%metadata: vector<8xi32>, %compdata: vector<32xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_32(<8 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v8i32.v64i16(<8 x i32> %0, <64 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x32
+ : vector<8xi32>, vector<32xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=64: metadata=16, compressed_data=64, data=128
+llvm.func @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_64(%metadata: vector<16xi32>, %compdata: vector<64xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp2to4_elmsz_b16_idxsz_b4_rpf_64(<16 x i32> %0, <64 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <64 x i32> %1 to <128 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i16> @llvm.nvvm.spdecompress.v256i16.v16i32.v128i16(<16 x i32> %0, <128 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 4)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i16> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to4 index_size = b4 elem_size = b16 rep_factor = x64
+ : vector<16xi32>, vector<64xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to8.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to8.mlir
new file mode 100644
index 00000000000000..89fb641e044b31
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp2to8.mlir
@@ -0,0 +1,184 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA_TRUNC:[0-9]+]] = trunc i32 %1 to i16
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i16 %[[CDATA_TRUNC]] to <2 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v2i8(i32 %0, <2 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_2(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=2, data=8
+llvm.func @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<2xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_4(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b8 rep_factor = x4
+ : i32, vector<2xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=8: metadata=2, compressed_data=4, data=16
+llvm.func @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_8(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_8(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b8 rep_factor = x8
+ : vector<2xi32>, vector<4xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=16: metadata=4, compressed_data=8, data=32
+llvm.func @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_16(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<4xi32>, vector<8xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=32: metadata=8, compressed_data=16, data=64
+llvm.func @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<8xi32>, %compdata: vector<16xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_32(<8 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v8i32.v64i8(<8 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<8xi32>, vector<16xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=64: metadata=16, compressed_data=32, data=128
+llvm.func @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_64(%metadata: vector<16xi32>, %compdata: vector<32xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp2to8_elmsz_b8_idxsz_b4_rpf_64(<16 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <128 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <512 x i8> @llvm.nvvm.spdecompress.v512i8.v16i32.v128i8(<16 x i32> %0, <128 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <512 x i8> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b8 rep_factor = x64
+ : vector<16xi32>, vector<32xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
+
+// rpf=1: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <2 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i16> @llvm.nvvm.spdecompress.v8i16.i32.v2i16(i32 %0, <2 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i16> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b16 rep_factor = x1
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=2, data=8
+llvm.func @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_2(%metadata: i32, %compdata: vector<2xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_2(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <4 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i16> @llvm.nvvm.spdecompress.v16i16.i32.v4i16(i32 %0, <4 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i16> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b16 rep_factor = x2
+ : i32, vector<2xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=4: metadata=1, compressed_data=4, data=16
+llvm.func @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_4(%metadata: i32, %compdata: vector<4xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_4(i32 %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <8 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i16> @llvm.nvvm.spdecompress.v32i16.i32.v8i16(i32 %0, <8 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i16> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b16 rep_factor = x4
+ : i32, vector<4xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=8: metadata=2, compressed_data=8, data=32
+llvm.func @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_8(%metadata: vector<2xi32>, %compdata: vector<8xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_8(<2 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <16 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i16> @llvm.nvvm.spdecompress.v64i16.v2i32.v16i16(<2 x i32> %0, <16 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i16> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b16 rep_factor = x8
+ : vector<2xi32>, vector<8xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=16: metadata=4, compressed_data=16, data=64
+llvm.func @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_16(%metadata: vector<4xi32>, %compdata: vector<16xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_16(<4 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <32 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i16> @llvm.nvvm.spdecompress.v128i16.v4i32.v32i16(<4 x i32> %0, <32 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i16> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b16 rep_factor = x16
+ : vector<4xi32>, vector<16xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=32: metadata=8, compressed_data=32, data=128
+llvm.func @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_32(%metadata: vector<8xi32>, %compdata: vector<32xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp2to8_elmsz_b16_idxsz_b4_rpf_32(<8 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <64 x i16>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i16> @llvm.nvvm.spdecompress.v256i16.v8i32.v64i16(<8 x i32> %0, <64 x i16> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i16> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp2to8 index_size = b4 elem_size = b16 rep_factor = x32
+ : vector<8xi32>, vector<32xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp4to16.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp4to16.mlir
new file mode 100644
index 00000000000000..420abe11bb9858
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp4to16.mlir
@@ -0,0 +1,85 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: metadata=1, compressed_data=1, data=4
+llvm.func @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to16 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=2, data=8
+llvm.func @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: vector<2xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_2(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to16 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, vector<2xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=4: metadata=2, compressed_data=4, data=16
+llvm.func @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_4(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_4(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to16 index_size = b4 elem_size = b8 rep_factor = x4
+ : vector<2xi32>, vector<4xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=8: metadata=4, compressed_data=8, data=32
+llvm.func @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_8(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_8(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to16 index_size = b4 elem_size = b8 rep_factor = x8
+ : vector<4xi32>, vector<8xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=16: metadata=8, compressed_data=16, data=64
+llvm.func @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<8xi32>, %compdata: vector<16xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_16(<8 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v8i32.v64i8(<8 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to16 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<8xi32>, vector<16xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=32: metadata=16, compressed_data=32, data=128
+llvm.func @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<16xi32>, %compdata: vector<32xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp4to16_elmsz_b8_idxsz_b4_rpf_32(<16 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <128 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <512 x i8> @llvm.nvvm.spdecompress.v512i8.v16i32.v128i8(<16 x i32> %0, <128 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 16)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <512 x i8> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to16 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<16xi32>, vector<32xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
diff --git a/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp4to8.mlir b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp4to8.mlir
new file mode 100644
index 00000000000000..7489b33472b463
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/spdecompress-sp4to8.mlir
@@ -0,0 +1,99 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// rpf=1: metadata=1, compressed_data=1, data=2
+llvm.func @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_1(%metadata: i32, %compdata: i32) -> vector<2xi32> {
+ // CHECK-LABEL: define <2 x i32> @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_1(i32 %0, i32 %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast i32 %1 to <4 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <8 x i8> @llvm.nvvm.spdecompress.v8i8.i32.v4i8(i32 %0, <4 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <8 x i8> %[[RES]] to <2 x i32>
+ // CHECK-NEXT: ret <2 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to8 index_size = b4 elem_size = b8 rep_factor = x1
+ : i32, i32 -> vector<2xi32>
+ llvm.return %res : vector<2xi32>
+}
+
+// rpf=2: metadata=1, compressed_data=2, data=4
+llvm.func @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_2(%metadata: i32, %compdata: vector<2xi32>) -> vector<4xi32> {
+ // CHECK-LABEL: define <4 x i32> @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_2(i32 %0, <2 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <2 x i32> %1 to <8 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <16 x i8> @llvm.nvvm.spdecompress.v16i8.i32.v8i8(i32 %0, <8 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <16 x i8> %[[RES]] to <4 x i32>
+ // CHECK-NEXT: ret <4 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to8 index_size = b4 elem_size = b8 rep_factor = x2
+ : i32, vector<2xi32> -> vector<4xi32>
+ llvm.return %res : vector<4xi32>
+}
+
+// rpf=4: metadata=2, compressed_data=4, data=8
+llvm.func @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_4(%metadata: vector<2xi32>, %compdata: vector<4xi32>) -> vector<8xi32> {
+ // CHECK-LABEL: define <8 x i32> @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_4(<2 x i32> %0, <4 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <4 x i32> %1 to <16 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <32 x i8> @llvm.nvvm.spdecompress.v32i8.v2i32.v16i8(<2 x i32> %0, <16 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <32 x i8> %[[RES]] to <8 x i32>
+ // CHECK-NEXT: ret <8 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to8 index_size = b4 elem_size = b8 rep_factor = x4
+ : vector<2xi32>, vector<4xi32> -> vector<8xi32>
+ llvm.return %res : vector<8xi32>
+}
+
+// rpf=8: metadata=4, compressed_data=8, data=16
+llvm.func @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_8(%metadata: vector<4xi32>, %compdata: vector<8xi32>) -> vector<16xi32> {
+ // CHECK-LABEL: define <16 x i32> @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_8(<4 x i32> %0, <8 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <8 x i32> %1 to <32 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <64 x i8> @llvm.nvvm.spdecompress.v64i8.v4i32.v32i8(<4 x i32> %0, <32 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <64 x i8> %[[RES]] to <16 x i32>
+ // CHECK-NEXT: ret <16 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to8 index_size = b4 elem_size = b8 rep_factor = x8
+ : vector<4xi32>, vector<8xi32> -> vector<16xi32>
+ llvm.return %res : vector<16xi32>
+}
+
+// rpf=16: metadata=8, compressed_data=16, data=32
+llvm.func @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_16(%metadata: vector<8xi32>, %compdata: vector<16xi32>) -> vector<32xi32> {
+ // CHECK-LABEL: define <32 x i32> @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_16(<8 x i32> %0, <16 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <16 x i32> %1 to <64 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <128 x i8> @llvm.nvvm.spdecompress.v128i8.v8i32.v64i8(<8 x i32> %0, <64 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <128 x i8> %[[RES]] to <32 x i32>
+ // CHECK-NEXT: ret <32 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to8 index_size = b4 elem_size = b8 rep_factor = x16
+ : vector<8xi32>, vector<16xi32> -> vector<32xi32>
+ llvm.return %res : vector<32xi32>
+}
+
+// rpf=32: metadata=16, compressed_data=32, data=64
+llvm.func @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_32(%metadata: vector<16xi32>, %compdata: vector<32xi32>) -> vector<64xi32> {
+ // CHECK-LABEL: define <64 x i32> @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_32(<16 x i32> %0, <32 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <32 x i32> %1 to <128 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <256 x i8> @llvm.nvvm.spdecompress.v256i8.v16i32.v128i8(<16 x i32> %0, <128 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <256 x i8> %[[RES]] to <64 x i32>
+ // CHECK-NEXT: ret <64 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to8 index_size = b4 elem_size = b8 rep_factor = x32
+ : vector<16xi32>, vector<32xi32> -> vector<64xi32>
+ llvm.return %res : vector<64xi32>
+}
+
+// rpf=64: metadata=32, compressed_data=64, data=128
+llvm.func @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_64(%metadata: vector<32xi32>, %compdata: vector<64xi32>) -> vector<128xi32> {
+ // CHECK-LABEL: define <128 x i32> @spdecompress_sp4to8_elmsz_b8_idxsz_b4_rpf_64(<32 x i32> %0, <64 x i32> %1) {
+ // CHECK-NEXT: %[[CDATA:[0-9]+]] = bitcast <64 x i32> %1 to <256 x i8>
+ // CHECK-NEXT: %[[RES:[0-9]+]] = call <512 x i8> @llvm.nvvm.spdecompress.v512i8.v32i32.v256i8(<32 x i32> %0, <256 x i8> %[[CDATA]], /* idx_size= */ i32 4, /* num_tgt= */ i32 8)
+ // CHECK-NEXT: %[[DATA:[0-9]+]] = bitcast <512 x i8> %[[RES]] to <128 x i32>
+ // CHECK-NEXT: ret <128 x i32> %[[DATA]]
+ // CHECK-NEXT: }
+ %res = nvvm.spdecompress %metadata, %compdata
+ comp_factor = sp4to8 index_size = b4 elem_size = b8 rep_factor = x64
+ : vector<32xi32>, vector<64xi32> -> vector<128xi32>
+ llvm.return %res : vector<128xi32>
+}
More information about the Mlir-commits
mailing list