[clang] 660a54c - [mlir][llvm] Fix metadata and alignment on masked memory intrinsics (#221879)
via cfe-commits
cfe-commits at lists.llvm.org
Wed Sep 9 07:14:08 PDT 2026
Author: Tobias Gysi
Date: 2026-09-09T16:14:03+02:00
New Revision: 660a54c87b3940c7eb1e51a5af523b9a24fc2b99
URL: https://github.com/llvm/llvm-project/commit/660a54c87b3940c7eb1e51a5af523b9a24fc2b99
DIFF: https://github.com/llvm/llvm-project/commit/660a54c87b3940c7eb1e51a5af523b9a24fc2b99.diff
LOG: [mlir][llvm] Fix metadata and alignment on masked memory intrinsics (#221879)
llvm.intr.masked.load, masked.store, masked.gather and masked.scatter
implemented neither AccessGroupOpInterface nor AliasAnalysisOpInterface,
so they could not carry access_groups, alias_scopes, noalias_scopes or
tbaa. Importing LLVM IR that attached such metadata to one of them
silently dropped it, while llvm.memcpy in the same module preserved it.
Also widen alignment from I32Attr to I64Attr, matching the rest of the
LLVM dialect and LLVM proper, and make it optional. LangRef gives the
default alignment of these four intrinsics as the ABI alignment of the
type, exactly as for llvm.load and llvm.store, so an absent align
parameter attribute now imports as an absent alignment attribute instead
of being materialised as align 1, and an absent alignment attribute
exports without an align parameter attribute. This changes the textual
IR, as alignment = 4 : i32 becomes alignment = 4 : i64. Downstream
updates should rewrite only lines that mention a masked intrinsic rather
than blanket-substituting i32 with i64; other i32 spellings must stay,
including llvm.intr.matrix.*'s rows and columns attributes and i32
vector and element types on the intrinsics themselves.
Co-Authored-By: Claude Opus 5
Added:
mlir/test/Target/LLVMIR/Import/masked-intrinsic-metadata.ll
mlir/test/Target/LLVMIR/masked-intrinsic-metadata.mlir
Modified:
clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp
mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir
mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
mlir/test/Dialect/LLVMIR/opaque-ptr.mlir
mlir/test/Dialect/LLVMIR/roundtrip.mlir
mlir/test/Target/LLVMIR/Import/intrinsic.ll
mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
mlir/test/Target/LLVMIR/llvmir-invalid.mlir
mlir/test/Target/LLVMIR/opaque-ptr.mlir
Removed:
################################################################################
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 5e0f9c8985685..6761da8be6491 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -2373,11 +2373,9 @@ cir::direct::CIRToLLVMVecMaskedLoadOpLowering::matchAndRewrite(
unsigned alignment =
(unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmResTy));
- mlir::IntegerAttr alignAttr = rewriter.getI32IntegerAttr(alignment);
-
auto newLoad = mlir::LLVM::MaskedLoadOp::create(
rewriter, op.getLoc(), llvmResTy, adaptor.getAddr(), adaptor.getMask(),
- adaptor.getPassThru(), alignAttr);
+ adaptor.getPassThru(), alignment);
rewriter.replaceOp(op, newLoad.getResult());
return mlir::success();
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index ea33df8dd828c..3cc6859deb8ef 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -17,6 +17,7 @@ class LLVM_UnaryIntrOpBase<string func, Type element,
list<Trait> traits = [], bit requiresFastmath = 0> :
LLVM_OneResultIntrOp<func, [], [0],
!listconcat([Pure, SameOperandsAndResultType], traits),
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
requiresFastmath> {
dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$in);
let assemblyFormat = "`(` operands `)` attr-dict `:` "
@@ -39,6 +40,7 @@ class LLVM_BinarySameArgsIntrOpBase<string func, Type element,
list<Trait> traits = [], bit requiresFastmath = 0> :
LLVM_OneResultIntrOp<func, [], [0],
!listconcat([Pure, SameOperandsAndResultType], traits),
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
requiresFastmath> {
dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$a,
LLVM_ScalarOrVectorOf<element>:$b);
@@ -63,6 +65,7 @@ class LLVM_TernarySameArgsIntrOpBase<string func, Type element,
list<Trait> traits = [], bit requiresFastmath = 0> :
LLVM_OneResultIntrOp<func, [], [0],
!listconcat([Pure, SameOperandsAndResultType], traits),
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
requiresFastmath> {
dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$a,
LLVM_ScalarOrVectorOf<element>:$b,
@@ -87,6 +90,7 @@ class LLVM_TernarySameArgsIntrOpF<string func, list<Trait> traits = []> :
class LLVM_CountZerosIntrOp<string func, list<Trait> traits = []> :
LLVM_OneResultIntrOp<func, [], [0],
!listconcat([Pure, SameOperandsAndResultType], traits),
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["is_zero_poison"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
@@ -94,6 +98,7 @@ class LLVM_CountZerosIntrOp<string func, list<Trait> traits = []> :
}
def LLVM_AbsOp : LLVM_OneResultIntrOp<"abs", [], [0], [Pure],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["is_int_min_poison"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
@@ -108,6 +113,7 @@ def LLVM_ArithmeticFenceOp
}
def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["bit"]> {
let arguments = (ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$in, I32Attr:$bit);
@@ -115,7 +121,8 @@ def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure],
class LLVM_PowFI<string func> :
LLVM_OneResultIntrOp<func, [], [0,1],
- [Pure], /*requiresFastmath=*/1> {
+ [Pure], /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
+ /*requiresFastmath=*/1> {
let arguments =
(ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$val,
AnySignlessInteger:$power,
@@ -391,6 +398,7 @@ def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">;
def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1],
[DeclareOpInterfaceMethods<PromotableOpInterface>],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[0], /*immArgAttrNames=*/["size"]> {
let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
@@ -444,6 +452,8 @@ class LLVM_ConstrainedIntr<string mnem, int numArgs,
!gt(hasRoundingMode, 0) : [DeclareOpInterfaceMethods<RoundingModeOpInterface>],
true : [])
# extraTraits,
+ /*requiresAccessGroup=*/0,
+ /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/1,
/*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[],
@@ -592,6 +602,7 @@ class LLVM_ConstrainedFCmpIntrBase<string mnem>
"operands", "arg_0", "res",
"::mlir::LLVM::getI1SameShape($_self)">,
DeclareOpInterfaceMethods<FPExceptionBehaviorOpInterface>],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/1> {
let arguments = (ins FCmpPredicate:$predicate,
LLVM_Type:$arg_0,
@@ -883,6 +894,7 @@ def LLVM_ExpectOp
def LLVM_ExpectWithProbabilityOp
: LLVM_OneResultIntrOp<"expect.with.probability", [], [0],
[Pure, AllTypesMatch<["val", "expected", "res"]>],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[2], /*immArgAttrNames=*/["prob"]> {
let arguments = (ins AnySignlessInteger:$val,
@@ -1187,6 +1199,7 @@ def LLVM_StackRestoreOp : LLVM_ZeroResultIntrOp<"stackrestore", [0]> {
class LLVM_VecReductionBase<string mnem, Type element, bit requiresFastmath=0>
: LLVM_OneResultIntrOp<"vector.reduce." # mnem, [], [0],
[Pure, SameOperandsAndResultElementType],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
requiresFastmath> {
dag commonArgs = (ins LLVM_VectorOf<element>:$in);
}
@@ -1213,6 +1226,8 @@ class LLVM_VecReductionAccBase<string mnem, Type element>
/*overloadedResults=*/[],
/*overloadedOperands=*/[1],
/*traits=*/[Pure, SameOperandsAndResultElementType],
+ /*requiresAccessGroup=*/0,
+ /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/1>,
Arguments<(ins element:$start_value,
LLVM_VectorOf<element>:$input,
@@ -1366,104 +1381,206 @@ def LLVM_GetActiveLaneMaskOp
}
/// Create a call to Masked Load intrinsic.
-def LLVM_MaskedLoadOp : LLVM_OneResultIntrOp<"masked.load"> {
- let arguments = (ins LLVM_AnyPointer:$data, LLVM_VectorOf<I1>:$mask,
- Optional<LLVM_AnyVector>:$pass_thru, I32Attr:$alignment,
- UnitAttr:$nontemporal);
+def LLVM_MaskedLoadOp
+ : LLVM_OneResultIntrOp<"masked.load", /*overloadedResults=*/[],
+ /*overloadedOperands=*/[], /*traits=*/[],
+ /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> {
+ dag args = (ins LLVM_AnyPointer:$data, LLVM_VectorOf<I1>:$mask,
+ Optional<LLVM_AnyVector>:$pass_thru,
+ OptionalAttr<I64Attr>:$alignment, UnitAttr:$nontemporal);
+ let arguments = !con(args, baseArgs);
let results = (outs LLVM_AnyVector:$res);
+ let builders = [
+ OpBuilder<(ins "Type":$res, "Value":$data, "Value":$mask,
+ "Value":$pass_thru, "uint64_t":$alignment,
+ CArg<"bool", "false">:$nontemporal), [{
+ build($_builder, $_state, res, data, mask, pass_thru,
+ $_builder.getI64IntegerAttr(alignment),
+ nontemporal ? $_builder.getUnitAttr() : nullptr,
+ /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+ /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
+ }]>
+ ];
let assemblyFormat =
"operands attr-dict `:` functional-type(operands, results)";
string llvmBuilder = [{
+ // An absent alignment attribute means the ABI alignment of the type, which
+ // LLVM spells as an absent align parameter attribute. The IRBuilder helper
+ // always attaches one, so remove it again in that case.
+ llvm::Align alignment =
+ $alignment ? llvm::Align(*$alignment) : llvm::Align(1);
auto *inst = $pass_thru ? builder.CreateMaskedLoad(
- $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru) :
+ $_resultType, $data, alignment, $mask, $pass_thru) :
builder.CreateMaskedLoad(
- $_resultType, $data, llvm::Align($alignment), $mask);
+ $_resultType, $data, alignment, $mask);
+ if (!$alignment)
+ inst->removeParamAttr(0, llvm::Attribute::Alignment);
$res = inst;
- }] #setNonTemporalMetadataCode;
+ }] # setNonTemporalMetadataCode
+ # setAccessGroupsMetadataCode
+ # setAliasAnalysisMetadataCode;
string mlirBuilder = [{
auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
bool nontemporal = intrinInst->hasMetadata(llvm::LLVMContext::MD_nontemporal);
- llvm::Align alignment = intrinInst->getParamAlign(0).valueOrOne();
+ llvm::MaybeAlign alignment = intrinInst->getParamAlign(0);
$res = LLVM::MaskedLoadOp::create($_builder, $_location,
$_resultType, $data, $mask, $pass_thru,
- $_builder.getI32IntegerAttr(alignment.value()),
- nontemporal ? $_builder.getUnitAttr() : nullptr);
+ alignment ? $_builder.getI64IntegerAttr(alignment->value())
+ : ::mlir::IntegerAttr(),
+ nontemporal ? $_builder.getUnitAttr() : nullptr,
+ /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+ /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}];
- list<int> llvmArgIndices = [0, 1, 2, -1, -1];
+ list<int> llvmArgIndices = [0, 1, 2, -1, -1, -1, -1, -1, -1];
}
/// Create a call to Masked Store intrinsic.
-def LLVM_MaskedStoreOp : LLVM_ZeroResultIntrOp<"masked.store"> {
- let arguments = (ins LLVM_AnyVector:$value, LLVM_AnyPointer:$data,
- LLVM_VectorOf<I1>:$mask, I32Attr:$alignment,
- UnitAttr:$nontemporal);
- let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
+def LLVM_MaskedStoreOp
+ : LLVM_ZeroResultIntrOp<"masked.store", /*overloadedOperands=*/[],
+ /*traits=*/[], /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> {
+ dag args = (ins LLVM_AnyVector:$value, LLVM_AnyPointer:$data,
+ LLVM_VectorOf<I1>:$mask, OptionalAttr<I64Attr>:$alignment,
+ UnitAttr:$nontemporal);
+ let arguments = !con(args, baseArgs);
+ let builders = [
+ LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder,
+ OpBuilder<(ins "Value":$value, "Value":$data, "Value":$mask,
+ "uint64_t":$alignment,
+ CArg<"bool", "false">:$nontemporal), [{
+ build($_builder, $_state, value, data, mask,
+ $_builder.getI64IntegerAttr(alignment),
+ nontemporal ? $_builder.getUnitAttr() : nullptr,
+ /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+ /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
+ }]>
+ ];
let assemblyFormat = "$value `,` $data `,` $mask attr-dict `:` "
"type($value) `,` type($mask) `into` qualified(type($data))";
string llvmBuilder = [{
- auto *inst = builder.CreateMaskedStore(
- $value, $data, llvm::Align($alignment), $mask);
- }] #setNonTemporalMetadataCode;
+ // An absent alignment attribute means the ABI alignment of the type, which
+ // LLVM spells as an absent align parameter attribute. The IRBuilder helper
+ // always attaches one, so remove it again in that case.
+ llvm::Align alignment =
+ $alignment ? llvm::Align(*$alignment) : llvm::Align(1);
+ auto *inst = builder.CreateMaskedStore($value, $data, alignment, $mask);
+ if (!$alignment)
+ inst->removeParamAttr(1, llvm::Attribute::Alignment);
+ }] # setNonTemporalMetadataCode
+ # setAccessGroupsMetadataCode
+ # setAliasAnalysisMetadataCode;
string mlirBuilder = [{
auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
- llvm::Align alignment = intrinInst->getParamAlign(1).valueOrOne();
+ llvm::MaybeAlign alignment = intrinInst->getParamAlign(1);
bool nontemporal = intrinInst->hasMetadata(llvm::LLVMContext::MD_nontemporal);
$_op = LLVM::MaskedStoreOp::create($_builder, $_location,
- $value, $data, $mask, $_builder.getI32IntegerAttr(alignment.value()),
- nontemporal ? $_builder.getUnitAttr() : nullptr);
+ $value, $data, $mask,
+ alignment ? $_builder.getI64IntegerAttr(alignment->value())
+ : ::mlir::IntegerAttr(),
+ nontemporal ? $_builder.getUnitAttr() : nullptr,
+ /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+ /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}];
- list<int> llvmArgIndices = [0, 1, 2, -1, -1];
+ list<int> llvmArgIndices = [0, 1, 2, -1, -1, -1, -1, -1, -1];
}
/// Create a call to Masked Gather intrinsic.
-def LLVM_masked_gather : LLVM_OneResultIntrOp<"masked.gather"> {
- let arguments = (ins LLVM_VectorOf<LLVM_AnyPointer>:$ptrs,
- LLVM_VectorOf<I1>:$mask, Variadic<LLVM_AnyVector>:$pass_thru,
- I32Attr:$alignment);
+def LLVM_masked_gather
+ : LLVM_OneResultIntrOp<"masked.gather", /*overloadedResults=*/[],
+ /*overloadedOperands=*/[], /*traits=*/[],
+ /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> {
+ dag args = (ins LLVM_VectorOf<LLVM_AnyPointer>:$ptrs,
+ LLVM_VectorOf<I1>:$mask, Variadic<LLVM_AnyVector>:$pass_thru,
+ OptionalAttr<I64Attr>:$alignment);
+ let arguments = !con(args, baseArgs);
let results = (outs LLVM_AnyVector:$res);
- let builders = [LLVM_OneResultOpBuilder];
+ let builders = [
+ LLVM_OneResultOpBuilder,
+ OpBuilder<(ins "Type":$res, "Value":$ptrs, "Value":$mask,
+ "ValueRange":$pass_thru, "uint64_t":$alignment), [{
+ build($_builder, $_state, res, ptrs, mask, pass_thru,
+ $_builder.getI64IntegerAttr(alignment),
+ /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+ /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
+ }]>
+ ];
let assemblyFormat =
"operands attr-dict `:` functional-type(operands, results)";
string llvmBuilder = [{
- $res = $pass_thru.empty() ? builder.CreateMaskedGather(
- $_resultType, $ptrs, llvm::Align($alignment), $mask) :
+ // An absent alignment attribute means the ABI alignment of the type, which
+ // LLVM spells as an absent align parameter attribute. The IRBuilder helper
+ // always attaches one, so remove it again in that case.
+ llvm::Align alignment =
+ $alignment ? llvm::Align(*$alignment) : llvm::Align(1);
+ auto *inst = $pass_thru.empty() ? builder.CreateMaskedGather(
+ $_resultType, $ptrs, alignment, $mask) :
builder.CreateMaskedGather(
- $_resultType, $ptrs, llvm::Align($alignment), $mask, $pass_thru[0]);
- }];
+ $_resultType, $ptrs, alignment, $mask, $pass_thru[0]);
+ if (!$alignment)
+ inst->removeParamAttr(0, llvm::Attribute::Alignment);
+ $res = inst;
+ }] # setAccessGroupsMetadataCode
+ # setAliasAnalysisMetadataCode;
string mlirBuilder = [{
auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
- llvm::Align alignment = intrinInst->getParamAlign(0).valueOrOne();
+ llvm::MaybeAlign alignment = intrinInst->getParamAlign(0);
$res = LLVM::masked_gather::create($_builder, $_location,
$_resultType, $ptrs, $mask, $pass_thru,
- $_builder.getI32IntegerAttr(alignment.value()));
+ alignment ? $_builder.getI64IntegerAttr(alignment->value())
+ : ::mlir::IntegerAttr(),
+ /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+ /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}];
- list<int> llvmArgIndices = [0, 1, 2, -1];
+ list<int> llvmArgIndices = [0, 1, 2, -1, -1, -1, -1, -1];
let hasVerifier = 1;
}
/// Create a call to Masked Scatter intrinsic.
-def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
- let arguments = (ins LLVM_AnyVector:$value, LLVM_VectorOf<LLVM_AnyPointer>:$ptrs,
- LLVM_VectorOf<I1>:$mask, I32Attr:$alignment);
- let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder];
+def LLVM_masked_scatter
+ : LLVM_ZeroResultIntrOp<"masked.scatter", /*overloadedOperands=*/[],
+ /*traits=*/[], /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> {
+ dag args = (ins LLVM_AnyVector:$value,
+ LLVM_VectorOf<LLVM_AnyPointer>:$ptrs,
+ LLVM_VectorOf<I1>:$mask, OptionalAttr<I64Attr>:$alignment);
+ let arguments = !con(args, baseArgs);
+ let builders = [
+ LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder,
+ OpBuilder<(ins "Value":$value, "Value":$ptrs, "Value":$mask,
+ "uint64_t":$alignment), [{
+ build($_builder, $_state, value, ptrs, mask,
+ $_builder.getI64IntegerAttr(alignment),
+ /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+ /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
+ }]>
+ ];
let assemblyFormat = "$value `,` $ptrs `,` $mask attr-dict `:` "
"type($value) `,` type($mask) `into` type($ptrs)";
string llvmBuilder = [{
- builder.CreateMaskedScatter(
- $value, $ptrs, llvm::Align($alignment), $mask);
- }];
+ // An absent alignment attribute means the ABI alignment of the type, which
+ // LLVM spells as an absent align parameter attribute. The IRBuilder helper
+ // always attaches one, so remove it again in that case.
+ llvm::Align alignment =
+ $alignment ? llvm::Align(*$alignment) : llvm::Align(1);
+ auto *inst = builder.CreateMaskedScatter($value, $ptrs, alignment, $mask);
+ if (!$alignment)
+ inst->removeParamAttr(1, llvm::Attribute::Alignment);
+ }] # setAccessGroupsMetadataCode
+ # setAliasAnalysisMetadataCode;
string mlirBuilder = [{
auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
- llvm::Align alignment = intrinInst->getParamAlign(1).valueOrOne();
+ llvm::MaybeAlign alignment = intrinInst->getParamAlign(1);
$_op = LLVM::masked_scatter::create($_builder, $_location,
- $value, $ptrs, $mask, $_builder.getI32IntegerAttr(alignment.value()));
+ $value, $ptrs, $mask,
+ alignment ? $_builder.getI64IntegerAttr(alignment->value())
+ : ::mlir::IntegerAttr(),
+ /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
+ /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}];
- list<int> llvmArgIndices = [0, 1, 2, -1];
+ list<int> llvmArgIndices = [0, 1, 2, -1, -1, -1, -1, -1];
let hasVerifier = 1;
}
@@ -1471,7 +1588,8 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
/// Create a call to Masked Expand Load intrinsic.
def LLVM_masked_expandload
: LLVM_OneResultIntrOp<"masked.expandload", [0], [0],
- /*traits=*/[], /*requiresFastMath=*/0, /*requiresArgAndResultAttrs=*/1,
+ /*traits=*/[], /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
+ /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/1,
/*immArgPositions=*/[], /*immArgAttrNames=*/[]> {
dag args = (ins LLVM_AnyPointer:$ptr,
LLVM_VectorOf<I1>:$mask,
@@ -1583,6 +1701,7 @@ def LLVM_vector_insert
PredOpTrait<"it is not inserting scalable into fixed-length vectors.",
CPred<"!isScalableVectorType($srcvec.getType()) || "
"isScalableVectorType($dstvec.getType())">>],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[2], /*immArgAttrNames=*/["pos"]> {
let arguments = (ins LLVM_AnyVector:$dstvec, LLVM_AnyVector:$srcvec,
@@ -1617,6 +1736,7 @@ def LLVM_vector_extract
PredOpTrait<"it is not extracting scalable from fixed-length vectors.",
CPred<"!isScalableVectorType($res.getType()) || "
"isScalableVectorType($srcvec.getType())">>],
+ /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
/*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
/*immArgPositions=*/[1], /*immArgAttrNames=*/["pos"]> {
let arguments = (ins LLVM_AnyVector:$srcvec, I64Attr:$pos);
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index a787840663171..c71f9d0a3f54f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -473,12 +473,14 @@ class LLVM_ZeroResultIntrOp<string mnem, list<int> overloadedOperands = [],
class LLVM_OneResultIntrOp<string mnem, list<int> overloadedResults = [],
list<int> overloadedOperands = [],
list<Trait> traits = [],
+ bit requiresAccessGroup = 0,
+ bit requiresAliasAnalysis = 0,
bit requiresFastmath = 0,
bit requiresArgAndResultAttrs = 0,
list<int> immArgPositions = [],
list<string> immArgAttrNames = []>
: LLVM_IntrOp<mnem, overloadedResults, overloadedOperands, traits, 1,
- /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
+ requiresAccessGroup, requiresAliasAnalysis,
requiresFastmath, requiresArgAndResultAttrs,
/*requiresOpBundles=*/0, immArgPositions,
immArgAttrNames>;
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 152c45b695cee..c1252b86cbcbb 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -345,7 +345,7 @@ class VectorGatherOpConversion
// Replace with the gather intrinsic.
rewriter.replaceOpWithNewOp<LLVM::masked_gather>(
gather, typeConverter->convertType(vType), ptrs, adaptor.getMask(),
- adaptor.getPassThru(), rewriter.getI32IntegerAttr(align));
+ adaptor.getPassThru(), align);
return success();
}
@@ -403,8 +403,7 @@ class VectorScatterOpConversion
// Replace with the scatter intrinsic.
rewriter.replaceOpWithNewOp<LLVM::masked_scatter>(
- scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(),
- rewriter.getI32IntegerAttr(align));
+ scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(), align);
return success();
}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp
index 406ccd6849972..44a9284f5a5ab 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp
@@ -116,6 +116,22 @@ SmallVector<Value> mlir::LLVM::MemsetInlineOp::getAccessedOperands() {
return {getDst()};
}
+SmallVector<Value> mlir::LLVM::MaskedLoadOp::getAccessedOperands() {
+ return {getData()};
+}
+
+SmallVector<Value> mlir::LLVM::MaskedStoreOp::getAccessedOperands() {
+ return {getData()};
+}
+
+SmallVector<Value> mlir::LLVM::masked_gather::getAccessedOperands() {
+ return {getPtrs()};
+}
+
+SmallVector<Value> mlir::LLVM::masked_scatter::getAccessedOperands() {
+ return {getPtrs()};
+}
+
SmallVector<Value> mlir::LLVM::CallOp::getAccessedOperands() {
// Note: This must not use `getArgOperands`, which excludes the variadic
// arguments of a call to a variadic callee. Those are passed to the callee
diff --git a/mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir b/mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir
index bc0bb9d69ec80..fd4a9eb765e1b 100644
--- a/mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir
@@ -72,8 +72,8 @@ func.func @masked_load(%base: memref<?xf32>, %mask: vector<16xi1>, %passthru: ve
// ALL-LABEL: func @masked_load
-// VEC-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 64 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
-// MEMREF-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+// VEC-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 64 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+// MEMREF-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
// -----
@@ -85,8 +85,8 @@ func.func @masked_load_with_alignment_attribute(%base: memref<?xf32>, %mask: vec
// ALL-LABEL: func @masked_load_with_alignment_attribute
-// VEC-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
-// MEMREF-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+// VEC-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+// MEMREF-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
// -----
@@ -102,8 +102,8 @@ func.func @masked_store(%base: memref<?xf32>, %mask: vector<16xi1>, %passthru: v
// ALL-LABEL: func @masked_store
-// VEC-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 64 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
-// MEMREF-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
+// VEC-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 64 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr
+// MEMREF-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr
// -----
@@ -115,8 +115,8 @@ func.func @masked_store_with_alignment_attribute(%base: memref<?xf32>, %mask: ve
// ALL-LABEL: func @masked_store_with_alignment_attribute
-// VEC-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
-// MEMREF-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
+// VEC-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr
+// MEMREF-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr
// -----
@@ -132,8 +132,8 @@ func.func @scatter(%base: memref<?xf32>, %index: vector<3xi32>, %mask: vector<3x
// ALL-LABEL: func @scatter
-// VEC-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 16 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
-// MEMREF-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
+// VEC-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 16 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
+// MEMREF-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
// -----
@@ -145,8 +145,8 @@ func.func @scatter_with_alignment_attribute(%base: memref<?xf32>, %index: vector
// ALL-LABEL: func @scatter_with_alignment_attribute
-// VEC-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
-// MEMREF-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
+// VEC-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
+// MEMREF-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
// -----
@@ -162,8 +162,8 @@ func.func @gather(%base: memref<?xf32>, %index: vector<3xi32>, %mask: vector<3xi
// ALL-LABEL: func @gather
-// VEC-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 16 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
-// MEMREF-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// VEC-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 16 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// MEMREF-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
// -----
@@ -175,5 +175,5 @@ func.func @gather_with_alignment_attribute(%base: memref<?xf32>, %index: vector<
// ALL-LABEL: func @gather_with_alignment_attribute
-// VEC-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
-// MEMREF-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// VEC-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// MEMREF-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 4b89bc60a3d76..f7c7929ef9605 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -1591,7 +1591,7 @@ func.func @masked_load(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
// CHECK: return %[[L]] : vector<16xf32>
// -----
@@ -1606,7 +1606,7 @@ func.func @masked_load_scalable(%arg0: memref<?xf32>, %arg1: vector<[16]xi1>, %a
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xf32>) -> vector<[16]xf32>
+// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xf32>) -> vector<[16]xf32>
// CHECK: return %[[L]] : vector<[16]xf32>
// -----
@@ -1617,7 +1617,7 @@ func.func @masked_load_index(%arg0: memref<?xindex>, %arg1: vector<16xi1>, %arg2
return %0 : vector<16xindex>
}
// CHECK-LABEL: func @masked_load_index
-// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xi64>) -> vector<16xi64>
+// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xi64>) -> vector<16xi64>
// -----
@@ -1627,7 +1627,7 @@ func.func @masked_load_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16]
return %0 : vector<[16]xindex>
}
// CHECK-LABEL: func @masked_load_index_scalable
-// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xi64>) -> vector<[16]xi64>
+// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xi64>) -> vector<[16]xi64>
// -----
@@ -1637,7 +1637,7 @@ func.func @masked_load_with_alignment(%arg0: memref<?xf32>, %arg1: vector<16xi1>
}
// CHECK-LABEL: func @masked_load_with_alignment
-// CHECK: llvm.intr.masked.load %{{.*}} {alignment = 2 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+// CHECK: llvm.intr.masked.load %{{.*}} {alignment = 2 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
// -----
@@ -1655,7 +1655,7 @@ func.func @masked_store(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vecto
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
+// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr
// -----
@@ -1669,7 +1669,7 @@ func.func @masked_store_scalable(%arg0: memref<?xf32>, %arg1: vector<[16]xi1>, %
// CHECK: %[[CO:.*]] = arith.constant 0 : index
// CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32
-// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[16]xf32>, vector<[16]xi1> into !llvm.ptr
+// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<[16]xf32>, vector<[16]xi1> into !llvm.ptr
// -----
@@ -1679,7 +1679,7 @@ func.func @masked_store_index(%arg0: memref<?xindex>, %arg1: vector<16xi1>, %arg
return
}
// CHECK-LABEL: func @masked_store_index
-// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<16xi64>, vector<16xi1> into !llvm.ptr
+// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<16xi64>, vector<16xi1> into !llvm.ptr
// -----
@@ -1689,7 +1689,7 @@ func.func @masked_store_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16
return
}
// CHECK-LABEL: func @masked_store_index_scalable
-// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<[16]xi64>, vector<[16]xi1> into !llvm.ptr
+// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<[16]xi64>, vector<[16]xi1> into !llvm.ptr
// -----
@@ -1699,7 +1699,7 @@ func.func @masked_store_with_alignment(%arg0: memref<?xf32>, %arg1: vector<16xi1
}
// CHECK-LABEL: func @masked_store_with_alignment
-// CHECK: llvm.intr.masked.store %{{.*}} {alignment = 2 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
+// CHECK: llvm.intr.masked.store %{{.*}} {alignment = 2 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr
// -----
@@ -1715,7 +1715,7 @@ func.func @gather(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1
// CHECK-LABEL: func @gather
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi32>) -> vector<3x!llvm.ptr>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
// CHECK: return %[[G]] : vector<3xf32>
// -----
@@ -1728,7 +1728,7 @@ func.func @gather_scalable(%arg0: memref<?xf32>, %arg1: vector<[3]xi32>, %arg2:
// CHECK-LABEL: func @gather_scalable
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi32>) -> vector<[3]x!llvm.ptr>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
// CHECK: return %[[G]] : vector<[3]xf32>
// -----
@@ -1741,7 +1741,7 @@ func.func @gather_global_memory(%arg0: memref<?xf32, 1>, %arg1: vector<3xi32>, %
// CHECK-LABEL: func @gather_global_memory
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, vector<3xi32>) -> vector<3x!llvm.ptr<1>>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr<1>>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr<1>>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
// CHECK: return %[[G]] : vector<3xf32>
// -----
@@ -1754,7 +1754,7 @@ func.func @gather_global_memory_scalable(%arg0: memref<?xf32, 1>, %arg1: vector<
// CHECK-LABEL: func @gather_global_memory_scalable
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, vector<[3]xi32>) -> vector<[3]x!llvm.ptr<1>>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[3]x!llvm.ptr<1>>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[3]x!llvm.ptr<1>>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
// CHECK: return %[[G]] : vector<[3]xf32>
// -----
@@ -1768,7 +1768,7 @@ func.func @gather_index(%arg0: memref<?xindex>, %arg1: vector<3xindex>, %arg2: v
// CHECK-LABEL: func @gather_index
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi64>) -> vector<3x!llvm.ptr>, i64
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xi64>) -> vector<3xi64>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xi64>) -> vector<3xi64>
// CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[G]] : vector<3xi64> to vector<3xindex>
// -----
@@ -1781,7 +1781,7 @@ func.func @gather_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[3]xindex
// CHECK-LABEL: func @gather_index_scalable
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi64>) -> vector<[3]x!llvm.ptr>, i64
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xi64>) -> vector<[3]xi64>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xi64>) -> vector<[3]xi64>
// CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[G]] : vector<[3]xi64> to vector<[3]xindex>
// -----
@@ -1795,7 +1795,7 @@ func.func @gather_1d_from_2d(%arg0: memref<4x4xf32>, %arg1: vector<4xi32>, %arg2
// CHECK-LABEL: func @gather_1d_from_2d
// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<4xi32>) -> vector<4x!llvm.ptr>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<4x!llvm.ptr>, vector<4xi1>, vector<4xf32>) -> vector<4xf32>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<4x!llvm.ptr>, vector<4xi1>, vector<4xf32>) -> vector<4xf32>
// CHECK: return %[[G]] : vector<4xf32>
// -----
@@ -1809,7 +1809,7 @@ func.func @gather_1d_from_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]x
// CHECK-LABEL: func @gather_1d_from_2d_scalable
// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<[4]xi32>) -> vector<[4]x!llvm.ptr>, f32
-// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[4]x!llvm.ptr>, vector<[4]xi1>, vector<[4]xf32>) -> vector<[4]xf32>
+// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[4]x!llvm.ptr>, vector<[4]xi1>, vector<[4]xf32>) -> vector<[4]xf32>
// CHECK: return %[[G]] : vector<[4]xf32>
// -----
@@ -1820,7 +1820,7 @@ func.func @gather_with_alignment(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %ar
}
// CHECK-LABEL: func @gather_with_alignment
-// CHECK: llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
// -----
@@ -1850,7 +1850,7 @@ func.func @scatter(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi
// CHECK-LABEL: func @scatter
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi32>) -> vector<3x!llvm.ptr>, f32
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
// -----
@@ -1862,7 +1862,7 @@ func.func @scatter_scalable(%arg0: memref<?xf32>, %arg1: vector<[3]xi32>, %arg2:
// CHECK-LABEL: func @scatter_scalable
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi32>) -> vector<[3]x!llvm.ptr>, f32
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[3]xf32>, vector<[3]xi1> into vector<[3]x!llvm.ptr>
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<[3]xf32>, vector<[3]xi1> into vector<[3]x!llvm.ptr>
// -----
@@ -1874,7 +1874,7 @@ func.func @scatter_index(%arg0: memref<?xindex>, %arg1: vector<3xindex>, %arg2:
// CHECK-LABEL: func @scatter_index
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi64>) -> vector<3x!llvm.ptr>, i64
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i32} : vector<3xi64>, vector<3xi1> into vector<3x!llvm.ptr>
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i64} : vector<3xi64>, vector<3xi1> into vector<3x!llvm.ptr>
// -----
@@ -1886,7 +1886,7 @@ func.func @scatter_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[3]xinde
// CHECK-LABEL: func @scatter_index_scalable
// CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi64>) -> vector<[3]x!llvm.ptr>, i64
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i32} : vector<[3]xi64>, vector<[3]xi1> into vector<[3]x!llvm.ptr>
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i64} : vector<[3]xi64>, vector<[3]xi1> into vector<[3]x!llvm.ptr>
// -----
@@ -1899,7 +1899,7 @@ func.func @scatter_1d_into_2d(%arg0: memref<4x4xf32>, %arg1: vector<4xi32>, %arg
// CHECK-LABEL: func @scatter_1d_into_2d
// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<4xi32>) -> vector<4x!llvm.ptr>, f32
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<4xf32>, vector<4xi1> into vector<4x!llvm.ptr>
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<4xf32>, vector<4xi1> into vector<4x!llvm.ptr>
// -----
@@ -1912,7 +1912,7 @@ func.func @scatter_1d_into_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]
// CHECK-LABEL: func @scatter_1d_into_2d_scalable
// CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32
// CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<[4]xi32>) -> vector<[4]x!llvm.ptr>, f32
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[4]xf32>, vector<[4]xi1> into vector<[4]x!llvm.ptr>
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<[4]xf32>, vector<[4]xi1> into vector<[4]x!llvm.ptr>
// -----
@@ -1922,7 +1922,7 @@ func.func @scatter_with_alignment(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %a
}
// CHECK-LABEL: func @scatter_with_alignment
-// CHECK: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
+// CHECK: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr>
// -----
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index e6dbb7b55d48d..6eba18fafff05 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -1606,8 +1606,8 @@ func.func @gather_with_mask(%arg0: memref<?xf32>, %arg1: vector<2x3xi32>, %arg2:
}
// CHECK-LABEL: func @gather_with_mask
-// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
-// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
+// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32>
// -----
@@ -1622,8 +1622,8 @@ func.func @gather_with_mask_scalable(%arg0: memref<?xf32>, %arg1: vector<2x[3]xi
}
// CHECK-LABEL: func @gather_with_mask_scalable
-// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
-// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
+// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
+// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32>
// -----
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
index 18deadd0d7a79..3b314830d58c1 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir
@@ -41,7 +41,7 @@ func.func @transfer_read_write_1d(%A : memref<?xf32>, %base: index) -> vector<17
//
// 6. Rewrite as a masked read.
// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %[[gep]], %[[mask]],
-// CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i32} :
+// CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i64} :
// CHECK-SAME: -> vector<17xf32>
//
// 1. Let dim be the memref dimension, compute the in-bound index (dim - offset)
@@ -62,7 +62,7 @@ func.func @transfer_read_write_1d(%A : memref<?xf32>, %base: index) -> vector<17
//
// 4. Rewrite as a masked write.
// CHECK: llvm.intr.masked.store %[[loaded]], %[[gep_b]], %[[mask_b]]
-// CHECK-SAME: {alignment = 4 : i32} :
+// CHECK-SAME: {alignment = 4 : i64} :
// CHECK-SAME: vector<17xf32>, vector<17xi1> into !llvm.ptr
func.func @transfer_read_write_1d_scalable(%A : memref<?xf32>, %base: index) -> vector<[17]xf32> {
@@ -105,7 +105,7 @@ func.func @transfer_read_write_1d_scalable(%A : memref<?xf32>, %base: index) ->
//
// 6. Rewrite as a masked read.
// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %[[gep]], %[[mask]],
-// CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i32} :
+// CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i64} :
// CHECK-SAME: -> vector<[17]xf32>
//
// 1. Let dim be the memref dimension, compute the in-bound index (dim - offset)
@@ -129,7 +129,7 @@ func.func @transfer_read_write_1d_scalable(%A : memref<?xf32>, %base: index) ->
//
// 5. Rewrite as a masked write.
// CHECK: llvm.intr.masked.store %[[loaded]], %[[gep_b]], %[[mask_b]]
-// CHECK-SAME: {alignment = 4 : i32} :
+// CHECK-SAME: {alignment = 4 : i64} :
// CHECK-SAME: vector<[17]xf32>, vector<[17]xi1> into !llvm.ptr
// -----
@@ -149,10 +149,10 @@ func.func @transfer_read_write_index_1d(%A : memref<?xindex>, %base: index) -> v
// CHECK: %[[SPLAT:.*]] = arith.constant dense<7> : vector<17xindex>
// CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[SPLAT]] : vector<17xindex> to vector<17xi64>
-// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} :
+// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} :
// CHECK-SAME: (!llvm.ptr, vector<17xi1>, vector<17xi64>) -> vector<17xi64>
-// CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i32} :
+// CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i64} :
// CHECK-SAME: vector<17xi64>, vector<17xi1> into !llvm.ptr
func.func @transfer_read_write_index_1d_scalable(%A : memref<?xindex>, %base: index) -> vector<[17]xindex> {
@@ -170,10 +170,10 @@ func.func @transfer_read_write_index_1d_scalable(%A : memref<?xindex>, %base: in
// CHECK: %[[SPLAT:.*]] = arith.constant dense<7> : vector<[17]xindex>
// CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[SPLAT]] : vector<[17]xindex> to vector<[17]xi64>
-// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} :
+// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} :
// CHECK-SAME: (!llvm.ptr, vector<[17]xi1>, vector<[17]xi64>) -> vector<[17]xi64>
-// CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i32} :
+// CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i64} :
// CHECK-SAME: vector<[17]xi64>, vector<[17]xi1> into !llvm.ptr
// -----
diff --git a/mlir/test/Dialect/LLVMIR/opaque-ptr.mlir b/mlir/test/Dialect/LLVMIR/opaque-ptr.mlir
index 82c3e5bf2e2db..8a4357e62a5cc 100644
--- a/mlir/test/Dialect/LLVMIR/opaque-ptr.mlir
+++ b/mlir/test/Dialect/LLVMIR/opaque-ptr.mlir
@@ -62,7 +62,7 @@ llvm.func @opaque_ptr_matrix_load_store(%ptr: !llvm.ptr, %stride: i64) -> vector
llvm.func @opaque_ptr_masked_load(%arg0: !llvm.ptr, %arg1: vector<7xi1>) -> vector<7xf32> {
// CHECK: = llvm.intr.masked.load
// CHECK: (!llvm.ptr, vector<7xi1>) -> vector<7xf32>
- %0 = llvm.intr.masked.load %arg0, %arg1 { alignment = 1: i32} :
+ %0 = llvm.intr.masked.load %arg0, %arg1 { alignment = 1: i64} :
(!llvm.ptr, vector<7xi1>) -> vector<7xf32>
llvm.return %0 : vector<7 x f32>
}
@@ -71,7 +71,7 @@ llvm.func @opaque_ptr_masked_load(%arg0: !llvm.ptr, %arg1: vector<7xi1>) -> vect
llvm.func @opaque_ptr_gather(%M: vector<7x!llvm.ptr>, %mask: vector<7xi1>) -> vector<7xf32> {
// CHECK: = llvm.intr.masked.gather
// CHECK: (vector<7x!llvm.ptr>, vector<7xi1>) -> vector<7xf32>
- %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i32} :
+ %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i64} :
(vector<7x!llvm.ptr>, vector<7xi1>) -> vector<7xf32>
llvm.return %a : vector<7xf32>
}
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 2ab49690656f1..0db3d897f58d0 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -1302,3 +1302,26 @@ llvm.func @repeated_function_metadata() attributes {
#llvm.func_metadata<"type", #llvm.md_node<#llvm.md_const<0 : i64>, #llvm.md_string<"typeid1">>>
]
}
+
+#rt_alias_scope_domain = #llvm.alias_scope_domain<id = distinct[4]<>, description = "rt domain">
+#rt_alias_scope = #llvm.alias_scope<id = distinct[5]<>, domain = #rt_alias_scope_domain>
+#rt_access_group = #llvm.access_group<id = distinct[6]<>>
+
+// CHECK-LABEL: @masked_intrinsic_metadata_roundtrip
+llvm.func @masked_intrinsic_metadata_roundtrip(%ptr: !llvm.ptr, %mask: vector<7xi1>) {
+ // CHECK: llvm.intr.masked.load
+ // CHECK-SAME: access_groups = [#{{[^]]*}}]
+ // CHECK-SAME: alias_scopes = [#{{[^]]*}}]
+ // CHECK-SAME: noalias_scopes = [#{{[^]]*}}]
+ %0 = llvm.intr.masked.load %ptr, %mask {
+ alignment = 4 : i64,
+ access_groups = [#rt_access_group],
+ alias_scopes = [#rt_alias_scope],
+ noalias_scopes = [#rt_alias_scope]} : (!llvm.ptr, vector<7xi1>) -> vector<7xf32>
+ // CHECK: llvm.intr.masked.store
+ // CHECK-SAME: access_groups = [#{{[^]]*}}]
+ llvm.intr.masked.store %0, %ptr, %mask {
+ alignment = 4 : i64,
+ access_groups = [#rt_access_group]} : vector<7xf32>, vector<7xi1> into !llvm.ptr
+ llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 543a563349ade..260eccb5dd751 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -541,20 +541,26 @@ define <7 x i1> @get_active_lane_mask(i64 %0, i64 %1) {
; CHECK-SAME: %[[VEC:[a-zA-Z0-9]+]]
; CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]]
define void @masked_load_store_intrinsics(ptr %vec, <7 x i1> %mask) {
- ; CHECK: %[[UNDEF:.+]] = llvm.mlir.undef
- ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i32}
+ ; CHECK: %[[POISON:.+]] = llvm.mlir.poison
+ ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[POISON]] {alignment = 1 : i64}
; CHECK-SAME: (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
- %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 1, <7 x i1> %mask, <7 x float> undef)
- ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32}
- %2 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1)
- ; CHECK: %[[VAL3:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32, nontemporal}
- %3 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1), !nontemporal !{i32 1}
- ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32}
+ %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 %vec, <7 x i1> %mask, <7 x float> poison)
+ ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i64}
+ %2 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 4 %vec, <7 x i1> %mask, <7 x float> %1)
+ ; CHECK: %[[VAL3:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i64, nontemporal}
+ %3 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 4 %vec, <7 x i1> %mask, <7 x float> %1), !nontemporal !{i32 1}
+ ; CHECK: %[[VAL4:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[POISON]] :
+ ; CHECK-SAME: (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>{{$}}
+ %4 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, <7 x i1> %mask, <7 x float> poison)
+ ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i64}
; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr
- call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr %vec, i32 8, <7 x i1> %mask)
- ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32, nontemporal}
- ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr
- call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr %vec, i32 8, <7 x i1> %mask), !nontemporal !{i32 1}
+ call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr align 8 %vec, <7 x i1> %mask)
+ ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i64, nontemporal}
+ ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr
+ call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr align 8 %vec, <7 x i1> %mask), !nontemporal !{i32 1}
+ ; CHECK: llvm.intr.masked.store %[[VAL4]], %[[VEC]], %[[MASK]] :
+ ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr{{$}}
+ call void @llvm.masked.store.v7f32.p0(<7 x float> %4, ptr %vec, <7 x i1> %mask)
ret void
}
@@ -562,15 +568,21 @@ define void @masked_load_store_intrinsics(ptr %vec, <7 x i1> %mask) {
; CHECK-SAME: %[[VEC:[a-zA-Z0-9]+]]
; CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]]
define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) {
- ; CHECK: %[[UNDEF:.+]] = llvm.mlir.undef
- ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i32}
+ ; CHECK: %[[POISON:.+]] = llvm.mlir.poison
+ ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[POISON]] {alignment = 1 : i64}
; CHECK-SAME: (vector<7x!llvm.ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
- %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, i32 1, <7 x i1> %mask, <7 x float> undef)
- ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32}
- %2 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, i32 4, <7 x i1> %mask, <7 x float> %1)
- ; CHECK: llvm.intr.masked.scatter %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32}
+ %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 1 %vec, <7 x i1> %mask, <7 x float> poison)
+ ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i64}
+ %2 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 4 %vec, <7 x i1> %mask, <7 x float> %1)
+ ; CHECK: %[[VAL3:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[POISON]] :
+ ; CHECK-SAME: (vector<7x!llvm.ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32>{{$}}
+ %3 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, <7 x i1> %mask, <7 x float> poison)
+ ; CHECK: llvm.intr.masked.scatter %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i64}
; CHECK-SAME: vector<7xf32>, vector<7xi1> into vector<7x!llvm.ptr>
- call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %2, <7 x ptr> %vec, i32 8, <7 x i1> %mask)
+ call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %2, <7 x ptr> align 8 %vec, <7 x i1> %mask)
+ ; CHECK: llvm.intr.masked.scatter %[[VAL3]], %[[VEC]], %[[MASK]] :
+ ; CHECK-SAME: vector<7xf32>, vector<7xi1> into vector<7x!llvm.ptr>{{$}}
+ call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %3, <7 x ptr> %vec, <7 x i1> %mask)
ret void
}
@@ -1845,10 +1857,10 @@ declare <48 x float> @llvm.matrix.transpose.v48f32(<48 x float>, i32 immarg, i32
declare <48 x float> @llvm.matrix.column.major.load.v48f32.i64(ptr nocapture, i64, i1 immarg, i32 immarg, i32 immarg)
declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg)
declare <7 x i1> @llvm.get.active.lane.mask.v7i1.i64(i64, i64)
-declare <7 x float> @llvm.masked.load.v7f32.p0(ptr, i32 immarg, <7 x i1>, <7 x float>)
-declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr, i32 immarg, <7 x i1>)
-declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, i32 immarg, <7 x i1>, <7 x float>)
-declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>)
+declare <7 x float> @llvm.masked.load.v7f32.p0(ptr, <7 x i1>, <7 x float>)
+declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr, <7 x i1>)
+declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, <7 x i1>, <7 x float>)
+declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, <7 x i1>)
declare <7 x float> @llvm.masked.expandload.v7f32(ptr, <7 x i1>, <7 x float>)
declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr, <7 x i1>)
declare void @llvm.var.annotation.p0.p0(ptr, ptr, ptr, i32, ptr)
diff --git a/mlir/test/Target/LLVMIR/Import/masked-intrinsic-metadata.ll b/mlir/test/Target/LLVMIR/Import/masked-intrinsic-metadata.ll
new file mode 100644
index 0000000000000..a94ca48c671df
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/masked-intrinsic-metadata.ll
@@ -0,0 +1,64 @@
+; RUN: mlir-translate -import-llvm %s | FileCheck %s
+
+; CHECK-DAG: #[[$AG:.+]] = #llvm.access_group<id = {{.*}}>
+; CHECK-DAG: #[[DOMAIN:.+]] = #llvm.alias_scope_domain<id = {{.*}}, description = "domain">
+; CHECK-DAG: #[[TBAA_ROOT:.+]] = #llvm.tbaa_root<id = "Simple C/C++ TBAA">
+; CHECK-DAG: #[[$SCOPE:.+]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]], description = "scope">
+; CHECK-DAG: #[[$NOALIAS:.+]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]], description = "noalias">
+; CHECK-DAG: #[[TBAA_CHAR:.+]] = #llvm.tbaa_type_desc<id = "omnipotent char", members = {<#[[TBAA_ROOT]], 0>}>
+; CHECK-DAG: #[[TBAA_INT:.+]] = #llvm.tbaa_type_desc<id = "int", members = {<#[[TBAA_CHAR]], 0>}>
+; CHECK-DAG: #[[$TBAA_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[TBAA_INT]], access_type = #[[TBAA_INT]], offset = 0>
+
+; CHECK-LABEL: @masked_load_store_metadata
+define void @masked_load_store_metadata(ptr %ptr, <7 x i1> %mask, <7 x float> %val) {
+ ; CHECK: llvm.intr.masked.load
+ ; CHECK-SAME: access_groups = [#[[$AG]]]
+ ; CHECK-SAME: alias_scopes = [#[[$SCOPE]]]
+ ; CHECK-SAME: alignment = 4 : i64
+ ; CHECK-SAME: noalias_scopes = [#[[$NOALIAS]]]
+ ; CHECK-SAME: tbaa = [#[[$TBAA_TAG]]]
+ %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 4 %ptr, <7 x i1> %mask, <7 x float> poison), !tbaa !0, !llvm.access.group !4, !alias.scope !5, !noalias !7
+ ; CHECK: llvm.intr.masked.store
+ ; CHECK-SAME: access_groups = [#[[$AG]]]
+ ; CHECK-SAME: alias_scopes = [#[[$SCOPE]]]
+ ; CHECK-SAME: alignment = 4 : i64
+ ; CHECK-SAME: noalias_scopes = [#[[$NOALIAS]]]
+ ; CHECK-SAME: tbaa = [#[[$TBAA_TAG]]]
+ call void @llvm.masked.store.v7f32.p0(<7 x float> %val, ptr align 4 %ptr, <7 x i1> %mask), !tbaa !0, !llvm.access.group !4, !alias.scope !5, !noalias !7
+ ret void
+}
+
+; CHECK-LABEL: @masked_gather_scatter_metadata
+define void @masked_gather_scatter_metadata(<7 x ptr> %ptrs, <7 x i1> %mask, <7 x float> %val) {
+ ; CHECK: llvm.intr.masked.gather
+ ; CHECK-SAME: access_groups = [#[[$AG]]]
+ ; CHECK-SAME: alias_scopes = [#[[$SCOPE]]]
+ ; CHECK-SAME: alignment = 4 : i64
+ ; CHECK-SAME: noalias_scopes = [#[[$NOALIAS]]]
+ ; CHECK-SAME: tbaa = [#[[$TBAA_TAG]]]
+ %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 4 %ptrs, <7 x i1> %mask, <7 x float> poison), !tbaa !0, !llvm.access.group !4, !alias.scope !5, !noalias !7
+ ; CHECK: llvm.intr.masked.scatter
+ ; CHECK-SAME: access_groups = [#[[$AG]]]
+ ; CHECK-SAME: alias_scopes = [#[[$SCOPE]]]
+ ; CHECK-SAME: alignment = 4 : i64
+ ; CHECK-SAME: noalias_scopes = [#[[$NOALIAS]]]
+ ; CHECK-SAME: tbaa = [#[[$TBAA_TAG]]]
+ call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %val, <7 x ptr> align 4 %ptrs, <7 x i1> %mask), !tbaa !0, !llvm.access.group !4, !alias.scope !5, !noalias !7
+ ret void
+}
+
+declare <7 x float> @llvm.masked.load.v7f32.p0(ptr, <7 x i1>, <7 x float>)
+declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr, <7 x i1>)
+declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, <7 x i1>, <7 x float>)
+declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, <7 x i1>)
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"int", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = distinct !{}
+!5 = !{!6}
+!6 = distinct !{!6, !9, !"scope"}
+!7 = !{!8}
+!8 = distinct !{!8, !9, !"noalias"}
+!9 = distinct !{!9, !"domain"}
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 66183397b8377..676e29e6b70c5 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -591,19 +591,25 @@ llvm.func @get_active_lane_mask(%base: i64, %n: i64) -> (vector<7xi1>) {
// CHECK-LABEL: @masked_load_store_intrinsics
llvm.func @masked_load_store_intrinsics(%A: !llvm.ptr, %mask: vector<7xi1>) {
// CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison)
- %a = llvm.intr.masked.load %A, %mask { alignment = 1: i32} :
+ %a = llvm.intr.masked.load %A, %mask { alignment = 1: i64} :
(!llvm.ptr, vector<7xi1>) -> vector<7xf32>
// CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison), !nontemporal !1
- %b = llvm.intr.masked.load %A, %mask { alignment = 1: i32, nontemporal} :
+ %b = llvm.intr.masked.load %A, %mask { alignment = 1: i64, nontemporal} :
(!llvm.ptr, vector<7xi1>) -> vector<7xf32>
// CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
- %c = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i32} :
+ %c = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i64} :
(!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison){{$}}
+ %d = llvm.intr.masked.load %A, %mask :
+ (!llvm.ptr, vector<7xi1>) -> vector<7xf32>
// CHECK: call void @llvm.masked.store.v7f32.p0(<7 x float> %{{.*}}, ptr align 1 %0, <7 x i1> %{{.*}})
- llvm.intr.masked.store %b, %A, %mask { alignment = 1: i32} :
+ llvm.intr.masked.store %b, %A, %mask { alignment = 1: i64} :
vector<7xf32>, vector<7xi1> into !llvm.ptr
// CHECK: call void @llvm.masked.store.v7f32.p0(<7 x float> %{{.*}}, ptr align 1 %0, <7 x i1> %{{.*}}), !nontemporal !{{.*}}
- llvm.intr.masked.store %b, %A, %mask { alignment = 1: i32, nontemporal} :
+ llvm.intr.masked.store %b, %A, %mask { alignment = 1: i64, nontemporal} :
+ vector<7xf32>, vector<7xi1> into !llvm.ptr
+ // CHECK: call void @llvm.masked.store.v7f32.p0(<7 x float> %{{.*}}, ptr %0, <7 x i1> %{{.*}}){{$}}
+ llvm.intr.masked.store %d, %A, %mask :
vector<7xf32>, vector<7xi1> into !llvm.ptr
llvm.return
}
@@ -611,13 +617,19 @@ llvm.func @masked_load_store_intrinsics(%A: !llvm.ptr, %mask: vector<7xi1>) {
// CHECK-LABEL: @masked_gather_scatter_intrinsics
llvm.func @masked_gather_scatter_intrinsics(%M: vector<7 x !llvm.ptr>, %mask: vector<7xi1>) {
// CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison)
- %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i32} :
+ %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i64} :
(vector<7 x !llvm.ptr>, vector<7xi1>) -> vector<7xf32>
// CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
- %b = llvm.intr.masked.gather %M, %mask, %a { alignment = 1: i32} :
+ %b = llvm.intr.masked.gather %M, %mask, %a { alignment = 1: i64} :
(vector<7 x !llvm.ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+ // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison){{$}}
+ %c = llvm.intr.masked.gather %M, %mask :
+ (vector<7 x !llvm.ptr>, vector<7xi1>) -> vector<7xf32>
// CHECK: call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %{{.*}}, <7 x ptr> align 1 %{{.*}}, <7 x i1> %{{.*}})
- llvm.intr.masked.scatter %b, %M, %mask { alignment = 1: i32} :
+ llvm.intr.masked.scatter %b, %M, %mask { alignment = 1: i64} :
+ vector<7xf32>, vector<7xi1> into vector<7 x !llvm.ptr>
+ // CHECK: call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %{{.*}}, <7 x ptr> %{{.*}}, <7 x i1> %{{.*}}){{$}}
+ llvm.intr.masked.scatter %c, %M, %mask :
vector<7xf32>, vector<7xi1> into vector<7 x !llvm.ptr>
llvm.return
}
diff --git a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir
index ef6681ca3821f..6dd74420e21aa 100644
--- a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir
@@ -198,7 +198,7 @@ llvm.func @active_lane_intr_wrong_type(%base : i64, %n : vector<7xi64>) -> vecto
llvm.func @masked_load_intr_wrong_type(%ptr : i64, %mask : vector<7xi1>) -> vector<7xf32> {
// expected-error @below{{op operand #0 must be LLVM pointer type, but got 'i64'}}
- %0 = llvm.intr.masked.load %ptr, %mask { alignment = 1: i32} : (i64, vector<7xi1>) -> vector<7xf32>
+ %0 = llvm.intr.masked.load %ptr, %mask { alignment = 1: i64} : (i64, vector<7xi1>) -> vector<7xf32>
llvm.return %0 : vector<7xf32>
}
@@ -206,7 +206,7 @@ llvm.func @masked_load_intr_wrong_type(%ptr : i64, %mask : vector<7xi1>) -> vect
llvm.func @masked_store_intr_wrong_type(%vec : vector<7xf32>, %ptr : !llvm.ptr, %mask : vector<7xi32>) {
// expected-error @below{{op operand #2 must be LLVM dialect-compatible vector of 1-bit signless integer, but got 'vector<7xi32>}}
- llvm.intr.masked.store %vec, %ptr, %mask { alignment = 1: i32} : vector<7xf32>, vector<7xi32> into !llvm.ptr
+ llvm.intr.masked.store %vec, %ptr, %mask { alignment = 1: i64} : vector<7xf32>, vector<7xi32> into !llvm.ptr
llvm.return
}
@@ -214,7 +214,7 @@ llvm.func @masked_store_intr_wrong_type(%vec : vector<7xf32>, %ptr : !llvm.ptr,
llvm.func @masked_gather_intr_wrong_type(%ptrs : vector<7xf32>, %mask : vector<7xi1>) -> vector<7xf32> {
// expected-error @below{{op operand #0 must be LLVM dialect-compatible vector of LLVM pointer type, but got 'vector<7xf32>'}}
- %0 = llvm.intr.masked.gather %ptrs, %mask { alignment = 1: i32} : (vector<7xf32>, vector<7xi1>) -> vector<7xf32>
+ %0 = llvm.intr.masked.gather %ptrs, %mask { alignment = 1: i64} : (vector<7xf32>, vector<7xi1>) -> vector<7xf32>
llvm.return %0 : vector<7xf32>
}
@@ -222,7 +222,7 @@ llvm.func @masked_gather_intr_wrong_type(%ptrs : vector<7xf32>, %mask : vector<7
llvm.func @masked_gather_intr_wrong_type_scalable(%ptrs : vector<7x!llvm.ptr>, %mask : vector<[7]xi1>) -> vector<[7]xf32> {
// expected-error @below{{expected operand #1 type to be 'vector<[7]x!llvm.ptr>'}}
- %0 = llvm.intr.masked.gather %ptrs, %mask { alignment = 1: i32} : (vector<7x!llvm.ptr>, vector<[7]xi1>) -> vector<[7]xf32>
+ %0 = llvm.intr.masked.gather %ptrs, %mask { alignment = 1: i64} : (vector<7x!llvm.ptr>, vector<[7]xi1>) -> vector<[7]xf32>
llvm.return %0 : vector<[7]xf32>
}
@@ -230,7 +230,7 @@ llvm.func @masked_gather_intr_wrong_type_scalable(%ptrs : vector<7x!llvm.ptr>, %
llvm.func @masked_scatter_intr_wrong_type(%vec : f32, %ptrs : vector<7x!llvm.ptr>, %mask : vector<7xi1>) {
// expected-error @below{{invalid kind of type specified: expected builtin.vector, but found 'f32'}}
- llvm.intr.masked.scatter %vec, %ptrs, %mask { alignment = 1: i32} : f32, vector<7xi1> into vector<7x!llvm.ptr>
+ llvm.intr.masked.scatter %vec, %ptrs, %mask { alignment = 1: i64} : f32, vector<7xi1> into vector<7x!llvm.ptr>
llvm.return
}
@@ -238,7 +238,7 @@ llvm.func @masked_scatter_intr_wrong_type(%vec : f32, %ptrs : vector<7x!llvm.ptr
llvm.func @masked_scatter_intr_wrong_type_scalable(%vec : vector<[7]xf32>, %ptrs : vector<7x!llvm.ptr>, %mask : vector<[7]xi1>) {
// expected-error @below{{expected operand #2 type to be 'vector<[7]x!llvm.ptr>'}}
- llvm.intr.masked.scatter %vec, %ptrs, %mask { alignment = 1: i32} : vector<[7]xf32>, vector<[7]xi1> into vector<7x!llvm.ptr>
+ llvm.intr.masked.scatter %vec, %ptrs, %mask { alignment = 1: i64} : vector<[7]xf32>, vector<[7]xi1> into vector<7x!llvm.ptr>
llvm.return
}
diff --git a/mlir/test/Target/LLVMIR/masked-intrinsic-metadata.mlir b/mlir/test/Target/LLVMIR/masked-intrinsic-metadata.mlir
new file mode 100644
index 0000000000000..354030afc5fdc
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/masked-intrinsic-metadata.mlir
@@ -0,0 +1,73 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+#alias_scope_domain = #llvm.alias_scope_domain<id = distinct[0]<>, description = "The domain">
+#alias_scope1 = #llvm.alias_scope<id = distinct[1]<>, domain = #alias_scope_domain, description = "The first scope">
+#alias_scope2 = #llvm.alias_scope<id = distinct[2]<>, domain = #alias_scope_domain, description = "The second scope">
+#access_group = #llvm.access_group<id = distinct[3]<>>
+#tbaa_root = #llvm.tbaa_root<id = "Simple C/C++ TBAA">
+#tbaa_type_desc = #llvm.tbaa_type_desc<id = "int", members = {<#tbaa_root, 0>}>
+#tbaa_tag = #llvm.tbaa_tag<base_type = #tbaa_type_desc, access_type = #tbaa_type_desc, offset = 0>
+
+// CHECK-LABEL: @masked_load_store_metadata
+llvm.func @masked_load_store_metadata(%ptr: !llvm.ptr, %mask: vector<7xi1>) {
+ // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0
+ // CHECK-SAME: !tbaa ![[$TBAA:[0-9]+]]
+ // CHECK-SAME: !alias.scope ![[$SCOPE1:[0-9]+]]
+ // CHECK-SAME: !noalias ![[$SCOPE2:[0-9]+]]
+ // CHECK-SAME: !llvm.access.group ![[$AG:[0-9]+]]
+ %0 = llvm.intr.masked.load %ptr, %mask {
+ alignment = 4 : i64,
+ access_groups = [#access_group],
+ alias_scopes = [#alias_scope1],
+ noalias_scopes = [#alias_scope2],
+ tbaa = [#tbaa_tag]} : (!llvm.ptr, vector<7xi1>) -> vector<7xf32>
+ // CHECK: call void @llvm.masked.store.v7f32.p0
+ // CHECK-SAME: !tbaa ![[$TBAA]]
+ // CHECK-SAME: !alias.scope ![[$SCOPE1]]
+ // CHECK-SAME: !noalias ![[$SCOPE2]]
+ // CHECK-SAME: !llvm.access.group ![[$AG]]
+ llvm.intr.masked.store %0, %ptr, %mask {
+ alignment = 4 : i64,
+ access_groups = [#access_group],
+ alias_scopes = [#alias_scope1],
+ noalias_scopes = [#alias_scope2],
+ tbaa = [#tbaa_tag]} : vector<7xf32>, vector<7xi1> into !llvm.ptr
+ llvm.return
+}
+
+// CHECK-LABEL: @masked_gather_scatter_metadata
+llvm.func @masked_gather_scatter_metadata(%ptrs: vector<7 x !llvm.ptr>, %mask: vector<7xi1>) {
+ // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0
+ // CHECK-SAME: !tbaa ![[$TBAA]]
+ // CHECK-SAME: !alias.scope ![[$SCOPE1]]
+ // CHECK-SAME: !noalias ![[$SCOPE2]]
+ // CHECK-SAME: !llvm.access.group ![[$AG]]
+ %0 = llvm.intr.masked.gather %ptrs, %mask {
+ alignment = 4 : i64,
+ access_groups = [#access_group],
+ alias_scopes = [#alias_scope1],
+ noalias_scopes = [#alias_scope2],
+ tbaa = [#tbaa_tag]} : (vector<7 x !llvm.ptr>, vector<7xi1>) -> vector<7xf32>
+ // CHECK: call void @llvm.masked.scatter.v7f32.v7p0
+ // CHECK-SAME: !tbaa ![[$TBAA]]
+ // CHECK-SAME: !alias.scope ![[$SCOPE1]]
+ // CHECK-SAME: !noalias ![[$SCOPE2]]
+ // CHECK-SAME: !llvm.access.group ![[$AG]]
+ llvm.intr.masked.scatter %0, %ptrs, %mask {
+ alignment = 4 : i64,
+ access_groups = [#access_group],
+ alias_scopes = [#alias_scope1],
+ noalias_scopes = [#alias_scope2],
+ tbaa = [#tbaa_tag]} : vector<7xf32>, vector<7xi1> into vector<7 x !llvm.ptr>
+ llvm.return
+}
+
+// CHECK-DAG: ![[$TBAA]] = !{![[TBAA_TYPE:[0-9]+]], ![[TBAA_TYPE]], i64 0}
+// CHECK-DAG: ![[TBAA_TYPE]] = !{!"int", ![[TBAA_ROOT:[0-9]+]], i64 0}
+// CHECK-DAG: ![[TBAA_ROOT]] = !{!"Simple C/C++ TBAA"}
+// CHECK-DAG: ![[$SCOPE1]] = !{![[SCOPE1_DECL:[0-9]+]]}
+// CHECK-DAG: ![[SCOPE1_DECL]] = distinct !{![[SCOPE1_DECL]], ![[DOMAIN:[0-9]+]], !"The first scope"}
+// CHECK-DAG: ![[DOMAIN]] = distinct !{![[DOMAIN]], !"The domain"}
+// CHECK-DAG: ![[$SCOPE2]] = !{![[SCOPE2_DECL:[0-9]+]]}
+// CHECK-DAG: ![[SCOPE2_DECL]] = distinct !{![[SCOPE2_DECL]], ![[DOMAIN]], !"The second scope"}
+// CHECK-DAG: ![[$AG]] = distinct !{}
diff --git a/mlir/test/Target/LLVMIR/opaque-ptr.mlir b/mlir/test/Target/LLVMIR/opaque-ptr.mlir
index cd7c31a0945a8..dee6f00f61c05 100644
--- a/mlir/test/Target/LLVMIR/opaque-ptr.mlir
+++ b/mlir/test/Target/LLVMIR/opaque-ptr.mlir
@@ -60,7 +60,7 @@ llvm.func @opaque_ptr_matrix_load_store(%ptr: !llvm.ptr, %stride: i64) -> vector
// CHECK-LABEL: @opaque_ptr_masked_load
llvm.func @opaque_ptr_masked_load(%arg0: !llvm.ptr, %arg1: vector<7xi1>) -> vector<7xf32> {
// CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr
- %0 = llvm.intr.masked.load %arg0, %arg1 { alignment = 1: i32} :
+ %0 = llvm.intr.masked.load %arg0, %arg1 { alignment = 1: i64} :
(!llvm.ptr, vector<7xi1>) -> vector<7xf32>
llvm.return %0 : vector<7 x f32>
}
@@ -68,7 +68,7 @@ llvm.func @opaque_ptr_masked_load(%arg0: !llvm.ptr, %arg1: vector<7xi1>) -> vect
// CHECK-LABEL: @opaque_ptr_gather
llvm.func @opaque_ptr_gather(%M: vector<7 x !llvm.ptr>, %mask: vector<7xi1>) -> vector<7xf32> {
// CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 1 {{.*}},
- %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i32} :
+ %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i64} :
(vector<7 x !llvm.ptr>, vector<7xi1>) -> vector<7xf32>
llvm.return %a : vector<7xf32>
}
More information about the cfe-commits
mailing list