[Mlir-commits] [mlir] e3add74 - [mlir][ArmNeon/ArmSVE] Reorganise .td and test files (NFC) (#236797)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Fri Oct 30 02:38:02 PDT 2026
Author: Andrzej Warzyński
Date: 2026-10-30T09:37:57Z
New Revision: e3add74734d6b054052a66d386817edc7faacee1
URL: https://github.com/llvm/llvm-project/commit/e3add74734d6b054052a66d386817edc7faacee1
DIFF: https://github.com/llvm/llvm-project/commit/e3add74734d6b054052a66d386817edc7faacee1.diff
LOG: [mlir][ArmNeon/ArmSVE] Reorganise .td and test files (NFC) (#236797)
Group related operation definitions and tests, and add comments to make
the organisation clearer and more consistent.
* Group DOT definitions and tests together.
* Group MMLA definitions and tests together.
* Clearly separate regular operations from intrinsic operations in
ArmSVE.td.
* Move tests that exercise scalable vectors without SVE-specific
operations
from mlir/test/Target/LLVMIR/arme-sve.mlir to
mlir/test/Target/LLVMIR/scalable-vectors.mlir.
* Add TODOs identifying potentially redundant tests for future removal.
This organisation should simplify maintenance and make it clearer where
new definitions and tests belong.
Added:
mlir/test/Target/LLVMIR/scalable-vectors.mlir
Modified:
mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
mlir/test/Dialect/ArmSVE/invalid.mlir
mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir
mlir/test/Target/LLVMIR/arm-neon-2d.mlir
mlir/test/Target/LLVMIR/arm-neon.mlir
mlir/test/Target/LLVMIR/arm-sve.mlir
Removed:
################################################################################
diff --git a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
index ce86ff2cfd9222..7e7aef62735d79 100644
--- a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
+++ b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
@@ -40,7 +40,7 @@ class NeonVectorOfLength<int length, Type elementType> : ShapedContainerType<
"::mlir::VectorType">;
//===----------------------------------------------------------------------===//
-// ArmNeon op definitions
+// ArmNeon Op classes
//===----------------------------------------------------------------------===//
// ArmNeon dialect op that corresponds (and is convertible to) an LLVM IR
@@ -72,6 +72,15 @@ class ArmNeon_OverloadedOperandsWithOneResultIntrOp<string mnemonic,
list<Trait> traits = []>
: ArmNeon_IntrOp<mnemonic, [0], overloadedOperands, 1, traits>;
+class ArmNeon_2dOp<string mnemonic, list<Trait> traits = []>
+ : Op</*dialect=*/ArmNeon_Dialect,
+ /*opName=*/"2d." # mnemonic,
+ /*traits=*/traits>;
+
+//===----------------------------------------------------------------------===//
+// SMullOp
+//===----------------------------------------------------------------------===//
+
def SMullOp : ArmNeon_OverloadedOneResultIntrOp<"smull", [
Pure,
AllTypesMatch<["a", "b"]>,
@@ -101,6 +110,10 @@ def SMullOp : ArmNeon_OverloadedOneResultIntrOp<"smull", [
"$a `,` $b attr-dict `:` type($a) `to` type($res)";
}
+//===----------------------------------------------------------------------===//
+// SdotOp
+//===----------------------------------------------------------------------===//
+
def SdotOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"sdot",[1], [
Pure,
AllTypesMatch<["b", "c"]>,
@@ -129,6 +142,10 @@ def SdotOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"sdot",[1], [
"$a `,` $b `,` $c attr-dict `:` type($b) `,` type($c) `to` type($res)";
}
+//===----------------------------------------------------------------------===//
+// SmmlaOp
+//===----------------------------------------------------------------------===//
+
def SmmlaOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"smmla",[1], [
Pure,
AllTypesMatch<["src1", "src2"]>,
@@ -159,6 +176,10 @@ def SmmlaOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"smmla",[1], [
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
}
+//===----------------------------------------------------------------------===//
+// UmmlaOp
+//===----------------------------------------------------------------------===//
+
def UmmlaOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"ummla",[1], [
Pure,
AllTypesMatch<["src1", "src2"]>,
@@ -190,6 +211,10 @@ def UmmlaOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"ummla",[1], [
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
}
+//===----------------------------------------------------------------------===//
+// UsmmlaOp
+//===----------------------------------------------------------------------===//
+
def UsmmlaOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"usmmla",[1], [
Pure,
AllTypesMatch<["src1", "src2"]>,
@@ -222,6 +247,10 @@ def UsmmlaOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"usmmla",[1], [
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
}
+//===----------------------------------------------------------------------===//
+// BfmmlaOp
+//===----------------------------------------------------------------------===//
+
def BfmmlaOp : ArmNeon_IntrOp<"bfmmla", [], [], 1, [
Pure,
AllTypesMatch<["src1", "src2"]>,
@@ -250,10 +279,9 @@ def BfmmlaOp : ArmNeon_IntrOp<"bfmmla", [], [], 1, [
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
}
-class ArmNeon_2dOp<string mnemonic, list<Trait> traits = []>
- : Op</*dialect=*/ArmNeon_Dialect,
- /*opName=*/"2d." # mnemonic,
- /*traits=*/traits>;
+//===----------------------------------------------------------------------===//
+// Sdot2dOp
+//===----------------------------------------------------------------------===//
def Sdot2dOp : ArmNeon_2dOp<"sdot", [
Pure,
diff --git a/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
index 2f23404991799e..b2e35e0c8be164 100644
--- a/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
+++ b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
@@ -68,8 +68,25 @@ def SVEVector : AnyTypeOf<[
Scalable1DVectorOfLength<16, [I8]>],
"an SVE vector with element size <= 64-bit">;
+// Inputs valid for the multi-vector zips (not including the 128-bit element zipqs)
+def ZipInputVectorType : AnyTypeOf<[
+ Scalable1DVectorOfLength<2, [I64, F64]>,
+ Scalable1DVectorOfLength<4, [I32, F32]>,
+ Scalable1DVectorOfLength<8, [I16, F16, BF16]>,
+ Scalable1DVectorOfLength<16, [I8]>],
+ "an SVE vector with element size <= 64-bit">;
+
+//===----------------------------------------------------------------------===//
+// ArmSVE constraints
+//===----------------------------------------------------------------------===//
+
+class SvboolTypeConstraint<string lhsArg, string rhsArg> : TypesMatchWith<
+ "expected corresponding svbool type widened to [16]xi1",
+ lhsArg, rhsArg,
+ "VectorType(VectorType::Builder(::llvm::cast<VectorType>($_self)).setDim(::llvm::cast<VectorType>($_self).getRank() - 1, 16))">;
+
//===----------------------------------------------------------------------===//
-// ArmSVE op definitions
+// ArmSVE Op classes
//===----------------------------------------------------------------------===//
class ArmSVE_Op<string mnemonic, list<Trait> traits = []> :
@@ -149,6 +166,10 @@ class ScalableMaskedIOp<string mnemonic, string op_description,
"$mask `,` $src1 `,` $src2 attr-dict `:` type($mask) `,` type($res)";
}
+//===----------------------------------------------------------------------===//
+// SdotOp
+//===----------------------------------------------------------------------===//
+
def SdotOp : ArmSVE_Op<"sdot", [Pure,
AllTypesMatch<["src1", "src2"]>,
AllTypesMatch<["acc", "dst"]>]> {
@@ -178,6 +199,43 @@ def SdotOp : ArmSVE_Op<"sdot", [Pure,
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($dst)";
}
+//===----------------------------------------------------------------------===//
+// UdotOp
+//===----------------------------------------------------------------------===//
+
+def UdotOp : ArmSVE_Op<"udot", [Pure,
+ AllTypesMatch<["src1", "src2"]>,
+ AllTypesMatch<["acc", "dst"]>]> {
+ let summary = "Vector-vector dot product and accumulate op";
+ let description = [{
+ UDOT: Unsigned integer addition of dot product.
+
+ This function maps to the UDOT instruction, and it takes signless integer
+ operands that the operation interprets as unsigned. It partitions the second
+ and third vector inputs into groups of four elements. They calculate the dot
+ product of each group (without loss of precision) and then add each result
+ to the overlapping element of the first vector input.
+
+ Source:
+ https://developer.arm.com/documentation/100987/0000
+ }];
+ // Supports either:
+ // (vector<16xi8>, vector<16xi8>) -> (vector<4xi32>)
+ // (vector<8xi16>. vector<8xi16>) -> (vector<2xi64>)
+ let arguments = (ins
+ ScalableVectorOfLengthAndType<[4, 2], [I32, I64]>:$acc,
+ ScalableVectorOfLengthAndType<[16, 8], [I8, I16]>:$src1,
+ ScalableVectorOfLengthAndType<[16, 8], [I8, I16]>:$src2
+ );
+ let results = (outs ScalableVectorOfLengthAndType<[4, 2], [I32, I64]>:$dst);
+ let assemblyFormat =
+ "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($dst)";
+}
+
+//===----------------------------------------------------------------------===//
+// SmmlaOp
+//===----------------------------------------------------------------------===//
+
def SmmlaOp : ArmSVE_Op<"smmla", [Pure,
AllTypesMatch<["src1", "src2"]>,
AllTypesMatch<["acc", "dst"]>]> {
@@ -208,34 +266,9 @@ def SmmlaOp : ArmSVE_Op<"smmla", [Pure,
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($dst)";
}
-def UdotOp : ArmSVE_Op<"udot", [Pure,
- AllTypesMatch<["src1", "src2"]>,
- AllTypesMatch<["acc", "dst"]>]> {
- let summary = "Vector-vector dot product and accumulate op";
- let description = [{
- UDOT: Unsigned integer addition of dot product.
-
- This function maps to the UDOT instruction, and it takes signless integer
- operands that the operation interprets as unsigned. It partitions the second
- and third vector inputs into groups of four elements. They calculate the dot
- product of each group (without loss of precision) and then add each result
- to the overlapping element of the first vector input.
-
- Source:
- https://developer.arm.com/documentation/100987/0000
- }];
- // Supports either:
- // (vector<16xi8>, vector<16xi8>) -> (vector<4xi32>)
- // (vector<8xi16>. vector<8xi16>) -> (vector<2xi64>)
- let arguments = (ins
- ScalableVectorOfLengthAndType<[4, 2], [I32, I64]>:$acc,
- ScalableVectorOfLengthAndType<[16, 8], [I8, I16]>:$src1,
- ScalableVectorOfLengthAndType<[16, 8], [I8, I16]>:$src2
- );
- let results = (outs ScalableVectorOfLengthAndType<[4, 2], [I32, I64]>:$dst);
- let assemblyFormat =
- "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($dst)";
-}
+//===----------------------------------------------------------------------===//
+// UmmlaOp
+//===----------------------------------------------------------------------===//
def UmmlaOp : ArmSVE_Op<"ummla", [Pure,
AllTypesMatch<["src1", "src2"]>,
@@ -267,6 +300,10 @@ def UmmlaOp : ArmSVE_Op<"ummla", [Pure,
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($dst)";
}
+//===----------------------------------------------------------------------===//
+// UsmmlaOp
+//===----------------------------------------------------------------------===//
+
def UsmmlaOp : ArmSVE_Op<"usmmla", [Pure,
AllTypesMatch<["src1", "src2"]>,
AllTypesMatch<["acc", "dst"]>]> {
@@ -295,6 +332,10 @@ def UsmmlaOp : ArmSVE_Op<"usmmla", [Pure,
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($dst)";
}
+//===----------------------------------------------------------------------===//
+// BfmmlaOp
+//===----------------------------------------------------------------------===//
+
def BfmmlaOp : ArmSVE_IntrOp<"bfmmla", [Pure,
AllTypesMatch<["src1", "src2"]>,
AllTypesMatch<["acc", "res"]>,
@@ -330,10 +371,9 @@ def BfmmlaOp : ArmSVE_IntrOp<"bfmmla", [Pure,
"$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
}
-class SvboolTypeConstraint<string lhsArg, string rhsArg> : TypesMatchWith<
- "expected corresponding svbool type widened to [16]xi1",
- lhsArg, rhsArg,
- "VectorType(VectorType::Builder(::llvm::cast<VectorType>($_self)).setDim(::llvm::cast<VectorType>($_self).getRank() - 1, 16))">;
+//===----------------------------------------------------------------------===//
+// ConvertFromSvboolOp
+//===----------------------------------------------------------------------===//
def ConvertFromSvboolOp : ArmSVE_Op<"convert_from_svbool",
[Pure,
@@ -369,6 +409,10 @@ def ConvertFromSvboolOp : ArmSVE_Op<"convert_from_svbool",
let assemblyFormat = "$source attr-dict `:` type($result)";
}
+//===----------------------------------------------------------------------===//
+// ConvertToSvboolOp
+//===----------------------------------------------------------------------===//
+
def ConvertToSvboolOp : ArmSVE_Op<"convert_to_svbool",
[Pure,
SvboolTypeConstraint<"source", "result">]> {
@@ -405,13 +449,9 @@ def ConvertToSvboolOp : ArmSVE_Op<"convert_to_svbool",
let assemblyFormat = "$source attr-dict `:` type($source)";
}
-// Inputs valid for the multi-vector zips (not including the 128-bit element zipqs)
-def ZipInputVectorType : AnyTypeOf<[
- Scalable1DVectorOfLength<2, [I64, F64]>,
- Scalable1DVectorOfLength<4, [I32, F32]>,
- Scalable1DVectorOfLength<8, [I16, F16, BF16]>,
- Scalable1DVectorOfLength<16, [I8]>],
- "an SVE vector with element size <= 64-bit">;
+//===----------------------------------------------------------------------===//
+// ZipX2Op
+//===----------------------------------------------------------------------===//
def ZipX2Op : ArmSVE_Op<"zip.x2", [Pure,
AllTypesMatch<["sourceV1", "sourceV2",
@@ -456,6 +496,10 @@ def ZipX2Op : ArmSVE_Op<"zip.x2", [Pure,
}];
}
+//===----------------------------------------------------------------------===//
+// ZipX4Op
+//===----------------------------------------------------------------------===//
+
def ZipX4Op
: ArmSVE_Op<"zip.x4",
[Pure,
@@ -518,6 +562,10 @@ def ZipX4Op
}];
}
+//===----------------------------------------------------------------------===//
+// PselOp
+//===----------------------------------------------------------------------===//
+
def PselOp : ArmSVE_Op<"psel", [Pure, AllTypesMatch<["p1", "result"]>]> {
let summary = "Predicate select";
@@ -552,29 +600,9 @@ def PselOp : ArmSVE_Op<"psel", [Pure, AllTypesMatch<["p1", "result"]>]> {
}];
}
-def ScalableMaskedAddIOp : ScalableMaskedIOp<"masked.addi", "addition",
- [Commutative]>;
-
-def ScalableMaskedAddFOp : ScalableMaskedFOp<"masked.addf", "addition",
- [Commutative]>;
-
-def ScalableMaskedSubIOp : ScalableMaskedIOp<"masked.subi", "subtraction">;
-
-def ScalableMaskedSubFOp : ScalableMaskedFOp<"masked.subf", "subtraction">;
-
-def ScalableMaskedMulIOp : ScalableMaskedIOp<"masked.muli", "multiplication",
- [Commutative]>;
-
-def ScalableMaskedMulFOp : ScalableMaskedFOp<"masked.mulf", "multiplication",
- [Commutative]>;
-
-def ScalableMaskedSDivIOp : ScalableMaskedIOp<"masked.divi_signed",
- "signed division">;
-
-def ScalableMaskedUDivIOp : ScalableMaskedIOp<"masked.divi_unsigned",
- "unsigned division">;
-
-def ScalableMaskedDivFOp : ScalableMaskedFOp<"masked.divf", "division">;
+//===----------------------------------------------------------------------===//
+// DupQLaneOp
+//===----------------------------------------------------------------------===//
def DupQLaneOp : ArmSVE_Op<"dupq_lane", [Pure, AllTypesMatch<["src", "dst"]>]> {
let summary = "Broadcast indexed 128-bit segment to vector";
@@ -615,6 +643,40 @@ def DupQLaneOp : ArmSVE_Op<"dupq_lane", [Pure, AllTypesMatch<["src", "dst"]>]> {
}];
}
+//===----------------------------------------------------------------------===//
+// Masked arithmetic Ops
+//
+// TODO: Audit - do we need these?
+//===----------------------------------------------------------------------===//
+
+def ScalableMaskedAddIOp : ScalableMaskedIOp<"masked.addi", "addition",
+ [Commutative]>;
+
+def ScalableMaskedAddFOp : ScalableMaskedFOp<"masked.addf", "addition",
+ [Commutative]>;
+
+def ScalableMaskedSubIOp : ScalableMaskedIOp<"masked.subi", "subtraction">;
+
+def ScalableMaskedSubFOp : ScalableMaskedFOp<"masked.subf", "subtraction">;
+
+def ScalableMaskedMulIOp : ScalableMaskedIOp<"masked.muli", "multiplication",
+ [Commutative]>;
+
+def ScalableMaskedMulFOp : ScalableMaskedFOp<"masked.mulf", "multiplication",
+ [Commutative]>;
+
+def ScalableMaskedSDivIOp : ScalableMaskedIOp<"masked.divi_signed",
+ "signed division">;
+
+def ScalableMaskedUDivIOp : ScalableMaskedIOp<"masked.divi_unsigned",
+ "unsigned division">;
+
+def ScalableMaskedDivFOp : ScalableMaskedFOp<"masked.divf", "division">;
+
+//===----------------------------------------------------------------------===//
+// Intrinsic ops
+//===----------------------------------------------------------------------===//
+
def UmmlaIntrOp :
ArmSVE_IntrBinaryOverloadedOp<"ummla">,
Arguments<(ins AnyScalableVectorOfAnyRank, AnyScalableVectorOfAnyRank, AnyScalableVectorOfAnyRank)>;
diff --git a/mlir/test/Dialect/ArmSVE/invalid.mlir b/mlir/test/Dialect/ArmSVE/invalid.mlir
index 1d2529be0560b7..67a7a9d48d234a 100644
--- a/mlir/test/Dialect/ArmSVE/invalid.mlir
+++ b/mlir/test/Dialect/ArmSVE/invalid.mlir
@@ -1,7 +1,73 @@
// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+//===----------------------------------------------------------------------===//
+// MMLA Ops
+//===----------------------------------------------------------------------===//
+
+func.func @bfmmla_invalid_element_type_lhs_rhs(%acc: vector<[4]xf32>,
+ %lhs: vector<[8]xf16>,
+ %rhs: vector<[8]xf16>) -> vector<[4]xf32> {
+ // expected-error at +1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<[8]xf16>'}}
+ %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xf16> to vector<[4]xf32>
+ return %0 : vector<[4]xf32>
+}
+
// -----
+func.func @bfmmla_invalid_dimension_lhs_rhs(%acc: vector<[4]xf32>,
+ %lhs: vector<[4]xbf16>,
+ %rhs: vector<[4]xbf16>) -> vector<[4]xf32> {
+ // expected-error at +1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<[4]xbf16>}}
+ %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[4]xbf16> to vector<[4]xf32>
+ return %0 : vector<[4]xf32>
+}
+
+// -----
+
+func.func @bfmmla_fixed_dimension_lhs_rhs(%acc: vector<[4]xf32>,
+ %lhs: vector<8xbf16>,
+ %rhs: vector<8xbf16>) -> vector<[4]xf32> {
+ // expected-error at +1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<8xbf16>}}
+ %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<8xbf16> to vector<[4]xf32>
+ return %0 : vector<[4]xf32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_element_type_acc(%acc: vector<[4]xi32>,
+ %lhs: vector<[8]xbf16>,
+ %rhs: vector<[8]xbf16>) -> vector<[4]xi32> {
+ // expected-error at +1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<[4]xi32>'}}
+ %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<[4]xi32>
+ return %0 : vector<[4]xi32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_dimension_acc(%acc: vector<[8]xf32>,
+ %lhs: vector<[8]xbf16>,
+ %rhs: vector<[8]xbf16>) -> vector<[8]xf32> {
+ // expected-error at +1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<[8]xf32>'}}
+ %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<[8]xf32>
+ return %0 : vector<[8]xf32>
+}
+
+// -----
+
+func.func @bfmmla_fixed_dimension_acc(%acc: vector<4xf32>,
+ %lhs: vector<[8]xbf16>,
+ %rhs: vector<[8]xbf16>) -> vector<4xf32> {
+ // expected-error at +1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<4xf32>'}}
+ %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<4xf32>
+ return %0 : vector<4xf32>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// From/To bool Ops
+//===----------------------------------------------------------------------===//
+
func.func @arm_sve_convert_from_svbool__bad_mask_type(%bool: vector<2x[16]xi1>) -> vector<2x[8]xi2> {
// expected-error at +1 {{'result' must be trailing scalable vector of 1-bit signless integer values with dim -1 having a size of {16, 8, 4, 2, 1}, but got 'vector<2x[8]xi2>'}}
%mask = arm_sve.convert_from_svbool %bool : vector<2x[8]xi2>
@@ -51,6 +117,10 @@ func.func @arm_sve_convert_to_svbool__bad_mask_scalability(%mask : vector<[4]x[8
// -----
+//===----------------------------------------------------------------------===//
+// Zip Ops
+//===----------------------------------------------------------------------===//
+
func.func @arm_sve_zip_x2_bad_vector_type(%a : vector<[7]xi8>) {
// expected-error at +1 {{op operand #0 must be an SVE vector with element size <= 64-bit, but got 'vector<[7]xi8>'}}
arm_sve.zip.x2 %a, %a : vector<[7]xi8>
@@ -72,63 +142,3 @@ func.func @arm_sve_psel_bad_vector_type(%a : vector<[7]xi1>, %index: index) {
arm_sve.psel %a, %a[%index] : vector<[7]xi1>, vector<[7]xi1>
return
}
-
-// -----
-
-func.func @bfmmla_invalid_element_type_lhs_rhs(%acc: vector<[4]xf32>,
- %lhs: vector<[8]xf16>,
- %rhs: vector<[8]xf16>) -> vector<[4]xf32> {
- // expected-error at +1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<[8]xf16>'}}
- %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xf16> to vector<[4]xf32>
- return %0 : vector<[4]xf32>
-}
-
-// -----
-
-func.func @bfmmla_invalid_dimension_lhs_rhs(%acc: vector<[4]xf32>,
- %lhs: vector<[4]xbf16>,
- %rhs: vector<[4]xbf16>) -> vector<[4]xf32> {
- // expected-error at +1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<[4]xbf16>}}
- %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[4]xbf16> to vector<[4]xf32>
- return %0 : vector<[4]xf32>
-}
-
-// -----
-
-func.func @bfmmla_fixed_dimension_lhs_rhs(%acc: vector<[4]xf32>,
- %lhs: vector<8xbf16>,
- %rhs: vector<8xbf16>) -> vector<[4]xf32> {
- // expected-error at +1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<8xbf16>}}
- %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<8xbf16> to vector<[4]xf32>
- return %0 : vector<[4]xf32>
-}
-
-// -----
-
-func.func @bfmmla_invalid_element_type_acc(%acc: vector<[4]xi32>,
- %lhs: vector<[8]xbf16>,
- %rhs: vector<[8]xbf16>) -> vector<[4]xi32> {
- // expected-error at +1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<[4]xi32>'}}
- %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<[4]xi32>
- return %0 : vector<[4]xi32>
-}
-
-// -----
-
-func.func @bfmmla_invalid_dimension_acc(%acc: vector<[8]xf32>,
- %lhs: vector<[8]xbf16>,
- %rhs: vector<[8]xbf16>) -> vector<[8]xf32> {
- // expected-error at +1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<[8]xf32>'}}
- %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<[8]xf32>
- return %0 : vector<[8]xf32>
-}
-
-// -----
-
-func.func @bfmmla_fixed_dimension_acc(%acc: vector<4xf32>,
- %lhs: vector<[8]xbf16>,
- %rhs: vector<[8]xbf16>) -> vector<4xf32> {
- // expected-error at +1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<4xf32>'}}
- %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<4xf32>
- return %0 : vector<4xf32>
-}
diff --git a/mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir b/mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir
index 8c658db009adf3..3650e61696394b 100644
--- a/mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir
+++ b/mlir/test/Dialect/ArmSVE/legalize-for-llvm.mlir
@@ -1,5 +1,9 @@
// RUN: mlir-opt -convert-vector-to-llvm="enable-arm-sve" -convert-func-to-llvm -convert-arith-to-llvm -cse -reconcile-unrealized-casts -split-input-file %s | FileCheck %s
+//===----------------------------------------------------------------------===//
+// Dot ops
+//===----------------------------------------------------------------------===//
+
func.func @arm_sve_sdot(%a: vector<[16]xi8>,
%b: vector<[16]xi8>,
%c: vector<[4]xi32>)
@@ -12,24 +16,28 @@ func.func @arm_sve_sdot(%a: vector<[16]xi8>,
// -----
-func.func @arm_sve_smmla(%a: vector<[16]xi8>,
- %b: vector<[16]xi8>,
- %c: vector<[4]xi32>)
+func.func @arm_sve_udot(%a: vector<[16]xi8>,
+ %b: vector<[16]xi8>,
+ %c: vector<[4]xi32>)
-> vector<[4]xi32> {
- // CHECK: arm_sve.intr.smmla
- %0 = arm_sve.smmla %c, %a, %b :
+ // CHECK: arm_sve.intr.udot
+ %0 = arm_sve.udot %c, %a, %b :
vector<[16]xi8> to vector<[4]xi32>
return %0 : vector<[4]xi32>
}
// -----
-func.func @arm_sve_udot(%a: vector<[16]xi8>,
- %b: vector<[16]xi8>,
- %c: vector<[4]xi32>)
+//===----------------------------------------------------------------------===//
+// MMLA Ops
+//===----------------------------------------------------------------------===//
+
+func.func @arm_sve_smmla(%a: vector<[16]xi8>,
+ %b: vector<[16]xi8>,
+ %c: vector<[4]xi32>)
-> vector<[4]xi32> {
- // CHECK: arm_sve.intr.udot
- %0 = arm_sve.udot %c, %a, %b :
+ // CHECK: arm_sve.intr.smmla
+ %0 = arm_sve.smmla %c, %a, %b :
vector<[16]xi8> to vector<[4]xi32>
return %0 : vector<[4]xi32>
}
@@ -60,6 +68,10 @@ func.func @arm_sve_usmmla(%a: vector<[16]xi8>,
// -----
+//===----------------------------------------------------------------------===//
+// Masked Arith Ops (integer)
+//===----------------------------------------------------------------------===//
+
func.func @arm_sve_arithi_masked(%a: vector<[4]xi32>,
%b: vector<[4]xi32>,
%c: vector<[4]xi32>,
@@ -87,6 +99,10 @@ func.func @arm_sve_arithi_masked(%a: vector<[4]xi32>,
// -----
+//===----------------------------------------------------------------------===//
+// Masked Arith Ops (fp)
+//===----------------------------------------------------------------------===//
+
func.func @arm_sve_arithf_masked(%a: vector<[4]xf32>,
%b: vector<[4]xf32>,
%c: vector<[4]xf32>,
@@ -111,6 +127,10 @@ func.func @arm_sve_arithf_masked(%a: vector<[4]xf32>,
// -----
+//===----------------------------------------------------------------------===//
+// TODO: This is a duplicate of other tests, remove
+//===----------------------------------------------------------------------===//
+
func.func @arm_sve_abs_
diff (%a: vector<[4]xi32>,
%b: vector<[4]xi32>)
-> vector<[4]xi32> {
@@ -137,6 +157,12 @@ func.func @arm_sve_abs_
diff (%a: vector<[4]xi32>,
// -----
+//===----------------------------------------------------------------------===//
+// vscale
+//
+// TODO: This is not specific to SVE, move somewhere more suitable.
+//===----------------------------------------------------------------------===//
+
func.func @get_vector_scale() -> index {
// CHECK: llvm.intr.vscale
%0 = vector.vscale
@@ -145,6 +171,10 @@ func.func @get_vector_scale() -> index {
// -----
+//===----------------------------------------------------------------------===//
+// From/To bool Ops
+//===----------------------------------------------------------------------===//
+
func.func @convert_1d_mask_to_svbool(%mask: vector<[4]xi1>) -> vector<[16]xi1>
{
// CHECK: "arm_sve.intr.convert.to.svbool"(%{{.*}}) : (vector<[4]xi1>) -> vector<[16]xi1>
@@ -202,6 +232,10 @@ func.func @convert_2d_mask_from_svbool(%svbool: vector<3x[16]xi1>) -> vector<3x[
// -----
+//===----------------------------------------------------------------------===//
+// Zip Ops
+//===----------------------------------------------------------------------===//
+
func.func @arm_sve_zip_x2(%a: vector<[8]xi16>, %b: vector<[8]xi16>)
-> (vector<[8]xi16>, vector<[8]xi16>)
{
@@ -226,6 +260,10 @@ func.func @arm_sve_zip_x4(
// -----
+//===----------------------------------------------------------------------===//
+// Mask Ops
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: @arm_sve_predicate_sized_create_masks(
// CHECK-SAME: %[[INDEX:.*]]: i64
func.func @arm_sve_predicate_sized_create_masks(%index: index) -> (vector<[2]xi1>, vector<[4]xi1>, vector<[8]xi1>, vector<[16]xi1>) {
@@ -243,8 +281,8 @@ func.func @arm_sve_predicate_sized_create_masks(%index: index) -> (vector<[2]xi1
// -----
-// CHECK-LABEL: @arm_sve_unsupported_create_masks
-func.func @arm_sve_unsupported_create_masks(%index: index) -> (vector<[1]xi1>, vector<[7]xi1>, vector<[32]xi1>) {
+// CHECK-LABEL: @negative_arm_sve_create_masks
+func.func @negative_arm_sve_create_masks(%index: index) -> (vector<[1]xi1>, vector<[7]xi1>, vector<[32]xi1>) {
// CHECK-NOT: arm_sve.intr.whilelt
%0 = vector.create_mask %index : vector<[1]xi1>
%1 = vector.create_mask %index : vector<[7]xi1>
@@ -254,6 +292,10 @@ func.func @arm_sve_unsupported_create_masks(%index: index) -> (vector<[1]xi1>, v
// -----
+//===----------------------------------------------------------------------===//
+// Psel Ops
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: @arm_sve_psel_matching_predicate_types(
// CHECK-SAME: %[[P0:[a-z0-9]+]]: vector<[4]xi1>,
// CHECK-SAME: %[[P1:[a-z0-9]+]]: vector<[4]xi1>,
@@ -286,6 +328,10 @@ func.func @arm_sve_psel_mixed_predicate_types(%p0: vector<[8]xi1>, %p1: vector<[
// -----
+//===----------------------------------------------------------------------===//
+// Dup Ops
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: @arm_sve_dupq_lane(
// CHECK-SAME: %[[A0:[a-z0-9]+]]: vector<[16]xi8>
// CHECK-SAME: %[[A1:[a-z0-9]+]]: vector<[8]xi16>
diff --git a/mlir/test/Target/LLVMIR/arm-neon-2d.mlir b/mlir/test/Target/LLVMIR/arm-neon-2d.mlir
index dfb4e4fd309a97..6cec0363df7cb6 100644
--- a/mlir/test/Target/LLVMIR/arm-neon-2d.mlir
+++ b/mlir/test/Target/LLVMIR/arm-neon-2d.mlir
@@ -1,5 +1,9 @@
// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(arm-neon-2d-to-intr))" %s | FileCheck %s
+//===----------------------------------------------------------------------===//
+// Dot ops
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: arm_neon_sdot2d_4x4_i8i8
func.func @arm_neon_sdot2d_4x4_i8i8(%a: vector<4xi32>, %b: vector<4x4xi8>, %c: vector<4x4xi8>) -> vector<4xi32> {
// CHECK: arm_neon.intr.sdot %{{.*}}, %{{.*}}, %{{.*}} : vector<16xi8>, vector<16xi8> to vector<4xi32>
diff --git a/mlir/test/Target/LLVMIR/arm-neon.mlir b/mlir/test/Target/LLVMIR/arm-neon.mlir
index e1328ad448f0a5..c84b315c8b1547 100644
--- a/mlir/test/Target/LLVMIR/arm-neon.mlir
+++ b/mlir/test/Target/LLVMIR/arm-neon.mlir
@@ -1,5 +1,33 @@
// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
+//===----------------------------------------------------------------------===//
+// Dot ops
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: arm_neon_sdot_8_i8i8
+llvm.func @arm_neon_sdot_8_i8i8(%a: vector<2xi32>, %b: vector<8xi8>, %c: vector<8xi8>) -> vector<2xi32> {
+ // CHECK: %[[V0:.*]] = call <2 x i32> @llvm.aarch64.neon.sdot.v2i32.v8i8(<2 x i32> %{{.*}}, <8 x i8> %{{.*}}, <8 x i8> %{{.*}})
+ // CHECK-NEXT: ret <2 x i32>
+ %0 = arm_neon.intr.sdot %a, %b, %c : vector<8xi8>, vector<8xi8> to vector<2xi32>
+ llvm.return %0 : vector<2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: arm_neon_sdot_16_i8i8
+llvm.func @arm_neon_sdot_16_i8i8(%a: vector<4xi32>, %b: vector<16xi8>, %c: vector<16xi8>) -> vector<4xi32> {
+ // CHECK: %[[V0:.*]] = call <4 x i32> @llvm.aarch64.neon.sdot.v4i32.v16i8(<4 x i32> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}})
+ // CHECK-NEXT: ret <4 x i32>
+ %0 = arm_neon.intr.sdot %a, %b, %c : vector<16xi8>, vector<16xi8> to vector<4xi32>
+ llvm.return %0 : vector<4xi32>
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// Smull Ops
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: arm_neon_smull
llvm.func @arm_neon_smull(%arg0: vector<8xi8>, %arg1: vector<8xi8>) -> !llvm.struct<(vector<8xi16>, vector<4xi32>, vector<2xi64>)> {
// CHECK: %[[V0:.*]] = call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %{{.*}}, <8 x i8> %{{.*}})
@@ -26,25 +54,9 @@ llvm.func @arm_neon_smull(%arg0: vector<8xi8>, %arg1: vector<8xi8>) -> !llvm.str
// -----
-// CHECK-LABEL: arm_neon_sdot_8_i8i8
-llvm.func @arm_neon_sdot_8_i8i8(%a: vector<2xi32>, %b: vector<8xi8>, %c: vector<8xi8>) -> vector<2xi32> {
- // CHECK: %[[V0:.*]] = call <2 x i32> @llvm.aarch64.neon.sdot.v2i32.v8i8(<2 x i32> %{{.*}}, <8 x i8> %{{.*}}, <8 x i8> %{{.*}})
- // CHECK-NEXT: ret <2 x i32>
- %0 = arm_neon.intr.sdot %a, %b, %c : vector<8xi8>, vector<8xi8> to vector<2xi32>
- llvm.return %0 : vector<2xi32>
-}
-
-// -----
-
-// CHECK-LABEL: arm_neon_sdot_16_i8i8
-llvm.func @arm_neon_sdot_16_i8i8(%a: vector<4xi32>, %b: vector<16xi8>, %c: vector<16xi8>) -> vector<4xi32> {
- // CHECK: %[[V0:.*]] = call <4 x i32> @llvm.aarch64.neon.sdot.v4i32.v16i8(<4 x i32> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}})
- // CHECK-NEXT: ret <4 x i32>
- %0 = arm_neon.intr.sdot %a, %b, %c : vector<16xi8>, vector<16xi8> to vector<4xi32>
- llvm.return %0 : vector<4xi32>
-}
-
-// -----
+//===----------------------------------------------------------------------===//
+// MMLA Ops
+//===----------------------------------------------------------------------===//
// CHECK-LABEL: arm_neon_smmla
llvm.func @arm_neon_smmla(%arg0: vector<16xi8>,
diff --git a/mlir/test/Target/LLVMIR/arm-sve.mlir b/mlir/test/Target/LLVMIR/arm-sve.mlir
index b365e555c0b782..0270e4ad657108 100644
--- a/mlir/test/Target/LLVMIR/arm-sve.mlir
+++ b/mlir/test/Target/LLVMIR/arm-sve.mlir
@@ -1,5 +1,9 @@
// RUN: mlir-translate --mlir-to-llvmir %s | FileCheck %s
+//===----------------------------------------------------------------------===//
+// Dot intrinsics
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_sdot
llvm.func @arm_sve_sdot(%arg0: vector<[16]xi8>,
%arg1: vector<[16]xi8>,
@@ -12,18 +16,6 @@ llvm.func @arm_sve_sdot(%arg0: vector<[16]xi8>,
llvm.return %0 : vector<[4]xi32>
}
-// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_smmla
-llvm.func @arm_sve_smmla(%arg0: vector<[16]xi8>,
- %arg1: vector<[16]xi8>,
- %arg2: vector<[4]xi32>)
- -> vector<[4]xi32> {
- // CHECK: call <vscale x 4 x i32> @llvm.aarch64.sve.smmla.nxv4i32(<vscale x 4
- %0 = "arm_sve.intr.smmla"(%arg2, %arg0, %arg1) :
- (vector<[4]xi32>, vector<[16]xi8>, vector<[16]xi8>)
- -> vector<[4]xi32>
- llvm.return %0 : vector<[4]xi32>
-}
-
// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_udot
llvm.func @arm_sve_udot(%arg0: vector<[16]xi8>,
%arg1: vector<[16]xi8>,
@@ -36,6 +28,22 @@ llvm.func @arm_sve_udot(%arg0: vector<[16]xi8>,
llvm.return %0 : vector<[4]xi32>
}
+//===----------------------------------------------------------------------===//
+// MMLA intrinsics
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_smmla
+llvm.func @arm_sve_smmla(%arg0: vector<[16]xi8>,
+ %arg1: vector<[16]xi8>,
+ %arg2: vector<[4]xi32>)
+ -> vector<[4]xi32> {
+ // CHECK: call <vscale x 4 x i32> @llvm.aarch64.sve.smmla.nxv4i32(<vscale x 4
+ %0 = "arm_sve.intr.smmla"(%arg2, %arg0, %arg1) :
+ (vector<[4]xi32>, vector<[16]xi8>, vector<[16]xi8>)
+ -> vector<[4]xi32>
+ llvm.return %0 : vector<[4]xi32>
+}
+
// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_ummla
llvm.func @arm_sve_ummla(%arg0: vector<[16]xi8>,
%arg1: vector<[16]xi8>,
@@ -72,29 +80,9 @@ llvm.func @arm_sve_bfmmla(%arg0: vector<[8]xbf16>,
llvm.return %0 : vector<[4]xf32>
}
-// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_arithi
-llvm.func @arm_sve_arithi(%arg0: vector<[4]xi32>,
- %arg1: vector<[4]xi32>,
- %arg2: vector<[4]xi32>)
- -> vector<[4]xi32> {
- // CHECK: mul <vscale x 4 x i32>
- %0 = llvm.mul %arg0, %arg1 : vector<[4]xi32>
- // CHECK: add <vscale x 4 x i32>
- %1 = llvm.add %0, %arg2 : vector<[4]xi32>
- llvm.return %1 : vector<[4]xi32>
-}
-
-// CHECK-LABEL: define <vscale x 4 x float> @arm_sve_arithf
-llvm.func @arm_sve_arithf(%arg0: vector<[4]xf32>,
- %arg1: vector<[4]xf32>,
- %arg2: vector<[4]xf32>)
- -> vector<[4]xf32> {
- // CHECK: fmul <vscale x 4 x float>
- %0 = llvm.fmul %arg0, %arg1 : vector<[4]xf32>
- // CHECK: fadd <vscale x 4 x float>
- %1 = llvm.fadd %0, %arg2 : vector<[4]xf32>
- llvm.return %1 : vector<[4]xf32>
-}
+//===----------------------------------------------------------------------===//
+// Masked Arith intrinsics (integer)
+//===----------------------------------------------------------------------===//
// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_arithi_masked
llvm.func @arm_sve_arithi_masked(%arg0: vector<[4]xi32>,
@@ -132,6 +120,10 @@ llvm.func @arm_sve_arithi_masked(%arg0: vector<[4]xi32>,
llvm.return %4 : vector<[4]xi32>
}
+//===----------------------------------------------------------------------===//
+// Masked Arith intrinsics (fp)
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: define <vscale x 4 x float> @arm_sve_arithf_masked
llvm.func @arm_sve_arithf_masked(%arg0: vector<[4]xf32>,
%arg1: vector<[4]xf32>,
@@ -163,23 +155,9 @@ llvm.func @arm_sve_arithf_masked(%arg0: vector<[4]xf32>,
llvm.return %3 : vector<[4]xf32>
}
-// CHECK-LABEL: define <vscale x 4 x i1> @arm_sve_mask_genf
-llvm.func @arm_sve_mask_genf(%arg0: vector<[4]xf32>,
- %arg1: vector<[4]xf32>)
- -> vector<[4]xi1> {
- // CHECK: fcmp oeq <vscale x 4 x float>
- %0 = llvm.fcmp "oeq" %arg0, %arg1 : vector<[4]xf32>
- llvm.return %0 : vector<[4]xi1>
-}
-
-// CHECK-LABEL: define <vscale x 4 x i1> @arm_sve_mask_geni
-llvm.func @arm_sve_mask_geni(%arg0: vector<[4]xi32>,
- %arg1: vector<[4]xi32>)
- -> vector<[4]xi1> {
- // CHECK: icmp uge <vscale x 4 x i32>
- %0 = llvm.icmp "uge" %arg0, %arg1 : vector<[4]xi32>
- llvm.return %0 : vector<[4]xi1>
-}
+//===----------------------------------------------------------------------===//
+// TODO: This is a duplicate of other tests, remove
+//===----------------------------------------------------------------------===//
// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_abs_
diff
llvm.func @arm_sve_abs_
diff (%arg0: vector<[4]xi32>,
@@ -214,86 +192,9 @@ llvm.func @arm_sve_abs_
diff (%arg0: vector<[4]xi32>,
llvm.return %6 : vector<[4]xi32>
}
-// CHECK-LABEL: define void @memcopy
-llvm.func @memcopy(%arg0: !llvm.ptr, %arg1: !llvm.ptr,
- %arg2: i64, %arg3: i64, %arg4: i64,
- %arg5: !llvm.ptr, %arg6: !llvm.ptr,
- %arg7: i64, %arg8: i64, %arg9: i64,
- %arg10: i64) {
- %0 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>, array<1 x i64>)>
- %1 = llvm.insertvalue %arg0, %0[0] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %2 = llvm.insertvalue %arg1, %1[1] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %3 = llvm.insertvalue %arg2, %2[2] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %4 = llvm.insertvalue %arg3, %3[3, 0] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %5 = llvm.insertvalue %arg4, %4[4, 0] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %6 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %7 = llvm.insertvalue %arg5, %6[0] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %8 = llvm.insertvalue %arg6, %7[1] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %9 = llvm.insertvalue %arg7, %8[2] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %10 = llvm.insertvalue %arg8, %9[3, 0] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %11 = llvm.insertvalue %arg9, %10[4, 0] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- %12 = llvm.mlir.constant(0 : i64) : i64
- %13 = llvm.mlir.constant(4 : i64) : i64
- // CHECK: [[VL:%[0-9]+]] = call i64 @llvm.vscale.i64()
- %14 = "llvm.intr.vscale"() : () -> i64
- // CHECK: mul i64 [[VL]], 4
- %15 = llvm.mul %14, %13 : i64
- llvm.br ^bb1(%12 : i64)
-^bb1(%16: i64):
- %17 = llvm.icmp "slt" %16, %arg10 : i64
- llvm.cond_br %17, ^bb2, ^bb3
-^bb2:
- // CHECK: extractvalue { ptr, ptr, i64, [1 x i64], [1 x i64] }
- %18 = llvm.extractvalue %5[1] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- // CHECK: getelementptr float, ptr
- %19 = llvm.getelementptr %18[%16] : (!llvm.ptr, i64) -> !llvm.ptr, f32
- // CHECK: load <vscale x 4 x float>, ptr
- %21 = llvm.load %19 : !llvm.ptr -> vector<[4]xf32>
- // CHECK: extractvalue { ptr, ptr, i64, [1 x i64], [1 x i64] }
- %22 = llvm.extractvalue %11[1] : !llvm.struct<(ptr, ptr, i64,
- array<1 x i64>,
- array<1 x i64>)>
- // CHECK: getelementptr float, ptr
- %23 = llvm.getelementptr %22[%16] : (!llvm.ptr, i64) -> !llvm.ptr, f32
- // CHECK: store <vscale x 4 x float> %{{[0-9]+}}, ptr %{{[0-9]+}}
- llvm.store %21, %23 : vector<[4]xf32>, !llvm.ptr
- %25 = llvm.add %16, %15 : i64
- llvm.br ^bb1(%25 : i64)
-^bb3:
- llvm.return
-}
-
-// CHECK-LABEL: define i64 @get_vector_scale()
-llvm.func @get_vector_scale() -> i64 {
- // CHECK: call i64 @llvm.vscale.i64()
- %0 = "llvm.intr.vscale"() : () -> i64
- llvm.return %0 : i64
-}
+//===----------------------------------------------------------------------===//
+// From/To bool intrinsics
+//===----------------------------------------------------------------------===//
// CHECK-LABEL: @arm_sve_convert_from_svbool(
// CHECK-SAME: <vscale x 16 x i1> %[[SVBOOL:[0-9]+]])
@@ -339,6 +240,10 @@ llvm.func @arm_sve_convert_to_svbool(
llvm.return
}
+//===----------------------------------------------------------------------===//
+// Zip intrinsics (fp)
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: arm_sve_zip_x2(
// CHECK-SAME: <vscale x 16 x i8> %[[V1:[0-9]+]],
// CHECK-SAME: <vscale x 8 x i16> %[[V2:[0-9]+]],
@@ -381,6 +286,10 @@ llvm.func @arm_sve_zip_x4(%nxv16i8: vector<[16]xi8>, %nxv8i16: vector<[8]xi16>,
llvm.return
}
+//===----------------------------------------------------------------------===//
+// While* intrinsics (fp)
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: arm_sve_whilelt(
// CHECK-SAME: i64 %[[BASE:[0-9]+]],
// CHECK-SAME: i64 %[[N:[0-9]+]]
@@ -396,6 +305,10 @@ llvm.func @arm_sve_whilelt(%base: i64, %n: i64) {
llvm.return
}
+//===----------------------------------------------------------------------===//
+// Psel intrinsics (fp)
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: arm_sve_psel(
// CHECK-SAME: <vscale x 16 x i1> %[[PN:[0-9]+]],
// CHECK-SAME: <vscale x 2 x i1> %[[P1:[0-9]+]],
@@ -415,6 +328,10 @@ llvm.func @arm_sve_psel(%pn: vector<[16]xi1>, %p1: vector<[2]xi1>, %p2: vector<[
llvm.return
}
+//===----------------------------------------------------------------------===//
+// Dupq intrinsics (fp)
+//===----------------------------------------------------------------------===//
+
// CHECK-LABEL: @arm_sve_dupq_lane
// CHECK-SAME: <vscale x 16 x i8> %[[V0:[0-9]+]]
// CHECK-SAME: <vscale x 8 x i16> %[[V1:[0-9]+]]
diff --git a/mlir/test/Target/LLVMIR/scalable-vectors.mlir b/mlir/test/Target/LLVMIR/scalable-vectors.mlir
new file mode 100644
index 00000000000000..56963ff47c8de7
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/scalable-vectors.mlir
@@ -0,0 +1,126 @@
+// RUN: mlir-translate --mlir-to-llvmir %s | FileCheck %s
+
+// CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_arithi
+llvm.func @arm_sve_arithi(%arg0: vector<[4]xi32>,
+ %arg1: vector<[4]xi32>,
+ %arg2: vector<[4]xi32>)
+ -> vector<[4]xi32> {
+ // CHECK: mul <vscale x 4 x i32>
+ %0 = llvm.mul %arg0, %arg1 : vector<[4]xi32>
+ // CHECK: add <vscale x 4 x i32>
+ %1 = llvm.add %0, %arg2 : vector<[4]xi32>
+ llvm.return %1 : vector<[4]xi32>
+}
+
+// CHECK-LABEL: define <vscale x 4 x float> @arm_sve_arithf
+llvm.func @arm_sve_arithf(%arg0: vector<[4]xf32>,
+ %arg1: vector<[4]xf32>,
+ %arg2: vector<[4]xf32>)
+ -> vector<[4]xf32> {
+ // CHECK: fmul <vscale x 4 x float>
+ %0 = llvm.fmul %arg0, %arg1 : vector<[4]xf32>
+ // CHECK: fadd <vscale x 4 x float>
+ %1 = llvm.fadd %0, %arg2 : vector<[4]xf32>
+ llvm.return %1 : vector<[4]xf32>
+}
+
+// CHECK-LABEL: define <vscale x 4 x i1> @arm_sve_mask_genf
+llvm.func @arm_sve_mask_genf(%arg0: vector<[4]xf32>,
+ %arg1: vector<[4]xf32>)
+ -> vector<[4]xi1> {
+ // CHECK: fcmp oeq <vscale x 4 x float>
+ %0 = llvm.fcmp "oeq" %arg0, %arg1 : vector<[4]xf32>
+ llvm.return %0 : vector<[4]xi1>
+}
+
+// CHECK-LABEL: define <vscale x 4 x i1> @arm_sve_mask_geni
+llvm.func @arm_sve_mask_geni(%arg0: vector<[4]xi32>,
+ %arg1: vector<[4]xi32>)
+ -> vector<[4]xi1> {
+ // CHECK: icmp uge <vscale x 4 x i32>
+ %0 = llvm.icmp "uge" %arg0, %arg1 : vector<[4]xi32>
+ llvm.return %0 : vector<[4]xi1>
+}
+
+// CHECK-LABEL: define i64 @get_vector_scale()
+llvm.func @get_vector_scale() -> i64 {
+ // CHECK: call i64 @llvm.vscale.i64()
+ %0 = "llvm.intr.vscale"() : () -> i64
+ llvm.return %0 : i64
+}
+
+// TODO: Delete the following - every op contained in this function is already
+// tested separately.
+// CHECK-LABEL: define void @memcopy
+llvm.func @memcopy(%arg0: !llvm.ptr, %arg1: !llvm.ptr,
+ %arg2: i64, %arg3: i64, %arg4: i64,
+ %arg5: !llvm.ptr, %arg6: !llvm.ptr,
+ %arg7: i64, %arg8: i64, %arg9: i64,
+ %arg10: i64) {
+ %0 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>, array<1 x i64>)>
+ %1 = llvm.insertvalue %arg0, %0[0] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %2 = llvm.insertvalue %arg1, %1[1] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %3 = llvm.insertvalue %arg2, %2[2] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %4 = llvm.insertvalue %arg3, %3[3, 0] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %5 = llvm.insertvalue %arg4, %4[4, 0] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %6 = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %7 = llvm.insertvalue %arg5, %6[0] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %8 = llvm.insertvalue %arg6, %7[1] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %9 = llvm.insertvalue %arg7, %8[2] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %10 = llvm.insertvalue %arg8, %9[3, 0] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %11 = llvm.insertvalue %arg9, %10[4, 0] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ %12 = llvm.mlir.constant(0 : i64) : i64
+ %13 = llvm.mlir.constant(4 : i64) : i64
+ // CHECK: [[VL:%[0-9]+]] = call i64 @llvm.vscale.i64()
+ %14 = "llvm.intr.vscale"() : () -> i64
+ // CHECK: mul i64 [[VL]], 4
+ %15 = llvm.mul %14, %13 : i64
+ llvm.br ^bb1(%12 : i64)
+^bb1(%16: i64):
+ %17 = llvm.icmp "slt" %16, %arg10 : i64
+ llvm.cond_br %17, ^bb2, ^bb3
+^bb2:
+ // CHECK: extractvalue { ptr, ptr, i64, [1 x i64], [1 x i64] }
+ %18 = llvm.extractvalue %5[1] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ // CHECK: getelementptr float, ptr
+ %19 = llvm.getelementptr %18[%16] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+ // CHECK: load <vscale x 4 x float>, ptr
+ %21 = llvm.load %19 : !llvm.ptr -> vector<[4]xf32>
+ // CHECK: extractvalue { ptr, ptr, i64, [1 x i64], [1 x i64] }
+ %22 = llvm.extractvalue %11[1] : !llvm.struct<(ptr, ptr, i64,
+ array<1 x i64>,
+ array<1 x i64>)>
+ // CHECK: getelementptr float, ptr
+ %23 = llvm.getelementptr %22[%16] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+ // CHECK: store <vscale x 4 x float> %{{[0-9]+}}, ptr %{{[0-9]+}}
+ llvm.store %21, %23 : vector<[4]xf32>, !llvm.ptr
+ %25 = llvm.add %16, %15 : i64
+ llvm.br ^bb1(%25 : i64)
+^bb3:
+ llvm.return
+}
More information about the Mlir-commits
mailing list