[clang] [llvm] [RISCV][VLS] Support RISCV VLS calling convention (PR #100346)

[Craig Topper via llvm-commits]

================
@@ -359,9 +405,158 @@ ABIArgInfo RISCVABIInfo::coerceAndExpandFPCCEligibleStruct(
   return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType);
 }
 
+bool RISCVABIInfo::detectVLSCCEligibleStruct(QualType Ty, unsigned ABIVLen,
+                                             llvm::Type *&VLSType) const {
+  // No riscv_vls_cc attribute.
+  if (ABIVLen == 1)
+    return false;
+
+  // Legal struct for VLS calling convention should fulfill following rules:
+  // 1. Struct element should be either "homogeneous fixed-length vectors" or "a
+  //    fixed-length vector array".
+  // 2. Number of struct elements or array elements should be greater or equal
+  //    to 1 and less or equal to 8
+  // 3. Total number of vector registers needed should not exceed 8.
+  //
+  // Examples: Assume ABI_VLEN = 128.
+  // These are legal structs:
+  //   a. Structs with 1~8 "same" fixed-length vectors, e.g.
+  //   struct {
+  //     __attribute__((vector_size(16))) int a;
+  //     __attribute__((vector_size(16))) int b;
+  //   }
+  //
+  //   b. Structs with "single" fixed-length vector array with lengh 1~8, e.g.
+  //   struct {
+  //     __attribute__((vector_size(16))) int a[3];
+  //   }
+  // These are illegal structs:
+  //   a. Structs with 9 fixed-length vectors, e.g.
+  //   struct {
+  //     __attribute__((vector_size(16))) int a;
+  //     __attribute__((vector_size(16))) int b;
+  //     __attribute__((vector_size(16))) int c;
+  //     __attribute__((vector_size(16))) int d;
+  //     __attribute__((vector_size(16))) int e;
+  //     __attribute__((vector_size(16))) int f;
+  //     __attribute__((vector_size(16))) int g;
+  //     __attribute__((vector_size(16))) int h;
+  //     __attribute__((vector_size(16))) int i;
+  //   }
+  //
+  //   b. Structs with "multiple" fixed-length vector array, e.g.
+  //   struct {
+  //     __attribute__((vector_size(16))) int a[2];
+  //     __attribute__((vector_size(16))) int b[2];
+  //   }
+  //
+  //   c. Vector registers needed exceeds 8, e.g.
+  //   struct {
+  //     // Registers needed for single fixed-length element:
+  //     // 64 * 8 / ABI_VLEN = 4
+  //     __attribute__((vector_size(64))) int a;
+  //     __attribute__((vector_size(64))) int b;
+  //     __attribute__((vector_size(64))) int c;
+  //     __attribute__((vector_size(64))) int d;
+  //   }
+  //
+  // Struct of 1 fixed-length vector is passed as a scalable vector.
+  // Struct of >1 fixed-length vectors are passed as vector tuple.
+  // Struct of 1 array of fixed-length vectors is passed as a scalable vector.
+  // Otherwise, pass the struct indirectly.
+
+  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(CGT.ConvertType(Ty))) {
+    int NumElts = STy->getStructNumElements();
----------------
topperc wrote:

int -> unsigned

https://github.com/llvm/llvm-project/pull/100346