[clang] [HLSL] Implement '__builtin_hlsl_is_intangible' type trait (PR #104544)

[Justin Bogner via cfe-commits]

================
@@ -1154,3 +1156,70 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   }
   return false;
 }
+
+static bool calculateIsIntangibleType(QualType Ty) {
+  Ty = Ty->getCanonicalTypeUnqualified();
+  if (Ty->isBuiltinType())
+    return Ty->isHLSLSpecificType();
+
+  llvm::SmallVector<QualType, 8> TypesToScan;
+  TypesToScan.push_back(Ty);
+  while (!TypesToScan.empty()) {
+    QualType T = TypesToScan.pop_back_val();
+    assert(T == T->getCanonicalTypeUnqualified() && "expected sugar-free type");
+    assert(!isa<MatrixType>(T) && "Matrix types not yet supported in HLSL");
+
+    if (const auto *AT = dyn_cast<ConstantArrayType>(T)) {
+      QualType ElTy = AT->getElementType()->getCanonicalTypeUnqualified();
+      if (ElTy->isBuiltinType())
+        return ElTy->isHLSLSpecificType();
+      TypesToScan.push_back(ElTy);
+      continue;
+    }
+
+    if (const auto *VT = dyn_cast<VectorType>(T)) {
+      QualType ElTy = VT->getElementType()->getCanonicalTypeUnqualified();
+      assert(ElTy->isBuiltinType() && "vectors can only contain builtin types");
+      if (ElTy->isHLSLSpecificType())
+        return true;
+      continue;
+    }
+
+    if (const auto *RT = dyn_cast<RecordType>(T)) {
+      const RecordDecl *RD = RT->getDecl();
+      for (const auto *FD : RD->fields()) {
+        QualType FieldTy = FD->getType()->getCanonicalTypeUnqualified();
+        if (FieldTy->isBuiltinType()) {
+          if (FieldTy->isHLSLSpecificType())
+            return true;
+        } else {
+          TypesToScan.push_back(FieldTy);
+        }
+      }
+
+      if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+        for (const CXXBaseSpecifier &B : CXXRD->bases()) {
+          TypesToScan.push_back(B.getType()->getCanonicalTypeUnqualified());
+        }
+      }
+      continue;
+    }
+  }
+  return false;
+}
+
+bool SemaHLSL::IsIntangibleType(const clang::QualType Ty) {
+  if (Ty.isNull())
+    return false;
+
+  const auto CachedEntry = IsIntangibleTypeCache.find(Ty.getTypePtr());
+  if (CachedEntry != IsIntangibleTypeCache.end()) {
+    assert(CachedEntry->second == calculateIsIntangibleType(Ty) &&
+           "IsIntangibleType mismatch");
+    return CachedEntry->second;
+  }
+
+  bool IsIntangible = calculateIsIntangibleType(Ty);
+  IsIntangibleTypeCache[Ty.getTypePtr()] = IsIntangible;
----------------
bogner wrote:

Should we skip the cache for results that are trivially computable? Specifically, I think we should probably pull the handling of builtin types into this function before we even consider the cache, since it's cheaper to re-compute that than to potentially store every builtin type in the cache.

https://github.com/llvm/llvm-project/pull/104544