[clang] [HLSL] Add bounds checks for the HLSL `fmod` vector arguments and return types (PR #131035)

[Kaitlin Peng via cfe-commits]

https://github.com/kmpeng created https://github.com/llvm/llvm-project/pull/131035

Fixes #131024.
- Fixes template for scalar and vector `fmod` intrinsic overloads
- Fixes `fmod` Sema test

>From b33ab9d6a3c87c59ca9b2d3e8e5001f1d79bb620 Mon Sep 17 00:00:00 2001
From: kmpeng <kaitlinpeng at microsoft.com>
Date: Tue, 11 Mar 2025 16:47:27 -0700
Subject: [PATCH] add bounds checks for the hlsl fmod vector arguments and
 return types

---
 .../lib/Headers/hlsl/hlsl_intrinsic_helpers.h |  4 +-
 clang/lib/Headers/hlsl/hlsl_intrinsics.h      | 22 ++++++++--
 clang/test/SemaHLSL/BuiltIns/fmod-errors.hlsl | 44 ++++++++++++++-----
 3 files changed, 53 insertions(+), 17 deletions(-)

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
index 5f7c047dbf340..89ab664e90ba9 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
@@ -58,9 +58,7 @@ constexpr vector<T, L> reflect_vec_impl(vector<T, L> I, vector<T, L> N) {
 #endif
 }
 
-template <typename T>
-constexpr enable_if_t<is_same<float, T>::value || is_same<half, T>::value, T>
-fmod_impl(T X, T Y) {
+template <typename T> constexpr T fmod_impl(T X, T Y) {
 #if !defined(__DIRECTX__)
   return __builtin_elementwise_fmod(X, Y);
 #else
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 5459cbeb34fd0..a48a8e998a015 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -129,19 +129,33 @@ const inline float distance(__detail::HLSL_FIXED_VECTOR<float, N> X,
 /// Return the floating-point remainder of the x parameter divided by the y
 /// parameter.
 
+template <typename T>
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-const inline half fmod(half X, half Y) { return __detail::fmod_impl(X, Y); }
+const inline __detail::enable_if_t<__detail::is_arithmetic<T>::Value &&
+                                       __detail::is_same<half, T>::value,
+                                   T> fmod(T X, T Y) {
+  return __detail::fmod_impl(X, Y);
+}
 
-const inline float fmod(float X, float Y) { return __detail::fmod_impl(X, Y); }
+template <typename T>
+const inline __detail::enable_if_t<
+    __detail::is_arithmetic<T>::Value && __detail::is_same<float, T>::value, T>
+fmod(T X, T Y) {
+  return __detail::fmod_impl(X, Y);
+}
 
 template <int N>
 _HLSL_16BIT_AVAILABILITY(shadermodel, 6.2)
-const inline vector<half, N> fmod(vector<half, N> X, vector<half, N> Y) {
+const inline __detail::HLSL_FIXED_VECTOR<half, N> fmod(
+    __detail::HLSL_FIXED_VECTOR<half, N> X,
+    __detail::HLSL_FIXED_VECTOR<half, N> Y) {
   return __detail::fmod_vec_impl(X, Y);
 }
 
 template <int N>
-const inline vector<float, N> fmod(vector<float, N> X, vector<float, N> Y) {
+const inline __detail::HLSL_FIXED_VECTOR<float, N>
+fmod(__detail::HLSL_FIXED_VECTOR<float, N> X,
+     __detail::HLSL_FIXED_VECTOR<float, N> Y) {
   return __detail::fmod_vec_impl(X, Y);
 }
 
diff --git a/clang/test/SemaHLSL/BuiltIns/fmod-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/fmod-errors.hlsl
index 86f5a6f7bea9c..fc931139e523d 100644
--- a/clang/test/SemaHLSL/BuiltIns/fmod-errors.hlsl
+++ b/clang/test/SemaHLSL/BuiltIns/fmod-errors.hlsl
@@ -3,8 +3,8 @@
 float test_no_second_arg(float2 p0) {
   return fmod(p0);
   // expected-error at -1 {{no matching function for call to 'fmod'}}
-  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 1 was provided}}
-  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 1 was provided}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
   // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
   // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 1 was provided}}
 }
@@ -12,22 +12,46 @@ float test_no_second_arg(float2 p0) {
 float test_too_many_arg(float2 p0) {
   return fmod(p0, p0, p0);
   // expected-error at -1 {{no matching function for call to 'fmod'}}
-  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 3 were provided}}
-  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function not viable: requires 2 arguments, but 3 were provided}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
   // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
   // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function template not viable: requires 2 arguments, but 3 were provided}}
 }
 
 float test_double_inputs(double p0, double p1) {
   return fmod(p0, p1);
-  // expected-error at -1  {{call to 'fmod' is ambiguous}}
-  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error at -1  {{no matching function for call to 'fmod'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
 }
 
 float test_int_inputs(int p0, int p1) {
   return fmod(p0, p1);
-  // expected-error at -1  {{call to 'fmod' is ambiguous}}
-  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function}}
-  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate function}}
+  // expected-error at -1  {{no matching function for call to 'fmod'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored}}
+}
+
+float1 test_vec1_inputs(float1 p0, float1 p1) {
+  return fmod(p0, p1);
+  // expected-error at -1  {{no matching function for call to 'fmod'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 1>>'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 1>>'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, half>'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 1]: no type named 'Type' in 'hlsl::__detail::enable_if<false, float>'}}
+}
+
+typedef float float5 __attribute__((ext_vector_type(5)));
+
+float5 test_vec5_inputs(float5 p0, float5 p1) {
+  return fmod(p0, p1);
+  // expected-error at -1  {{no matching function for call to 'fmod'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 5>>'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with T = float5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, vector<float, 5>>'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, half>'}}
+  // expected-note at hlsl/hlsl_intrinsics.h:* {{candidate template ignored: substitution failure [with N = 5]: no type named 'Type' in 'hlsl::__detail::enable_if<false, float>'}}
 }