[clang] [HLSL] fix D3DCOLORtoUBYTE4 return type to be int (PR #151353)

[Farzon Lotfi via cfe-commits]

https://github.com/farzonl created https://github.com/llvm/llvm-project/pull/151353

fixes #150673
fixes #150678

The issue was we were using the wrong return type.

>From f89ea77874da5f5964a857f6871476058a9bc673 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi at microsoft.com>
Date: Wed, 30 Jul 2025 11:32:38 -0400
Subject: [PATCH] [HLSL] fix D3DCOLORtoUBYTE4 return type to be int

fixes #150673
fixes #150678

The issue was we were using the wrong return type.
---
 clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h       | 2 +-
 clang/lib/Headers/hlsl/hlsl_intrinsics.h              | 2 +-
 clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl | 8 ++++++++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
index e8ccccb489815..61919d9a17c6b 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
@@ -12,7 +12,7 @@
 namespace hlsl {
 namespace __detail {
 
-constexpr vector<uint, 4> d3d_color_to_ubyte4_impl(vector<float, 4> V) {
+constexpr vector<int, 4> d3d_color_to_ubyte4_impl(vector<float, 4> V) {
   // Use the same scaling factor used by FXC, and DXC for DXIL
   // (i.e., 255.001953)
   // https://github.com/microsoft/DirectXShaderCompiler/blob/070d0d5a2beacef9eeb51037a9b04665716fd6f3/lib/HLSL/HLOperationLower.cpp#L666C1-L697C2
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 499a05328ee4f..d9d87c827e6a4 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -418,7 +418,7 @@ const inline float4 lit(float NDotL, float NDotH, float M) {
 /// This function swizzles and scales components of the \a x parameter. Use this
 /// function to compensate for the lack of UBYTE4 support in some hardware.
 
-constexpr vector<uint, 4> D3DCOLORtoUBYTE4(vector<float, 4> V) {
+constexpr int4 D3DCOLORtoUBYTE4(float4 V) {
   return __detail::d3d_color_to_ubyte4_impl(V);
 }
 
diff --git a/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl b/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
index 990f0aa910f30..b78a007b67117 100644
--- a/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
@@ -10,3 +10,11 @@ int4 test_D3DCOLORtoUBYTE4(float4 p1) {
   // CHECK: ret [[INT_TYPE]] %[[SHUFFLED]]
   return D3DCOLORtoUBYTE4(p1);
 }
+
+// Note this test confirms issue 150673 is fixed 
+// by confirming the negative does not become a poison
+// CHECK-LABEL: test_constant_inputs
+int4 test_constant_inputs() {
+  // CHECK: ret <4 x i32> <i32 -12877, i32 2833, i32 0, i32 25500>
+  return D3DCOLORtoUBYTE4(float4(0, 11.11, -50.5, 100));
+}