[clang] [HLSL] fix D3DCOLORtoUBYTE4 return type to be int (PR #151353)

[Farzon Lotfi via cfe-commits]

https://github.com/farzonl updated https://github.com/llvm/llvm-project/pull/151353

>From b5f34ee30c375fb90fb2d4d1e4cdbb417a30812b Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi at microsoft.com>
Date: Wed, 30 Jul 2025 11:32:38 -0400
Subject: [PATCH 1/3] [HLSL] fix D3DCOLORtoUBYTE4 return type to be int

fixes #150673
fixes #150678

The issue was we were using the wrong return type.
---
 clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h       | 2 +-
 clang/lib/Headers/hlsl/hlsl_intrinsics.h              | 2 +-
 clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl | 8 ++++++++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
index e8ccccb489815..61919d9a17c6b 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
@@ -12,7 +12,7 @@
 namespace hlsl {
 namespace __detail {
 
-constexpr vector<uint, 4> d3d_color_to_ubyte4_impl(vector<float, 4> V) {
+constexpr vector<int, 4> d3d_color_to_ubyte4_impl(vector<float, 4> V) {
   // Use the same scaling factor used by FXC, and DXC for DXIL
   // (i.e., 255.001953)
   // https://github.com/microsoft/DirectXShaderCompiler/blob/070d0d5a2beacef9eeb51037a9b04665716fd6f3/lib/HLSL/HLOperationLower.cpp#L666C1-L697C2
diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
index 499a05328ee4f..d9d87c827e6a4 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsics.h
@@ -418,7 +418,7 @@ const inline float4 lit(float NDotL, float NDotH, float M) {
 /// This function swizzles and scales components of the \a x parameter. Use this
 /// function to compensate for the lack of UBYTE4 support in some hardware.
 
-constexpr vector<uint, 4> D3DCOLORtoUBYTE4(vector<float, 4> V) {
+constexpr int4 D3DCOLORtoUBYTE4(float4 V) {
   return __detail::d3d_color_to_ubyte4_impl(V);
 }
 
diff --git a/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl b/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
index 990f0aa910f30..b78a007b67117 100644
--- a/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
@@ -10,3 +10,11 @@ int4 test_D3DCOLORtoUBYTE4(float4 p1) {
   // CHECK: ret [[INT_TYPE]] %[[SHUFFLED]]
   return D3DCOLORtoUBYTE4(p1);
 }
+
+// Note this test confirms issue 150673 is fixed 
+// by confirming the negative does not become a poison
+// CHECK-LABEL: test_constant_inputs
+int4 test_constant_inputs() {
+  // CHECK: ret <4 x i32> <i32 -12877, i32 2833, i32 0, i32 25500>
+  return D3DCOLORtoUBYTE4(float4(0, 11.11, -50.5, 100));
+}

>From 06a751a14a75c4a23d2e81f0ea1065352b33cbdf Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi at microsoft.com>
Date: Thu, 31 Jul 2025 10:13:19 -0400
Subject: [PATCH 2/3] fixup syntactic sugar

---
 clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
index 61919d9a17c6b..c877234479ad1 100644
--- a/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
+++ b/clang/lib/Headers/hlsl/hlsl_intrinsic_helpers.h
@@ -12,7 +12,7 @@
 namespace hlsl {
 namespace __detail {
 
-constexpr vector<int, 4> d3d_color_to_ubyte4_impl(vector<float, 4> V) {
+constexpr int4 d3d_color_to_ubyte4_impl(float4 V) {
   // Use the same scaling factor used by FXC, and DXC for DXIL
   // (i.e., 255.001953)
   // https://github.com/microsoft/DirectXShaderCompiler/blob/070d0d5a2beacef9eeb51037a9b04665716fd6f3/lib/HLSL/HLOperationLower.cpp#L666C1-L697C2

>From 8566e2cceef88b0bc5ee12c204eb051d54be88e6 Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi at microsoft.com>
Date: Thu, 31 Jul 2025 11:37:17 -0400
Subject: [PATCH 3/3] fix test case

---
 clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl b/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
index b78a007b67117..3c9e35a2543a7 100644
--- a/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/D3DCOLORtoUBYTE4.hlsl
@@ -5,7 +5,7 @@
 // CHECK-LABEL: D3DCOLORtoUBYTE4
 int4 test_D3DCOLORtoUBYTE4(float4 p1) {
   // CHECK: %[[SCALED:.*]] = fmul [[FMFLAGS:.*]][[FLOAT_TYPE:<4 x float>]] %{{.*}}, splat (float 0x406FE01000000000)
-  // CHECK: %[[CONVERTED:.*]] = fptoui [[FLOAT_TYPE]] %[[SCALED]] to [[INT_TYPE:<4 x i32>]]
+  // CHECK: %[[CONVERTED:.*]] = fptosi [[FLOAT_TYPE]] %[[SCALED]] to [[INT_TYPE:<4 x i32>]]
   // CHECK: %[[SHUFFLED:.*]] = shufflevector [[INT_TYPE]] %[[CONVERTED]], [[INT_TYPE]] poison, <4 x i32> <i32 2, i32 1, i32 0, i32 3>
   // CHECK: ret [[INT_TYPE]] %[[SHUFFLED]]
   return D3DCOLORtoUBYTE4(p1);