[llvm] [DAG] SimplifySetCC - adjust setcc result type when peeking through truncates (PR #226422)

[Simon Pilgrim via llvm-commits]

https://github.com/RKSimon created https://github.com/llvm/llvm-project/pull/226422

For the "(setcc (trunc x) (trunc y)) -> (setcc x y)" fold with non vXi1 cmp result type, make sure we adjust the type and then trunc/extend it correctly using BooleanContent.

Fixes #226365

>From 3c1ba80b829eea8d74d20faea0330ef1c26d71df Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Fri, 25 Sep 2026 10:50:46 +0100
Subject: [PATCH] [DAG] SimplifySetCC - adjust setcc result type when peeking
 through truncates

For the "(setcc (trunc x) (trunc y)) -> (setcc x y)" fold with non vXi1 cmp result type, make sure we adjust the type and then trunc/extend it correctly using BooleanContent.

Fixes #226365
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 13 ++++-
 llvm/test/CodeGen/X86/vec_setcc-3.ll          | 53 +++++++++++++++++++
 2 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/X86/vec_setcc-3.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 7c97dfb2c806a1..db9a86aecfa8b6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -5882,7 +5882,18 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
        (!ISD::isUnsignedIntSetCC(Cond) && N0->getFlags().hasNoSignedWrap() &&
         N1->getFlags().hasNoSignedWrap())) &&
       isTypeDesirableForOp(ISD::SETCC, N0.getOperand(0).getValueType())) {
-    return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
+    if (VT.getScalarType() == MVT::i1)
+      return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
+    // For (legal) non vXi1 cases - ensure we adjust the cmp and result types.
+    EVT OldCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+                                     N0.getValueType());
+    if (VT == OldCCVT) {
+      EVT NewCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+                                       N0.getOperand(0).getValueType());
+      return DAG.getBoolExtOrTrunc(
+          DAG.getSetCC(dl, NewCCVT, N0.getOperand(0), N1.getOperand(0), Cond),
+          dl, VT, N0.getOperand(0).getValueType());
+    }
   }
 
   // Fold (setcc (sub nsw a, b), zero, s??) -> (setcc a, b, s??)
diff --git a/llvm/test/CodeGen/X86/vec_setcc-3.ll b/llvm/test/CodeGen/X86/vec_setcc-3.ll
new file mode 100644
index 00000000000000..b7b1dd6c465a87
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vec_setcc-3.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE42
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512
+
+; ensure that the comparison result type is adjusted when peeking through truncations
+define <8 x i1> @PR226365(<8 x i32> %a0, <8 x i32> %a1) {
+; SSE2-LABEL: PR226365:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslld $16, %xmm1
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    pslld $16, %xmm0
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-NEXT:    pslld $16, %xmm3
+; SSE2-NEXT:    psrad $16, %xmm3
+; SSE2-NEXT:    pslld $16, %xmm2
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-NEXT:    psubusw %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqw %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE42-LABEL: PR226365:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    packusdw %xmm1, %xmm0
+; SSE42-NEXT:    packusdw %xmm3, %xmm2
+; SSE42-NEXT:    pminuw %xmm0, %xmm2
+; SSE42-NEXT:    pcmpeqw %xmm2, %xmm0
+; SSE42-NEXT:    retq
+;
+; AVX2-LABEL: PR226365:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: PR226365:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpcmpleud %ymm1, %ymm0, %k0
+; AVX512-NEXT:    vpmovm2w %k0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %t0 = trunc nuw <8 x i32> %a0 to <8 x i16>
+  %t1 = trunc nuw <8 x i32> %a1 to <8 x i16>
+  %c = icmp ule <8 x i16> %t0, %t1
+  ret <8 x i1> %c
+}