[llvm] [X86] Don't form pmaddwd for shift by 15 (PR #206473)

[Nikita Popov via llvm-commits]

https://github.com/nikic created https://github.com/llvm/llvm-project/pull/206473

I believe the transform is not correct for shifts by 15, because the multiplier gets sign extended, turning a multiply by 2^15 into one by -2^15.

>From 980caf15264da6106296f0380204efd98964d4d6 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Mon, 29 Jun 2026 14:34:04 +0200
Subject: [PATCH 1/2] Add test for miscompile

---
 llvm/test/CodeGen/X86/madd.ll | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 4ca0b05dbe92b..032f086213cb6 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -3882,3 +3882,21 @@ define <3 x i32> @oddvector_shl(<12 x i16> %A) {
    %ret = add <3 x i32> %odd, %even
    ret <3 x i32> %ret
 }
+
+define <4 x i32> @shl15(<8 x i16> %x) {
+; SSE-LABEL: shl15:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: shl15:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32768,32768,32768,32768,32768,32768,32768,32768]
+; AVX-NEXT:    retq
+  %s = sext <8 x i16> %x to <8 x i32>
+  %m = shl <8 x i32> %s, splat (i32 15)
+  %even = shufflevector <8 x i32> %m, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %odd = shufflevector <8 x i32> %m, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %r = add <4 x i32> %even, %odd
+  ret <4 x i32> %r
+}

>From 8a00d3ffb6ba4f74c9b8076b5e37146d2dc35b26 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Mon, 29 Jun 2026 14:36:45 +0200
Subject: [PATCH 2/2] Fix

---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  4 +-
 llvm/test/CodeGen/X86/madd.ll           | 52 +++++++++++++++++++++----
 2 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b95ac78f50049..5575b68ed7c72 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59681,9 +59681,9 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDNode *N,
     if (N0.getValueType().getScalarType() != MVT::i16)
       return SDValue();
 
-    // A shift by more than 15 would overflow an i16.
+    // A shift by more 15 or more would overflow a signed i16.
     if (!ISD::matchUnaryPredicate(Mul.getOperand(1), [](ConstantSDNode *C) {
-          return C->getAPIntValue().ule(15);
+          return C->getAPIntValue().ult(15);
         }))
       return SDValue();
 
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 032f086213cb6..4ee77c0bdf70c 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -3883,16 +3883,52 @@ define <3 x i32> @oddvector_shl(<12 x i16> %A) {
    ret <3 x i32> %ret
 }
 
+; Don't form pmaddwd for shift by 15, because this would get interpreted as
+; a multiply by -2^15 rather than +2^15.
 define <4 x i32> @shl15(<8 x i16> %x) {
-; SSE-LABEL: shl15:
-; SSE:       # %bb.0:
-; SSE-NEXT:    pmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE-NEXT:    retq
+; SSE2-LABEL: shl15:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psrad $16, %xmm0
+; SSE2-NEXT:    pslld $15, %xmm0
+; SSE2-NEXT:    pslld $15, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm0[0,2]
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,3],xmm0[1,3]
+; SSE2-NEXT:    paddd %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
 ;
-; AVX-LABEL: shl15:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX-NEXT:    retq
+; SSE42-LABEL: shl15:
+; SSE42:       # %bb.0:
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT:    pmovsxwd %xmm1, %xmm1
+; SSE42-NEXT:    pmovsxwd %xmm0, %xmm0
+; SSE42-NEXT:    pslld $15, %xmm0
+; SSE42-NEXT:    pslld $15, %xmm1
+; SSE42-NEXT:    phaddd %xmm1, %xmm0
+; SSE42-NEXT:    retq
+;
+; AVX1-LABEL: shl15:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $15, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $15, %xmm1, %xmm1
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX256-LABEL: shl15:
+; AVX256:       # %bb.0:
+; AVX256-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX256-NEXT:    vpslld $15, %ymm0, %ymm0
+; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX256-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX256-NEXT:    vzeroupper
+; AVX256-NEXT:    retq
   %s = sext <8 x i16> %x to <8 x i32>
   %m = shl <8 x i32> %s, splat (i32 15)
   %even = shufflevector <8 x i32> %m, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>