[llvm] [SelectionDAG] Fix assertion failure on inline asm register type mismatch (PR #166615)

[Ahmed Nour via llvm-commits]

https://github.com/ahmednoursphinx created https://github.com/llvm/llvm-project/pull/166615

Resolves https://github.com/llvm/llvm-project/issues/166057

>From 684e005c76d7db5500fb0925547b58327c993d84 Mon Sep 17 00:00:00 2001
From: ahmed <ahmednour.mohamed2012 at gmail.com>
Date: Wed, 5 Nov 2025 20:52:13 +0200
Subject: [PATCH] [SelectionDAG] Fix assertion failure on inline asm register
 type mismatch

---
 .../SelectionDAG/SelectionDAGBuilder.cpp      | 12 ++++++++++-
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  2 ++
 .../inline-asm-y-constraint-size-mismatch.ll  | 20 +++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/X86/inline-asm-y-constraint-size-mismatch.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index fa0c899dfcc27..9a6a76f73f8fe 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -365,7 +365,17 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
 
     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
     NumParts = NumRegs; // Silence a compiler warning.
-    assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
+    
+    // Check if the register type matches the part type
+    if (RegisterVT != PartVT) {
+      diagnosePossiblyInvalidConstraint(
+          *DAG.getContext(), V,
+          "register type (" + EVT(RegisterVT).getEVTString() +
+              ") doesn't match operand type (" + EVT(PartVT).getEVTString() +
+              ")");
+      return DAG.getUNDEF(ValueVT);
+    }
+    
     assert(RegisterVT.getSizeInBits() ==
            Parts[0].getSimpleValueType().getSizeInBits() &&
            "Part type sizes don't match!");
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6edf0185df813..bf432f1d56d4a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -61825,6 +61825,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       break;
     case 'y':   // MMX_REGS if MMX allowed.
       if (!Subtarget.hasMMX()) break;
+      // MMX registers are 64-bit only
+      if (VT.getSizeInBits() != 64) break;
       return std::make_pair(0U, &X86::VR64RegClass);
     case 'v':
     case 'x':   // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
diff --git a/llvm/test/CodeGen/X86/inline-asm-y-constraint-size-mismatch.ll b/llvm/test/CodeGen/X86/inline-asm-y-constraint-size-mismatch.ll
new file mode 100644
index 0000000000000..9f4ed28a4d312
--- /dev/null
+++ b/llvm/test/CodeGen/X86/inline-asm-y-constraint-size-mismatch.ll
@@ -0,0 +1,20 @@
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
+
+; Test that using MMX register constraint 'y' (64-bit) with a 256-bit vector
+; produces a proper error message instead of an assertion failure.
+
+; CHECK: error: couldn't allocate output register for constraint 'y'
+
+define <8 x i32> @test_mmx_constraint_size_mismatch() {
+entry:
+  %out = tail call <8 x i32> asm "something $0", "=y"()
+  ret <8 x i32> %out
+}
+
+; Also test with a different vector size
+define <4 x i32> @test_mmx_constraint_128bit() {
+entry:
+  %out = tail call <4 x i32> asm "something $0", "=y"()
+  ret <4 x i32> %out
+}
+