[llvm] [SelectionDAG] Fix assertion failure on inline asm register type mismatch (PR #166615)

[Ahmed Nour via llvm-commits]

https://github.com/ahmednoursphinx updated https://github.com/llvm/llvm-project/pull/166615

>From 684e005c76d7db5500fb0925547b58327c993d84 Mon Sep 17 00:00:00 2001
From: ahmed <ahmednour.mohamed2012 at gmail.com>
Date: Wed, 5 Nov 2025 20:52:13 +0200
Subject: [PATCH 1/2] [SelectionDAG] Fix assertion failure on inline asm
 register type mismatch

---
 .../SelectionDAG/SelectionDAGBuilder.cpp      | 12 ++++++++++-
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  2 ++
 .../inline-asm-y-constraint-size-mismatch.ll  | 20 +++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/X86/inline-asm-y-constraint-size-mismatch.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index fa0c899dfcc27..9a6a76f73f8fe 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -365,7 +365,17 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
 
     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
     NumParts = NumRegs; // Silence a compiler warning.
-    assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
+    
+    // Check if the register type matches the part type
+    if (RegisterVT != PartVT) {
+      diagnosePossiblyInvalidConstraint(
+          *DAG.getContext(), V,
+          "register type (" + EVT(RegisterVT).getEVTString() +
+              ") doesn't match operand type (" + EVT(PartVT).getEVTString() +
+              ")");
+      return DAG.getUNDEF(ValueVT);
+    }
+    
     assert(RegisterVT.getSizeInBits() ==
            Parts[0].getSimpleValueType().getSizeInBits() &&
            "Part type sizes don't match!");
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6edf0185df813..bf432f1d56d4a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -61825,6 +61825,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
       break;
     case 'y':   // MMX_REGS if MMX allowed.
       if (!Subtarget.hasMMX()) break;
+      // MMX registers are 64-bit only
+      if (VT.getSizeInBits() != 64) break;
       return std::make_pair(0U, &X86::VR64RegClass);
     case 'v':
     case 'x':   // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
diff --git a/llvm/test/CodeGen/X86/inline-asm-y-constraint-size-mismatch.ll b/llvm/test/CodeGen/X86/inline-asm-y-constraint-size-mismatch.ll
new file mode 100644
index 0000000000000..9f4ed28a4d312
--- /dev/null
+++ b/llvm/test/CodeGen/X86/inline-asm-y-constraint-size-mismatch.ll
@@ -0,0 +1,20 @@
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
+
+; Test that using MMX register constraint 'y' (64-bit) with a 256-bit vector
+; produces a proper error message instead of an assertion failure.
+
+; CHECK: error: couldn't allocate output register for constraint 'y'
+
+define <8 x i32> @test_mmx_constraint_size_mismatch() {
+entry:
+  %out = tail call <8 x i32> asm "something $0", "=y"()
+  ret <8 x i32> %out
+}
+
+; Also test with a different vector size
+define <4 x i32> @test_mmx_constraint_128bit() {
+entry:
+  %out = tail call <4 x i32> asm "something $0", "=y"()
+  ret <4 x i32> %out
+}
+

>From 36a7461181238819e0775958493cff4305e235de Mon Sep 17 00:00:00 2001
From: ahmed <ahmednour.mohamed2012 at gmail.com>
Date: Wed, 5 Nov 2025 20:54:34 +0200
Subject: [PATCH 2/2] chore: Fix formatting

---
 .../CodeGen/SelectionDAG/SelectionDAGBuilder.cpp   | 14 +++++++-------
 llvm/lib/Target/X86/X86ISelLowering.cpp            |  3 ++-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 9a6a76f73f8fe..ecdc3750e6e02 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -365,17 +365,17 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
 
     assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
     NumParts = NumRegs; // Silence a compiler warning.
-    
+
     // Check if the register type matches the part type
     if (RegisterVT != PartVT) {
-      diagnosePossiblyInvalidConstraint(
-          *DAG.getContext(), V,
-          "register type (" + EVT(RegisterVT).getEVTString() +
-              ") doesn't match operand type (" + EVT(PartVT).getEVTString() +
-              ")");
+      diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
+                                        "register type (" +
+                                            EVT(RegisterVT).getEVTString() +
+                                            ") doesn't match operand type (" +
+                                            EVT(PartVT).getEVTString() + ")");
       return DAG.getUNDEF(ValueVT);
     }
-    
+
     assert(RegisterVT.getSizeInBits() ==
            Parts[0].getSimpleValueType().getSizeInBits() &&
            "Part type sizes don't match!");
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index bf432f1d56d4a..2a02f12b18181 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -61826,7 +61826,8 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     case 'y':   // MMX_REGS if MMX allowed.
       if (!Subtarget.hasMMX()) break;
       // MMX registers are 64-bit only
-      if (VT.getSizeInBits() != 64) break;
+      if (VT.getSizeInBits() != 64)
+        break;
       return std::make_pair(0U, &X86::VR64RegClass);
     case 'v':
     case 'x':   // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed