[llvm] Fix EXTEND_VECTOR_INREG widening when input > result size (PR #177095)

[via llvm-commits]

https://github.com/nataliakokoromyti updated https://github.com/llvm/llvm-project/pull/177095

>From 30f5b7e2f9a811c2f7b29dae55d85e13c5817e83 Mon Sep 17 00:00:00 2001
From: Natalia Kokoromyti <knatalia at yost-cm-01-imme.stanford.edu>
Date: Tue, 20 Jan 2026 21:44:16 -0800
Subject: [PATCH 1/2] Fix EXTEND_VECTOR_INREG widening when input > result size

---
 .../SelectionDAG/LegalizeVectorTypes.cpp      | 29 +++++++++++++++++--
 .../AMDGPU/extend-vector-inreg-nonpow2.ll     | 23 +++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/extend-vector-inreg-nonpow2.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 951120e0fe073..b05e29a8fb96c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -7702,10 +7702,35 @@ SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
-  SDValue InOp = GetWidenedVector(N->getOperand(0));
-  return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), InOp);
+  SDLoc DL(N);
+  EVT ResVT = N->getValueType(0);
+
+  // Widen the input as requested by the legalizer.
+  SDValue WideInOp = GetWidenedVector(N->getOperand(0));
+  EVT WideInVT = WideInOp.getValueType();
+
+  // EXTEND_VECTOR_INREG requires input bits <= result bits.
+  // If widening makes the input larger than the original result, widen the
+  // result to match, then extract back down.
+  if (WideInVT.getSizeInBits() > ResVT.getSizeInBits()) {
+    assert(ResVT.isVector() && "Expected vector result for EXTEND_VECTOR_INREG");
+    EVT ResEltVT = ResVT.getVectorElementType();
+    unsigned EltBits = ResEltVT.getSizeInBits();
+    assert(EltBits && (WideInVT.getSizeInBits() % EltBits) == 0 &&
+           "Widened input size must be a multiple of result element size");
+
+    unsigned WideNumElts = WideInVT.getSizeInBits() / EltBits;
+    EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), ResEltVT, WideNumElts);
+
+    SDValue WideRes = DAG.getNode(N->getOpcode(), DL, WideResVT, WideInOp);
+    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, WideRes,
+                       DAG.getVectorIdxConstant(0, DL));
+  }
+
+  return DAG.getNode(N->getOpcode(), DL, ResVT, WideInOp);
 }
 
+
 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
   // We have to widen the value, but we want only to store the original
   // vector type.
diff --git a/llvm/test/CodeGen/AMDGPU/extend-vector-inreg-nonpow2.ll b/llvm/test/CodeGen/AMDGPU/extend-vector-inreg-nonpow2.ll
new file mode 100644
index 0000000000000..4f2875dfb0c28
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/extend-vector-inreg-nonpow2.ll
@@ -0,0 +1,23 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o /dev/null %s
+;
+; Regression test for https://github.com/llvm/llvm-project/issues/176966
+; Ensures we don't crash in DAG type legalization when widening
+; EXTEND_VECTOR_INREG where the widened input becomes larger than the result.
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgcn-amd-amdhsa"
+
+define <32 x i8> @backsmith_pure_2() {
+entry:
+  %0 = load <6 x i8>, ptr addrspace(5) null, align 8
+  %shuffle5 = shufflevector <6 x i8> %0, <6 x i8> zeroinitializer,
+              <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison,
+                          i32 poison, i32 poison, i32 poison, i32 poison,
+                          i32 poison, i32 poison, i32 poison, i32 poison,
+                          i32 poison, i32 poison, i32 poison, i32 poison,
+                          i32 poison, i32 poison, i32 poison, i32 poison,
+                          i32 5, i32 poison, i32 poison, i32 poison,
+                          i32 poison, i32 poison, i32 poison, i32 poison,
+                          i32 poison, i32 poison, i32 poison, i32 poison>
+  ret <32 x i8> %shuffle5
+}

>From 75b677209bbcca2e7902171a9fa343cb49b6544a Mon Sep 17 00:00:00 2001
From: Natalia Kokoromyti <knatalia at yost-cm-01-imme.stanford.edu>
Date: Tue, 20 Jan 2026 22:11:30 -0800
Subject: [PATCH 2/2] fix clang-format

---
 llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index b05e29a8fb96c..7c99740134673 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -7713,7 +7713,8 @@ SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
   // If widening makes the input larger than the original result, widen the
   // result to match, then extract back down.
   if (WideInVT.getSizeInBits() > ResVT.getSizeInBits()) {
-    assert(ResVT.isVector() && "Expected vector result for EXTEND_VECTOR_INREG");
+    assert(ResVT.isVector() &&
+           "Expected vector result for EXTEND_VECTOR_INREG");
     EVT ResEltVT = ResVT.getVectorElementType();
     unsigned EltBits = ResEltVT.getSizeInBits();
     assert(EltBits && (WideInVT.getSizeInBits() % EltBits) == 0 &&
@@ -7730,7 +7731,6 @@ SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) {
   return DAG.getNode(N->getOpcode(), DL, ResVT, WideInOp);
 }
 
-
 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
   // We have to widen the value, but we want only to store the original
   // vector type.