[llvm] [WebAssembly] Fall back to SelectionDAG for vector compares in FastISel (PR #227368)

[Patrick Rachford via llvm-commits]

https://github.com/rachfop updated https://github.com/llvm/llvm-project/pull/227368

>From 8deb64fa14ca4a486e07c63daa8ceede13ae91f3 Mon Sep 17 00:00:00 2001
From: rachfop <prachford at icloud.com>
Date: Tue, 29 Sep 2026 09:31:07 -0700
Subject: [PATCH] [WebAssembly] Fall back to SelectionDAG for vector compares
 in FastISel

FastISel's selectICmp and selectFCmp each classify compare operands
with a single scalar test: anything but i64 is treated as i32, and
anything but f64 as f32, vectors included. A vector compare therefore
emits a scalar i32.eq/i32.ne (or f32.eq) that consumes the two v128
registers holding the operands, and the resulting module fails
validation ("type mismatch: expected i32, found v128").

Return false for vector operands in both selectors so the SelectionDAG
lowers them to SIMD compares.

The vector compare only reaches this path when the rest of the block
does not bail out, which is why it went unnoticed: the common shapes
(a vector icmp feeding extractelement, a bitcast, an intrinsic, or a
branch condition) all miss somewhere in FastISel and the SelectionDAG
revisits the block, replacing the bad compare. A vector compare
feeding an ordinary call is the shape that survives, because FastISel
lowers calls fine and nothing misses.

Found compiling wasm64 programs, where vector compares stay live
through call boundaries; the module was rejected by wasmtime.
---
 .../WebAssembly/WebAssemblyFastISel.cpp       | 14 ++++++
 .../CodeGen/WebAssembly/fast-isel-simd128.ll  | 45 +++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index db03f5874234c..eb630ee3af5b7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -1173,6 +1173,13 @@ bool WebAssemblyFastISel::selectSExt(const Instruction *I) {
 bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
   const auto *ICmp = cast<ICmpInst>(I);
 
+  // The I32 test below classifies every non-i64 type as i32, so a vector
+  // compare would emit a scalar compare over v128 registers and produce
+  // an invalid module. The SelectionDAG lowers vector compares to SIMD
+  // compares.
+  if (ICmp->getOperand(0)->getType()->isVectorTy())
+    return false;
+
   bool I32 = getSimpleType(ICmp->getOperand(0)->getType()) != MVT::i64;
   unsigned Opc;
   bool IsSigned = false;
@@ -1234,6 +1241,13 @@ bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
 bool WebAssemblyFastISel::selectFCmp(const Instruction *I) {
   const auto *FCmp = cast<FCmpInst>(I);
 
+  // The F32 test below classifies every non-f64 type as f32, so a vector
+  // compare would emit a scalar compare over v128 registers and produce
+  // an invalid module. The SelectionDAG lowers vector compares to SIMD
+  // compares.
+  if (FCmp->getOperand(0)->getType()->isVectorTy())
+    return false;
+
   Register LHS = getRegForValue(FCmp->getOperand(0));
   if (LHS == 0)
     return false;
diff --git a/llvm/test/CodeGen/WebAssembly/fast-isel-simd128.ll b/llvm/test/CodeGen/WebAssembly/fast-isel-simd128.ll
index df14e1054d91b..3f5c911e2afd4 100644
--- a/llvm/test/CodeGen/WebAssembly/fast-isel-simd128.ll
+++ b/llvm/test/CodeGen/WebAssembly/fast-isel-simd128.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
 ; RUN: llc < %s -fast-isel -fast-isel-abort=0 -mattr=+simd128 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=wasm64 -fast-isel -fast-isel-abort=0 -mattr=+simd128 -verify-machineinstrs | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
@@ -21,3 +22,47 @@ cond.true:                                        ; preds = %entry
   %vecext = extractelement <4 x i8> %conv, i32 0
   ret i8 %vecext
 }
+
+; selectICmp classifies any non-i64 operand as i32 and selectFCmp any
+; non-f64 operand as f32, so without the vector bail-out a vector
+; compare takes the FastISel path and emits a scalar i32.eq/f32.eq over
+; v128 registers, failing module validation. FastISel has no vector
+; compare lowering of its own, so a valid SIMD compare can only come
+; from the SelectionDAG fallback; the wasm64 RUN covers memory64
+; addressing. The scalar eq_i32 guards that the bail-out is vector-only.
+
+declare i1 @any16(<16 x i1>)
+declare i1 @any4(<4 x i1>)
+
+; CHECK-LABEL: eq_i8x16:
+define i32 @eq_i8x16(ptr %pa, ptr %pb) {
+entry:
+  %a = load <16 x i8>, ptr %pa
+  %b = load <16 x i8>, ptr %pb
+  %c = icmp ne <16 x i8> %a, %b
+  %r = call i1 @any16(<16 x i1> %c)
+  %z = zext i1 %r to i32
+  ret i32 %z
+}
+; CHECK: i8x16.ne{{ *$}}
+
+; CHECK-LABEL: eq_f32x4:
+define i32 @eq_f32x4(ptr %pa, ptr %pb) {
+entry:
+  %a = load <4 x float>, ptr %pa
+  %b = load <4 x float>, ptr %pb
+  %c = fcmp oeq <4 x float> %a, %b
+  %r = call i1 @any4(<4 x i1> %c)
+  %z = zext i1 %r to i32
+  ret i32 %z
+}
+; CHECK: f32x4.eq{{ *$}}
+
+; CHECK-LABEL: eq_i32:
+define i32 @eq_i32(i32 %a, i32 %b) {
+entry:
+  %c = icmp eq i32 %a, %b
+  %z = zext i1 %c to i32
+  ret i32 %z
+}
+; CHECK: i32.eq{{ *$}}