[llvm] [WebAssembly] Fall back to SelectionDAG for vector compares in FastISel (PR #227368)

[Patrick Rachford via llvm-commits]

https://github.com/rachfop updated https://github.com/llvm/llvm-project/pull/227368

>From d870bad37a6552145b867ccebe40d74ecd46643e Mon Sep 17 00:00:00 2001
From: rachfop <prachford at icloud.com>
Date: Tue, 29 Sep 2026 09:31:07 -0700
Subject: [PATCH] [WebAssembly] Fall back to SelectionDAG for vector compares
 in FastISel

FastISel's selectICmp and selectFCmp each classify compare operands
with a single scalar test: anything but i64 is treated as i32, and
anything but f64 as f32, vectors included. A vector compare therefore
emits a scalar i32.eq/i32.ne (or f32.eq) that consumes the two v128
registers holding the operands, and the resulting module fails
validation ("type mismatch: expected i32, found v128").

Return false for vector operands in both selectors so the SelectionDAG
lowers them to SIMD compares.

The vector compare only reaches this path when the rest of the block
does not bail out, which is why it went unnoticed: the common shapes
(a vector icmp feeding extractelement, a bitcast, an intrinsic, or a
branch condition) all miss somewhere in FastISel and the SelectionDAG
revisits the block, replacing the bad compare. A vector compare
feeding an ordinary call is the shape that survives, because FastISel
lowers calls fine and nothing misses.

Detected compiling Mojo programs for wasm64, where vector compares
stay live through call boundaries; the module was rejected by
wasmtime.
---
 .../WebAssembly/WebAssemblyFastISel.cpp       | 11 +++++
 .../WebAssembly/fast-isel-vector-cmp.ll       | 44 +++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 llvm/test/CodeGen/WebAssembly/fast-isel-vector-cmp.ll

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index db03f5874234c..c0c5dad3d3139 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -1173,6 +1173,13 @@ bool WebAssemblyFastISel::selectSExt(const Instruction *I) {
 bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
   const auto *ICmp = cast<ICmpInst>(I);
 
+  // The I32 test below classifies every non-i64 type as i32, so a vector
+  // compare would emit a scalar compare over v128 registers and produce
+  // an invalid module. The SelectionDAG lowers vector compares to SIMD
+  // compares.
+  if (ICmp->getOperand(0)->getType()->isVectorTy())
+    return false;
+
   bool I32 = getSimpleType(ICmp->getOperand(0)->getType()) != MVT::i64;
   unsigned Opc;
   bool IsSigned = false;
@@ -1234,6 +1241,10 @@ bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
 bool WebAssemblyFastISel::selectFCmp(const Instruction *I) {
   const auto *FCmp = cast<FCmpInst>(I);
 
+  // Same as selectICmp.
+  if (FCmp->getOperand(0)->getType()->isVectorTy())
+    return false;
+
   Register LHS = getRegForValue(FCmp->getOperand(0));
   if (LHS == 0)
     return false;
diff --git a/llvm/test/CodeGen/WebAssembly/fast-isel-vector-cmp.ll b/llvm/test/CodeGen/WebAssembly/fast-isel-vector-cmp.ll
new file mode 100644
index 0000000000000..0602bfc1984f0
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/fast-isel-vector-cmp.ll
@@ -0,0 +1,44 @@
+; RUN: llc %s -mtriple=wasm32 -O0 -fast-isel -mattr=+simd128 -verify-machineinstrs -o - | FileCheck %s
+; RUN: llc %s -mtriple=wasm64 -O0 -fast-isel -mattr=+simd128 -verify-machineinstrs -o - | FileCheck %s
+; RUN: llc %s -mtriple=wasm32 -O0 -fast-isel=false -mattr=+simd128 -verify-machineinstrs -o - | FileCheck %s
+
+; selectICmp classifies any non-i64 operand as i32 and selectFCmp any
+; non-f64 operand as f32, so a vector compare takes the FastISel path
+; and emits a scalar i32.eq/f32.eq over v128 registers, failing module
+; validation. The -fast-isel=false RUN exercises the SelectionDAG path.
+
+declare i1 @any16(<16 x i1>)
+declare i1 @any4(<4 x i1>)
+
+; CHECK-LABEL: eq_i8x16:
+define i32 @eq_i8x16(ptr %pa, ptr %pb) {
+entry:
+  %a = load <16 x i8>, ptr %pa
+  %b = load <16 x i8>, ptr %pb
+  %c = icmp ne <16 x i8> %a, %b
+  %r = call i1 @any16(<16 x i1> %c)
+  %z = zext i1 %r to i32
+  ret i32 %z
+}
+; CHECK: i8x16.ne{{ *$}}
+
+; CHECK-LABEL: eq_f32x4:
+define i32 @eq_f32x4(ptr %pa, ptr %pb) {
+entry:
+  %a = load <4 x float>, ptr %pa
+  %b = load <4 x float>, ptr %pb
+  %c = fcmp oeq <4 x float> %a, %b
+  %r = call i1 @any4(<4 x i1> %c)
+  %z = zext i1 %r to i32
+  ret i32 %z
+}
+; CHECK: f32x4.eq{{ *$}}
+
+; CHECK-LABEL: eq_i32:
+define i32 @eq_i32(i32 %a, i32 %b) {
+entry:
+  %c = icmp eq i32 %a, %b
+  %z = zext i1 %c to i32
+  ret i32 %z
+}
+; CHECK: i32.eq{{ *$}}