[llvm] [RISCV] Fold vmand of mask comparisons into a masked comparison (PR #216264)
Pengcheng Wang via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 02:58:42 PDT 2026
https://github.com/wangpc-pp updated https://github.com/llvm/llvm-project/pull/216264
>From c0e83497f4fdb40ad869c91aceb34e1dd932406a Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 14 Aug 2026 11:46:31 +0800
Subject: [PATCH 1/5] [RISCV] Add tests for vmand of mask comparisons
Precommit tests for folding vmand(vmsxx(a, b), vmsxx(c, d)) into a
mask-undisturbed masked comparison. These currently show the unoptimized
vmslt + vmslt + vmand.mm sequence.
Assisted-by: TRAE CLI (Opus 4.8)
---
.../RISCV/rvv/vmand-to-masked-compare.ll | 58 ++++
.../RISCV/rvv/vmand-to-masked-compare.mir | 247 ++++++++++++++++++
2 files changed, 305 insertions(+)
create mode 100644 llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
create mode 100644 llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
new file mode 100644
index 0000000000000..4cd4940571fc9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh -verify-machineinstrs | FileCheck %s
+
+; Test that vmand(vmsxx(a, b), vmsxx(c, d)) is folded into a mask-undisturbed
+; masked comparison, saving a vmand.mm. See RISCVVectorPeephole.
+
+declare <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>, <vscale x 2 x i32>, i64)
+declare <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float>, <vscale x 2 x float>, i64)
+declare <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1>, <vscale x 2 x i1>, i64)
+
+; (a < b) && (b < c)
+define <vscale x 2 x i1> @and_icmp(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl) {
+; CHECK-LABEL: and_icmp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT: vmslt.vv v8, v8, v9
+; CHECK-NEXT: vmslt.vv v9, v9, v10
+; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: ret
+ %m1 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, i64 %vl)
+ %m2 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl)
+ %r = call <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1> %m1, <vscale x 2 x i1> %m2, i64 %vl)
+ ret <vscale x 2 x i1> %r
+}
+
+; Quiet floating-point comparisons can be folded too.
+define <vscale x 2 x i1> @and_fcmp(<vscale x 2 x float> %a, <vscale x 2 x float> %b, <vscale x 2 x float> %c, i64 %vl) {
+; CHECK-LABEL: and_fcmp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT: vmflt.vv v8, v8, v9
+; CHECK-NEXT: vmflt.vv v9, v9, v10
+; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: ret
+ %m1 = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float> %a, <vscale x 2 x float> %b, i64 %vl)
+ %m2 = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float> %b, <vscale x 2 x float> %c, i64 %vl)
+ %r = call <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1> %m1, <vscale x 2 x i1> %m2, i64 %vl)
+ ret <vscale x 2 x i1> %r
+}
+
+; (a < b) && (b < c) && (c < d) chains through the mask.
+define <vscale x 2 x i1> @and_icmp_chain(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, <vscale x 2 x i32> %d, i64 %vl) {
+; CHECK-LABEL: and_icmp_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT: vmslt.vv v12, v9, v10
+; CHECK-NEXT: vmslt.vv v8, v8, v9
+; CHECK-NEXT: vmslt.vv v9, v10, v11
+; CHECK-NEXT: vmand.mm v8, v8, v12
+; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: ret
+ %m1 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, i64 %vl)
+ %m2 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl)
+ %m3 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %c, <vscale x 2 x i32> %d, i64 %vl)
+ %and1 = call <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1> %m1, <vscale x 2 x i1> %m2, i64 %vl)
+ %and2 = call <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1> %and1, <vscale x 2 x i1> %m3, i64 %vl)
+ ret <vscale x 2 x i1> %and2
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
new file mode 100644
index 0000000000000..cdaf4832526b0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
@@ -0,0 +1,247 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc %s -o - -mtriple=riscv64 -mattr=+v -run-pass=riscv-vector-peephole -verify-machineinstrs | FileCheck %s
+
+# vmand(vmslt(a, b), vmslt(c, d)) can be folded to a mask-undisturbed masked
+# compare, where the first comparison provides both the mask and passthru:
+# vmslt.vv v0, va, vb; vmslt.vv v0, vc, vd, v0.t
+
+---
+name: fold_vv
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: fold_vv
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# The comparison can be either operand of the vmand since it is commutative.
+---
+name: fold_vv_commuted
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: fold_vv_commuted
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp2, killed %cmp1, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp2, killed %cmp1, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# A quiet floating-point compare (nofpexcept) can be folded.
+---
+name: fold_fp_nofpexcept
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: fold_fp_nofpexcept
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = nofpexcept PseudoVMFLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = nofpexcept PseudoVMFLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = nofpexcept PseudoVMFLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = nofpexcept PseudoVMFLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# A floating-point compare that may raise exceptions must not be folded, since
+# masking would skip the exceptions on the inactive elements.
+---
+name: no_fold_fp_may_raise
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: no_fold_fp_may_raise
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMFLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMFLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMFLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMFLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# For LMUL >= 2 the masked comparison's dest is earlyclobbered into vrnov0 and
+# cannot share v0 with the mask, so folding would introduce extra vmv1r.v moves.
+# Don't fold in that case.
+---
+name: no_fold_m2
+body: |
+ bb.0:
+ liveins: $x10, $v8m2, $v10m2, $v12m2
+ ; CHECK-LABEL: name: no_fold_m2
+ ; CHECK: liveins: $x10, $v8m2, $v10m2, $v12m2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vrm2 = COPY $v12m2
+ ; CHECK-NEXT: %b:vrm2 = COPY $v10m2
+ ; CHECK-NEXT: %a:vrm2 = COPY $v8m2
+ ; CHECK-NEXT: early-clobber %cmp1:vr = PseudoVMSLT_VV_M2 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: early-clobber %cmp2:vr = PseudoVMSLT_VV_M2 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B16 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vrm2 = COPY $v12m2
+ %b:vrm2 = COPY $v10m2
+ %a:vrm2 = COPY $v8m2
+ %cmp1:vr = PseudoVMSLT_VV_M2 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M2 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B16 killed %cmp1, killed %cmp2, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# If both comparisons have multiple uses then neither can be consumed by the
+# fold without leaving an unmasked comparison behind, so don't fold.
+---
+name: no_fold_multiple_uses_of_compare
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: no_fold_multiple_uses_of_compare
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 %cmp1, %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %or:vr = PseudoVMOR_MM_B32 %cmp1, %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %res:vr = PseudoVMAND_MM_B32 %and, %or, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %res
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 %cmp1, %cmp2, %avl, 0 /* e8 */
+ %or:vr = PseudoVMOR_MM_B32 %cmp1, %cmp2, %avl, 0 /* e8 */
+ %res:vr = PseudoVMAND_MM_B32 %and, %or, %avl, 0 /* e8 */
+ $v0 = COPY %res
+ PseudoRET implicit $v0
+...
+
+# If the vmand result has multiple uses it is an interior mask value, and
+# requiring it to live in v0 tends to add moves, so don't fold.
+---
+name: no_fold_multiple_uses_of_result
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: no_fold_multiple_uses_of_result
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 %cmp1, %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %andn:vr = PseudoVMANDN_MM_B32 %cmp1, %and, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %or:vr = PseudoVMOR_MM_B32 %and, %andn, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %or
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 %cmp1, %cmp2, %avl, 0 /* e8 */
+ %andn:vr = PseudoVMANDN_MM_B32 %cmp1, %and, %avl, 0 /* e8 */
+ %or:vr = PseudoVMOR_MM_B32 %and, %andn, %avl, 0 /* e8 */
+ $v0 = COPY %or
+ PseudoRET implicit $v0
+...
+
+# vmsbf.m shares the operand shape of a compare but its result depends on the
+# mask, so it must not be folded.
+---
+name: no_fold_vmsbf
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9
+ ; CHECK-LABEL: name: no_fold_vmsbf
+ ; CHECK: liveins: $x10, $v8, $v9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: early-clobber %sbf:vr = PseudoVMSBF_M_B32 %a, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %sbf, %b, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %sbf:vr = PseudoVMSBF_M_B32 %a, %avl, 0 /* e8 */
+ %and:vr = PseudoVMAND_MM_B32 killed %sbf, %b, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
>From 921cb0ec9fc13be92e1578025077f676f95d6b51 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 14 Aug 2026 11:48:35 +0800
Subject: [PATCH 2/5] [RISCV] Fold vmand of mask comparisons into a masked
comparison
Fold `vmand(vmsxx(a, b), vmsxx(c, d))` into a single mask-undisturbed
masked comparison in RISCVVectorPeephole:
```
vmsxx ma, va, vb
vmsxx mb, vc, vd
vmand v0, ma, mb
->
vmsxx.vv v0, va, vb
vmsxx.vv v0, vc, vd, v0.t
```
The fold is guarded for correctness and profitability:
- Only comparisons are folded, identified as masked-pseudo-table entries
whose unmasked form has no passthru or policy operand. Mask-dependent
ops like vmsbf.m/viota.m share that shape but are excluded via
elementsDependOnMask, since masking would change their result.
- Comparisons that may raise FP exceptions are skipped, since masking
would drop the exceptions on the inactive elements.
- Only LMUL <= 1 is folded: for larger LMUL the masked comparison's dest
is earlyclobbered into vrnov0 and cannot share v0 with the mask, which
would introduce extra vmv1r.v moves and regress.
- Only single-use vmand results are folded, so the value coalesces onto
v0 for free rather than needing moves for an interior mask value.
Fixes https://github.com/llvm/llvm-project/issues/215832.
Assisted-by: TRAE CLI (Opus 4.8)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 166 ++++++++++++++++++
.../RISCV/rvv/fixed-vectors-fp-setcc.ll | 81 +++++----
.../rvv/fixed-vectors-reduction-int-vp.ll | 30 ++--
.../RISCV/rvv/fixed-vectors-setcc-fp-vp.ll | 23 ++-
.../RISCV/rvv/intrinsic-vector-match.ll | 5 +-
llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll | 102 +++++------
.../RISCV/rvv/vmand-to-masked-compare.ll | 24 ++-
.../RISCV/rvv/vmand-to-masked-compare.mir | 27 +--
llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll | 5 +-
9 files changed, 305 insertions(+), 158 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 2f1b3c409ff2b..a34e4b111a531 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -74,6 +74,7 @@ class RISCVVectorPeephole : public MachineFunctionPass {
bool foldUndefPassthruVMV_V_V(MachineInstr &MI);
bool foldVMV_V_V(MachineInstr &MI);
bool foldVMergeToMask(MachineInstr &MI) const;
+ bool foldVMANDToMaskedCompare(MachineInstr &MI) const;
bool hasSameEEW(const MachineInstr &User, const MachineInstr &Src) const;
bool isAllOnesMask(const MachineInstr *MaskDef) const;
@@ -787,6 +788,168 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
return true;
}
+/// Fold a mask-register AND of a mask comparison into a mask-undisturbed
+/// masked comparison, saving an instruction:
+///
+/// %cmp1 = PseudoVMSLT_VV_M1 %a, %b, %vl, %sew
+/// %cmp2 = PseudoVMSLT_VV_M1 %c, %d, %vl, %sew
+/// %and = PseudoVMAND_MM %cmp1, %cmp2, %vl, 0
+/// ->
+/// %cmp1 = PseudoVMSLT_VV_M1 %a, %b, %vl, %sew
+/// %and = PseudoVMSLT_VV_M1_MASK %cmp1, %c, %d, %cmp1, %vl, %sew, mu
+///
+/// This works because for a mask-undisturbed masked compare whose passthru is
+/// the same register as its mask %m, the result is %m[i] ? (c cmp d)[i] : %m[i],
+/// which is exactly %m[i] & (c cmp d)[i], i.e. vmand(%m, vmscmp(c, d)).
+///
+/// Since vmand is commutative it's enough for either operand to be a foldable
+/// comparison; the other operand becomes both the mask and the passthru.
+bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
+ if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMAND_MM)
+ return false;
+
+ // The masked comparison we create needs its mask (and passthru) in v0, which
+ // the original vmand did not require. If the vmand's result has more than one
+ // use then it is an interior mask value rather than a final result feeding v0,
+ // and introducing the v0 requirement tends to add vmv1r.v moves. Only fold
+ // single-use results, where the value coalesces onto v0 for free.
+ if (!MRI->hasOneUse(MI.getOperand(0).getReg()))
+ return false;
+
+ // Try each operand as the comparison to be masked; the other becomes the
+ // mask/passthru.
+ for (unsigned CmpIdx : {1, 2}) {
+ unsigned MaskIdx = CmpIdx == 1 ? 2 : 1;
+
+ // The comparison must be single use so that folding it into MI doesn't
+ // leave an extra unmasked comparison behind.
+ SmallVector<MachineInstr *, 4> CmpCopies;
+ Register CmpReg = lookThruCopies(MI.getOperand(CmpIdx).getReg(),
+ /*OneUseOnly=*/true, &CmpCopies);
+ if (!CmpReg.isVirtual() || !MRI->hasOneUse(CmpReg))
+ continue;
+ MachineInstr &Cmp = *MRI->getUniqueVRegDef(CmpReg);
+ if (Cmp.getParent() != MI.getParent())
+ continue;
+
+ // Only fold comparisons: entries in the masked pseudo table whose unmasked
+ // form has neither a passthru nor a policy operand. This excludes ops like
+ // vmsbf.m/viota.m which share that shape but whose active elements depend on
+ // the mask, so masking them would change their result.
+ const RISCV::RISCVMaskedPseudoInfo *Info =
+ RISCV::lookupMaskedIntrinsicByUnmasked(Cmp.getOpcode());
+ if (!Info)
+ continue;
+ const MCInstrDesc &UnmaskedDesc = Cmp.getDesc();
+ if (RISCVII::isFirstDefTiedToFirstUse(UnmaskedDesc) ||
+ RISCVII::hasVecPolicyOp(UnmaskedDesc.TSFlags))
+ continue;
+ unsigned CmpMCOpc = RISCV::getRVVMCOpcode(Cmp.getOpcode());
+ if (RISCVII::elementsDependOnMask(TII->get(CmpMCOpc).TSFlags))
+ continue;
+
+ // The EEW of the comparison's dest must match vmand's SEW.
+ if (!hasSameEEW(MI, Cmp))
+ continue;
+
+ // Masking restricts the comparison to the mask's active elements, so any FP
+ // exceptions raised on inactive elements would be lost.
+ if (Cmp.hasUnmodeledSideEffects() || Cmp.mayRaiseFPException())
+ continue;
+
+ // Use the smaller of the two VLs for the result. The comparison's inactive
+ // (and tail) elements will be filled from the mask, so if vmand's VL is the
+ // smaller one we still need the comparison's original VL for the elements
+ // between them to come from the mask rather than the raw comparison.
+ const MachineOperand &CmpVL =
+ Cmp.getOperand(RISCVII::getVLOpNum(Cmp.getDesc()));
+ const MachineOperand &MIVL =
+ MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));
+ MachineOperand MinVL = MachineOperand::CreateImm(0);
+ if (RISCV::isVLKnownLE(CmpVL, MIVL))
+ MinVL = CmpVL;
+ else if (RISCV::isVLKnownLE(MIVL, CmpVL))
+ MinVL = MIVL;
+ else
+ continue;
+
+ const MachineOperand &MaskOp = MI.getOperand(MaskIdx);
+ Register MaskReg = MaskOp.getReg();
+
+ unsigned MaskedOpc = Info->MaskedPseudo;
+ const MCInstrDesc &MaskedDesc = TII->get(MaskedOpc);
+ unsigned SEW = Cmp.getOperand(RISCVII::getSEWOpNum(Cmp.getDesc())).getImm();
+
+ // Only fold if the masked comparison's dest can live in v0. Its mask
+ // operand must be v0, and we reuse the mask as the passthru, so if the dest
+ // can also be v0 the whole thing coalesces onto v0 and we save the vmand for
+ // free. For LMUL >= 2 the dest is earlyclobbered into vrnov0, which would
+ // force extra vmv1r.v moves for the mask and result and make this a
+ // regression, so bail out in that case. This check must happen before we
+ // mutate any instructions below.
+ if (!TII->getRegClass(MaskedDesc, 0)->contains(RISCV::V0))
+ continue;
+
+ // Make sure the mask and VL dominate the comparison, sinking it if needed.
+ if (!ensureDominates({&MaskOp, &MinVL}, Cmp))
+ continue;
+
+ // The masked comparison's mask operand lives in the VMV0 (v0) class, and its
+ // passthru operand shares the dest's class. Copy the vmand mask into both;
+ // the coalescer collapses these back onto v0, matching the two-instruction
+ // ideal.
+ Register MaskV0Reg = MRI->createVirtualRegister(&RISCV::VMV0RegClass);
+ BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
+ TII->get(TargetOpcode::COPY), MaskV0Reg)
+ .addReg(MaskReg);
+ Register PassthruReg =
+ MRI->createVirtualRegister(TII->getRegClass(MaskedDesc, 1));
+ BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
+ TII->get(TargetOpcode::COPY), PassthruReg)
+ .addReg(MaskReg);
+
+ // Build the masked comparison. Its dest reuses vmand's dest; the passthru
+ // (tied to the dest) and mask are both the other vmand operand. Preserve the
+ // source comparison's MI flags (e.g. nofpexcept), which still hold since the
+ // masked comparison operates on a subset of the original active elements.
+ Register DestReg = MI.getOperand(0).getReg();
+ MachineInstr *Masked =
+ BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(), MaskedDesc)
+ .addDef(DestReg)
+ .addReg(PassthruReg)
+ .add(Cmp.getOperand(1))
+ .add(Cmp.getOperand(2))
+ .addReg(MaskV0Reg)
+ .add(MinVL)
+ .addImm(SEW)
+ // The result is a mask register, whose tail is always agnostic, so
+ // we only need mask-undisturbed (MASK_AGNOSTIC clear) to preserve
+ // the inactive elements from the mask/passthru.
+ .addImm(RISCVVType::TAIL_AGNOSTIC)
+ .setMIFlags(Cmp.getFlags());
+
+ // Now that the comparison is masked, constrain its operands to the masked
+ // pseudo's register classes (e.g. vr -> vrnov0 for LMUL >= 2).
+ for (MachineOperand &MO : Masked->explicit_operands()) {
+ if (!MO.isReg() || !MO.getReg().isVirtual())
+ continue;
+ if (const TargetRegisterClass *RC =
+ Masked->getRegClassConstraint(MO.getOperandNo(), TII, TRI))
+ MRI->constrainRegClass(MO.getReg(), RC);
+ }
+ MRI->clearKillFlags(MaskReg);
+ MI.eraseFromParent();
+ Cmp.eraseFromParent();
+ for (MachineInstr *CmpCopy : CmpCopies)
+ if (MRI->use_nodbg_empty(CmpCopy->getOperand(0).getReg()))
+ CmpCopy->eraseFromParent();
+
+ return true;
+ }
+
+ return false;
+}
+
bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -806,6 +969,9 @@ bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
for (MachineInstr &MI : make_early_inc_range(MBB))
Changed |= foldVMergeToMask(MI);
+ for (MachineInstr &MI : make_early_inc_range(MBB))
+ Changed |= foldVMANDToMaskedCompare(MI);
+
for (MachineInstr &MI : make_early_inc_range(MBB)) {
Changed |= convertToVLMAX(MI);
Changed |= convertToUnmasked(MI);
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-setcc.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-setcc.ll
index 65f9e17428b60..269527663ca0c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-setcc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-setcc.ll
@@ -225,13 +225,12 @@ define void @fcmp_ord_vv_v8bf16(ptr %x, ptr %y, ptr %z) {
;
; ZVFBFA-LABEL: fcmp_ord_vv_v8bf16:
; ZVFBFA: # %bb.0:
-; ZVFBFA-NEXT: vsetivli zero, 8, e16alt, m1, ta, ma
+; ZVFBFA-NEXT: vsetivli zero, 8, e16alt, m1, ta, mu
; ZVFBFA-NEXT: vle16.v v8, (a1)
-; ZVFBFA-NEXT: vle16.v v9, (a0)
-; ZVFBFA-NEXT: vmfeq.vv v8, v8, v8
-; ZVFBFA-NEXT: vmfeq.vv v9, v9, v9
-; ZVFBFA-NEXT: vmand.mm v8, v9, v8
-; ZVFBFA-NEXT: vsm.v v8, (a2)
+; ZVFBFA-NEXT: vmfeq.vv v0, v8, v8
+; ZVFBFA-NEXT: vle16.v v8, (a0)
+; ZVFBFA-NEXT: vmfeq.vv v0, v8, v8, v0.t
+; ZVFBFA-NEXT: vsm.v v0, (a2)
; ZVFBFA-NEXT: ret
%a = load <8 x bfloat>, ptr %x
%b = load <8 x bfloat>, ptr %y
@@ -3060,12 +3059,11 @@ define void @fcmp_one_vv_v8f64_nonans(ptr %x, ptr %y, ptr %z) {
define void @fcmp_ord_vv_v4f16(ptr %x, ptr %y, ptr %z) {
; ZVFH-LABEL: fcmp_ord_vv_v4f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, mu
; ZVFH-NEXT: vle16.v v8, (a1)
-; ZVFH-NEXT: vle16.v v9, (a0)
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmand.mm v0, v9, v8
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8
+; ZVFH-NEXT: vle16.v v8, (a0)
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFH-NEXT: vmv.v.i v8, 0
; ZVFH-NEXT: vmerge.vim v8, v8, 1, v0
@@ -3082,13 +3080,14 @@ define void @fcmp_ord_vv_v4f16(ptr %x, ptr %y, ptr %z) {
; ZVFHMIN: # %bb.0:
; ZVFHMIN-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; ZVFHMIN-NEXT: vle16.v v8, (a1)
-; ZVFHMIN-NEXT: vle16.v v9, (a0)
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT: vmfeq.vv v9, v10, v10
-; ZVFHMIN-NEXT: vmfeq.vv v8, v8, v8
-; ZVFHMIN-NEXT: vmand.mm v0, v8, v9
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT: vle16.v v8, (a0)
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFHMIN-NEXT: vmv.v.i v8, 0
; ZVFHMIN-NEXT: vmerge.vim v8, v8, 1, v0
@@ -5091,12 +5090,11 @@ define void @fcmp_one_vf_v8f64_nonans(ptr %x, double %y, ptr %z) {
define void @fcmp_ord_vf_v4f16(ptr %x, half %y, ptr %z) {
; ZVFH-LABEL: fcmp_ord_vf_v4f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, mu
; ZVFH-NEXT: vfmv.v.f v8, fa0
-; ZVFH-NEXT: vle16.v v9, (a0)
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmfeq.vf v8, v8, fa0
-; ZVFH-NEXT: vmand.mm v0, v9, v8
+; ZVFH-NEXT: vmfeq.vf v0, v8, fa0
+; ZVFH-NEXT: vle16.v v8, (a0)
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFH-NEXT: vmv.v.i v8, 0
; ZVFH-NEXT: vmerge.vim v8, v8, 1, v0
@@ -5113,14 +5111,15 @@ define void @fcmp_ord_vf_v4f16(ptr %x, half %y, ptr %z) {
; ZVFHMIN: # %bb.0:
; ZVFHMIN-NEXT: fmv.x.h a2, fa0
; ZVFHMIN-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
-; ZVFHMIN-NEXT: vle16.v v8, (a0)
-; ZVFHMIN-NEXT: vmv.v.x v9, a2
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT: vmv.v.x v8, a2
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT: vmfeq.vv v9, v10, v10
-; ZVFHMIN-NEXT: vmfeq.vv v8, v8, v8
-; ZVFHMIN-NEXT: vmand.mm v0, v9, v8
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT: vle16.v v8, (a0)
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFHMIN-NEXT: vmv.v.i v8, 0
; ZVFHMIN-NEXT: vmerge.vim v8, v8, 1, v0
@@ -7126,12 +7125,11 @@ define void @fcmp_one_fv_v8f64_nonans(ptr %x, double %y, ptr %z) {
define void @fcmp_ord_fv_v4f16(ptr %x, half %y, ptr %z) {
; ZVFH-LABEL: fcmp_ord_fv_v4f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, mu
+; ZVFH-NEXT: vle16.v v8, (a0)
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8
; ZVFH-NEXT: vfmv.v.f v8, fa0
-; ZVFH-NEXT: vle16.v v9, (a0)
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmfeq.vf v8, v8, fa0
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vmfeq.vf v0, v8, fa0, v0.t
; ZVFH-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFH-NEXT: vmv.v.i v8, 0
; ZVFH-NEXT: vmerge.vim v8, v8, 1, v0
@@ -7146,16 +7144,17 @@ define void @fcmp_ord_fv_v4f16(ptr %x, half %y, ptr %z) {
;
; ZVFHMIN-LABEL: fcmp_ord_fv_v4f16:
; ZVFHMIN: # %bb.0:
-; ZVFHMIN-NEXT: fmv.x.h a2, fa0
; ZVFHMIN-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; ZVFHMIN-NEXT: vle16.v v8, (a0)
-; ZVFHMIN-NEXT: vmv.v.x v9, a2
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT: fmv.x.h a0, fa0
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vmv.v.x v8, a0
; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT: vmfeq.vv v9, v10, v10
-; ZVFHMIN-NEXT: vmfeq.vv v8, v8, v8
-; ZVFHMIN-NEXT: vmand.mm v0, v8, v9
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFHMIN-NEXT: vmv.v.i v8, 0
; ZVFHMIN-NEXT: vmerge.vim v8, v8, 1, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
index 4f137a200dea8..742c5282bb847 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
@@ -1204,10 +1204,9 @@ define i8 @vpreduce_mul_v1i8(i8 %s, <1 x i8> %v, <1 x i1> %m, i32 zeroext %evl)
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset ra, -4
-; RV32-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e32, mf2, ta, mu
; RV32-NEXT: vmv.s.x v9, a1
-; RV32-NEXT: vmsne.vi v9, v9, 0
-; RV32-NEXT: vmand.mm v0, v9, v0
+; RV32-NEXT: vmsne.vi v0, v9, 0, v0.t
; RV32-NEXT: vmv.v.i v9, 1
; RV32-NEXT: vsetvli zero, zero, e8, mf8, ta, ma
; RV32-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1228,10 +1227,9 @@ define i8 @vpreduce_mul_v1i8(i8 %s, <1 x i8> %v, <1 x i1> %m, i32 zeroext %evl)
; RV64-NEXT: .cfi_def_cfa_offset 16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset ra, -8
-; RV64-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-NEXT: vsetivli zero, 1, e32, mf2, ta, mu
; RV64-NEXT: vmv.s.x v9, a1
-; RV64-NEXT: vmsne.vi v9, v9, 0
-; RV64-NEXT: vmand.mm v0, v9, v0
+; RV64-NEXT: vmsne.vi v0, v9, 0, v0.t
; RV64-NEXT: vmv.v.i v9, 1
; RV64-NEXT: vsetvli zero, zero, e8, mf8, ta, ma
; RV64-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1256,10 +1254,9 @@ define signext i8 @vpreduce_mul_v2i8(i8 signext %s, <2 x i8> %v, <2 x i1> %m, i3
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset ra, -4
-; RV32-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT: vsetivli zero, 2, e32, mf2, ta, mu
; RV32-NEXT: vid.v v9
-; RV32-NEXT: vmsltu.vx v9, v9, a1
-; RV32-NEXT: vmand.mm v0, v9, v0
+; RV32-NEXT: vmsltu.vx v0, v9, a1, v0.t
; RV32-NEXT: vsetvli zero, zero, e8, mf8, ta, ma
; RV32-NEXT: vmv.v.i v9, 1
; RV32-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1284,10 +1281,9 @@ define signext i8 @vpreduce_mul_v2i8(i8 signext %s, <2 x i8> %v, <2 x i1> %m, i3
; RV64-NEXT: .cfi_def_cfa_offset 16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset ra, -8
-; RV64-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-NEXT: vsetivli zero, 2, e32, mf2, ta, mu
; RV64-NEXT: vid.v v9
-; RV64-NEXT: vmsltu.vx v9, v9, a1
-; RV64-NEXT: vmand.mm v0, v9, v0
+; RV64-NEXT: vmsltu.vx v0, v9, a1, v0.t
; RV64-NEXT: vsetvli zero, zero, e8, mf8, ta, ma
; RV64-NEXT: vmv.v.i v9, 1
; RV64-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1316,10 +1312,9 @@ define signext i8 @vpreduce_mul_v4i8(i8 signext %s, <4 x i8> %v, <4 x i1> %m, i3
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset ra, -4
-; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, mu
; RV32-NEXT: vid.v v9
-; RV32-NEXT: vmsltu.vx v9, v9, a1
-; RV32-NEXT: vmand.mm v0, v9, v0
+; RV32-NEXT: vmsltu.vx v0, v9, a1, v0.t
; RV32-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; RV32-NEXT: vmv.v.i v9, 1
; RV32-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1346,10 +1341,9 @@ define signext i8 @vpreduce_mul_v4i8(i8 signext %s, <4 x i8> %v, <4 x i1> %m, i3
; RV64-NEXT: .cfi_def_cfa_offset 16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset ra, -8
-; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, mu
; RV64-NEXT: vid.v v9
-; RV64-NEXT: vmsltu.vx v9, v9, a1
-; RV64-NEXT: vmand.mm v0, v9, v0
+; RV64-NEXT: vmsltu.vx v0, v9, a1, v0.t
; RV64-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; RV64-NEXT: vmv.v.i v9, 1
; RV64-NEXT: vmerge.vvm v8, v9, v8, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
index f012e8d01a5b0..183433165b9b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
@@ -432,10 +432,9 @@ define <8 x i1> @fcmp_one_vf_swap_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i3
define <8 x i1> @fcmp_ord_vv_v8f16(<8 x half> %va, <8 x half> %vb, <8 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vv_v8f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, mu
+; ZVFH-NEXT: vmfeq.vv v0, v9, v9
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vv_v8f16:
@@ -457,11 +456,10 @@ define <8 x i1> @fcmp_ord_vv_v8f16(<8 x half> %va, <8 x half> %vb, <8 x i1> %m,
define <8 x i1> @fcmp_ord_vf_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vf_v8f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, mu
; ZVFH-NEXT: vfmv.v.f v9, fa0
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vf v9, v9, fa0
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vmfeq.vf v0, v9, fa0
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vf_v8f16:
@@ -487,11 +485,10 @@ define <8 x i1> @fcmp_ord_vf_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i32 zer
define <8 x i1> @fcmp_ord_vf_swap_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vf_swap_v8f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; ZVFH-NEXT: vfmv.v.f v9, fa0
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vf v9, v9, fa0
-; ZVFH-NEXT: vmand.mm v0, v9, v8
+; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, mu
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8
+; ZVFH-NEXT: vfmv.v.f v8, fa0
+; ZVFH-NEXT: vmfeq.vf v0, v8, fa0, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vf_swap_v8f16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
index 6febcd22e3288..5165ba8355197 100644
--- a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
@@ -143,10 +143,9 @@ define <vscale x 16 x i1> @match_nxv16i8_v16i8(<vscale x 16 x i8> %op1, <16 x i8
define <16 x i1> @match_v16i8_v1i8(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask) {
; CHECK-LABEL: match_v16i8_v1i8:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, mu
; CHECK-NEXT: vrgather.vi v10, v9, 0
-; CHECK-NEXT: vmseq.vv v8, v8, v10
-; CHECK-NEXT: vmand.mm v0, v8, v0
+; CHECK-NEXT: vmseq.vv v0, v8, v10, v0.t
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index fceb9030891ff..579286382d346 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -281,11 +281,12 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8
; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT: vmfeq.vv v8, v10, v10
-; CHECK-NEXT: vmfeq.vv v9, v9, v9
-; CHECK-NEXT: vmand.mm v0, v9, v8
+; CHECK-NEXT: vmfeq.vv v0, v10, v10
+; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v9, v9, v0.t
; CHECK-NEXT: ret
%v = call <vscale x 1 x i1> @llvm.vp.fcmp.nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, metadata !"ord", <vscale x 1 x i1> %m, i32 %evl)
ret <vscale x 1 x i1> %v
@@ -295,13 +296,13 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
; CHECK-LABEL: fcmp_ord_vf_nxv1bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT: vfmv.v.f v9, fa5
+; CHECK-NEXT: vmfeq.vf v0, v9, fa5
+; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT: vfmv.v.f v8, fa5
-; CHECK-NEXT: vmfeq.vv v9, v9, v9
-; CHECK-NEXT: vmfeq.vf v8, v8, fa5
-; CHECK-NEXT: vmand.mm v0, v9, v8
+; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v9, v9, v0.t
; CHECK-NEXT: ret
%elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
%vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -315,11 +316,10 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, b
; CHECK-NEXT: fcvt.s.bf16 fa5, fa0
; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v9, v9
; CHECK-NEXT: vfmv.v.f v8, fa5
-; CHECK-NEXT: vmfeq.vv v9, v9, v9
-; CHECK-NEXT: vmfeq.vf v8, v8, fa5
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vmfeq.vf v0, v8, fa5, v0.t
; CHECK-NEXT: ret
%elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
%vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -1795,21 +1795,21 @@ define <vscale x 1 x i1> @fcmp_one_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vv_nxv1f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, mu
+; ZVFH-NEXT: vmfeq.vv v0, v9, v9
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vv_nxv1f16:
; ZVFHMIN: # %bb.0:
; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
; ZVFHMIN-NEXT: vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT: vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT: vmfeq.vv v9, v9, v9
-; ZVFHMIN-NEXT: vmand.mm v0, v9, v8
+; ZVFHMIN-NEXT: vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: ret
%v = call <vscale x 1 x i1> @llvm.vp.fcmp.nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, metadata !"ord", <vscale x 1 x i1> %m, i32 %evl)
ret <vscale x 1 x i1> %v
@@ -1818,23 +1818,22 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x
define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vf_nxv1f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, mu
; ZVFH-NEXT: vfmv.v.f v9, fa0
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vf v9, v9, fa0
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vmfeq.vf v0, v9, fa0
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vf_nxv1f16:
; ZVFHMIN: # %bb.0:
; ZVFHMIN-NEXT: fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT: vfmv.v.f v9, fa5
+; ZVFHMIN-NEXT: vmfeq.vf v0, v9, fa5
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma
; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT: vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT: vmfeq.vv v9, v9, v9
-; ZVFHMIN-NEXT: vmfeq.vf v8, v8, fa5
-; ZVFHMIN-NEXT: vmand.mm v0, v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: ret
%elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
%vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1845,11 +1844,10 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vf_swap_nxv1f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
-; ZVFH-NEXT: vfmv.v.f v9, fa0
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vf v9, v9, fa0
-; ZVFH-NEXT: vmand.mm v0, v9, v8
+; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, mu
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8
+; ZVFH-NEXT: vfmv.v.f v8, fa0
+; ZVFH-NEXT: vmfeq.vf v0, v8, fa0, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vf_swap_nxv1f16:
@@ -1857,11 +1855,10 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
; ZVFHMIN-NEXT: fcvt.s.h fa5, fa0
; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9
; ZVFHMIN-NEXT: vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT: vmfeq.vv v9, v9, v9
-; ZVFHMIN-NEXT: vmfeq.vf v8, v8, fa5
-; ZVFHMIN-NEXT: vmand.mm v0, v8, v9
+; ZVFHMIN-NEXT: vmfeq.vf v0, v8, fa5, v0.t
; ZVFHMIN-NEXT: ret
%elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
%vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -3634,10 +3631,9 @@ define <vscale x 1 x i1> @fcmp_one_vf_swap_nxv1f64(<vscale x 1 x double> %va, do
define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; CHECK-LABEL: fcmp_ord_vv_nxv1f64:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT: vmfeq.vv v9, v9, v9
-; CHECK-NEXT: vmfeq.vv v8, v8, v8
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v9, v9
+; CHECK-NEXT: vmfeq.vv v0, v8, v8, v0.t
; CHECK-NEXT: ret
%v = call <vscale x 1 x i1> @llvm.vp.fcmp.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, metadata !"ord", <vscale x 1 x i1> %m, i32 %evl)
ret <vscale x 1 x i1> %v
@@ -3646,11 +3642,10 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f64(<vscale x 1 x double> %va, <vscale
define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; CHECK-LABEL: fcmp_ord_vf_nxv1f64:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, mu
; CHECK-NEXT: vfmv.v.f v9, fa0
-; CHECK-NEXT: vmfeq.vv v8, v8, v8
-; CHECK-NEXT: vmfeq.vf v9, v9, fa0
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vmfeq.vf v0, v9, fa0
+; CHECK-NEXT: vmfeq.vv v0, v8, v8, v0.t
; CHECK-NEXT: ret
%elt.head = insertelement <vscale x 1 x double> poison, double %b, i32 0
%vb = shufflevector <vscale x 1 x double> %elt.head, <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer
@@ -3661,11 +3656,10 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f64(<vscale x 1 x double> %va, double
define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; CHECK-LABEL: fcmp_ord_vf_swap_nxv1f64:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT: vfmv.v.f v9, fa0
-; CHECK-NEXT: vmfeq.vv v8, v8, v8
-; CHECK-NEXT: vmfeq.vf v9, v9, fa0
-; CHECK-NEXT: vmand.mm v0, v9, v8
+; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v8, v8
+; CHECK-NEXT: vfmv.v.f v8, fa0
+; CHECK-NEXT: vmfeq.vf v0, v8, fa0, v0.t
; CHECK-NEXT: ret
%elt.head = insertelement <vscale x 1 x double> poison, double %b, i32 0
%vb = shufflevector <vscale x 1 x double> %elt.head, <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
index 4cd4940571fc9..ef4b2bff7bea7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
@@ -12,10 +12,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1>, <vscal
define <vscale x 2 x i1> @and_icmp(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl) {
; CHECK-LABEL: and_icmp:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT: vmslt.vv v8, v8, v9
-; CHECK-NEXT: vmslt.vv v9, v9, v10
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, mu
+; CHECK-NEXT: vmslt.vv v0, v9, v10
+; CHECK-NEXT: vmslt.vv v0, v8, v9, v0.t
; CHECK-NEXT: ret
%m1 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, i64 %vl)
%m2 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl)
@@ -27,10 +26,9 @@ define <vscale x 2 x i1> @and_icmp(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b,
define <vscale x 2 x i1> @and_fcmp(<vscale x 2 x float> %a, <vscale x 2 x float> %b, <vscale x 2 x float> %c, i64 %vl) {
; CHECK-LABEL: and_fcmp:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT: vmflt.vv v8, v8, v9
-; CHECK-NEXT: vmflt.vv v9, v9, v10
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, mu
+; CHECK-NEXT: vmflt.vv v0, v9, v10
+; CHECK-NEXT: vmflt.vv v0, v8, v9, v0.t
; CHECK-NEXT: ret
%m1 = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float> %a, <vscale x 2 x float> %b, i64 %vl)
%m2 = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float> %b, <vscale x 2 x float> %c, i64 %vl)
@@ -42,12 +40,10 @@ define <vscale x 2 x i1> @and_fcmp(<vscale x 2 x float> %a, <vscale x 2 x float>
define <vscale x 2 x i1> @and_icmp_chain(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, <vscale x 2 x i32> %d, i64 %vl) {
; CHECK-LABEL: and_icmp_chain:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT: vmslt.vv v12, v9, v10
-; CHECK-NEXT: vmslt.vv v8, v8, v9
-; CHECK-NEXT: vmslt.vv v9, v10, v11
-; CHECK-NEXT: vmand.mm v8, v8, v12
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, mu
+; CHECK-NEXT: vmslt.vv v0, v9, v10
+; CHECK-NEXT: vmslt.vv v0, v8, v9, v0.t
+; CHECK-NEXT: vmslt.vv v0, v10, v11, v0.t
; CHECK-NEXT: ret
%m1 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, i64 %vl)
%m2 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
index cdaf4832526b0..2537ffd7339b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
@@ -15,11 +15,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %avl:gprnox0 = COPY $x10
; CHECK-NEXT: %c:vr = COPY $v10
- ; CHECK-NEXT: %b:vr = COPY $v9
- ; CHECK-NEXT: %a:vr = COPY $v8
- ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
+ ; CHECK-NEXT: %a:vrnov0 = COPY $v8
; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
- ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %cmp2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %cmp2
+ ; CHECK-NEXT: %and:vr = PseudoVMSLT_VV_M1_MASK [[COPY1]], %a, %b, [[COPY]], %avl /* vl */, 5 /* e32 */, 1 /* ta, mu */
; CHECK-NEXT: $v0 = COPY %and
; CHECK-NEXT: PseudoRET implicit $v0
%avl:gprnox0 = COPY $x10
@@ -43,12 +44,13 @@ body: |
; CHECK: liveins: $x10, $v8, $v9, $v10
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %avl:gprnox0 = COPY $x10
- ; CHECK-NEXT: %c:vr = COPY $v10
- ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %c:vrnov0 = COPY $v10
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
; CHECK-NEXT: %a:vr = COPY $v8
; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
- ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
- ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp2, killed %cmp1, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %cmp1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %cmp1
+ ; CHECK-NEXT: %and:vr = PseudoVMSLT_VV_M1_MASK [[COPY1]], %b, %c, [[COPY]], %avl /* vl */, 5 /* e32 */, 1 /* ta, mu */
; CHECK-NEXT: $v0 = COPY %and
; CHECK-NEXT: PseudoRET implicit $v0
%avl:gprnox0 = COPY $x10
@@ -73,11 +75,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %avl:gprnox0 = COPY $x10
; CHECK-NEXT: %c:vr = COPY $v10
- ; CHECK-NEXT: %b:vr = COPY $v9
- ; CHECK-NEXT: %a:vr = COPY $v8
- ; CHECK-NEXT: %cmp1:vr = nofpexcept PseudoVMFLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
+ ; CHECK-NEXT: %a:vrnov0 = COPY $v8
; CHECK-NEXT: %cmp2:vr = nofpexcept PseudoVMFLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
- ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %cmp2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %cmp2
+ ; CHECK-NEXT: %and:vr = nofpexcept PseudoVMFLT_VV_M1_MASK [[COPY1]], %a, %b, [[COPY]], %avl /* vl */, 5 /* e32 */, 1 /* ta, mu */
; CHECK-NEXT: $v0 = COPY %and
; CHECK-NEXT: PseudoRET implicit $v0
%avl:gprnox0 = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index c1a7435849221..8bf644969e907 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -72,9 +72,8 @@ entry:
define <vscale x 1 x i1> @test5(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %avl) nounwind {
; CHECK-LABEL: test5:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT: vmseq.vv v8, v8, v9
-; CHECK-NEXT: vmand.mm v0, v8, v0
+; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, mu
+; CHECK-NEXT: vmseq.vv v0, v8, v9, v0.t
; CHECK-NEXT: ret
entry:
%vl = tail call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 0)
>From 16179dfc2f796e88a90d1f7f3791fe19d3e2e2f2 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 14 Aug 2026 15:41:45 +0800
Subject: [PATCH 3/5] clang-format
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 35 ++++++++++---------
1 file changed, 18 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index a34e4b111a531..dbc66984de5d4 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -799,8 +799,8 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
/// %and = PseudoVMSLT_VV_M1_MASK %cmp1, %c, %d, %cmp1, %vl, %sew, mu
///
/// This works because for a mask-undisturbed masked compare whose passthru is
-/// the same register as its mask %m, the result is %m[i] ? (c cmp d)[i] : %m[i],
-/// which is exactly %m[i] & (c cmp d)[i], i.e. vmand(%m, vmscmp(c, d)).
+/// the same register as its mask %m, the result is %m[i] ? (c cmp d)[i] :
+/// %m[i], which is exactly %m[i] & (c cmp d)[i], i.e. vmand(%m, vmscmp(c, d)).
///
/// Since vmand is commutative it's enough for either operand to be a foldable
/// comparison; the other operand becomes both the mask and the passthru.
@@ -810,9 +810,9 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
// The masked comparison we create needs its mask (and passthru) in v0, which
// the original vmand did not require. If the vmand's result has more than one
- // use then it is an interior mask value rather than a final result feeding v0,
- // and introducing the v0 requirement tends to add vmv1r.v moves. Only fold
- // single-use results, where the value coalesces onto v0 for free.
+ // use then it is an interior mask value rather than a final result feeding
+ // v0, and introducing the v0 requirement tends to add vmv1r.v moves. Only
+ // fold single-use results, where the value coalesces onto v0 for free.
if (!MRI->hasOneUse(MI.getOperand(0).getReg()))
return false;
@@ -834,8 +834,8 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
// Only fold comparisons: entries in the masked pseudo table whose unmasked
// form has neither a passthru nor a policy operand. This excludes ops like
- // vmsbf.m/viota.m which share that shape but whose active elements depend on
- // the mask, so masking them would change their result.
+ // vmsbf.m/viota.m which share that shape but whose active elements depend
+ // on the mask, so masking them would change their result.
const RISCV::RISCVMaskedPseudoInfo *Info =
RISCV::lookupMaskedIntrinsicByUnmasked(Cmp.getOpcode());
if (!Info)
@@ -882,9 +882,9 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
// Only fold if the masked comparison's dest can live in v0. Its mask
// operand must be v0, and we reuse the mask as the passthru, so if the dest
- // can also be v0 the whole thing coalesces onto v0 and we save the vmand for
- // free. For LMUL >= 2 the dest is earlyclobbered into vrnov0, which would
- // force extra vmv1r.v moves for the mask and result and make this a
+ // can also be v0 the whole thing coalesces onto v0 and we save the vmand
+ // for free. For LMUL >= 2 the dest is earlyclobbered into vrnov0, which
+ // would force extra vmv1r.v moves for the mask and result and make this a
// regression, so bail out in that case. This check must happen before we
// mutate any instructions below.
if (!TII->getRegClass(MaskedDesc, 0)->contains(RISCV::V0))
@@ -894,10 +894,10 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
if (!ensureDominates({&MaskOp, &MinVL}, Cmp))
continue;
- // The masked comparison's mask operand lives in the VMV0 (v0) class, and its
- // passthru operand shares the dest's class. Copy the vmand mask into both;
- // the coalescer collapses these back onto v0, matching the two-instruction
- // ideal.
+ // The masked comparison's mask operand lives in the VMV0 (v0) class, and
+ // its passthru operand shares the dest's class. Copy the vmand mask into
+ // both; the coalescer collapses these back onto v0, matching the
+ // two-instruction ideal.
Register MaskV0Reg = MRI->createVirtualRegister(&RISCV::VMV0RegClass);
BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
TII->get(TargetOpcode::COPY), MaskV0Reg)
@@ -909,9 +909,10 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
.addReg(MaskReg);
// Build the masked comparison. Its dest reuses vmand's dest; the passthru
- // (tied to the dest) and mask are both the other vmand operand. Preserve the
- // source comparison's MI flags (e.g. nofpexcept), which still hold since the
- // masked comparison operates on a subset of the original active elements.
+ // (tied to the dest) and mask are both the other vmand operand. Preserve
+ // the source comparison's MI flags (e.g. nofpexcept), which still hold
+ // since the masked comparison operates on a subset of the original active
+ // elements.
Register DestReg = MI.getOperand(0).getReg();
MachineInstr *Masked =
BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(), MaskedDesc)
>From 962b1f7d4a48b53100835a8cde8c36af6d47f5f1 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Mon, 17 Aug 2026 14:33:08 +0800
Subject: [PATCH 4/5] [RISCV] Use BuildMI dest/MIMetadata overload in
foldVMANDToMaskedCompare (NFC)
Address review feedback: build the masked comparison via the
BuildMI(..., MIMetadata(Cmp), MaskedDesc, DestReg) overload instead of a
separate .addDef(DestReg), and use MIMetadata(Cmp) so the new
instruction also inherits the source comparison's PC sections / MMRA
metadata rather than only its debug location.
Assisted-by: TRAE CLI (Opus 4.8)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index dbc66984de5d4..4c42a58442d25 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -915,8 +915,7 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
// elements.
Register DestReg = MI.getOperand(0).getReg();
MachineInstr *Masked =
- BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(), MaskedDesc)
- .addDef(DestReg)
+ BuildMI(*MI.getParent(), Cmp, MIMetadata(Cmp), MaskedDesc, DestReg)
.addReg(PassthruReg)
.add(Cmp.getOperand(1))
.add(Cmp.getOperand(2))
>From dfd21b07acf621af7bf1692ebd753d9394ec2079 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Thu, 20 Aug 2026 17:56:54 +0800
Subject: [PATCH 5/5] [RISCV] Hardcode passthru register class in
foldVMANDToMaskedCompare (NFC)
Address review feedback: the masked comparison's passthru shares the
dest's register class, and the earlier V0 containment check already
restricts folding to LMUL <= 1, so the passthru is always a single
vector register. Hardcode VRRegClass instead of looking it up via
TII->getRegClass.
Assisted-by: TRAE CLI (Opus 4.8)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 4c42a58442d25..2314fddaaa874 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -902,8 +902,9 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
TII->get(TargetOpcode::COPY), MaskV0Reg)
.addReg(MaskReg);
- Register PassthruReg =
- MRI->createVirtualRegister(TII->getRegClass(MaskedDesc, 1));
+ // The passthru shares the dest's class, which the V0 check above restricts
+ // to LMUL <= 1, so it is always a single vector register.
+ Register PassthruReg = MRI->createVirtualRegister(&RISCV::VRRegClass);
BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
TII->get(TargetOpcode::COPY), PassthruReg)
.addReg(MaskReg);
More information about the llvm-commits
mailing list