[llvm] [RISCV] Fold vmand of mask comparisons into a masked comparison (PR #216264)
Pengcheng Wang via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 23:32:31 PDT 2026
https://github.com/wangpc-pp updated https://github.com/llvm/llvm-project/pull/216264
>From c0e83497f4fdb40ad869c91aceb34e1dd932406a Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 14 Aug 2026 11:46:31 +0800
Subject: [PATCH 01/11] [RISCV] Add tests for vmand of mask comparisons
Precommit tests for folding vmand(vmsxx(a, b), vmsxx(c, d)) into a
mask-undisturbed masked comparison. These currently show the unoptimized
vmslt + vmslt + vmand.mm sequence.
Assisted-by: TRAE CLI (Opus 4.8)
---
.../RISCV/rvv/vmand-to-masked-compare.ll | 58 ++++
.../RISCV/rvv/vmand-to-masked-compare.mir | 247 ++++++++++++++++++
2 files changed, 305 insertions(+)
create mode 100644 llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
create mode 100644 llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
new file mode 100644
index 0000000000000..4cd4940571fc9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh -verify-machineinstrs | FileCheck %s
+
+; Test that vmand(vmsxx(a, b), vmsxx(c, d)) is folded into a mask-undisturbed
+; masked comparison, saving a vmand.mm. See RISCVVectorPeephole.
+
+declare <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>, <vscale x 2 x i32>, i64)
+declare <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float>, <vscale x 2 x float>, i64)
+declare <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1>, <vscale x 2 x i1>, i64)
+
+; (a < b) && (b < c)
+define <vscale x 2 x i1> @and_icmp(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl) {
+; CHECK-LABEL: and_icmp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT: vmslt.vv v8, v8, v9
+; CHECK-NEXT: vmslt.vv v9, v9, v10
+; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: ret
+ %m1 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, i64 %vl)
+ %m2 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl)
+ %r = call <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1> %m1, <vscale x 2 x i1> %m2, i64 %vl)
+ ret <vscale x 2 x i1> %r
+}
+
+; Quiet floating-point comparisons can be folded too.
+define <vscale x 2 x i1> @and_fcmp(<vscale x 2 x float> %a, <vscale x 2 x float> %b, <vscale x 2 x float> %c, i64 %vl) {
+; CHECK-LABEL: and_fcmp:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT: vmflt.vv v8, v8, v9
+; CHECK-NEXT: vmflt.vv v9, v9, v10
+; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: ret
+ %m1 = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float> %a, <vscale x 2 x float> %b, i64 %vl)
+ %m2 = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float> %b, <vscale x 2 x float> %c, i64 %vl)
+ %r = call <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1> %m1, <vscale x 2 x i1> %m2, i64 %vl)
+ ret <vscale x 2 x i1> %r
+}
+
+; (a < b) && (b < c) && (c < d) chains through the mask.
+define <vscale x 2 x i1> @and_icmp_chain(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, <vscale x 2 x i32> %d, i64 %vl) {
+; CHECK-LABEL: and_icmp_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT: vmslt.vv v12, v9, v10
+; CHECK-NEXT: vmslt.vv v8, v8, v9
+; CHECK-NEXT: vmslt.vv v9, v10, v11
+; CHECK-NEXT: vmand.mm v8, v8, v12
+; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: ret
+ %m1 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, i64 %vl)
+ %m2 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl)
+ %m3 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %c, <vscale x 2 x i32> %d, i64 %vl)
+ %and1 = call <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1> %m1, <vscale x 2 x i1> %m2, i64 %vl)
+ %and2 = call <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1> %and1, <vscale x 2 x i1> %m3, i64 %vl)
+ ret <vscale x 2 x i1> %and2
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
new file mode 100644
index 0000000000000..cdaf4832526b0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
@@ -0,0 +1,247 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc %s -o - -mtriple=riscv64 -mattr=+v -run-pass=riscv-vector-peephole -verify-machineinstrs | FileCheck %s
+
+# vmand(vmslt(a, b), vmslt(c, d)) can be folded to a mask-undisturbed masked
+# compare, where the first comparison provides both the mask and passthru:
+# vmslt.vv v0, va, vb; vmslt.vv v0, vc, vd, v0.t
+
+---
+name: fold_vv
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: fold_vv
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# The comparison can be either operand of the vmand since it is commutative.
+---
+name: fold_vv_commuted
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: fold_vv_commuted
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp2, killed %cmp1, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp2, killed %cmp1, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# A quiet floating-point compare (nofpexcept) can be folded.
+---
+name: fold_fp_nofpexcept
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: fold_fp_nofpexcept
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = nofpexcept PseudoVMFLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = nofpexcept PseudoVMFLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = nofpexcept PseudoVMFLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = nofpexcept PseudoVMFLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# A floating-point compare that may raise exceptions must not be folded, since
+# masking would skip the exceptions on the inactive elements.
+---
+name: no_fold_fp_may_raise
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: no_fold_fp_may_raise
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMFLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMFLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMFLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMFLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# For LMUL >= 2 the masked comparison's dest is earlyclobbered into vrnov0 and
+# cannot share v0 with the mask, so folding would introduce extra vmv1r.v moves.
+# Don't fold in that case.
+---
+name: no_fold_m2
+body: |
+ bb.0:
+ liveins: $x10, $v8m2, $v10m2, $v12m2
+ ; CHECK-LABEL: name: no_fold_m2
+ ; CHECK: liveins: $x10, $v8m2, $v10m2, $v12m2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vrm2 = COPY $v12m2
+ ; CHECK-NEXT: %b:vrm2 = COPY $v10m2
+ ; CHECK-NEXT: %a:vrm2 = COPY $v8m2
+ ; CHECK-NEXT: early-clobber %cmp1:vr = PseudoVMSLT_VV_M2 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: early-clobber %cmp2:vr = PseudoVMSLT_VV_M2 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B16 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vrm2 = COPY $v12m2
+ %b:vrm2 = COPY $v10m2
+ %a:vrm2 = COPY $v8m2
+ %cmp1:vr = PseudoVMSLT_VV_M2 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M2 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B16 killed %cmp1, killed %cmp2, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# If both comparisons have multiple uses then neither can be consumed by the
+# fold without leaving an unmasked comparison behind, so don't fold.
+---
+name: no_fold_multiple_uses_of_compare
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: no_fold_multiple_uses_of_compare
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 %cmp1, %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %or:vr = PseudoVMOR_MM_B32 %cmp1, %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %res:vr = PseudoVMAND_MM_B32 %and, %or, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %res
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 %cmp1, %cmp2, %avl, 0 /* e8 */
+ %or:vr = PseudoVMOR_MM_B32 %cmp1, %cmp2, %avl, 0 /* e8 */
+ %res:vr = PseudoVMAND_MM_B32 %and, %or, %avl, 0 /* e8 */
+ $v0 = COPY %res
+ PseudoRET implicit $v0
+...
+
+# If the vmand result has multiple uses it is an interior mask value, and
+# requiring it to live in v0 tends to add moves, so don't fold.
+---
+name: no_fold_multiple_uses_of_result
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: no_fold_multiple_uses_of_result
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %c:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 %cmp1, %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %andn:vr = PseudoVMANDN_MM_B32 %cmp1, %and, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %or:vr = PseudoVMOR_MM_B32 %and, %andn, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %or
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %c:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 %cmp1, %cmp2, %avl, 0 /* e8 */
+ %andn:vr = PseudoVMANDN_MM_B32 %cmp1, %and, %avl, 0 /* e8 */
+ %or:vr = PseudoVMOR_MM_B32 %and, %andn, %avl, 0 /* e8 */
+ $v0 = COPY %or
+ PseudoRET implicit $v0
+...
+
+# vmsbf.m shares the operand shape of a compare but its result depends on the
+# mask, so it must not be folded.
+---
+name: no_fold_vmsbf
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9
+ ; CHECK-LABEL: name: no_fold_vmsbf
+ ; CHECK: liveins: $x10, $v8, $v9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: early-clobber %sbf:vr = PseudoVMSBF_M_B32 %a, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %sbf, %b, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %sbf:vr = PseudoVMSBF_M_B32 %a, %avl, 0 /* e8 */
+ %and:vr = PseudoVMAND_MM_B32 killed %sbf, %b, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
>From 921cb0ec9fc13be92e1578025077f676f95d6b51 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 14 Aug 2026 11:48:35 +0800
Subject: [PATCH 02/11] [RISCV] Fold vmand of mask comparisons into a masked
comparison
Fold `vmand(vmsxx(a, b), vmsxx(c, d))` into a single mask-undisturbed
masked comparison in RISCVVectorPeephole:
```
vmsxx ma, va, vb
vmsxx mb, vc, vd
vmand v0, ma, mb
->
vmsxx.vv v0, va, vb
vmsxx.vv v0, vc, vd, v0.t
```
The fold is guarded for correctness and profitability:
- Only comparisons are folded, identified as masked-pseudo-table entries
whose unmasked form has no passthru or policy operand. Mask-dependent
ops like vmsbf.m/viota.m share that shape but are excluded via
elementsDependOnMask, since masking would change their result.
- Comparisons that may raise FP exceptions are skipped, since masking
would drop the exceptions on the inactive elements.
- Only LMUL <= 1 is folded: for larger LMUL the masked comparison's dest
is earlyclobbered into vrnov0 and cannot share v0 with the mask, which
would introduce extra vmv1r.v moves and regress.
- Only single-use vmand results are folded, so the value coalesces onto
v0 for free rather than needing moves for an interior mask value.
Fixes https://github.com/llvm/llvm-project/issues/215832.
Assisted-by: TRAE CLI (Opus 4.8)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 166 ++++++++++++++++++
.../RISCV/rvv/fixed-vectors-fp-setcc.ll | 81 +++++----
.../rvv/fixed-vectors-reduction-int-vp.ll | 30 ++--
.../RISCV/rvv/fixed-vectors-setcc-fp-vp.ll | 23 ++-
.../RISCV/rvv/intrinsic-vector-match.ll | 5 +-
llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll | 102 +++++------
.../RISCV/rvv/vmand-to-masked-compare.ll | 24 ++-
.../RISCV/rvv/vmand-to-masked-compare.mir | 27 +--
llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll | 5 +-
9 files changed, 305 insertions(+), 158 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 2f1b3c409ff2b..a34e4b111a531 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -74,6 +74,7 @@ class RISCVVectorPeephole : public MachineFunctionPass {
bool foldUndefPassthruVMV_V_V(MachineInstr &MI);
bool foldVMV_V_V(MachineInstr &MI);
bool foldVMergeToMask(MachineInstr &MI) const;
+ bool foldVMANDToMaskedCompare(MachineInstr &MI) const;
bool hasSameEEW(const MachineInstr &User, const MachineInstr &Src) const;
bool isAllOnesMask(const MachineInstr *MaskDef) const;
@@ -787,6 +788,168 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
return true;
}
+/// Fold a mask-register AND of a mask comparison into a mask-undisturbed
+/// masked comparison, saving an instruction:
+///
+/// %cmp1 = PseudoVMSLT_VV_M1 %a, %b, %vl, %sew
+/// %cmp2 = PseudoVMSLT_VV_M1 %c, %d, %vl, %sew
+/// %and = PseudoVMAND_MM %cmp1, %cmp2, %vl, 0
+/// ->
+/// %cmp1 = PseudoVMSLT_VV_M1 %a, %b, %vl, %sew
+/// %and = PseudoVMSLT_VV_M1_MASK %cmp1, %c, %d, %cmp1, %vl, %sew, mu
+///
+/// This works because for a mask-undisturbed masked compare whose passthru is
+/// the same register as its mask %m, the result is %m[i] ? (c cmp d)[i] : %m[i],
+/// which is exactly %m[i] & (c cmp d)[i], i.e. vmand(%m, vmscmp(c, d)).
+///
+/// Since vmand is commutative it's enough for either operand to be a foldable
+/// comparison; the other operand becomes both the mask and the passthru.
+bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
+ if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMAND_MM)
+ return false;
+
+ // The masked comparison we create needs its mask (and passthru) in v0, which
+ // the original vmand did not require. If the vmand's result has more than one
+ // use then it is an interior mask value rather than a final result feeding v0,
+ // and introducing the v0 requirement tends to add vmv1r.v moves. Only fold
+ // single-use results, where the value coalesces onto v0 for free.
+ if (!MRI->hasOneUse(MI.getOperand(0).getReg()))
+ return false;
+
+ // Try each operand as the comparison to be masked; the other becomes the
+ // mask/passthru.
+ for (unsigned CmpIdx : {1, 2}) {
+ unsigned MaskIdx = CmpIdx == 1 ? 2 : 1;
+
+ // The comparison must be single use so that folding it into MI doesn't
+ // leave an extra unmasked comparison behind.
+ SmallVector<MachineInstr *, 4> CmpCopies;
+ Register CmpReg = lookThruCopies(MI.getOperand(CmpIdx).getReg(),
+ /*OneUseOnly=*/true, &CmpCopies);
+ if (!CmpReg.isVirtual() || !MRI->hasOneUse(CmpReg))
+ continue;
+ MachineInstr &Cmp = *MRI->getUniqueVRegDef(CmpReg);
+ if (Cmp.getParent() != MI.getParent())
+ continue;
+
+ // Only fold comparisons: entries in the masked pseudo table whose unmasked
+ // form has neither a passthru nor a policy operand. This excludes ops like
+ // vmsbf.m/viota.m which share that shape but whose active elements depend on
+ // the mask, so masking them would change their result.
+ const RISCV::RISCVMaskedPseudoInfo *Info =
+ RISCV::lookupMaskedIntrinsicByUnmasked(Cmp.getOpcode());
+ if (!Info)
+ continue;
+ const MCInstrDesc &UnmaskedDesc = Cmp.getDesc();
+ if (RISCVII::isFirstDefTiedToFirstUse(UnmaskedDesc) ||
+ RISCVII::hasVecPolicyOp(UnmaskedDesc.TSFlags))
+ continue;
+ unsigned CmpMCOpc = RISCV::getRVVMCOpcode(Cmp.getOpcode());
+ if (RISCVII::elementsDependOnMask(TII->get(CmpMCOpc).TSFlags))
+ continue;
+
+ // The EEW of the comparison's dest must match vmand's SEW.
+ if (!hasSameEEW(MI, Cmp))
+ continue;
+
+ // Masking restricts the comparison to the mask's active elements, so any FP
+ // exceptions raised on inactive elements would be lost.
+ if (Cmp.hasUnmodeledSideEffects() || Cmp.mayRaiseFPException())
+ continue;
+
+ // Use the smaller of the two VLs for the result. The comparison's inactive
+ // (and tail) elements will be filled from the mask, so if vmand's VL is the
+ // smaller one we still need the comparison's original VL for the elements
+ // between them to come from the mask rather than the raw comparison.
+ const MachineOperand &CmpVL =
+ Cmp.getOperand(RISCVII::getVLOpNum(Cmp.getDesc()));
+ const MachineOperand &MIVL =
+ MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));
+ MachineOperand MinVL = MachineOperand::CreateImm(0);
+ if (RISCV::isVLKnownLE(CmpVL, MIVL))
+ MinVL = CmpVL;
+ else if (RISCV::isVLKnownLE(MIVL, CmpVL))
+ MinVL = MIVL;
+ else
+ continue;
+
+ const MachineOperand &MaskOp = MI.getOperand(MaskIdx);
+ Register MaskReg = MaskOp.getReg();
+
+ unsigned MaskedOpc = Info->MaskedPseudo;
+ const MCInstrDesc &MaskedDesc = TII->get(MaskedOpc);
+ unsigned SEW = Cmp.getOperand(RISCVII::getSEWOpNum(Cmp.getDesc())).getImm();
+
+ // Only fold if the masked comparison's dest can live in v0. Its mask
+ // operand must be v0, and we reuse the mask as the passthru, so if the dest
+ // can also be v0 the whole thing coalesces onto v0 and we save the vmand for
+ // free. For LMUL >= 2 the dest is earlyclobbered into vrnov0, which would
+ // force extra vmv1r.v moves for the mask and result and make this a
+ // regression, so bail out in that case. This check must happen before we
+ // mutate any instructions below.
+ if (!TII->getRegClass(MaskedDesc, 0)->contains(RISCV::V0))
+ continue;
+
+ // Make sure the mask and VL dominate the comparison, sinking it if needed.
+ if (!ensureDominates({&MaskOp, &MinVL}, Cmp))
+ continue;
+
+ // The masked comparison's mask operand lives in the VMV0 (v0) class, and its
+ // passthru operand shares the dest's class. Copy the vmand mask into both;
+ // the coalescer collapses these back onto v0, matching the two-instruction
+ // ideal.
+ Register MaskV0Reg = MRI->createVirtualRegister(&RISCV::VMV0RegClass);
+ BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
+ TII->get(TargetOpcode::COPY), MaskV0Reg)
+ .addReg(MaskReg);
+ Register PassthruReg =
+ MRI->createVirtualRegister(TII->getRegClass(MaskedDesc, 1));
+ BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
+ TII->get(TargetOpcode::COPY), PassthruReg)
+ .addReg(MaskReg);
+
+ // Build the masked comparison. Its dest reuses vmand's dest; the passthru
+ // (tied to the dest) and mask are both the other vmand operand. Preserve the
+ // source comparison's MI flags (e.g. nofpexcept), which still hold since the
+ // masked comparison operates on a subset of the original active elements.
+ Register DestReg = MI.getOperand(0).getReg();
+ MachineInstr *Masked =
+ BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(), MaskedDesc)
+ .addDef(DestReg)
+ .addReg(PassthruReg)
+ .add(Cmp.getOperand(1))
+ .add(Cmp.getOperand(2))
+ .addReg(MaskV0Reg)
+ .add(MinVL)
+ .addImm(SEW)
+ // The result is a mask register, whose tail is always agnostic, so
+ // we only need mask-undisturbed (MASK_AGNOSTIC clear) to preserve
+ // the inactive elements from the mask/passthru.
+ .addImm(RISCVVType::TAIL_AGNOSTIC)
+ .setMIFlags(Cmp.getFlags());
+
+ // Now that the comparison is masked, constrain its operands to the masked
+ // pseudo's register classes (e.g. vr -> vrnov0 for LMUL >= 2).
+ for (MachineOperand &MO : Masked->explicit_operands()) {
+ if (!MO.isReg() || !MO.getReg().isVirtual())
+ continue;
+ if (const TargetRegisterClass *RC =
+ Masked->getRegClassConstraint(MO.getOperandNo(), TII, TRI))
+ MRI->constrainRegClass(MO.getReg(), RC);
+ }
+ MRI->clearKillFlags(MaskReg);
+ MI.eraseFromParent();
+ Cmp.eraseFromParent();
+ for (MachineInstr *CmpCopy : CmpCopies)
+ if (MRI->use_nodbg_empty(CmpCopy->getOperand(0).getReg()))
+ CmpCopy->eraseFromParent();
+
+ return true;
+ }
+
+ return false;
+}
+
bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -806,6 +969,9 @@ bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
for (MachineInstr &MI : make_early_inc_range(MBB))
Changed |= foldVMergeToMask(MI);
+ for (MachineInstr &MI : make_early_inc_range(MBB))
+ Changed |= foldVMANDToMaskedCompare(MI);
+
for (MachineInstr &MI : make_early_inc_range(MBB)) {
Changed |= convertToVLMAX(MI);
Changed |= convertToUnmasked(MI);
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-setcc.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-setcc.ll
index 65f9e17428b60..269527663ca0c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-setcc.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-setcc.ll
@@ -225,13 +225,12 @@ define void @fcmp_ord_vv_v8bf16(ptr %x, ptr %y, ptr %z) {
;
; ZVFBFA-LABEL: fcmp_ord_vv_v8bf16:
; ZVFBFA: # %bb.0:
-; ZVFBFA-NEXT: vsetivli zero, 8, e16alt, m1, ta, ma
+; ZVFBFA-NEXT: vsetivli zero, 8, e16alt, m1, ta, mu
; ZVFBFA-NEXT: vle16.v v8, (a1)
-; ZVFBFA-NEXT: vle16.v v9, (a0)
-; ZVFBFA-NEXT: vmfeq.vv v8, v8, v8
-; ZVFBFA-NEXT: vmfeq.vv v9, v9, v9
-; ZVFBFA-NEXT: vmand.mm v8, v9, v8
-; ZVFBFA-NEXT: vsm.v v8, (a2)
+; ZVFBFA-NEXT: vmfeq.vv v0, v8, v8
+; ZVFBFA-NEXT: vle16.v v8, (a0)
+; ZVFBFA-NEXT: vmfeq.vv v0, v8, v8, v0.t
+; ZVFBFA-NEXT: vsm.v v0, (a2)
; ZVFBFA-NEXT: ret
%a = load <8 x bfloat>, ptr %x
%b = load <8 x bfloat>, ptr %y
@@ -3060,12 +3059,11 @@ define void @fcmp_one_vv_v8f64_nonans(ptr %x, ptr %y, ptr %z) {
define void @fcmp_ord_vv_v4f16(ptr %x, ptr %y, ptr %z) {
; ZVFH-LABEL: fcmp_ord_vv_v4f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, mu
; ZVFH-NEXT: vle16.v v8, (a1)
-; ZVFH-NEXT: vle16.v v9, (a0)
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmand.mm v0, v9, v8
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8
+; ZVFH-NEXT: vle16.v v8, (a0)
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFH-NEXT: vmv.v.i v8, 0
; ZVFH-NEXT: vmerge.vim v8, v8, 1, v0
@@ -3082,13 +3080,14 @@ define void @fcmp_ord_vv_v4f16(ptr %x, ptr %y, ptr %z) {
; ZVFHMIN: # %bb.0:
; ZVFHMIN-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; ZVFHMIN-NEXT: vle16.v v8, (a1)
-; ZVFHMIN-NEXT: vle16.v v9, (a0)
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT: vmfeq.vv v9, v10, v10
-; ZVFHMIN-NEXT: vmfeq.vv v8, v8, v8
-; ZVFHMIN-NEXT: vmand.mm v0, v8, v9
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT: vle16.v v8, (a0)
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFHMIN-NEXT: vmv.v.i v8, 0
; ZVFHMIN-NEXT: vmerge.vim v8, v8, 1, v0
@@ -5091,12 +5090,11 @@ define void @fcmp_one_vf_v8f64_nonans(ptr %x, double %y, ptr %z) {
define void @fcmp_ord_vf_v4f16(ptr %x, half %y, ptr %z) {
; ZVFH-LABEL: fcmp_ord_vf_v4f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, mu
; ZVFH-NEXT: vfmv.v.f v8, fa0
-; ZVFH-NEXT: vle16.v v9, (a0)
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmfeq.vf v8, v8, fa0
-; ZVFH-NEXT: vmand.mm v0, v9, v8
+; ZVFH-NEXT: vmfeq.vf v0, v8, fa0
+; ZVFH-NEXT: vle16.v v8, (a0)
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFH-NEXT: vmv.v.i v8, 0
; ZVFH-NEXT: vmerge.vim v8, v8, 1, v0
@@ -5113,14 +5111,15 @@ define void @fcmp_ord_vf_v4f16(ptr %x, half %y, ptr %z) {
; ZVFHMIN: # %bb.0:
; ZVFHMIN-NEXT: fmv.x.h a2, fa0
; ZVFHMIN-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
-; ZVFHMIN-NEXT: vle16.v v8, (a0)
-; ZVFHMIN-NEXT: vmv.v.x v9, a2
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT: vmv.v.x v8, a2
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT: vmfeq.vv v9, v10, v10
-; ZVFHMIN-NEXT: vmfeq.vv v8, v8, v8
-; ZVFHMIN-NEXT: vmand.mm v0, v9, v8
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT: vle16.v v8, (a0)
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFHMIN-NEXT: vmv.v.i v8, 0
; ZVFHMIN-NEXT: vmerge.vim v8, v8, 1, v0
@@ -7126,12 +7125,11 @@ define void @fcmp_one_fv_v8f64_nonans(ptr %x, double %y, ptr %z) {
define void @fcmp_ord_fv_v4f16(ptr %x, half %y, ptr %z) {
; ZVFH-LABEL: fcmp_ord_fv_v4f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFH-NEXT: vsetivli zero, 4, e16, mf2, ta, mu
+; ZVFH-NEXT: vle16.v v8, (a0)
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8
; ZVFH-NEXT: vfmv.v.f v8, fa0
-; ZVFH-NEXT: vle16.v v9, (a0)
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmfeq.vf v8, v8, fa0
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vmfeq.vf v0, v8, fa0, v0.t
; ZVFH-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFH-NEXT: vmv.v.i v8, 0
; ZVFH-NEXT: vmerge.vim v8, v8, 1, v0
@@ -7146,16 +7144,17 @@ define void @fcmp_ord_fv_v4f16(ptr %x, half %y, ptr %z) {
;
; ZVFHMIN-LABEL: fcmp_ord_fv_v4f16:
; ZVFHMIN: # %bb.0:
-; ZVFHMIN-NEXT: fmv.x.h a2, fa0
; ZVFHMIN-NEXT: vsetivli zero, 4, e16, mf2, ta, ma
; ZVFHMIN-NEXT: vle16.v v8, (a0)
-; ZVFHMIN-NEXT: vmv.v.x v9, a2
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v10, v8
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v8, v9
+; ZVFHMIN-NEXT: fmv.x.h a0, fa0
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vmv.v.x v8, a0
; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, ma
-; ZVFHMIN-NEXT: vmfeq.vv v9, v10, v10
-; ZVFHMIN-NEXT: vmfeq.vv v8, v8, v8
-; ZVFHMIN-NEXT: vmand.mm v0, v8, v9
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf2, ta, ma
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m1, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; ZVFHMIN-NEXT: vmv.v.i v8, 0
; ZVFHMIN-NEXT: vmerge.vim v8, v8, 1, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
index 4f137a200dea8..742c5282bb847 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
@@ -1204,10 +1204,9 @@ define i8 @vpreduce_mul_v1i8(i8 %s, <1 x i8> %v, <1 x i1> %m, i32 zeroext %evl)
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset ra, -4
-; RV32-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT: vsetivli zero, 1, e32, mf2, ta, mu
; RV32-NEXT: vmv.s.x v9, a1
-; RV32-NEXT: vmsne.vi v9, v9, 0
-; RV32-NEXT: vmand.mm v0, v9, v0
+; RV32-NEXT: vmsne.vi v0, v9, 0, v0.t
; RV32-NEXT: vmv.v.i v9, 1
; RV32-NEXT: vsetvli zero, zero, e8, mf8, ta, ma
; RV32-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1228,10 +1227,9 @@ define i8 @vpreduce_mul_v1i8(i8 %s, <1 x i8> %v, <1 x i1> %m, i32 zeroext %evl)
; RV64-NEXT: .cfi_def_cfa_offset 16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset ra, -8
-; RV64-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-NEXT: vsetivli zero, 1, e32, mf2, ta, mu
; RV64-NEXT: vmv.s.x v9, a1
-; RV64-NEXT: vmsne.vi v9, v9, 0
-; RV64-NEXT: vmand.mm v0, v9, v0
+; RV64-NEXT: vmsne.vi v0, v9, 0, v0.t
; RV64-NEXT: vmv.v.i v9, 1
; RV64-NEXT: vsetvli zero, zero, e8, mf8, ta, ma
; RV64-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1256,10 +1254,9 @@ define signext i8 @vpreduce_mul_v2i8(i8 signext %s, <2 x i8> %v, <2 x i1> %m, i3
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset ra, -4
-; RV32-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT: vsetivli zero, 2, e32, mf2, ta, mu
; RV32-NEXT: vid.v v9
-; RV32-NEXT: vmsltu.vx v9, v9, a1
-; RV32-NEXT: vmand.mm v0, v9, v0
+; RV32-NEXT: vmsltu.vx v0, v9, a1, v0.t
; RV32-NEXT: vsetvli zero, zero, e8, mf8, ta, ma
; RV32-NEXT: vmv.v.i v9, 1
; RV32-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1284,10 +1281,9 @@ define signext i8 @vpreduce_mul_v2i8(i8 signext %s, <2 x i8> %v, <2 x i1> %m, i3
; RV64-NEXT: .cfi_def_cfa_offset 16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset ra, -8
-; RV64-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-NEXT: vsetivli zero, 2, e32, mf2, ta, mu
; RV64-NEXT: vid.v v9
-; RV64-NEXT: vmsltu.vx v9, v9, a1
-; RV64-NEXT: vmand.mm v0, v9, v0
+; RV64-NEXT: vmsltu.vx v0, v9, a1, v0.t
; RV64-NEXT: vsetvli zero, zero, e8, mf8, ta, ma
; RV64-NEXT: vmv.v.i v9, 1
; RV64-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1316,10 +1312,9 @@ define signext i8 @vpreduce_mul_v4i8(i8 signext %s, <4 x i8> %v, <4 x i1> %m, i3
; RV32-NEXT: .cfi_def_cfa_offset 16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
; RV32-NEXT: .cfi_offset ra, -4
-; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT: vsetivli zero, 4, e32, m1, ta, mu
; RV32-NEXT: vid.v v9
-; RV32-NEXT: vmsltu.vx v9, v9, a1
-; RV32-NEXT: vmand.mm v0, v9, v0
+; RV32-NEXT: vmsltu.vx v0, v9, a1, v0.t
; RV32-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; RV32-NEXT: vmv.v.i v9, 1
; RV32-NEXT: vmerge.vvm v8, v9, v8, v0
@@ -1346,10 +1341,9 @@ define signext i8 @vpreduce_mul_v4i8(i8 signext %s, <4 x i8> %v, <4 x i1> %m, i3
; RV64-NEXT: .cfi_def_cfa_offset 16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
; RV64-NEXT: .cfi_offset ra, -8
-; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT: vsetivli zero, 4, e32, m1, ta, mu
; RV64-NEXT: vid.v v9
-; RV64-NEXT: vmsltu.vx v9, v9, a1
-; RV64-NEXT: vmand.mm v0, v9, v0
+; RV64-NEXT: vmsltu.vx v0, v9, a1, v0.t
; RV64-NEXT: vsetvli zero, zero, e8, mf4, ta, ma
; RV64-NEXT: vmv.v.i v9, 1
; RV64-NEXT: vmerge.vvm v8, v9, v8, v0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
index f012e8d01a5b0..183433165b9b8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
@@ -432,10 +432,9 @@ define <8 x i1> @fcmp_one_vf_swap_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i3
define <8 x i1> @fcmp_ord_vv_v8f16(<8 x half> %va, <8 x half> %vb, <8 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vv_v8f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, mu
+; ZVFH-NEXT: vmfeq.vv v0, v9, v9
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vv_v8f16:
@@ -457,11 +456,10 @@ define <8 x i1> @fcmp_ord_vv_v8f16(<8 x half> %va, <8 x half> %vb, <8 x i1> %m,
define <8 x i1> @fcmp_ord_vf_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vf_v8f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, mu
; ZVFH-NEXT: vfmv.v.f v9, fa0
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vf v9, v9, fa0
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vmfeq.vf v0, v9, fa0
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vf_v8f16:
@@ -487,11 +485,10 @@ define <8 x i1> @fcmp_ord_vf_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i32 zer
define <8 x i1> @fcmp_ord_vf_swap_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vf_swap_v8f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; ZVFH-NEXT: vfmv.v.f v9, fa0
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vf v9, v9, fa0
-; ZVFH-NEXT: vmand.mm v0, v9, v8
+; ZVFH-NEXT: vsetivli zero, 8, e16, m1, ta, mu
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8
+; ZVFH-NEXT: vfmv.v.f v8, fa0
+; ZVFH-NEXT: vmfeq.vf v0, v8, fa0, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vf_swap_v8f16:
diff --git a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
index 6febcd22e3288..5165ba8355197 100644
--- a/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/intrinsic-vector-match.ll
@@ -143,10 +143,9 @@ define <vscale x 16 x i1> @match_nxv16i8_v16i8(<vscale x 16 x i8> %op1, <16 x i8
define <16 x i1> @match_v16i8_v1i8(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask) {
; CHECK-LABEL: match_v16i8_v1i8:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, mu
; CHECK-NEXT: vrgather.vi v10, v9, 0
-; CHECK-NEXT: vmseq.vv v8, v8, v10
-; CHECK-NEXT: vmand.mm v0, v8, v0
+; CHECK-NEXT: vmseq.vv v0, v8, v10, v0.t
; CHECK-NEXT: ret
%r = tail call <16 x i1> @llvm.experimental.vector.match(<16 x i8> %op1, <1 x i8> %op2, <16 x i1> %mask)
ret <16 x i1> %r
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index fceb9030891ff..579286382d346 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -281,11 +281,12 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1bf16(<vscale x 1 x bfloat> %va, <vscal
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v10, v9
-; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8
; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT: vmfeq.vv v8, v10, v10
-; CHECK-NEXT: vmfeq.vv v9, v9, v9
-; CHECK-NEXT: vmand.mm v0, v9, v8
+; CHECK-NEXT: vmfeq.vv v0, v10, v10
+; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma
+; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8
+; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v9, v9, v0.t
; CHECK-NEXT: ret
%v = call <vscale x 1 x i1> @llvm.vp.fcmp.nxv1bf16(<vscale x 1 x bfloat> %va, <vscale x 1 x bfloat> %vb, metadata !"ord", <vscale x 1 x i1> %m, i32 %evl)
ret <vscale x 1 x i1> %v
@@ -295,13 +296,13 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
; CHECK-LABEL: fcmp_ord_vf_nxv1bf16:
; CHECK: # %bb.0:
; CHECK-NEXT: fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
+; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
+; CHECK-NEXT: vfmv.v.f v9, fa5
+; CHECK-NEXT: vmfeq.vf v0, v9, fa5
+; CHECK-NEXT: vsetvli zero, zero, e16, mf4, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
-; CHECK-NEXT: vfmv.v.f v8, fa5
-; CHECK-NEXT: vmfeq.vv v9, v9, v9
-; CHECK-NEXT: vmfeq.vf v8, v8, fa5
-; CHECK-NEXT: vmand.mm v0, v9, v8
+; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v9, v9, v0.t
; CHECK-NEXT: ret
%elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
%vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -315,11 +316,10 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, b
; CHECK-NEXT: fcvt.s.bf16 fa5, fa0
; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
; CHECK-NEXT: vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
+; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v9, v9
; CHECK-NEXT: vfmv.v.f v8, fa5
-; CHECK-NEXT: vmfeq.vv v9, v9, v9
-; CHECK-NEXT: vmfeq.vf v8, v8, fa5
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vmfeq.vf v0, v8, fa5, v0.t
; CHECK-NEXT: ret
%elt.head = insertelement <vscale x 1 x bfloat> poison, bfloat %b, i32 0
%vb = shufflevector <vscale x 1 x bfloat> %elt.head, <vscale x 1 x bfloat> poison, <vscale x 1 x i32> zeroinitializer
@@ -1795,21 +1795,21 @@ define <vscale x 1 x i1> @fcmp_one_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vv_nxv1f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
-; ZVFH-NEXT: vmfeq.vv v9, v9, v9
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, mu
+; ZVFH-NEXT: vmfeq.vv v0, v9, v9
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vv_nxv1f16:
; ZVFHMIN: # %bb.0:
; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
; ZVFHMIN-NEXT: vfwcvt.f.f.v v10, v9
-; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT: vmfeq.vv v8, v10, v10
-; ZVFHMIN-NEXT: vmfeq.vv v9, v9, v9
-; ZVFHMIN-NEXT: vmand.mm v0, v9, v8
+; ZVFHMIN-NEXT: vmfeq.vv v0, v10, v10
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: ret
%v = call <vscale x 1 x i1> @llvm.vp.fcmp.nxv1f16(<vscale x 1 x half> %va, <vscale x 1 x half> %vb, metadata !"ord", <vscale x 1 x i1> %m, i32 %evl)
ret <vscale x 1 x i1> %v
@@ -1818,23 +1818,22 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x
define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vf_nxv1f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, mu
; ZVFH-NEXT: vfmv.v.f v9, fa0
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vf v9, v9, fa0
-; ZVFH-NEXT: vmand.mm v0, v8, v9
+; ZVFH-NEXT: vmfeq.vf v0, v9, fa0
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vf_nxv1f16:
; ZVFHMIN: # %bb.0:
; ZVFHMIN-NEXT: fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
+; ZVFHMIN-NEXT: vsetvli a0, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT: vfmv.v.f v9, fa5
+; ZVFHMIN-NEXT: vmfeq.vf v0, v9, fa5
+; ZVFHMIN-NEXT: vsetvli zero, zero, e16, mf4, ta, ma
; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
-; ZVFHMIN-NEXT: vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT: vmfeq.vv v9, v9, v9
-; ZVFHMIN-NEXT: vmfeq.vf v8, v8, fa5
-; ZVFHMIN-NEXT: vmand.mm v0, v9, v8
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9, v0.t
; ZVFHMIN-NEXT: ret
%elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
%vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -1845,11 +1844,10 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; ZVFH-LABEL: fcmp_ord_vf_swap_nxv1f16:
; ZVFH: # %bb.0:
-; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
-; ZVFH-NEXT: vfmv.v.f v9, fa0
-; ZVFH-NEXT: vmfeq.vv v8, v8, v8
-; ZVFH-NEXT: vmfeq.vf v9, v9, fa0
-; ZVFH-NEXT: vmand.mm v0, v9, v8
+; ZVFH-NEXT: vsetvli a0, zero, e16, mf4, ta, mu
+; ZVFH-NEXT: vmfeq.vv v0, v8, v8
+; ZVFH-NEXT: vfmv.v.f v8, fa0
+; ZVFH-NEXT: vmfeq.vf v0, v8, fa0, v0.t
; ZVFH-NEXT: ret
;
; ZVFHMIN-LABEL: fcmp_ord_vf_swap_nxv1f16:
@@ -1857,11 +1855,10 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
; ZVFHMIN-NEXT: fcvt.s.h fa5, fa0
; ZVFHMIN-NEXT: vsetvli a0, zero, e16, mf4, ta, ma
; ZVFHMIN-NEXT: vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT: vsetvli zero, zero, e32, mf2, ta, mu
+; ZVFHMIN-NEXT: vmfeq.vv v0, v9, v9
; ZVFHMIN-NEXT: vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT: vmfeq.vv v9, v9, v9
-; ZVFHMIN-NEXT: vmfeq.vf v8, v8, fa5
-; ZVFHMIN-NEXT: vmand.mm v0, v8, v9
+; ZVFHMIN-NEXT: vmfeq.vf v0, v8, fa5, v0.t
; ZVFHMIN-NEXT: ret
%elt.head = insertelement <vscale x 1 x half> poison, half %b, i32 0
%vb = shufflevector <vscale x 1 x half> %elt.head, <vscale x 1 x half> poison, <vscale x 1 x i32> zeroinitializer
@@ -3634,10 +3631,9 @@ define <vscale x 1 x i1> @fcmp_one_vf_swap_nxv1f64(<vscale x 1 x double> %va, do
define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; CHECK-LABEL: fcmp_ord_vv_nxv1f64:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT: vmfeq.vv v9, v9, v9
-; CHECK-NEXT: vmfeq.vv v8, v8, v8
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v9, v9
+; CHECK-NEXT: vmfeq.vv v0, v8, v8, v0.t
; CHECK-NEXT: ret
%v = call <vscale x 1 x i1> @llvm.vp.fcmp.nxv1f64(<vscale x 1 x double> %va, <vscale x 1 x double> %vb, metadata !"ord", <vscale x 1 x i1> %m, i32 %evl)
ret <vscale x 1 x i1> %v
@@ -3646,11 +3642,10 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f64(<vscale x 1 x double> %va, <vscale
define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; CHECK-LABEL: fcmp_ord_vf_nxv1f64:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, mu
; CHECK-NEXT: vfmv.v.f v9, fa0
-; CHECK-NEXT: vmfeq.vv v8, v8, v8
-; CHECK-NEXT: vmfeq.vf v9, v9, fa0
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vmfeq.vf v0, v9, fa0
+; CHECK-NEXT: vmfeq.vv v0, v8, v8, v0.t
; CHECK-NEXT: ret
%elt.head = insertelement <vscale x 1 x double> poison, double %b, i32 0
%vb = shufflevector <vscale x 1 x double> %elt.head, <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer
@@ -3661,11 +3656,10 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f64(<vscale x 1 x double> %va, double
define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
; CHECK-LABEL: fcmp_ord_vf_swap_nxv1f64:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT: vfmv.v.f v9, fa0
-; CHECK-NEXT: vmfeq.vv v8, v8, v8
-; CHECK-NEXT: vmfeq.vf v9, v9, fa0
-; CHECK-NEXT: vmand.mm v0, v9, v8
+; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, mu
+; CHECK-NEXT: vmfeq.vv v0, v8, v8
+; CHECK-NEXT: vfmv.v.f v8, fa0
+; CHECK-NEXT: vmfeq.vf v0, v8, fa0, v0.t
; CHECK-NEXT: ret
%elt.head = insertelement <vscale x 1 x double> poison, double %b, i32 0
%vb = shufflevector <vscale x 1 x double> %elt.head, <vscale x 1 x double> poison, <vscale x 1 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
index 4cd4940571fc9..ef4b2bff7bea7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
@@ -12,10 +12,9 @@ declare <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1>, <vscal
define <vscale x 2 x i1> @and_icmp(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl) {
; CHECK-LABEL: and_icmp:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT: vmslt.vv v8, v8, v9
-; CHECK-NEXT: vmslt.vv v9, v9, v10
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, mu
+; CHECK-NEXT: vmslt.vv v0, v9, v10
+; CHECK-NEXT: vmslt.vv v0, v8, v9, v0.t
; CHECK-NEXT: ret
%m1 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, i64 %vl)
%m2 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl)
@@ -27,10 +26,9 @@ define <vscale x 2 x i1> @and_icmp(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b,
define <vscale x 2 x i1> @and_fcmp(<vscale x 2 x float> %a, <vscale x 2 x float> %b, <vscale x 2 x float> %c, i64 %vl) {
; CHECK-LABEL: and_fcmp:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT: vmflt.vv v8, v8, v9
-; CHECK-NEXT: vmflt.vv v9, v9, v10
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, mu
+; CHECK-NEXT: vmflt.vv v0, v9, v10
+; CHECK-NEXT: vmflt.vv v0, v8, v9, v0.t
; CHECK-NEXT: ret
%m1 = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float> %a, <vscale x 2 x float> %b, i64 %vl)
%m2 = call <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float> %b, <vscale x 2 x float> %c, i64 %vl)
@@ -42,12 +40,10 @@ define <vscale x 2 x i1> @and_fcmp(<vscale x 2 x float> %a, <vscale x 2 x float>
define <vscale x 2 x i1> @and_icmp_chain(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, <vscale x 2 x i32> %d, i64 %vl) {
; CHECK-LABEL: and_icmp_chain:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT: vmslt.vv v12, v9, v10
-; CHECK-NEXT: vmslt.vv v8, v8, v9
-; CHECK-NEXT: vmslt.vv v9, v10, v11
-; CHECK-NEXT: vmand.mm v8, v8, v12
-; CHECK-NEXT: vmand.mm v0, v8, v9
+; CHECK-NEXT: vsetvli zero, a0, e32, m1, ta, mu
+; CHECK-NEXT: vmslt.vv v0, v9, v10
+; CHECK-NEXT: vmslt.vv v0, v8, v9, v0.t
+; CHECK-NEXT: vmslt.vv v0, v10, v11, v0.t
; CHECK-NEXT: ret
%m1 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, i64 %vl)
%m2 = call <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
index cdaf4832526b0..2537ffd7339b0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
@@ -15,11 +15,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %avl:gprnox0 = COPY $x10
; CHECK-NEXT: %c:vr = COPY $v10
- ; CHECK-NEXT: %b:vr = COPY $v9
- ; CHECK-NEXT: %a:vr = COPY $v8
- ; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
+ ; CHECK-NEXT: %a:vrnov0 = COPY $v8
; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
- ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %cmp2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %cmp2
+ ; CHECK-NEXT: %and:vr = PseudoVMSLT_VV_M1_MASK [[COPY1]], %a, %b, [[COPY]], %avl /* vl */, 5 /* e32 */, 1 /* ta, mu */
; CHECK-NEXT: $v0 = COPY %and
; CHECK-NEXT: PseudoRET implicit $v0
%avl:gprnox0 = COPY $x10
@@ -43,12 +44,13 @@ body: |
; CHECK: liveins: $x10, $v8, $v9, $v10
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %avl:gprnox0 = COPY $x10
- ; CHECK-NEXT: %c:vr = COPY $v10
- ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %c:vrnov0 = COPY $v10
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
; CHECK-NEXT: %a:vr = COPY $v8
; CHECK-NEXT: %cmp1:vr = PseudoVMSLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
- ; CHECK-NEXT: %cmp2:vr = PseudoVMSLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
- ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp2, killed %cmp1, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %cmp1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %cmp1
+ ; CHECK-NEXT: %and:vr = PseudoVMSLT_VV_M1_MASK [[COPY1]], %b, %c, [[COPY]], %avl /* vl */, 5 /* e32 */, 1 /* ta, mu */
; CHECK-NEXT: $v0 = COPY %and
; CHECK-NEXT: PseudoRET implicit $v0
%avl:gprnox0 = COPY $x10
@@ -73,11 +75,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %avl:gprnox0 = COPY $x10
; CHECK-NEXT: %c:vr = COPY $v10
- ; CHECK-NEXT: %b:vr = COPY $v9
- ; CHECK-NEXT: %a:vr = COPY $v8
- ; CHECK-NEXT: %cmp1:vr = nofpexcept PseudoVMFLT_VV_M1 %a, %b, %avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
+ ; CHECK-NEXT: %a:vrnov0 = COPY $v8
; CHECK-NEXT: %cmp2:vr = nofpexcept PseudoVMFLT_VV_M1 %b, %c, %avl /* vl */, 5 /* e32 */
- ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp1, killed %cmp2, %avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %cmp2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %cmp2
+ ; CHECK-NEXT: %and:vr = nofpexcept PseudoVMFLT_VV_M1_MASK [[COPY1]], %a, %b, [[COPY]], %avl /* vl */, 5 /* e32 */, 1 /* ta, mu */
; CHECK-NEXT: $v0 = COPY %and
; CHECK-NEXT: PseudoRET implicit $v0
%avl:gprnox0 = COPY $x10
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index c1a7435849221..8bf644969e907 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -72,9 +72,8 @@ entry:
define <vscale x 1 x i1> @test5(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1, <vscale x 1 x i1> %2, i64 %avl) nounwind {
; CHECK-LABEL: test5:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT: vmseq.vv v8, v8, v9
-; CHECK-NEXT: vmand.mm v0, v8, v0
+; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, mu
+; CHECK-NEXT: vmseq.vv v0, v8, v9, v0.t
; CHECK-NEXT: ret
entry:
%vl = tail call i64 @llvm.riscv.vsetvli(i64 %avl, i64 3, i64 0)
>From 16179dfc2f796e88a90d1f7f3791fe19d3e2e2f2 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 14 Aug 2026 15:41:45 +0800
Subject: [PATCH 03/11] clang-format
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 35 ++++++++++---------
1 file changed, 18 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index a34e4b111a531..dbc66984de5d4 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -799,8 +799,8 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
/// %and = PseudoVMSLT_VV_M1_MASK %cmp1, %c, %d, %cmp1, %vl, %sew, mu
///
/// This works because for a mask-undisturbed masked compare whose passthru is
-/// the same register as its mask %m, the result is %m[i] ? (c cmp d)[i] : %m[i],
-/// which is exactly %m[i] & (c cmp d)[i], i.e. vmand(%m, vmscmp(c, d)).
+/// the same register as its mask %m, the result is %m[i] ? (c cmp d)[i] :
+/// %m[i], which is exactly %m[i] & (c cmp d)[i], i.e. vmand(%m, vmscmp(c, d)).
///
/// Since vmand is commutative it's enough for either operand to be a foldable
/// comparison; the other operand becomes both the mask and the passthru.
@@ -810,9 +810,9 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
// The masked comparison we create needs its mask (and passthru) in v0, which
// the original vmand did not require. If the vmand's result has more than one
- // use then it is an interior mask value rather than a final result feeding v0,
- // and introducing the v0 requirement tends to add vmv1r.v moves. Only fold
- // single-use results, where the value coalesces onto v0 for free.
+ // use then it is an interior mask value rather than a final result feeding
+ // v0, and introducing the v0 requirement tends to add vmv1r.v moves. Only
+ // fold single-use results, where the value coalesces onto v0 for free.
if (!MRI->hasOneUse(MI.getOperand(0).getReg()))
return false;
@@ -834,8 +834,8 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
// Only fold comparisons: entries in the masked pseudo table whose unmasked
// form has neither a passthru nor a policy operand. This excludes ops like
- // vmsbf.m/viota.m which share that shape but whose active elements depend on
- // the mask, so masking them would change their result.
+ // vmsbf.m/viota.m which share that shape but whose active elements depend
+ // on the mask, so masking them would change their result.
const RISCV::RISCVMaskedPseudoInfo *Info =
RISCV::lookupMaskedIntrinsicByUnmasked(Cmp.getOpcode());
if (!Info)
@@ -882,9 +882,9 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
// Only fold if the masked comparison's dest can live in v0. Its mask
// operand must be v0, and we reuse the mask as the passthru, so if the dest
- // can also be v0 the whole thing coalesces onto v0 and we save the vmand for
- // free. For LMUL >= 2 the dest is earlyclobbered into vrnov0, which would
- // force extra vmv1r.v moves for the mask and result and make this a
+ // can also be v0 the whole thing coalesces onto v0 and we save the vmand
+ // for free. For LMUL >= 2 the dest is earlyclobbered into vrnov0, which
+ // would force extra vmv1r.v moves for the mask and result and make this a
// regression, so bail out in that case. This check must happen before we
// mutate any instructions below.
if (!TII->getRegClass(MaskedDesc, 0)->contains(RISCV::V0))
@@ -894,10 +894,10 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
if (!ensureDominates({&MaskOp, &MinVL}, Cmp))
continue;
- // The masked comparison's mask operand lives in the VMV0 (v0) class, and its
- // passthru operand shares the dest's class. Copy the vmand mask into both;
- // the coalescer collapses these back onto v0, matching the two-instruction
- // ideal.
+ // The masked comparison's mask operand lives in the VMV0 (v0) class, and
+ // its passthru operand shares the dest's class. Copy the vmand mask into
+ // both; the coalescer collapses these back onto v0, matching the
+ // two-instruction ideal.
Register MaskV0Reg = MRI->createVirtualRegister(&RISCV::VMV0RegClass);
BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
TII->get(TargetOpcode::COPY), MaskV0Reg)
@@ -909,9 +909,10 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
.addReg(MaskReg);
// Build the masked comparison. Its dest reuses vmand's dest; the passthru
- // (tied to the dest) and mask are both the other vmand operand. Preserve the
- // source comparison's MI flags (e.g. nofpexcept), which still hold since the
- // masked comparison operates on a subset of the original active elements.
+ // (tied to the dest) and mask are both the other vmand operand. Preserve
+ // the source comparison's MI flags (e.g. nofpexcept), which still hold
+ // since the masked comparison operates on a subset of the original active
+ // elements.
Register DestReg = MI.getOperand(0).getReg();
MachineInstr *Masked =
BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(), MaskedDesc)
>From 962b1f7d4a48b53100835a8cde8c36af6d47f5f1 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Mon, 17 Aug 2026 14:33:08 +0800
Subject: [PATCH 04/11] [RISCV] Use BuildMI dest/MIMetadata overload in
foldVMANDToMaskedCompare (NFC)
Address review feedback: build the masked comparison via the
BuildMI(..., MIMetadata(Cmp), MaskedDesc, DestReg) overload instead of a
separate .addDef(DestReg), and use MIMetadata(Cmp) so the new
instruction also inherits the source comparison's PC sections / MMRA
metadata rather than only its debug location.
Assisted-by: TRAE CLI (Opus 4.8)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index dbc66984de5d4..4c42a58442d25 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -915,8 +915,7 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
// elements.
Register DestReg = MI.getOperand(0).getReg();
MachineInstr *Masked =
- BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(), MaskedDesc)
- .addDef(DestReg)
+ BuildMI(*MI.getParent(), Cmp, MIMetadata(Cmp), MaskedDesc, DestReg)
.addReg(PassthruReg)
.add(Cmp.getOperand(1))
.add(Cmp.getOperand(2))
>From dfd21b07acf621af7bf1692ebd753d9394ec2079 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Thu, 20 Aug 2026 17:56:54 +0800
Subject: [PATCH 05/11] [RISCV] Hardcode passthru register class in
foldVMANDToMaskedCompare (NFC)
Address review feedback: the masked comparison's passthru shares the
dest's register class, and the earlier V0 containment check already
restricts folding to LMUL <= 1, so the passthru is always a single
vector register. Hardcode VRRegClass instead of looking it up via
TII->getRegClass.
Assisted-by: TRAE CLI (Opus 4.8)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 4c42a58442d25..2314fddaaa874 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -902,8 +902,9 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
TII->get(TargetOpcode::COPY), MaskV0Reg)
.addReg(MaskReg);
- Register PassthruReg =
- MRI->createVirtualRegister(TII->getRegClass(MaskedDesc, 1));
+ // The passthru shares the dest's class, which the V0 check above restricts
+ // to LMUL <= 1, so it is always a single vector register.
+ Register PassthruReg = MRI->createVirtualRegister(&RISCV::VRRegClass);
BuildMI(*MI.getParent(), Cmp, Cmp.getDebugLoc(),
TII->get(TargetOpcode::COPY), PassthruReg)
.addReg(MaskReg);
>From 3d4749569f80ed2e09604ecc9abfa8f6eedd098d Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 21 Aug 2026 15:14:42 +0800
Subject: [PATCH 06/11] [RISCV] Add VL and copy-chain tests for vmand compare
fold
Add precommit coverage for unequal and unrelated VLs, including the unsafe case where the comparison VL is smaller than the vmand VL. Also cover folding through and cleaning up a multi-level COPY chain.
Assisted-by: TRAE CLI (GPT-5)
---
.../RISCV/rvv/vmand-to-masked-compare.mir | 113 ++++++++++++++++++
1 file changed, 113 insertions(+)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
index 2537ffd7339b0..249237a841f57 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
@@ -64,6 +64,119 @@ body: |
PseudoRET implicit $v0
...
+# The vmand VL can be smaller than the comparison VL. The folded comparison
+# uses the vmand VL.
+---
+name: vmand_vl_lt_cmp_vl
+body: |
+ bb.0:
+ liveins: $v8, $v9, $v10
+ ; CHECK-LABEL: name: vmand_vl_lt_cmp_vl
+ ; CHECK: liveins: $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %mask:vr = COPY $v10
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
+ ; CHECK-NEXT: %a:vrnov0 = COPY $v8
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %mask
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %mask
+ ; CHECK-NEXT: %and:vr = PseudoVMSLT_VV_M1_MASK [[COPY1]], %a, %b, [[COPY]], 2 /* vl */, 5 /* e32 */, 1 /* ta, mu */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %mask:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp:vr = PseudoVMSLT_VV_M1 %a, %b, 4, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp, killed %mask, 2, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# Keep the current behavior when the comparison VL is smaller than the vmand
+# VL.
+---
+name: cmp_vl_lt_vmand_vl
+body: |
+ bb.0:
+ liveins: $v8, $v9, $v10
+ ; CHECK-LABEL: name: cmp_vl_lt_vmand_vl
+ ; CHECK: liveins: $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %mask:vr = COPY $v10
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
+ ; CHECK-NEXT: %a:vrnov0 = COPY $v8
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %mask
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %mask
+ ; CHECK-NEXT: %and:vr = PseudoVMSLT_VV_M1_MASK [[COPY1]], %a, %b, [[COPY]], 2 /* vl */, 5 /* e32 */, 1 /* ta, mu */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %mask:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp:vr = PseudoVMSLT_VV_M1 %a, %b, 2, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp, killed %mask, 4, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# Don't fold when the relative order of the VLs is unknown.
+---
+name: unrelated_vls
+body: |
+ bb.0:
+ liveins: $x10, $x11, $v8, $v9, $v10
+ ; CHECK-LABEL: name: unrelated_vls
+ ; CHECK: liveins: $x10, $x11, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %cmp_avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %and_avl:gprnox0 = COPY $x11
+ ; CHECK-NEXT: %mask:vr = COPY $v10
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp:vr = PseudoVMSLT_VV_M1 %a, %b, %cmp_avl /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp, killed %mask, %and_avl /* vl */, 0 /* e8 */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %cmp_avl:gprnox0 = COPY $x10
+ %and_avl:gprnox0 = COPY $x11
+ %mask:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp:vr = PseudoVMSLT_VV_M1 %a, %b, %cmp_avl, 5 /* e32 */
+ %and:vr = PseudoVMAND_MM_B32 killed %cmp, killed %mask, %and_avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
+# Fold through a chain of single-use copies and remove the copies.
+---
+name: fold_through_copies
+body: |
+ bb.0:
+ liveins: $x10, $v8, $v9, $v10
+ ; CHECK-LABEL: name: fold_through_copies
+ ; CHECK: liveins: $x10, $v8, $v9, $v10
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %avl:gprnox0 = COPY $x10
+ ; CHECK-NEXT: %mask:vr = COPY $v10
+ ; CHECK-NEXT: %b:vrnov0 = COPY $v9
+ ; CHECK-NEXT: %a:vrnov0 = COPY $v8
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %mask
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %mask
+ ; CHECK-NEXT: %and:vr = PseudoVMSLT_VV_M1_MASK [[COPY1]], %a, %b, [[COPY]], %avl /* vl */, 5 /* e32 */, 1 /* ta, mu */
+ ; CHECK-NEXT: $v0 = COPY %and
+ ; CHECK-NEXT: PseudoRET implicit $v0
+ %avl:gprnox0 = COPY $x10
+ %mask:vr = COPY $v10
+ %b:vr = COPY $v9
+ %a:vr = COPY $v8
+ %cmp:vr = PseudoVMSLT_VV_M1 %a, %b, %avl, 5 /* e32 */
+ %copy1:vr = COPY %cmp
+ %copy2:vr = COPY %copy1
+ %and:vr = PseudoVMAND_MM_B32 killed %copy2, killed %mask, %avl, 0 /* e8 */
+ $v0 = COPY %and
+ PseudoRET implicit $v0
+...
+
# A quiet floating-point compare (nofpexcept) can be folded.
---
name: fold_fp_nofpexcept
>From b89b92ff7e17652dc60bd9510f6240cc446bc75a Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 21 Aug 2026 15:16:55 +0800
Subject: [PATCH 07/11] [RISCV] Require vmand VL to be no larger than compare
VL
Mask results are always tail agnostic, so reducing a masked comparison to the original comparison VL cannot preserve lanes that remain active in a larger vmand. Only fold when every active vmand lane is also active in the comparison, and use the vmand VL for the folded instruction.
Assisted-by: TRAE CLI (GPT-5)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 19 +++++++------------
.../RISCV/rvv/vmand-to-masked-compare.mir | 13 ++++++-------
2 files changed, 13 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 2314fddaaa874..6d180fe51014c 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -857,20 +857,15 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
if (Cmp.hasUnmodeledSideEffects() || Cmp.mayRaiseFPException())
continue;
- // Use the smaller of the two VLs for the result. The comparison's inactive
- // (and tail) elements will be filled from the mask, so if vmand's VL is the
- // smaller one we still need the comparison's original VL for the elements
- // between them to come from the mask rather than the raw comparison.
+ // All active elements of vmand must also be active in the comparison. If
+ // the comparison's VL were smaller, elements in between the two VLs would
+ // become tail elements of the masked comparison and could not be preserved
+ // from the mask because mask results are always tail agnostic.
const MachineOperand &CmpVL =
Cmp.getOperand(RISCVII::getVLOpNum(Cmp.getDesc()));
const MachineOperand &MIVL =
MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));
- MachineOperand MinVL = MachineOperand::CreateImm(0);
- if (RISCV::isVLKnownLE(CmpVL, MIVL))
- MinVL = CmpVL;
- else if (RISCV::isVLKnownLE(MIVL, CmpVL))
- MinVL = MIVL;
- else
+ if (!RISCV::isVLKnownLE(MIVL, CmpVL))
continue;
const MachineOperand &MaskOp = MI.getOperand(MaskIdx);
@@ -891,7 +886,7 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
continue;
// Make sure the mask and VL dominate the comparison, sinking it if needed.
- if (!ensureDominates({&MaskOp, &MinVL}, Cmp))
+ if (!ensureDominates({&MaskOp, &MIVL}, Cmp))
continue;
// The masked comparison's mask operand lives in the VMV0 (v0) class, and
@@ -921,7 +916,7 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
.add(Cmp.getOperand(1))
.add(Cmp.getOperand(2))
.addReg(MaskV0Reg)
- .add(MinVL)
+ .add(MIVL)
.addImm(SEW)
// The result is a mask register, whose tail is always agnostic, so
// we only need mask-undisturbed (MASK_AGNOSTIC clear) to preserve
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
index 249237a841f57..8a1efffa75358 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.mir
@@ -91,8 +91,8 @@ body: |
PseudoRET implicit $v0
...
-# Keep the current behavior when the comparison VL is smaller than the vmand
-# VL.
+# Don't fold when the comparison VL is smaller than the vmand VL. Elements
+# between the two VLs are active in vmand but tail agnostic in the comparison.
---
name: cmp_vl_lt_vmand_vl
body: |
@@ -102,11 +102,10 @@ body: |
; CHECK: liveins: $v8, $v9, $v10
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %mask:vr = COPY $v10
- ; CHECK-NEXT: %b:vrnov0 = COPY $v9
- ; CHECK-NEXT: %a:vrnov0 = COPY $v8
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vmv0 = COPY %mask
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY %mask
- ; CHECK-NEXT: %and:vr = PseudoVMSLT_VV_M1_MASK [[COPY1]], %a, %b, [[COPY]], 2 /* vl */, 5 /* e32 */, 1 /* ta, mu */
+ ; CHECK-NEXT: %b:vr = COPY $v9
+ ; CHECK-NEXT: %a:vr = COPY $v8
+ ; CHECK-NEXT: %cmp:vr = PseudoVMSLT_VV_M1 %a, %b, 2 /* vl */, 5 /* e32 */
+ ; CHECK-NEXT: %and:vr = PseudoVMAND_MM_B32 killed %cmp, killed %mask, 4 /* vl */, 0 /* e8 */
; CHECK-NEXT: $v0 = COPY %and
; CHECK-NEXT: PseudoRET implicit $v0
%mask:vr = COPY $v10
>From 5a201cf82e0eb2fddc2004a3ccd173d82b90c980 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 21 Aug 2026 15:19:27 +0800
Subject: [PATCH 08/11] [RISCV] Add an explicit RVV compare predicate
Classify integer, floating-point, and alternate-format vector comparison pseudos with a generated instruction predicate. Use it in foldVMANDToMaskedCompare so the fold does not rely on the current masked-pseudo shape implicitly identifying comparisons.
Assisted-by: TRAE CLI (GPT-5)
---
llvm/lib/Target/RISCV/RISCVInstrPredicates.td | 11 +++++++++++
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 8 ++++----
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrPredicates.td b/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
index cff834150cfeb..d3707718abf90 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
@@ -173,6 +173,17 @@ def isVSlideInstr
!instances<Pseudo>("^PseudoVSLIDEUP_VI.*")
])>>>;
+// Returns true if this is an integer or floating-point vector comparison.
+def isRVVCompare
+ : TIIPredicate<"isRVVCompare",
+ MCReturnStatement<
+ CheckOpcode<!listflatten([
+ !instances<Pseudo>(
+ "^PseudoVMS(EQ|NE|LTU?|LEU?|GTU?)_.*"),
+ !instances<Pseudo>(
+ "^PseudoVMF(EQ|NE|LT|LE|GT|GE)(_ALT)?_.*")
+ ])>>>;
+
def isFaultOnlyFirstLoad
: TIIPredicate<"isFaultOnlyFirstLoad",
MCReturnStatement<
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 6d180fe51014c..8b84ba150ee8f 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -831,11 +831,11 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
MachineInstr &Cmp = *MRI->getUniqueVRegDef(CmpReg);
if (Cmp.getParent() != MI.getParent())
continue;
+ if (!RISCVInstrInfo::isRVVCompare(Cmp))
+ continue;
- // Only fold comparisons: entries in the masked pseudo table whose unmasked
- // form has neither a passthru nor a policy operand. This excludes ops like
- // vmsbf.m/viota.m which share that shape but whose active elements depend
- // on the mask, so masking them would change their result.
+ // Only fold unmasked comparisons whose active elements don't depend on the
+ // mask.
const RISCV::RISCVMaskedPseudoInfo *Info =
RISCV::lookupMaskedIntrinsicByUnmasked(Cmp.getOpcode());
if (!Info)
>From 76a75a8ea52f5947172f689e322930f9d29e2f98 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 21 Aug 2026 15:20:50 +0800
Subject: [PATCH 09/11] [RISCV] Unconditionally remove traversed compare copies
(NFC)
lookThruCopies records each copy only after proving its destination has one non-debug use. After erasing vmand, deleting the copies in traversal order removes that sole use from the next copy, so the use-empty check is redundant.
Assisted-by: TRAE CLI (GPT-5)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 8b84ba150ee8f..1d01888473d19 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -937,8 +937,7 @@ bool RISCVVectorPeephole::foldVMANDToMaskedCompare(MachineInstr &MI) const {
MI.eraseFromParent();
Cmp.eraseFromParent();
for (MachineInstr *CmpCopy : CmpCopies)
- if (MRI->use_nodbg_empty(CmpCopy->getOperand(0).getReg()))
- CmpCopy->eraseFromParent();
+ CmpCopy->eraseFromParent();
return true;
}
>From 835e299a49b3641fe4d72e88c3b7a2f4437556e1 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Fri, 21 Aug 2026 15:21:22 +0800
Subject: [PATCH 10/11] [RISCV] Simplify vmand compare fold test (NFC)
Remove unnecessary intrinsic declarations and the unused Zvfh feature from the test. The test only uses f32 operations, which are already available with V.
Assisted-by: TRAE CLI (GPT-5)
---
llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
index ef4b2bff7bea7..3eba1f6514c35 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmand-to-masked-compare.ll
@@ -1,13 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v -verify-machineinstrs | FileCheck %s
; Test that vmand(vmsxx(a, b), vmsxx(c, d)) is folded into a mask-undisturbed
; masked comparison, saving a vmand.mm. See RISCVVectorPeephole.
-declare <vscale x 2 x i1> @llvm.riscv.vmslt.nxv2i32.nxv2i32.i64(<vscale x 2 x i32>, <vscale x 2 x i32>, i64)
-declare <vscale x 2 x i1> @llvm.riscv.vmflt.nxv2f32.nxv2f32.i64(<vscale x 2 x float>, <vscale x 2 x float>, i64)
-declare <vscale x 2 x i1> @llvm.riscv.vmand.nxv2i1.i64(<vscale x 2 x i1>, <vscale x 2 x i1>, i64)
-
; (a < b) && (b < c)
define <vscale x 2 x i1> @and_icmp(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, i64 %vl) {
; CHECK-LABEL: and_icmp:
>From b158a5044f9ce5dd11e8e2fdd218e4afe003bc2e Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Mon, 24 Aug 2026 12:47:40 +0800
Subject: [PATCH 11/11] [RISCV] Remove redundant compare shape checks (NFC)
The explicit isRVVCompare predicate restricts this fold to vector comparison pseudos. Their unmasked forms have no passthru or policy operand and their elements do not depend on the mask, so the subsequent shape and element-dependence checks are redundant.
Assisted-by: TRAE CLI (GPT-5)
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 10 +---------
1 file changed, 1 insertion(+), 9 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 89f0338bb22af..a07f662b6ff06 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -843,19 +843,11 @@ bool RISCVVectorPeepholeImpl::foldVMANDToMaskedCompare(MachineInstr &MI) const {
if (!RISCVInstrInfo::isRVVCompare(Cmp))
continue;
- // Only fold unmasked comparisons whose active elements don't depend on the
- // mask.
+ // Find the masked pseudo corresponding to the unmasked comparison.
const RISCV::RISCVMaskedPseudoInfo *Info =
RISCV::lookupMaskedIntrinsicByUnmasked(Cmp.getOpcode());
if (!Info)
continue;
- const MCInstrDesc &UnmaskedDesc = Cmp.getDesc();
- if (RISCVII::isFirstDefTiedToFirstUse(UnmaskedDesc) ||
- RISCVII::hasVecPolicyOp(UnmaskedDesc.TSFlags))
- continue;
- unsigned CmpMCOpc = RISCV::getRVVMCOpcode(Cmp.getOpcode());
- if (RISCVII::elementsDependOnMask(TII->get(CmpMCOpc).TSFlags))
- continue;
// The EEW of the comparison's dest must match vmand's SEW.
if (!hasSameEEW(MI, Cmp))
More information about the llvm-commits
mailing list