[llvm] [RISCV] Add missing COPY code motion when folding vmerge into mask (PR #176077)
Min-Yih Hsu via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 14 18:12:13 PST 2026
https://github.com/mshockwave updated https://github.com/llvm/llvm-project/pull/176077
>From 301b066b6386e24999be38794d0f13a4e25d4335 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Wed, 14 Jan 2026 16:46:19 -0800
Subject: [PATCH 1/3] [RISCV] Add missing COPY code motion when folding vmerge
into mask
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 28 +++++++++++----
.../CodeGen/RISCV/rvv/vmerge-peephole.mir | 34 +++++++++++++++++++
2 files changed, 55 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index a5385be0c011c..9878efe1b76d1 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -73,7 +73,9 @@ class RISCVVectorPeephole : public MachineFunctionPass {
bool isAllOnesMask(const MachineInstr *MaskDef) const;
std::optional<unsigned> getConstant(const MachineOperand &VL) const;
bool ensureDominates(const MachineOperand &Use, MachineInstr &Src) const;
- Register lookThruCopies(Register Reg, bool OneUseOnly = false) const;
+ Register
+ lookThruCopies(Register Reg, bool OneUseOnly = false,
+ SmallVectorImpl<MachineInstr *> *Copies = nullptr) const;
};
} // namespace
@@ -389,8 +391,9 @@ bool RISCVVectorPeephole::convertAllOnesVMergeToVMv(MachineInstr &MI) const {
// If \p Reg is defined by one or more COPYs of virtual registers, traverses
// the chain and returns the root non-COPY source.
-Register RISCVVectorPeephole::lookThruCopies(Register Reg,
- bool OneUseOnly) const {
+Register RISCVVectorPeephole::lookThruCopies(
+ Register Reg, bool OneUseOnly,
+ SmallVectorImpl<MachineInstr *> *Copies) const {
while (MachineInstr *Def = MRI->getUniqueVRegDef(Reg)) {
if (!Def->isFullCopy())
break;
@@ -399,6 +402,8 @@ Register RISCVVectorPeephole::lookThruCopies(Register Reg,
break;
if (OneUseOnly && !MRI->hasOneNonDBGUse(Reg))
break;
+ if (Copies)
+ Copies->push_back(Def);
Reg = Src;
}
return Reg;
@@ -735,10 +740,13 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMERGE_VVM)
return false;
+ // Collect chain of COPYs on True's result in case we need to do code motion
+ // later.
+ SmallVector<MachineInstr *, 4> TrueCopies;
Register PassthruReg = lookThruCopies(MI.getOperand(1).getReg());
Register FalseReg = lookThruCopies(MI.getOperand(2).getReg());
- Register TrueReg =
- lookThruCopies(MI.getOperand(3).getReg(), /*OneUseOnly=*/true);
+ Register TrueReg = lookThruCopies(MI.getOperand(3).getReg(),
+ /*OneUseOnly=*/true, &TrueCopies);
if (!TrueReg.isVirtual() || !MRI->hasOneUse(TrueReg))
return false;
MachineInstr &True = *MRI->getUniqueVRegDef(TrueReg);
@@ -821,8 +829,14 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
assert(RISCVII::hasVecPolicyOp(True.getDesc().TSFlags) &&
"Foldable unmasked pseudo should have a policy op already");
- // Make sure the mask dominates True, otherwise move down True so it does.
- // VL will always dominate since if it's a register they need to be the same.
+ // Make sure the mask dominates True and its copies, otherwise move down True
+ // so it does. VL will always dominate since if it's a register they need to
+ // be the same.
+ for (MachineInstr *TrueCopy : TrueCopies) {
+ [[maybe_unused]] bool CanDominate = ensureDominates(MaskOp, *TrueCopy);
+ assert(CanDominate &&
+ "Mask should always be able to dominate copies of True");
+ }
if (!ensureDominates(MaskOp, True))
return false;
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
index bc78a7732c15a..2cf8ceb9d6b92 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
@@ -181,3 +181,37 @@ body: |
%mask:vmv0 = COPY $v0
PseudoVSE8_V_M1 %copy, $noreg, %avl, 5 /* e8 */
%y:vrnov0 = PseudoVMERGE_VVM_M1 %passthru, %passthru, %copy, %mask, %avl, 5 /* e32 */
+...
+---
+name: true_copy_code_motion
+body: |
+ ; CHECK-LABEL: name: true_copy_code_motion
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF
+ ; CHECK-NEXT: PseudoBR %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vmv0 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber %5:vrm8nov0 = PseudoVZEXT_VF8_M8_MASK undef [[DEF1]], killed undef [[DEF]], [[DEF2]], 16, 6 /* e64 */, 1 /* ta, mu */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vrm8nov0 = COPY %5
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vrm8 = COPY killed %5
+ ; CHECK-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1
+
+ %326:vr = IMPLICIT_DEF
+ %327:vrm8nov0 = IMPLICIT_DEF
+ PseudoBR %bb.1
+
+ bb.1:
+
+ early-clobber %102:vrm8 = PseudoVZEXT_VF8_M8 undef $noreg, killed undef %326, 16, 6 /* e64 */, 3 /* ta, ma */
+ %123:vrm8nov0 = COPY %102
+ %124:vmv0 = IMPLICIT_DEF
+ %121:vrm8nov0 = PseudoVMERGE_VVM_M8 undef $noreg, killed undef %327, killed undef %123, undef %124, 16, 6 /* e64 */
+ %125:vrm8 = COPY killed %121
+ PseudoRET
+...
>From ba79a00ef868932d64a3e8c4d9bf1bf312992301 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Wed, 14 Jan 2026 18:01:01 -0800
Subject: [PATCH 2/3] fixup! Swap the sinking order
So that there won't be any lingering COPYs if we fail to sink True
instruction after mask.
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 9878efe1b76d1..78cf02e020362 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -832,13 +832,13 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
// Make sure the mask dominates True and its copies, otherwise move down True
// so it does. VL will always dominate since if it's a register they need to
// be the same.
- for (MachineInstr *TrueCopy : TrueCopies) {
- [[maybe_unused]] bool CanDominate = ensureDominates(MaskOp, *TrueCopy);
- assert(CanDominate &&
- "Mask should always be able to dominate copies of True");
- }
if (!ensureDominates(MaskOp, True))
return false;
+ for (MachineInstr *TrueCopy : TrueCopies) {
+ [[maybe_unused]] bool CanDominate =
+ ensureDominates(True.getOperand(0), *TrueCopy);
+ assert(CanDominate && "True should always be able to dominate its COPYs");
+ }
if (NeedsCommute) {
auto [OpIdx1, OpIdx2] = *NeedsCommute;
>From 8952aaa992168d9c3acc937d14337d63dd167915 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Wed, 14 Jan 2026 18:11:54 -0800
Subject: [PATCH 3/3] fixup! Insert more COPY into the test case
---
llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
index 2cf8ceb9d6b92..a9a4fc135dde1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
@@ -195,9 +195,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1:
; CHECK-NEXT: [[DEF2:%[0-9]+]]:vmv0 = IMPLICIT_DEF
- ; CHECK-NEXT: early-clobber %5:vrm8nov0 = PseudoVZEXT_VF8_M8_MASK undef [[DEF1]], killed undef [[DEF]], [[DEF2]], 16, 6 /* e64 */, 1 /* ta, mu */
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vrm8nov0 = COPY %5
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vrm8 = COPY killed %5
+ ; CHECK-NEXT: early-clobber %7:vrm8nov0 = PseudoVZEXT_VF8_M8_MASK undef [[DEF1]], killed undef [[DEF]], [[DEF2]], 16, 6 /* e64 */, 1 /* ta, mu */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vrm8nov0 = COPY %7
+ ; CHECK-NEXT: %a123:vrm8 = COPY [[COPY]]
+ ; CHECK-NEXT: %b123:vrm8nov0 = COPY %a123
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vrm8 = COPY killed %7
; CHECK-NEXT: PseudoRET
bb.0:
successors: %bb.1
@@ -210,8 +212,10 @@ body: |
early-clobber %102:vrm8 = PseudoVZEXT_VF8_M8 undef $noreg, killed undef %326, 16, 6 /* e64 */, 3 /* ta, ma */
%123:vrm8nov0 = COPY %102
+ %a123:vrm8 = COPY %123
+ %b123:vrm8nov0 = COPY %a123
%124:vmv0 = IMPLICIT_DEF
- %121:vrm8nov0 = PseudoVMERGE_VVM_M8 undef $noreg, killed undef %327, killed undef %123, undef %124, 16, 6 /* e64 */
+ %121:vrm8nov0 = PseudoVMERGE_VVM_M8 undef $noreg, killed undef %327, killed undef %b123, undef %124, 16, 6 /* e64 */
%125:vrm8 = COPY killed %121
PseudoRET
...
More information about the llvm-commits
mailing list