[llvm] [RISCV] Add missing COPY code motion when folding vmerge into mask (PR #176077)
Min-Yih Hsu via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 14 18:01:59 PST 2026
https://github.com/mshockwave created https://github.com/llvm/llvm-project/pull/176077
Found in #176001 (but unfortunately unrelated to its miscompilation), the following snippet
```
early-clobber %102:vrm8 = PseudoVZEXT_VF8_M8 undef $noreg, killed undef %326, 16, 6 /* e64 */, 3 /* ta, ma */
%123:vrm8nov0 = COPY %102
%124:vmv0 = IMPLICIT_DEF
%121:vrm8nov0 = PseudoVMERGE_VVM_M8 undef $noreg, killed undef %327, killed undef %123, undef %124, 16, 6 /* e64 */
%125:vrm8 = COPY killed %121
BEQ killed undef %325, $x0, %bb.8
PseudoBR %bb.7
```
is turned into
```
%123:vrm8nov0 = COPY %121:vrm8nov0
%124:vmv0 = IMPLICIT_DEF
early-clobber %121:vrm8nov0 = PseudoVZEXT_VF8_M8_MASK undef %327:vrm8nov0(tied-def 0), killed undef %326:vr, %124:vmv0, 16, 6, 1
%125:vrm8 = COPY killed %121:vrm8nov0
BEQ killed undef %325:gpr, $x0, %bb.8
PseudoBR %bb.7
```
by RISC-V Vector Peephole's vmerge folding. This is problematic because `%121` is used before its definition. This was caused by the fact that vector peephole try to sink the new instruction -- in this case `PseudoVZEXT_VF8_M8_MASK` -- until it's dominated by the mask. But we forgot to sink all the COPYs of its result like `%123` as well.
This patch fixes this by sinking those COPYs along with the new instruction when needed. It is worth noting that all the COPYs we collected in this situation are guaranteed to be one-use, so we can sink them easily without violating other def-use relationships.
>From 301b066b6386e24999be38794d0f13a4e25d4335 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Wed, 14 Jan 2026 16:46:19 -0800
Subject: [PATCH 1/2] [RISCV] Add missing COPY code motion when folding vmerge
into mask
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 28 +++++++++++----
.../CodeGen/RISCV/rvv/vmerge-peephole.mir | 34 +++++++++++++++++++
2 files changed, 55 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index a5385be0c011c..9878efe1b76d1 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -73,7 +73,9 @@ class RISCVVectorPeephole : public MachineFunctionPass {
bool isAllOnesMask(const MachineInstr *MaskDef) const;
std::optional<unsigned> getConstant(const MachineOperand &VL) const;
bool ensureDominates(const MachineOperand &Use, MachineInstr &Src) const;
- Register lookThruCopies(Register Reg, bool OneUseOnly = false) const;
+ Register
+ lookThruCopies(Register Reg, bool OneUseOnly = false,
+ SmallVectorImpl<MachineInstr *> *Copies = nullptr) const;
};
} // namespace
@@ -389,8 +391,9 @@ bool RISCVVectorPeephole::convertAllOnesVMergeToVMv(MachineInstr &MI) const {
// If \p Reg is defined by one or more COPYs of virtual registers, traverses
// the chain and returns the root non-COPY source.
-Register RISCVVectorPeephole::lookThruCopies(Register Reg,
- bool OneUseOnly) const {
+Register RISCVVectorPeephole::lookThruCopies(
+ Register Reg, bool OneUseOnly,
+ SmallVectorImpl<MachineInstr *> *Copies) const {
while (MachineInstr *Def = MRI->getUniqueVRegDef(Reg)) {
if (!Def->isFullCopy())
break;
@@ -399,6 +402,8 @@ Register RISCVVectorPeephole::lookThruCopies(Register Reg,
break;
if (OneUseOnly && !MRI->hasOneNonDBGUse(Reg))
break;
+ if (Copies)
+ Copies->push_back(Def);
Reg = Src;
}
return Reg;
@@ -735,10 +740,13 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMERGE_VVM)
return false;
+ // Collect chain of COPYs on True's result in case we need to do code motion
+ // later.
+ SmallVector<MachineInstr *, 4> TrueCopies;
Register PassthruReg = lookThruCopies(MI.getOperand(1).getReg());
Register FalseReg = lookThruCopies(MI.getOperand(2).getReg());
- Register TrueReg =
- lookThruCopies(MI.getOperand(3).getReg(), /*OneUseOnly=*/true);
+ Register TrueReg = lookThruCopies(MI.getOperand(3).getReg(),
+ /*OneUseOnly=*/true, &TrueCopies);
if (!TrueReg.isVirtual() || !MRI->hasOneUse(TrueReg))
return false;
MachineInstr &True = *MRI->getUniqueVRegDef(TrueReg);
@@ -821,8 +829,14 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
assert(RISCVII::hasVecPolicyOp(True.getDesc().TSFlags) &&
"Foldable unmasked pseudo should have a policy op already");
- // Make sure the mask dominates True, otherwise move down True so it does.
- // VL will always dominate since if it's a register they need to be the same.
+ // Make sure the mask dominates True and its copies, otherwise move down True
+ // so it does. VL will always dominate since if it's a register they need to
+ // be the same.
+ for (MachineInstr *TrueCopy : TrueCopies) {
+ [[maybe_unused]] bool CanDominate = ensureDominates(MaskOp, *TrueCopy);
+ assert(CanDominate &&
+ "Mask should always be able to dominate copies of True");
+ }
if (!ensureDominates(MaskOp, True))
return false;
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
index bc78a7732c15a..2cf8ceb9d6b92 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
@@ -181,3 +181,37 @@ body: |
%mask:vmv0 = COPY $v0
PseudoVSE8_V_M1 %copy, $noreg, %avl, 5 /* e8 */
%y:vrnov0 = PseudoVMERGE_VVM_M1 %passthru, %passthru, %copy, %mask, %avl, 5 /* e32 */
+...
+---
+name: true_copy_code_motion
+body: |
+ ; CHECK-LABEL: name: true_copy_code_motion
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF
+ ; CHECK-NEXT: PseudoBR %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vmv0 = IMPLICIT_DEF
+ ; CHECK-NEXT: early-clobber %5:vrm8nov0 = PseudoVZEXT_VF8_M8_MASK undef [[DEF1]], killed undef [[DEF]], [[DEF2]], 16, 6 /* e64 */, 1 /* ta, mu */
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vrm8nov0 = COPY %5
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vrm8 = COPY killed %5
+ ; CHECK-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1
+
+ %326:vr = IMPLICIT_DEF
+ %327:vrm8nov0 = IMPLICIT_DEF
+ PseudoBR %bb.1
+
+ bb.1:
+
+ early-clobber %102:vrm8 = PseudoVZEXT_VF8_M8 undef $noreg, killed undef %326, 16, 6 /* e64 */, 3 /* ta, ma */
+ %123:vrm8nov0 = COPY %102
+ %124:vmv0 = IMPLICIT_DEF
+ %121:vrm8nov0 = PseudoVMERGE_VVM_M8 undef $noreg, killed undef %327, killed undef %123, undef %124, 16, 6 /* e64 */
+ %125:vrm8 = COPY killed %121
+ PseudoRET
+...
>From ba79a00ef868932d64a3e8c4d9bf1bf312992301 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Wed, 14 Jan 2026 18:01:01 -0800
Subject: [PATCH 2/2] fixup! Swap the sinking order
So that there won't be any lingering COPYs if we fail to sink True
instruction after mask.
---
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 9878efe1b76d1..78cf02e020362 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -832,13 +832,13 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
// Make sure the mask dominates True and its copies, otherwise move down True
// so it does. VL will always dominate since if it's a register they need to
// be the same.
- for (MachineInstr *TrueCopy : TrueCopies) {
- [[maybe_unused]] bool CanDominate = ensureDominates(MaskOp, *TrueCopy);
- assert(CanDominate &&
- "Mask should always be able to dominate copies of True");
- }
if (!ensureDominates(MaskOp, True))
return false;
+ for (MachineInstr *TrueCopy : TrueCopies) {
+ [[maybe_unused]] bool CanDominate =
+ ensureDominates(True.getOperand(0), *TrueCopy);
+ assert(CanDominate && "True should always be able to dominate its COPYs");
+ }
if (NeedsCommute) {
auto [OpIdx1, OpIdx2] = *NeedsCommute;
More information about the llvm-commits
mailing list