[llvm] [RISCV] Fold vp.reverse of vp.load through binary ops (PR #205529)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 24 04:39:42 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Luke Lau (lukel97)
<details>
<summary>Changes</summary>
InstCombine canonicalizes reverses, including vp.reverses, by pulling them through binary ops: https://godbolt.org/z/cs4M1TsE3
We have a combine that converts vp.reverses of vp.loads into vp.strided.loads with a stride of -1, but it only matches the pattern `vp.reverse(vp.load)` directly.
This PR teaches the combine to look through binary ops for vp.loads so it can match the canonicalized form when there's a binary op that uses the vp.reverse.
All leaves must be either a vp.load or a binary op. It's worth noting with this PR we may now convert one reverse to two or more reverse strided loads, but I'm assuming this is still profitable over a vrgather.
---
Full diff: https://github.com/llvm/llvm-project/pull/205529.diff
2 Files Affected:
- (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (+67-43)
- (modified) llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll (+126)
``````````diff
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 7a2b9611683c6..8b8134158379d 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -19649,61 +19649,85 @@ static SDValue performReverseEVLCombine(SDNode *N, SelectionDAG &DAG,
//
// splice.right(reverse(vp.load(ADDR, REVMASK, EVL)), poison, EVL)
// -> vp.strided.load(ADDR, -1, MASK, EVL)
-
- // Check if its first operand is a vp.load.
+ //
+ // vp.reverse(binop(vp.load(ADDR, REVMASK, EVL), splat), EVL)
+ // -> binop(vp.strided.load(ADDR, -1, MASK, EVL), splat)
using namespace SDPatternMatch;
SDValue Op, EVL;
- if (!sd_match(N,
- m_ReverseEVL(m_OneUse(m_Value(Op, m_SpecificOpc(ISD::VP_LOAD))),
- m_Value(EVL))))
+ if (!sd_match(N, m_ReverseEVL(m_Value(Op), m_Value(EVL))))
return SDValue();
- auto *VPLoad = cast<VPLoadSDNode>(Op);
-
- EVT LoadVT = VPLoad->getValueType(0);
- // We do not have a strided_load version for masks, and the evl of vp.reverse
- // and vp.load should always be the same.
- if (!LoadVT.getVectorElementType().isByteSized() ||
- EVL != VPLoad->getVectorLength())
- return SDValue();
-
- SDValue LoadMask = VPLoad->getMask();
- // If Mask is all ones, then load is unmasked and can be reversed.
- if (!isOneOrOneSplat(LoadMask)) {
- // If the mask is not all ones, we can reverse the load if the mask was also
- // reversed by a vp.reverse with the same EVL.
- SDValue OrigMask;
- if (!sd_match(LoadMask, m_ReverseEVL(m_Value(OrigMask), m_Specific(EVL))))
+ // Check that all leaves are splats or vp_loads, and collect the latter.
+ SmallVector<SDValue> Worklist = {Op};
+ SmallVector<VPLoadSDNode *> VPLoads;
+ while (!Worklist.empty()) {
+ SDValue X = Worklist.pop_back_val();
+ if (!X.hasOneUse())
+ return SDValue();
+ if (auto *VPLoad = dyn_cast<VPLoadSDNode>(X))
+ VPLoads.push_back(VPLoad);
+ else if (DAG.isSplatValue(X))
+ continue;
+ else if (DAG.getTargetLoweringInfo().isBinOp(X.getOpcode()))
+ append_range(Worklist, X->op_values());
+ else
return SDValue();
- LoadMask = OrigMask;
}
- // Base = LoadAddr + (NumElem - 1) * ElemWidthByte
- SDLoc DL(N);
- MVT XLenVT = Subtarget.getXLenVT();
- SDValue NumElem = VPLoad->getVectorLength();
- uint64_t ElemWidthByte = VPLoad->getValueType(0).getScalarSizeInBits() / 8;
+ SmallVector<SDValue> LoadMasks;
+ for (auto *VPLoad : VPLoads) {
+ EVT LoadVT = VPLoad->getValueType(0);
+ // We do not have a strided_load version for masks, and the evl of
+ // vp.reverse and vp.load should always be the same.
+ if (!LoadVT.getVectorElementType().isByteSized() ||
+ EVL != VPLoad->getVectorLength())
+ return SDValue();
- SDValue Temp1 = DAG.getNode(ISD::SUB, DL, XLenVT, NumElem,
- DAG.getConstant(1, DL, XLenVT));
- SDValue Temp2 = DAG.getNode(ISD::MUL, DL, XLenVT, Temp1,
- DAG.getConstant(ElemWidthByte, DL, XLenVT));
- SDValue Base = DAG.getNode(ISD::ADD, DL, XLenVT, VPLoad->getBasePtr(), Temp2);
- SDValue Stride = DAG.getSignedConstant(-ElemWidthByte, DL, XLenVT);
+ SDValue LoadMask = VPLoad->getMask();
+ // If Mask is all ones, then load is unmasked and can be reversed.
+ if (isOneOrOneSplat(LoadMask))
+ LoadMasks.push_back(LoadMask);
+ else {
+ // If the mask is not all ones, we can reverse the load if the mask was
+ // also reversed by a vp.reverse with the same EVL.
+ SDValue OrigMask;
+ if (!sd_match(LoadMask, m_ReverseEVL(m_Value(OrigMask), m_Specific(EVL))))
+ return SDValue();
+ LoadMasks.push_back(OrigMask);
+ }
+ }
- MachineFunction &MF = DAG.getMachineFunction();
- MachinePointerInfo PtrInfo(VPLoad->getAddressSpace());
- MachineMemOperand *MMO = MF.getMachineMemOperand(
- PtrInfo, VPLoad->getMemOperand()->getFlags(),
- LocationSize::beforeOrAfterPointer(), VPLoad->getAlign());
+ // Reverse the vp_loads.
+ for (auto [VPLoad, LoadMask] : zip_equal(VPLoads, LoadMasks)) {
+ // Base = LoadAddr + (NumElem - 1) * ElemWidthByte
+ SDLoc DL(N);
+ MVT XLenVT = Subtarget.getXLenVT();
+ SDValue NumElem = VPLoad->getVectorLength();
+ uint64_t ElemWidthByte = VPLoad->getValueType(0).getScalarSizeInBits() / 8;
+
+ SDValue Temp1 = DAG.getNode(ISD::SUB, DL, XLenVT, NumElem,
+ DAG.getConstant(1, DL, XLenVT));
+ SDValue Temp2 = DAG.getNode(ISD::MUL, DL, XLenVT, Temp1,
+ DAG.getConstant(ElemWidthByte, DL, XLenVT));
+ SDValue Base =
+ DAG.getNode(ISD::ADD, DL, XLenVT, VPLoad->getBasePtr(), Temp2);
+ SDValue Stride = DAG.getSignedConstant(-ElemWidthByte, DL, XLenVT);
- SDValue Ret = DAG.getStridedLoadVP(
- LoadVT, DL, VPLoad->getChain(), Base, Stride, LoadMask,
- VPLoad->getVectorLength(), MMO, VPLoad->isExpandingLoad());
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachinePointerInfo PtrInfo(VPLoad->getAddressSpace());
+ MachineMemOperand *MMO = MF.getMachineMemOperand(
+ PtrInfo, VPLoad->getMemOperand()->getFlags(),
+ LocationSize::beforeOrAfterPointer(), VPLoad->getAlign());
- DAG.ReplaceAllUsesOfValueWith(SDValue(VPLoad, 1), Ret.getValue(1));
+ SDValue Ret = DAG.getStridedLoadVP(
+ VPLoad->getValueType(0), DL, VPLoad->getChain(), Base, Stride, LoadMask,
+ VPLoad->getVectorLength(), MMO, VPLoad->isExpandingLoad());
+ DAG.ReplaceAllUsesWith(VPLoad, Ret.getNode());
+ }
- return Ret;
+ // Remove the top level reverse.
+ (void)sd_match(N, m_ReverseEVL(m_Value(Op), m_Specific(EVL)));
+ return Op;
}
// Fold (i32 (bitcast (v4i8/v2i16 const_splat))) to a scalar i32 constant
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll b/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
index 9bf38753e5054..ba2725af8250d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
@@ -161,3 +161,129 @@ define <vscale x 2 x float> @test_different_evl_splice(ptr %ptr, i32 zeroext %ev
%splice = call <vscale x 2 x float> @llvm.vector.splice.right(<vscale x 2 x float> %rev, <vscale x 2 x float> poison, i32 %evl2)
ret <vscale x 2 x float> %splice
}
+
+define <vscale x 2 x float> @binop(ptr %ptr, i32 zeroext %evl) {
+; CHECK-LABEL: binop:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a2, a1, 2
+; CHECK-NEXT: add a0, a2, a0
+; CHECK-NEXT: addi a0, a0, -4
+; CHECK-NEXT: li a2, -4
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vlse32.v v8, (a0), a2
+; CHECK-NEXT: lui a0, 260096
+; CHECK-NEXT: fmv.w.x fa5, a0
+; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vf v8, v8, fa5
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load, splat (float 1.0)
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+define <vscale x 2 x float> @binop_nested(ptr %ptr, i32 zeroext %evl) {
+; CHECK-LABEL: binop_nested:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a2, a1, 2
+; CHECK-NEXT: add a0, a2, a0
+; CHECK-NEXT: addi a0, a0, -4
+; CHECK-NEXT: li a2, -4
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vlse32.v v8, (a0), a2
+; CHECK-NEXT: lui a0, 263168
+; CHECK-NEXT: fmv.w.x fa5, a0
+; CHECK-NEXT: lui a0, 260096
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfmul.vf v8, v8, fa5
+; CHECK-NEXT: fmv.w.x fa5, a0
+; CHECK-NEXT: vfadd.vf v8, v8, fa5
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fmul = fmul <vscale x 2 x float> %load, splat (float 3.0)
+ %fadd = fadd <vscale x 2 x float> %fmul, splat (float 1.0)
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+define <vscale x 2 x float> @binop_2loads(ptr %ptr1, ptr %ptr2, i32 zeroext %evl) {
+; CHECK-LABEL: binop_2loads:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a3, a2, 2
+; CHECK-NEXT: addi a3, a3, -4
+; CHECK-NEXT: li a4, -4
+; CHECK-NEXT: add a1, a1, a3
+; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT: vlse32.v v8, (a1), a4
+; CHECK-NEXT: add a0, a0, a3
+; CHECK-NEXT: vlse32.v v9, (a0), a4
+; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vv v8, v9, v8
+; CHECK-NEXT: ret
+ %load1 = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr1, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %load2 = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr2, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load1, %load2
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+define <vscale x 2 x float> @binop_2splats(float %f1, float %f2, i32 zeroext %evl) {
+; CHECK-LABEL: binop_2splats:
+; CHECK: # %bb.0:
+; CHECK-NEXT: fadd.s fa5, fa0, fa1
+; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfmv.v.f v8, fa5
+; CHECK-NEXT: ret
+ %splat1.head = insertelement <vscale x 2 x float> poison, float %f1, i32 0
+ %splat1 = shufflevector <vscale x 2 x float> %splat1.head, <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer
+ %splat2.head = insertelement <vscale x 2 x float> poison, float %f2, i32 0
+ %splat2 = shufflevector <vscale x 2 x float> %splat2.head, <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer
+ %fadd = fadd <vscale x 2 x float> %splat1, %splat2
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+; Negative test, can't combine because a leaf isn't a splat.
+
+define <vscale x 2 x float> @binop_nonsplat(ptr %ptr, <vscale x 2 x float> %v, i32 zeroext %evl) {
+; CHECK-LABEL: binop_nonsplat:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v9
+; CHECK-NEXT: vle32.v v10, (a0)
+; CHECK-NEXT: addi a0, a1, -1
+; CHECK-NEXT: vsetvli a2, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vv v10, v10, v8
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vrsub.vx v9, v9, a0
+; CHECK-NEXT: vrgather.vv v8, v10, v9
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load, %v
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+; Negative test, can't combine because binary op has multiple uses.
+define <vscale x 2 x float> @binop_multiuse(ptr %ptr, i32 zeroext %evl) {
+; CHECK-LABEL: binop_multiuse:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a2, 260096
+; CHECK-NEXT: fmv.w.x fa5, a2
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vle32.v v9, (a0)
+; CHECK-NEXT: addi a2, a1, -1
+; CHECK-NEXT: vsetvli a3, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vf v9, v9, fa5
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vrsub.vx v10, v8, a2
+; CHECK-NEXT: vrgather.vv v8, v9, v10
+; CHECK-NEXT: vs1r.v v9, (a0)
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load, splat (float 1.0)
+ store <vscale x 2 x float> %fadd, ptr %ptr
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/205529
More information about the llvm-commits
mailing list