[llvm] b9347e0 - [RISCV] Fold vp.reverse of vp.load through binary ops (#205529)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 04:24:30 PDT 2026
Author: Luke Lau
Date: 2026-07-06T11:24:25Z
New Revision: b9347e0f1bcfa92387fdb57954a081196c655601
URL: https://github.com/llvm/llvm-project/commit/b9347e0f1bcfa92387fdb57954a081196c655601
DIFF: https://github.com/llvm/llvm-project/commit/b9347e0f1bcfa92387fdb57954a081196c655601.diff
LOG: [RISCV] Fold vp.reverse of vp.load through binary ops (#205529)
InstCombine canonicalizes reverses, including vp.reverses, by pulling
them through binary ops: https://godbolt.org/z/cs4M1TsE3
We have a combine that converts vp.reverses of vp.loads into
vp.strided.loads with a stride of -1, but it only matches the pattern
`vp.reverse(vp.load)` directly.
This PR teaches the combine to look through binary ops for vp.loads so
it can match the canonicalized form when there's a binary op that uses
the vp.reverse.
All leaves must be either a vp.load or a splat, but it still will only
convert one vp.load at a time. This combine isn't always profitable so
we should eventually remove it when it's done somewhere that's cost
model driven like the loop vectorizer or vector combine.
Added:
Modified:
llvm/include/llvm/CodeGen/SelectionDAGNodes.h
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index b280bd48c94e9..990b649fafe19 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -234,8 +234,13 @@ class SDValue {
/// Return true if there are no nodes using value ResNo of Node.
inline bool use_empty() const;
- /// Return true if there is exactly one node using value ResNo of Node.
+ /// Return true if there is exactly one node using value ResNo of Node, in
+ /// exactly one operand.
inline bool hasOneUse() const;
+
+ /// Return true if there is exactly one node using value ResNo of Node, in
+ /// potentially multiple operands.
+ inline bool hasOneUser() const;
};
template <> struct DenseMapInfo<SDValue> {
@@ -1318,6 +1323,13 @@ inline bool SDValue::hasOneUse() const {
return Node->hasNUsesOfValue(1, ResNo);
}
+inline bool SDValue::hasOneUser() const {
+ auto Uses = make_filter_range(Node->uses(),
+ [this](SDUse &U) { return U.get() == *this; });
+ auto Users = map_range(Uses, [](SDUse &U) { return U.getUser(); });
+ return all_equal(Users);
+}
+
inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 0c510b0ed74c7..65878eb032541 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -19752,6 +19752,9 @@ static auto m_ReverseEVL = [](auto X, auto EVL) {
m_Node(ISD::EXPERIMENTAL_VP_REVERSE, X, m_Value(), EVL));
};
+// TODO: A vlse.v is not necessarily faster than a vrgather.vv on all uarchs.
+// Remove once a cost model driven transform is implemented in the loop
+// vectorizer.
static SDValue performReverseEVLCombine(SDNode *N, SelectionDAG &DAG,
const RISCVSubtarget &Subtarget) {
// Fold:
@@ -19760,16 +19763,35 @@ static SDValue performReverseEVLCombine(SDNode *N, SelectionDAG &DAG,
//
// splice.right(reverse(vp.load(ADDR, REVMASK, EVL)), poison, EVL)
// -> vp.strided.load(ADDR, -1, MASK, EVL)
-
- // Check if its first operand is a vp.load.
+ //
+ // vp.reverse(binop(vp.load(ADDR, REVMASK, EVL), splat), EVL)
+ // -> binop(vp.strided.load(ADDR, -1, MASK, EVL), splat)
using namespace SDPatternMatch;
SDValue Op, EVL;
- if (!sd_match(N,
- m_ReverseEVL(m_OneUse(m_Value(Op, m_SpecificOpc(ISD::VP_LOAD))),
- m_Value(EVL))))
+ if (!sd_match(N, m_ReverseEVL(m_Value(Op), m_Value(EVL))))
return SDValue();
- auto *VPLoad = cast<VPLoadSDNode>(Op);
+ VPLoadSDNode *VPLoad = nullptr;
+ // Find the single vp_load and check all other leaves are splats.
+ SmallVector<SDValue> Worklist = {Op};
+ while (!Worklist.empty()) {
+ SDValue X = Worklist.pop_back_val();
+ if (!X.hasOneUser())
+ return SDValue();
+ if (auto *VPL = dyn_cast<VPLoadSDNode>(X)) {
+ if (VPLoad && VPLoad != VPL)
+ return SDValue();
+ VPLoad = VPL;
+ } else if (DAG.isSplatValue(X))
+ continue;
+ else if (DAG.getTargetLoweringInfo().isBinOp(X.getOpcode()) &&
+ X->getNumValues() == 1)
+ append_range(Worklist, X->op_values());
+ else
+ return SDValue();
+ }
+ if (!VPLoad)
+ return SDValue();
EVT LoadVT = VPLoad->getValueType(0);
// We do not have a strided_load version for masks, and the evl of vp.reverse
@@ -19812,9 +19834,11 @@ static SDValue performReverseEVLCombine(SDNode *N, SelectionDAG &DAG,
LoadVT, DL, VPLoad->getChain(), Base, Stride, LoadMask,
VPLoad->getVectorLength(), MMO, VPLoad->isExpandingLoad());
- DAG.ReplaceAllUsesOfValueWith(SDValue(VPLoad, 1), Ret.getValue(1));
+ DAG.ReplaceAllUsesWith(VPLoad, Ret.getNode());
- return Ret;
+ // Remove the top level reverse.
+ (void)sd_match(N, m_ReverseEVL(m_Value(Op), m_Value()));
+ return Op;
}
// Fold (i32 (bitcast (v4i8/v2i16 const_splat))) to a scalar i32 constant
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll b/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
index 9bf38753e5054..b7235b78f6175 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-combine-reverse-load.ll
@@ -161,3 +161,155 @@ define <vscale x 2 x float> @test_
diff erent_evl_splice(ptr %ptr, i32 zeroext %ev
%splice = call <vscale x 2 x float> @llvm.vector.splice.right(<vscale x 2 x float> %rev, <vscale x 2 x float> poison, i32 %evl2)
ret <vscale x 2 x float> %splice
}
+
+define <vscale x 2 x float> @binop(ptr %ptr, i32 zeroext %evl) {
+; CHECK-LABEL: binop:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a2, a1, 2
+; CHECK-NEXT: add a0, a2, a0
+; CHECK-NEXT: addi a0, a0, -4
+; CHECK-NEXT: li a2, -4
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vlse32.v v8, (a0), a2
+; CHECK-NEXT: lui a0, 260096
+; CHECK-NEXT: fmv.w.x fa5, a0
+; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vf v8, v8, fa5
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load, splat (float 1.0)
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+define <vscale x 2 x float> @binop_2uses_1user(ptr %ptr, i32 zeroext %evl) {
+; CHECK-LABEL: binop_2uses_1user:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a2, a1, 2
+; CHECK-NEXT: add a0, a2, a0
+; CHECK-NEXT: addi a0, a0, -4
+; CHECK-NEXT: li a2, -4
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vlse32.v v8, (a0), a2
+; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vv v8, v8, v8
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load, %load
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+define <vscale x 2 x float> @binop_chain(ptr %ptr, i32 zeroext %evl) {
+; CHECK-LABEL: binop_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a2, a1, 2
+; CHECK-NEXT: add a2, a2, a0
+; CHECK-NEXT: addi a2, a2, -4
+; CHECK-NEXT: li a3, -4
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vlse32.v v8, (a2), a3
+; CHECK-NEXT: lui a1, 260096
+; CHECK-NEXT: fmv.w.x fa5, a1
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vf v8, v8, fa5
+; CHECK-NEXT: vs1r.v v8, (a0)
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load, splat (float 1.0)
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ store <vscale x 2 x float> %rev, ptr %ptr
+ ret <vscale x 2 x float> %rev
+}
+
+define <vscale x 2 x float> @binop_nested(ptr %ptr, i32 zeroext %evl) {
+; CHECK-LABEL: binop_nested:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a2, a1, 2
+; CHECK-NEXT: add a0, a2, a0
+; CHECK-NEXT: addi a0, a0, -4
+; CHECK-NEXT: li a2, -4
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vlse32.v v8, (a0), a2
+; CHECK-NEXT: lui a0, 263168
+; CHECK-NEXT: fmv.w.x fa5, a0
+; CHECK-NEXT: lui a0, 260096
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfmul.vf v8, v8, fa5
+; CHECK-NEXT: fmv.w.x fa5, a0
+; CHECK-NEXT: vfadd.vf v8, v8, fa5
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fmul = fmul <vscale x 2 x float> %load, splat (float 3.0)
+ %fadd = fadd <vscale x 2 x float> %fmul, splat (float 1.0)
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+; Negative test, we don't want to create more than one vlse.v
+
+define <vscale x 2 x float> @binop_2loads(ptr %ptr1, ptr %ptr2, i32 zeroext %evl) {
+; CHECK-LABEL: binop_2loads:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT: vle32.v v8, (a0)
+; CHECK-NEXT: vid.v v9
+; CHECK-NEXT: vle32.v v10, (a1)
+; CHECK-NEXT: addi a0, a2, -1
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vv v10, v8, v10
+; CHECK-NEXT: vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT: vrsub.vx v9, v9, a0
+; CHECK-NEXT: vrgather.vv v8, v10, v9
+; CHECK-NEXT: ret
+ %load1 = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr1, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %load2 = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr2, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load1, %load2
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+; Negative test, can't combine because a leaf isn't a splat.
+
+define <vscale x 2 x float> @binop_nonsplat(ptr %ptr, <vscale x 2 x float> %v, i32 zeroext %evl) {
+; CHECK-LABEL: binop_nonsplat:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v9
+; CHECK-NEXT: vle32.v v10, (a0)
+; CHECK-NEXT: addi a0, a1, -1
+; CHECK-NEXT: vsetvli a2, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vv v10, v10, v8
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vrsub.vx v9, v9, a0
+; CHECK-NEXT: vrgather.vv v8, v10, v9
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load, %v
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
+
+; Negative test, can't combine because binary op has multiple uses.
+define <vscale x 2 x float> @binop_multiuse(ptr %ptr, i32 zeroext %evl) {
+; CHECK-LABEL: binop_multiuse:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a2, 260096
+; CHECK-NEXT: fmv.w.x fa5, a2
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vle32.v v9, (a0)
+; CHECK-NEXT: addi a2, a1, -1
+; CHECK-NEXT: vsetvli a3, zero, e32, m1, ta, ma
+; CHECK-NEXT: vfadd.vf v9, v9, fa5
+; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT: vrsub.vx v10, v8, a2
+; CHECK-NEXT: vrgather.vv v8, v9, v10
+; CHECK-NEXT: vs1r.v v9, (a0)
+; CHECK-NEXT: ret
+ %load = call <vscale x 2 x float> @llvm.vp.load(ptr %ptr, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ %fadd = fadd <vscale x 2 x float> %load, splat (float 1.0)
+ store <vscale x 2 x float> %fadd, ptr %ptr
+ %rev = call <vscale x 2 x float> @llvm.experimental.vp.reverse(<vscale x 2 x float> %fadd, <vscale x 2 x i1> splat (i1 true), i32 %evl)
+ ret <vscale x 2 x float> %rev
+}
More information about the llvm-commits
mailing list