[llvm] [RISCV] Reassociate scalar accumulator chains of vector reductions (PR #206471)
Pengcheng Wang via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 20:03:30 PDT 2026
https://github.com/wangpc-pp updated https://github.com/llvm/llvm-project/pull/206471
>From 697620ac6c7ccef1f99295d84b965cf90a1be087 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Mon, 29 Jun 2026 20:42:26 +0800
Subject: [PATCH 1/3] Precommit tests
---
.../RISCV/rvv/fixed-vectors-zvdot4a8i.ll | 151 ++++++++++++++++++
.../CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll | 104 ++++++++++++
2 files changed, 255 insertions(+)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
index 92809b3cd6905..d17ad175c8b08 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
@@ -1923,6 +1923,157 @@ entry:
ret i32 %sum
}
+; Two reductions added through a scalar accumulator chain should reassociate
+; so the dot products accumulate into one vector register and only a single
+; reduction remains.
+define i32 @vdot4au_vv_scalar_add_chain2(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x i8> %b1, i32 %acc) {
+; NODOT-LABEL: vdot4au_vv_scalar_add_chain2:
+; NODOT: # %bb.0: # %entry
+; NODOT-NEXT: vsetivli zero, 16, e32, m1, ta, ma
+; NODOT-NEXT: vmv.s.x v14, a0
+; NODOT-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT-NEXT: vwmulu.vv v12, v8, v9
+; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
+; NODOT-NEXT: vwredsumu.vs v12, v12, v14
+; NODOT-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; NODOT-NEXT: vwmulu.vv v8, v10, v11
+; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
+; NODOT-NEXT: vwredsumu.vs v8, v8, v12
+; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT-NEXT: vmv.x.s a0, v8
+; NODOT-NEXT: ret
+;
+; DOT-LABEL: vdot4au_vv_scalar_add_chain2:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vmv.v.i v13, 0
+; DOT-NEXT: vdot4au.vv v13, v8, v9
+; DOT-NEXT: vmv.s.x v8, a0
+; DOT-NEXT: vdot4au.vv v12, v10, v11
+; DOT-NEXT: vredsum.vs v8, v13, v8
+; DOT-NEXT: vredsum.vs v8, v12, v8
+; DOT-NEXT: vmv.x.s a0, v8
+; DOT-NEXT: ret
+entry:
+ %a0.zext = zext <16 x i8> %a0 to <16 x i32>
+ %b0.zext = zext <16 x i8> %b0 to <16 x i32>
+ %mul0 = mul <16 x i32> %a0.zext, %b0.zext
+ %r0 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul0)
+ %a1.zext = zext <16 x i8> %a1 to <16 x i32>
+ %b1.zext = zext <16 x i8> %b1 to <16 x i32>
+ %mul1 = mul <16 x i32> %a1.zext, %b1.zext
+ %r1 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul1)
+ %s0 = add i32 %r0, %acc
+ %s1 = add i32 %r1, %s0
+ ret i32 %s1
+}
+
+; Signed variant exercising the vdot4a.vv path.
+define i32 @vdot4a_vv_scalar_add_chain2(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x i8> %b1, i32 %acc) {
+; NODOT-LABEL: vdot4a_vv_scalar_add_chain2:
+; NODOT: # %bb.0: # %entry
+; NODOT-NEXT: vsetivli zero, 16, e32, m1, ta, ma
+; NODOT-NEXT: vmv.s.x v20, a0
+; NODOT-NEXT: vsetivli zero, 16, e16, m2, ta, ma
+; NODOT-NEXT: vsext.vf2 v16, v8
+; NODOT-NEXT: vsext.vf2 v18, v9
+; NODOT-NEXT: vwmul.vv v12, v16, v18
+; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT-NEXT: vredsum.vs v16, v12, v20
+; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
+; NODOT-NEXT: vsext.vf2 v12, v10
+; NODOT-NEXT: vsext.vf2 v14, v11
+; NODOT-NEXT: vwmul.vv v8, v12, v14
+; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT-NEXT: vredsum.vs v8, v8, v16
+; NODOT-NEXT: vmv.x.s a0, v8
+; NODOT-NEXT: ret
+;
+; DOT-LABEL: vdot4a_vv_scalar_add_chain2:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v12, 0
+; DOT-NEXT: vmv.v.i v13, 0
+; DOT-NEXT: vdot4a.vv v13, v8, v9
+; DOT-NEXT: vmv.s.x v8, a0
+; DOT-NEXT: vdot4a.vv v12, v10, v11
+; DOT-NEXT: vredsum.vs v8, v13, v8
+; DOT-NEXT: vredsum.vs v8, v12, v8
+; DOT-NEXT: vmv.x.s a0, v8
+; DOT-NEXT: ret
+entry:
+ %a0.sext = sext <16 x i8> %a0 to <16 x i32>
+ %b0.sext = sext <16 x i8> %b0 to <16 x i32>
+ %mul0 = mul <16 x i32> %a0.sext, %b0.sext
+ %r0 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul0)
+ %a1.sext = sext <16 x i8> %a1 to <16 x i32>
+ %b1.sext = sext <16 x i8> %b1 to <16 x i32>
+ %mul1 = mul <16 x i32> %a1.sext, %b1.sext
+ %r1 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul1)
+ %s0 = add i32 %r0, %acc
+ %s1 = add i32 %r1, %s0
+ ret i32 %s1
+}
+
+; Four-term chain without an external accumulator (x264 SAD shape).
+define i32 @vdot4au_vv_chain4(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x i8> %b1, <16 x i8> %a2, <16 x i8> %b2, <16 x i8> %a3, <16 x i8> %b3) {
+; NODOT-LABEL: vdot4au_vv_chain4:
+; NODOT: # %bb.0: # %entry
+; NODOT-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; NODOT-NEXT: vwmulu.vv v16, v10, v11
+; NODOT-NEXT: vwmulu.vv v18, v8, v9
+; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
+; NODOT-NEXT: vwaddu.vv v8, v16, v18
+; NODOT-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; NODOT-NEXT: vwmulu.vv v16, v12, v13
+; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
+; NODOT-NEXT: vwaddu.wv v8, v8, v16
+; NODOT-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; NODOT-NEXT: vwmulu.vv v12, v14, v15
+; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
+; NODOT-NEXT: vwaddu.wv v8, v8, v12
+; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT-NEXT: vmv.s.x v12, zero
+; NODOT-NEXT: vredsum.vs v8, v8, v12
+; NODOT-NEXT: vmv.x.s a0, v8
+; NODOT-NEXT: ret
+;
+; DOT-LABEL: vdot4au_vv_chain4:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; DOT-NEXT: vmv.v.i v16, 0
+; DOT-NEXT: vdot4au.vv v16, v10, v11
+; DOT-NEXT: vdot4au.vv v16, v8, v9
+; DOT-NEXT: vdot4au.vv v16, v12, v13
+; DOT-NEXT: vdot4au.vv v16, v14, v15
+; DOT-NEXT: vmv.s.x v8, zero
+; DOT-NEXT: vredsum.vs v8, v16, v8
+; DOT-NEXT: vmv.x.s a0, v8
+; DOT-NEXT: ret
+entry:
+ %a0.zext = zext <16 x i8> %a0 to <16 x i32>
+ %b0.zext = zext <16 x i8> %b0 to <16 x i32>
+ %mul0 = mul <16 x i32> %a0.zext, %b0.zext
+ %r0 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul0)
+ %a1.zext = zext <16 x i8> %a1 to <16 x i32>
+ %b1.zext = zext <16 x i8> %b1 to <16 x i32>
+ %mul1 = mul <16 x i32> %a1.zext, %b1.zext
+ %r1 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul1)
+ %a2.zext = zext <16 x i8> %a2 to <16 x i32>
+ %b2.zext = zext <16 x i8> %b2 to <16 x i32>
+ %mul2 = mul <16 x i32> %a2.zext, %b2.zext
+ %r2 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul2)
+ %a3.zext = zext <16 x i8> %a3 to <16 x i32>
+ %b3.zext = zext <16 x i8> %b3 to <16 x i32>
+ %mul3 = mul <16 x i32> %a3.zext, %b3.zext
+ %r3 = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %mul3)
+ %s0 = add i32 %r1, %r0
+ %s1 = add i32 %r2, %s0
+ %s2 = add i32 %r3, %s1
+ ret i32 %s2
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; DOT32: {{.*}}
; DOT64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
index 6fe0e3c5ad57f..42d006e650f0b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
@@ -1059,6 +1059,110 @@ entry:
ret <vscale x 2 x i32> %res
}
+; Two reductions added through a scalar accumulator chain should reassociate
+; so the dot products accumulate into one vector register and only a single
+; reduction remains.
+define i32 @vdot4au_vv_scalar_add_chain2(<vscale x 16 x i8> %a0, <vscale x 16 x i8> %b0, <vscale x 16 x i8> %a1, <vscale x 16 x i8> %b1, i32 %acc) {
+; NODOT-LABEL: vdot4au_vv_scalar_add_chain2:
+; NODOT: # %bb.0: # %entry
+; NODOT-NEXT: vsetvli a1, zero, e8, m2, ta, ma
+; NODOT-NEXT: vwmulu.vv v16, v8, v10
+; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma
+; NODOT-NEXT: vmv.s.x v8, a0
+; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
+; NODOT-NEXT: vwredsumu.vs v16, v16, v8
+; NODOT-NEXT: vsetvli zero, zero, e8, m2, ta, ma
+; NODOT-NEXT: vwmulu.vv v8, v12, v14
+; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
+; NODOT-NEXT: vwredsumu.vs v8, v8, v16
+; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma
+; NODOT-NEXT: vmv.x.s a0, v8
+; NODOT-NEXT: ret
+;
+; DOT-LABEL: vdot4au_vv_scalar_add_chain2:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a1, zero, e32, m2, ta, ma
+; DOT-NEXT: vmv.v.i v16, 0
+; DOT-NEXT: vmv.v.i v18, 0
+; DOT-NEXT: vdot4au.vv v18, v8, v10
+; DOT-NEXT: vmv.s.x v8, a0
+; DOT-NEXT: vredsum.vs v8, v18, v8
+; DOT-NEXT: vdot4au.vv v16, v12, v14
+; DOT-NEXT: vredsum.vs v8, v16, v8
+; DOT-NEXT: vmv.x.s a0, v8
+; DOT-NEXT: ret
+entry:
+ %a0.zext = zext <vscale x 16 x i8> %a0 to <vscale x 16 x i32>
+ %b0.zext = zext <vscale x 16 x i8> %b0 to <vscale x 16 x i32>
+ %mul0 = mul <vscale x 16 x i32> %a0.zext, %b0.zext
+ %r0 = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul0)
+ %a1.zext = zext <vscale x 16 x i8> %a1 to <vscale x 16 x i32>
+ %b1.zext = zext <vscale x 16 x i8> %b1 to <vscale x 16 x i32>
+ %mul1 = mul <vscale x 16 x i32> %a1.zext, %b1.zext
+ %r1 = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul1)
+ %s0 = add i32 %r0, %acc
+ %s1 = add i32 %r1, %s0
+ ret i32 %s1
+}
+
+; Four-term chain without an external accumulator (x264 SAD shape).
+define i32 @vdot4au_vv_chain4(<vscale x 16 x i8> %a0, <vscale x 16 x i8> %b0, <vscale x 16 x i8> %a1, <vscale x 16 x i8> %b1, <vscale x 16 x i8> %a2, <vscale x 16 x i8> %b2, <vscale x 16 x i8> %a3, <vscale x 16 x i8> %b3) {
+; NODOT-LABEL: vdot4au_vv_chain4:
+; NODOT: # %bb.0: # %entry
+; NODOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma
+; NODOT-NEXT: vwmulu.vv v24, v12, v14
+; NODOT-NEXT: vwmulu.vv v28, v8, v10
+; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
+; NODOT-NEXT: vwaddu.vv v8, v24, v28
+; NODOT-NEXT: vsetvli zero, zero, e8, m2, ta, ma
+; NODOT-NEXT: vwmulu.vv v24, v16, v18
+; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
+; NODOT-NEXT: vwaddu.wv v8, v8, v24
+; NODOT-NEXT: vsetvli zero, zero, e8, m2, ta, ma
+; NODOT-NEXT: vwmulu.vv v16, v20, v22
+; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
+; NODOT-NEXT: vwaddu.wv v8, v8, v16
+; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma
+; NODOT-NEXT: vmv.s.x v16, zero
+; NODOT-NEXT: vredsum.vs v8, v8, v16
+; NODOT-NEXT: vmv.x.s a0, v8
+; NODOT-NEXT: ret
+;
+; DOT-LABEL: vdot4au_vv_chain4:
+; DOT: # %bb.0: # %entry
+; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma
+; DOT-NEXT: vmv.v.i v24, 0
+; DOT-NEXT: vdot4au.vv v24, v12, v14
+; DOT-NEXT: vdot4au.vv v24, v8, v10
+; DOT-NEXT: vdot4au.vv v24, v16, v18
+; DOT-NEXT: vdot4au.vv v24, v20, v22
+; DOT-NEXT: vmv.s.x v8, zero
+; DOT-NEXT: vredsum.vs v8, v24, v8
+; DOT-NEXT: vmv.x.s a0, v8
+; DOT-NEXT: ret
+entry:
+ %a0.zext = zext <vscale x 16 x i8> %a0 to <vscale x 16 x i32>
+ %b0.zext = zext <vscale x 16 x i8> %b0 to <vscale x 16 x i32>
+ %mul0 = mul <vscale x 16 x i32> %a0.zext, %b0.zext
+ %r0 = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul0)
+ %a1.zext = zext <vscale x 16 x i8> %a1 to <vscale x 16 x i32>
+ %b1.zext = zext <vscale x 16 x i8> %b1 to <vscale x 16 x i32>
+ %mul1 = mul <vscale x 16 x i32> %a1.zext, %b1.zext
+ %r1 = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul1)
+ %a2.zext = zext <vscale x 16 x i8> %a2 to <vscale x 16 x i32>
+ %b2.zext = zext <vscale x 16 x i8> %b2 to <vscale x 16 x i32>
+ %mul2 = mul <vscale x 16 x i32> %a2.zext, %b2.zext
+ %r2 = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul2)
+ %a3.zext = zext <vscale x 16 x i8> %a3 to <vscale x 16 x i32>
+ %b3.zext = zext <vscale x 16 x i8> %b3 to <vscale x 16 x i32>
+ %mul3 = mul <vscale x 16 x i32> %a3.zext, %b3.zext
+ %r3 = tail call i32 @llvm.vector.reduce.add.v16i32(<vscale x 16 x i32> %mul3)
+ %s0 = add i32 %r1, %r0
+ %s1 = add i32 %r2, %s0
+ %s2 = add i32 %r3, %s1
+ ret i32 %s2
+}
+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; DOT32: {{.*}}
; DOT64: {{.*}}
>From 3afa8a3ebe7c21d394f67eaf7ab5ca3432064ceb Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Mon, 29 Jun 2026 20:42:33 +0800
Subject: [PATCH 2/3] [RISCV] Reassociate scalar accumulator chains of vector
reductions
A left-leaning scalar add chain of vector reductions, e.g.
`add(reduce(x0), add(reduce(x1), add(reduce(x2), acc)))`, is only partially
collapsed by the generic `DAGCombiner::reassociateReduction`, which folds a
single `add(vecreduce(x), vecreduce(y)) -> vecreduce(add(x, y))`. The cascade
breaks for the chains SLP produces (e.g. x264's SAD), leaving one reduction
per term, and with `Zvdot4a8i` one `vdot4a + vredsum` per term.
In this PR, we add a RISC-V combine that reassociates:
```
add(vecreduce_add(X), add(vecreduce_add(Y), Z))
-> add(vecreduce_add(add(X, Y)), Z)
```
This collapses the whole chain into a single reduction, after which
`foldReduceOperandViaVDOT4A` and `combineVdot4aAccum` chain all dot
products into one accumulator with a single `vredsum`, and the external
accumulator folds in as the reduction start value.
Fixes #168047.
Assisted-by: TraeCli (AI assistant)
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 48 +++++
.../CodeGen/RISCV/rvv/fixed-vectors-sad.ll | 176 +++++++++---------
.../RISCV/rvv/fixed-vectors-zvdot4a8i.ll | 67 +++----
.../CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll | 41 ++--
4 files changed, 184 insertions(+), 148 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e5ce36afb7b2f..37f75be08493f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -16633,6 +16633,52 @@ static SDValue combineAddMulh(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(RISCVISD::MULHSU, DL, VT, X, Mulh.getOperand(1));
}
+// Reassociate a scalar accumulator chain of vector reductions so the
+// reductions become adjacent and can be merged:
+// add(vecreduce_add(X), add(vecreduce_add(Y), Z))
+// -> add(vecreduce_add(add(X, Y)), Z)
+// The generic DAGCombiner only folds add(vecreduce(x), vecreduce(y)) ->
+// vecreduce(add(x, y)) when both operands of a single add are reductions
+// (DAGCombiner::reassociateReduction). SLP often produces a left-leaning
+// scalar chain add(reduce_i, acc) where the cascade breaks, leaving one
+// reduction (and, with Zvdot4a8i, one vredsum) per term. Applied to fixpoint
+// by the combiner worklist, this collapses the whole chain into
+// add(vecreduce_add(BigSum), acc), after which the generic fold and the
+// vdot4a accumulator chaining produce a single reduction.
+static SDValue combineReduceAddChain(SDNode *N, SelectionDAG &DAG,
+ const RISCVSubtarget &Subtarget) {
+ using namespace SDPatternMatch;
+ EVT VT = N->getValueType(0);
+ // Only scalar integer adds; vector reductions feed a scalar result.
+ if (!Subtarget.hasVInstructions() || !VT.isScalarInteger())
+ return SDValue();
+
+ // Match add(vecreduce_add(X), Tail) where Tail is add(vecreduce_add(Y), Z).
+ SDValue X, Y, Z;
+ if (!sd_match(N, m_Add(m_OneUse(m_UnaryOp(ISD::VECREDUCE_ADD, m_Value(X))),
+ m_OneUse(m_Add(m_OneUse(m_UnaryOp(ISD::VECREDUCE_ADD,
+ m_Value(Y))),
+ m_Value(Z))))))
+ return SDValue();
+
+ // The two reductions must be over the same vector type to merge them.
+ EVT SrcVT = X.getValueType();
+ if (SrcVT != Y.getValueType())
+ return SDValue();
+
+ // The add on the source vector type must be available.
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ if (!TLI.isOperationLegalOrCustom(ISD::ADD, SrcVT))
+ return SDValue();
+
+ // Reassociating the wrapping reduction sum invalidates the per-step nsw/nuw
+ // facts, so build the new nodes without flags.
+ SDLoc DL(N);
+ SDValue Sum = DAG.getNode(ISD::ADD, DL, SrcVT, X, Y);
+ SDValue Red = DAG.getNode(ISD::VECREDUCE_ADD, DL, VT, Sum);
+ return DAG.getNode(ISD::ADD, DL, VT, Red, Z);
+}
+
static SDValue performADDCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const RISCVSubtarget &Subtarget) {
@@ -16647,6 +16693,8 @@ static SDValue performADDCombine(SDNode *N,
if (SDValue V = combineShlAddIAdd(N, DAG, Subtarget))
return V;
}
+ if (SDValue V = combineReduceAddChain(N, DAG, Subtarget))
+ return V;
if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
return V;
if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
index 3afa4f18e05dd..85b090ee356f2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
@@ -141,40 +141,40 @@ entry:
define signext i32 @sad_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stridea, i32 signext %strideb) {
; CHECK-LABEL: sad_2block_16xi8_as_i32:
; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: add a4, a0, a2
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vle8.v v9, (a1)
-; CHECK-NEXT: add a0, a0, a2
-; CHECK-NEXT: vle8.v v10, (a0)
-; CHECK-NEXT: add a1, a1, a3
-; CHECK-NEXT: vminu.vv v11, v8, v9
-; CHECK-NEXT: vle8.v v12, (a1)
+; CHECK-NEXT: vle8.v v8, (a4)
+; CHECK-NEXT: add a5, a1, a3
+; CHECK-NEXT: vle8.v v9, (a5)
+; CHECK-NEXT: vminu.vv v10, v8, v9
; CHECK-NEXT: vmaxu.vv v8, v8, v9
-; CHECK-NEXT: add a0, a0, a2
-; CHECK-NEXT: vle8.v v9, (a0)
-; CHECK-NEXT: vsub.vv v8, v8, v11
-; CHECK-NEXT: add a1, a1, a3
-; CHECK-NEXT: vle8.v v11, (a1)
-; CHECK-NEXT: vminu.vv v13, v10, v12
-; CHECK-NEXT: vmaxu.vv v10, v10, v12
-; CHECK-NEXT: vminu.vv v12, v9, v11
-; CHECK-NEXT: vmaxu.vv v9, v9, v11
-; CHECK-NEXT: vsub.vv v10, v10, v13
-; CHECK-NEXT: vsub.vv v9, v9, v12
-; CHECK-NEXT: vwaddu.vv v12, v10, v8
+; CHECK-NEXT: add a4, a4, a2
+; CHECK-NEXT: vle8.v v9, (a4)
+; CHECK-NEXT: add a2, a4, a2
+; CHECK-NEXT: vsub.vv v8, v8, v10
+; CHECK-NEXT: vle8.v v10, (a2)
+; CHECK-NEXT: add a5, a5, a3
; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT: vzext.vf2 v14, v9
-; CHECK-NEXT: add a0, a0, a2
-; CHECK-NEXT: vle8.v v16, (a0)
-; CHECK-NEXT: vwaddu.vv v8, v14, v12
-; CHECK-NEXT: add a1, a1, a3
-; CHECK-NEXT: vle8.v v12, (a1)
+; CHECK-NEXT: vzext.vf2 v12, v8
+; CHECK-NEXT: vle8.v v8, (a5)
+; CHECK-NEXT: add a3, a5, a3
+; CHECK-NEXT: vle8.v v11, (a3)
; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT: vminu.vv v13, v16, v12
-; CHECK-NEXT: vmaxu.vv v12, v16, v12
-; CHECK-NEXT: vsub.vv v14, v12, v13
+; CHECK-NEXT: vminu.vv v14, v9, v8
+; CHECK-NEXT: vmaxu.vv v8, v9, v8
+; CHECK-NEXT: vminu.vv v9, v10, v11
+; CHECK-NEXT: vmaxu.vv v10, v10, v11
+; CHECK-NEXT: vle8.v v11, (a0)
+; CHECK-NEXT: vsub.vv v8, v8, v14
+; CHECK-NEXT: vle8.v v14, (a1)
+; CHECK-NEXT: vsub.vv v9, v10, v9
+; CHECK-NEXT: vminu.vv v10, v11, v14
+; CHECK-NEXT: vmaxu.vv v11, v11, v14
+; CHECK-NEXT: vwaddu.vv v14, v9, v8
+; CHECK-NEXT: vsub.vv v16, v11, v10
; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT: vzext.vf2 v12, v14
+; CHECK-NEXT: vwaddu.vv v8, v14, v12
+; CHECK-NEXT: vzext.vf2 v12, v16
; CHECK-NEXT: vwaddu.wv v8, v8, v12
; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT: vmv.s.x v12, zero
@@ -184,29 +184,31 @@ define signext i32 @sad_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stridea
;
; ZVABD-LABEL: sad_2block_16xi8_as_i32:
; ZVABD: # %bb.0: # %entry
+; ZVABD-NEXT: add a4, a0, a2
; ZVABD-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; ZVABD-NEXT: vle8.v v8, (a0)
-; ZVABD-NEXT: vle8.v v9, (a1)
-; ZVABD-NEXT: add a0, a0, a2
-; ZVABD-NEXT: vle8.v v10, (a0)
+; ZVABD-NEXT: vle8.v v8, (a4)
+; ZVABD-NEXT: add a5, a1, a3
+; ZVABD-NEXT: vle8.v v9, (a5)
; ZVABD-NEXT: vabdu.vv v8, v8, v9
-; ZVABD-NEXT: add a0, a0, a2
-; ZVABD-NEXT: vle8.v v9, (a0)
-; ZVABD-NEXT: add a1, a1, a3
-; ZVABD-NEXT: add a4, a1, a3
+; ZVABD-NEXT: add a4, a4, a2
+; ZVABD-NEXT: vle8.v v9, (a4)
+; ZVABD-NEXT: add a5, a5, a3
; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vzext.vf2 v12, v8
-; ZVABD-NEXT: vle8.v v8, (a4)
-; ZVABD-NEXT: vle8.v v11, (a1)
+; ZVABD-NEXT: vle8.v v8, (a5)
; ZVABD-NEXT: vsetvli zero, zero, e8, m1, ta, ma
; ZVABD-NEXT: vabdu.vv v8, v9, v8
-; ZVABD-NEXT: vwabdau.vv v12, v10, v11
+; ZVABD-NEXT: add a2, a4, a2
+; ZVABD-NEXT: vle8.v v9, (a2)
+; ZVABD-NEXT: add a3, a5, a3
; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vzext.vf2 v14, v8
+; ZVABD-NEXT: vle8.v v8, (a3)
+; ZVABD-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; ZVABD-NEXT: vwabdau.vv v14, v9, v8
+; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vwaddu.vv v8, v14, v12
-; ZVABD-NEXT: add a3, a4, a3
-; ZVABD-NEXT: vle8.v v14, (a3)
-; ZVABD-NEXT: add a0, a0, a2
+; ZVABD-NEXT: vle8.v v14, (a1)
; ZVABD-NEXT: vzext.vf2 v12, v14
; ZVABD-NEXT: vle8.v v16, (a0)
; ZVABD-NEXT: vzext.vf2 v14, v16
@@ -262,40 +264,40 @@ entry:
define signext i32 @sadu_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stridea, i32 signext %strideb) {
; CHECK-LABEL: sadu_2block_16xi8_as_i32:
; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: add a4, a0, a2
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT: vle8.v v8, (a0)
-; CHECK-NEXT: vle8.v v9, (a1)
-; CHECK-NEXT: add a0, a0, a2
-; CHECK-NEXT: vle8.v v10, (a0)
-; CHECK-NEXT: add a1, a1, a3
-; CHECK-NEXT: vmin.vv v11, v8, v9
-; CHECK-NEXT: vle8.v v12, (a1)
+; CHECK-NEXT: vle8.v v8, (a4)
+; CHECK-NEXT: add a5, a1, a3
+; CHECK-NEXT: vle8.v v9, (a5)
+; CHECK-NEXT: vmin.vv v10, v8, v9
; CHECK-NEXT: vmax.vv v8, v8, v9
-; CHECK-NEXT: add a0, a0, a2
-; CHECK-NEXT: vle8.v v9, (a0)
-; CHECK-NEXT: vsub.vv v8, v8, v11
-; CHECK-NEXT: add a1, a1, a3
-; CHECK-NEXT: vle8.v v11, (a1)
-; CHECK-NEXT: vmin.vv v13, v10, v12
-; CHECK-NEXT: vmax.vv v10, v10, v12
-; CHECK-NEXT: vmin.vv v12, v9, v11
-; CHECK-NEXT: vmax.vv v9, v9, v11
-; CHECK-NEXT: vsub.vv v10, v10, v13
-; CHECK-NEXT: vsub.vv v9, v9, v12
-; CHECK-NEXT: vwaddu.vv v12, v10, v8
+; CHECK-NEXT: add a4, a4, a2
+; CHECK-NEXT: vle8.v v9, (a4)
+; CHECK-NEXT: add a2, a4, a2
+; CHECK-NEXT: vsub.vv v8, v8, v10
+; CHECK-NEXT: vle8.v v10, (a2)
+; CHECK-NEXT: add a5, a5, a3
; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT: vzext.vf2 v14, v9
-; CHECK-NEXT: add a0, a0, a2
-; CHECK-NEXT: vle8.v v16, (a0)
-; CHECK-NEXT: vwaddu.vv v8, v14, v12
-; CHECK-NEXT: add a1, a1, a3
-; CHECK-NEXT: vle8.v v12, (a1)
+; CHECK-NEXT: vzext.vf2 v12, v8
+; CHECK-NEXT: vle8.v v8, (a5)
+; CHECK-NEXT: add a3, a5, a3
+; CHECK-NEXT: vle8.v v11, (a3)
; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT: vmin.vv v13, v16, v12
-; CHECK-NEXT: vmax.vv v12, v16, v12
-; CHECK-NEXT: vsub.vv v14, v12, v13
+; CHECK-NEXT: vmin.vv v14, v9, v8
+; CHECK-NEXT: vmax.vv v8, v9, v8
+; CHECK-NEXT: vmin.vv v9, v10, v11
+; CHECK-NEXT: vmax.vv v10, v10, v11
+; CHECK-NEXT: vle8.v v11, (a0)
+; CHECK-NEXT: vsub.vv v8, v8, v14
+; CHECK-NEXT: vle8.v v14, (a1)
+; CHECK-NEXT: vsub.vv v9, v10, v9
+; CHECK-NEXT: vmin.vv v10, v11, v14
+; CHECK-NEXT: vmax.vv v11, v11, v14
+; CHECK-NEXT: vwaddu.vv v14, v9, v8
+; CHECK-NEXT: vsub.vv v16, v11, v10
; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT: vzext.vf2 v12, v14
+; CHECK-NEXT: vwaddu.vv v8, v14, v12
+; CHECK-NEXT: vzext.vf2 v12, v16
; CHECK-NEXT: vwaddu.wv v8, v8, v12
; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT: vmv.s.x v12, zero
@@ -305,29 +307,31 @@ define signext i32 @sadu_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stride
;
; ZVABD-LABEL: sadu_2block_16xi8_as_i32:
; ZVABD: # %bb.0: # %entry
+; ZVABD-NEXT: add a4, a0, a2
; ZVABD-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; ZVABD-NEXT: vle8.v v8, (a0)
-; ZVABD-NEXT: vle8.v v9, (a1)
-; ZVABD-NEXT: add a0, a0, a2
-; ZVABD-NEXT: vle8.v v10, (a0)
+; ZVABD-NEXT: vle8.v v8, (a4)
+; ZVABD-NEXT: add a5, a1, a3
+; ZVABD-NEXT: vle8.v v9, (a5)
; ZVABD-NEXT: vabd.vv v8, v8, v9
-; ZVABD-NEXT: add a0, a0, a2
-; ZVABD-NEXT: vle8.v v9, (a0)
-; ZVABD-NEXT: add a1, a1, a3
-; ZVABD-NEXT: add a4, a1, a3
+; ZVABD-NEXT: add a4, a4, a2
+; ZVABD-NEXT: vle8.v v9, (a4)
+; ZVABD-NEXT: add a5, a5, a3
; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vzext.vf2 v12, v8
-; ZVABD-NEXT: vle8.v v8, (a4)
-; ZVABD-NEXT: vle8.v v11, (a1)
+; ZVABD-NEXT: vle8.v v8, (a5)
; ZVABD-NEXT: vsetvli zero, zero, e8, m1, ta, ma
; ZVABD-NEXT: vabd.vv v8, v9, v8
-; ZVABD-NEXT: vwabda.vv v12, v10, v11
+; ZVABD-NEXT: add a2, a4, a2
+; ZVABD-NEXT: vle8.v v9, (a2)
+; ZVABD-NEXT: add a3, a5, a3
; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vzext.vf2 v14, v8
+; ZVABD-NEXT: vle8.v v8, (a3)
+; ZVABD-NEXT: vsetvli zero, zero, e8, m1, ta, ma
+; ZVABD-NEXT: vwabda.vv v14, v9, v8
+; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vwaddu.vv v8, v14, v12
-; ZVABD-NEXT: add a3, a4, a3
-; ZVABD-NEXT: vle8.v v14, (a3)
-; ZVABD-NEXT: add a0, a0, a2
+; ZVABD-NEXT: vle8.v v14, (a1)
; ZVABD-NEXT: vsext.vf2 v12, v14
; ZVABD-NEXT: vle8.v v16, (a0)
; ZVABD-NEXT: vsext.vf2 v14, v16
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
index d17ad175c8b08..e49c9a7e6136e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
@@ -1929,17 +1929,14 @@ entry:
define i32 @vdot4au_vv_scalar_add_chain2(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x i8> %b1, i32 %acc) {
; NODOT-LABEL: vdot4au_vv_scalar_add_chain2:
; NODOT: # %bb.0: # %entry
-; NODOT-NEXT: vsetivli zero, 16, e32, m1, ta, ma
-; NODOT-NEXT: vmv.s.x v14, a0
; NODOT-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; NODOT-NEXT: vwmulu.vv v12, v8, v9
-; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vwredsumu.vs v12, v12, v14
-; NODOT-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; NODOT-NEXT: vwmulu.vv v8, v10, v11
+; NODOT-NEXT: vwmulu.vv v12, v10, v11
+; NODOT-NEXT: vwmulu.vv v14, v8, v9
; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vwredsumu.vs v8, v8, v12
+; NODOT-NEXT: vwaddu.vv v8, v12, v14
; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
+; NODOT-NEXT: vmv.s.x v12, a0
+; NODOT-NEXT: vredsum.vs v8, v8, v12
; NODOT-NEXT: vmv.x.s a0, v8
; NODOT-NEXT: ret
;
@@ -1947,11 +1944,9 @@ define i32 @vdot4au_vv_scalar_add_chain2(<16 x i8> %a0, <16 x i8> %b0, <16 x i8>
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT: vmv.v.i v12, 0
-; DOT-NEXT: vmv.v.i v13, 0
-; DOT-NEXT: vdot4au.vv v13, v8, v9
-; DOT-NEXT: vmv.s.x v8, a0
; DOT-NEXT: vdot4au.vv v12, v10, v11
-; DOT-NEXT: vredsum.vs v8, v13, v8
+; DOT-NEXT: vdot4au.vv v12, v8, v9
+; DOT-NEXT: vmv.s.x v8, a0
; DOT-NEXT: vredsum.vs v8, v12, v8
; DOT-NEXT: vmv.x.s a0, v8
; DOT-NEXT: ret
@@ -1973,20 +1968,16 @@ entry:
define i32 @vdot4a_vv_scalar_add_chain2(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x i8> %b1, i32 %acc) {
; NODOT-LABEL: vdot4a_vv_scalar_add_chain2:
; NODOT: # %bb.0: # %entry
-; NODOT-NEXT: vsetivli zero, 16, e32, m1, ta, ma
-; NODOT-NEXT: vmv.s.x v20, a0
; NODOT-NEXT: vsetivli zero, 16, e16, m2, ta, ma
-; NODOT-NEXT: vsext.vf2 v16, v8
-; NODOT-NEXT: vsext.vf2 v18, v9
-; NODOT-NEXT: vwmul.vv v12, v16, v18
-; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; NODOT-NEXT: vredsum.vs v16, v12, v20
-; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vsext.vf2 v12, v10
-; NODOT-NEXT: vsext.vf2 v14, v11
-; NODOT-NEXT: vwmul.vv v8, v12, v14
+; NODOT-NEXT: vsext.vf2 v16, v10
+; NODOT-NEXT: vsext.vf2 v18, v8
+; NODOT-NEXT: vsext.vf2 v20, v9
+; NODOT-NEXT: vwmul.vv v12, v18, v20
+; NODOT-NEXT: vsext.vf2 v8, v11
+; NODOT-NEXT: vwmacc.vv v12, v16, v8
; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; NODOT-NEXT: vredsum.vs v8, v8, v16
+; NODOT-NEXT: vmv.s.x v8, a0
+; NODOT-NEXT: vredsum.vs v8, v12, v8
; NODOT-NEXT: vmv.x.s a0, v8
; NODOT-NEXT: ret
;
@@ -1994,11 +1985,9 @@ define i32 @vdot4a_vv_scalar_add_chain2(<16 x i8> %a0, <16 x i8> %b0, <16 x i8>
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT: vmv.v.i v12, 0
-; DOT-NEXT: vmv.v.i v13, 0
-; DOT-NEXT: vdot4a.vv v13, v8, v9
-; DOT-NEXT: vmv.s.x v8, a0
; DOT-NEXT: vdot4a.vv v12, v10, v11
-; DOT-NEXT: vredsum.vs v8, v13, v8
+; DOT-NEXT: vdot4a.vv v12, v8, v9
+; DOT-NEXT: vmv.s.x v8, a0
; DOT-NEXT: vredsum.vs v8, v12, v8
; DOT-NEXT: vmv.x.s a0, v8
; DOT-NEXT: ret
@@ -2021,21 +2010,21 @@ define i32 @vdot4au_vv_chain4(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x
; NODOT-LABEL: vdot4au_vv_chain4:
; NODOT: # %bb.0: # %entry
; NODOT-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; NODOT-NEXT: vwmulu.vv v16, v10, v11
-; NODOT-NEXT: vwmulu.vv v18, v8, v9
+; NODOT-NEXT: vwmulu.vv v16, v14, v15
+; NODOT-NEXT: vwmulu.vv v18, v12, v13
; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vwaddu.vv v8, v16, v18
+; NODOT-NEXT: vwaddu.vv v12, v16, v18
; NODOT-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; NODOT-NEXT: vwmulu.vv v16, v12, v13
+; NODOT-NEXT: vwmulu.vv v16, v10, v11
; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vwaddu.wv v8, v8, v16
+; NODOT-NEXT: vwaddu.wv v12, v12, v16
; NODOT-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; NODOT-NEXT: vwmulu.vv v12, v14, v15
+; NODOT-NEXT: vwmulu.vv v10, v8, v9
; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vwaddu.wv v8, v8, v12
+; NODOT-NEXT: vwaddu.wv v12, v12, v10
; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; NODOT-NEXT: vmv.s.x v12, zero
-; NODOT-NEXT: vredsum.vs v8, v8, v12
+; NODOT-NEXT: vmv.s.x v8, zero
+; NODOT-NEXT: vredsum.vs v8, v12, v8
; NODOT-NEXT: vmv.x.s a0, v8
; NODOT-NEXT: ret
;
@@ -2043,10 +2032,10 @@ define i32 @vdot4au_vv_chain4(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT: vmv.v.i v16, 0
+; DOT-NEXT: vdot4au.vv v16, v14, v15
+; DOT-NEXT: vdot4au.vv v16, v12, v13
; DOT-NEXT: vdot4au.vv v16, v10, v11
; DOT-NEXT: vdot4au.vv v16, v8, v9
-; DOT-NEXT: vdot4au.vv v16, v12, v13
-; DOT-NEXT: vdot4au.vv v16, v14, v15
; DOT-NEXT: vmv.s.x v8, zero
; DOT-NEXT: vredsum.vs v8, v16, v8
; DOT-NEXT: vmv.x.s a0, v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
index 42d006e650f0b..88dd44b6165f6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
@@ -1066,16 +1066,13 @@ define i32 @vdot4au_vv_scalar_add_chain2(<vscale x 16 x i8> %a0, <vscale x 16 x
; NODOT-LABEL: vdot4au_vv_scalar_add_chain2:
; NODOT: # %bb.0: # %entry
; NODOT-NEXT: vsetvli a1, zero, e8, m2, ta, ma
-; NODOT-NEXT: vwmulu.vv v16, v8, v10
-; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma
-; NODOT-NEXT: vmv.s.x v8, a0
-; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
-; NODOT-NEXT: vwredsumu.vs v16, v16, v8
-; NODOT-NEXT: vsetvli zero, zero, e8, m2, ta, ma
-; NODOT-NEXT: vwmulu.vv v8, v12, v14
+; NODOT-NEXT: vwmulu.vv v16, v12, v14
+; NODOT-NEXT: vwmulu.vv v20, v8, v10
; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
-; NODOT-NEXT: vwredsumu.vs v8, v8, v16
+; NODOT-NEXT: vwaddu.vv v8, v16, v20
; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma
+; NODOT-NEXT: vmv.s.x v16, a0
+; NODOT-NEXT: vredsum.vs v8, v8, v16
; NODOT-NEXT: vmv.x.s a0, v8
; NODOT-NEXT: ret
;
@@ -1083,11 +1080,9 @@ define i32 @vdot4au_vv_scalar_add_chain2(<vscale x 16 x i8> %a0, <vscale x 16 x
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a1, zero, e32, m2, ta, ma
; DOT-NEXT: vmv.v.i v16, 0
-; DOT-NEXT: vmv.v.i v18, 0
-; DOT-NEXT: vdot4au.vv v18, v8, v10
-; DOT-NEXT: vmv.s.x v8, a0
-; DOT-NEXT: vredsum.vs v8, v18, v8
; DOT-NEXT: vdot4au.vv v16, v12, v14
+; DOT-NEXT: vdot4au.vv v16, v8, v10
+; DOT-NEXT: vmv.s.x v8, a0
; DOT-NEXT: vredsum.vs v8, v16, v8
; DOT-NEXT: vmv.x.s a0, v8
; DOT-NEXT: ret
@@ -1110,21 +1105,21 @@ define i32 @vdot4au_vv_chain4(<vscale x 16 x i8> %a0, <vscale x 16 x i8> %b0, <v
; NODOT-LABEL: vdot4au_vv_chain4:
; NODOT: # %bb.0: # %entry
; NODOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma
-; NODOT-NEXT: vwmulu.vv v24, v12, v14
-; NODOT-NEXT: vwmulu.vv v28, v8, v10
+; NODOT-NEXT: vwmulu.vv v24, v20, v22
+; NODOT-NEXT: vwmulu.vv v28, v16, v18
; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
-; NODOT-NEXT: vwaddu.vv v8, v24, v28
+; NODOT-NEXT: vwaddu.vv v16, v24, v28
; NODOT-NEXT: vsetvli zero, zero, e8, m2, ta, ma
-; NODOT-NEXT: vwmulu.vv v24, v16, v18
+; NODOT-NEXT: vwmulu.vv v24, v12, v14
; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
-; NODOT-NEXT: vwaddu.wv v8, v8, v24
+; NODOT-NEXT: vwaddu.wv v16, v16, v24
; NODOT-NEXT: vsetvli zero, zero, e8, m2, ta, ma
-; NODOT-NEXT: vwmulu.vv v16, v20, v22
+; NODOT-NEXT: vwmulu.vv v12, v8, v10
; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
-; NODOT-NEXT: vwaddu.wv v8, v8, v16
+; NODOT-NEXT: vwaddu.wv v16, v16, v12
; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma
-; NODOT-NEXT: vmv.s.x v16, zero
-; NODOT-NEXT: vredsum.vs v8, v8, v16
+; NODOT-NEXT: vmv.s.x v8, zero
+; NODOT-NEXT: vredsum.vs v8, v16, v8
; NODOT-NEXT: vmv.x.s a0, v8
; NODOT-NEXT: ret
;
@@ -1132,10 +1127,10 @@ define i32 @vdot4au_vv_chain4(<vscale x 16 x i8> %a0, <vscale x 16 x i8> %b0, <v
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma
; DOT-NEXT: vmv.v.i v24, 0
+; DOT-NEXT: vdot4au.vv v24, v20, v22
+; DOT-NEXT: vdot4au.vv v24, v16, v18
; DOT-NEXT: vdot4au.vv v24, v12, v14
; DOT-NEXT: vdot4au.vv v24, v8, v10
-; DOT-NEXT: vdot4au.vv v24, v16, v18
-; DOT-NEXT: vdot4au.vv v24, v20, v22
; DOT-NEXT: vmv.s.x v8, zero
; DOT-NEXT: vredsum.vs v8, v24, v8
; DOT-NEXT: vmv.x.s a0, v8
>From 46e18138560a408859f19df22188f69a0d2aaf25 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp at bytedance.com>
Date: Wed, 1 Jul 2026 10:42:31 +0800
Subject: [PATCH 3/3] [DAGCombiner] Reassociate chains of vector reductions
reassociateReduction already folds add(vecreduce(x), vecreduce(y)) into
vecreduce(add(x, y)), and the balanced-tree form
add(add(vecreduce(a), b), add(vecreduce(c), d)). It does not, however,
handle a linear chain of reductions like the one SLP emits for x264's
SAD:
add(reduce(X0), add(reduce(X1), add(reduce(X2), acc)))
Only the innermost pair can ever be merged; the cascade breaks and every
reduction survives to lowering, giving one vredsum (or one uadalp step
etc.) per term.
Add a third form to reassociateReduction:
op(vecreduce(X), op(vecreduce(Y), Z)) -> op(vecreduce(op(X, Y)), Z)
Applied to fixpoint by the combiner worklist this collapses any-length
chain into add(vecreduce(BigSum), acc), which the existing target
combines can then fold into a single reduction with the external
accumulator folded in as the reduction start value. On RISC-V with
Zvdot4a8i this recovers the desired single-vredsum, N-dot codegen for
x264 SAD.
Gated on the existing shouldReassociateReduction target hook and only
enabled for FADD/FMUL when all involved nodes carry the reassoc
fast-math flag, matching the existing two forms.
This supersedes the RISC-V-local combineReduceAddChain added in the
previous commit; it is removed here.
Fixes #168047.
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 38 ++++
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 48 -----
llvm/test/CodeGen/AArch64/vecreduce-add.ll | 63 +++----
llvm/test/CodeGen/AArch64/vecreduce-fadd.ll | 26 ++-
llvm/test/CodeGen/AArch64/vecreduce-fmul.ll | 9 +-
.../CodeGen/RISCV/rvv/fixed-vectors-sad.ll | 164 ++++++++----------
.../RISCV/rvv/fixed-vectors-zvdot4a8i.ll | 24 ++-
.../CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll | 22 ++-
8 files changed, 186 insertions(+), 208 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 1b9a62211deaf..39a904ab2cdb2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -1401,6 +1401,44 @@ SDValue DAGCombiner::reassociateReduction(unsigned RedOpc, unsigned Opc,
SDValue Op2 = DAG.getNode(Opc, DL, VT, B, D);
return DAG.getNode(Opc, DL, VT, Red, Op2);
}
+
+ // Reassociate a reduction chain so two reductions become adjacent and the
+ // folds above can merge them:
+ // op(vecreduce(X), op(vecreduce(Y), Z))
+ // -> op(vecreduce(op(X, Y)), Z)
+ // Applied to fixpoint by the combiner worklist, this collapses an
+ // arbitrarily long chain of reductions (such as the left-leaning chain SLP
+ // emits) into a single reduction.
+ auto FoldReductionChain = [&](SDValue Red0, SDValue Chain) -> SDValue {
+ SDValue X, Y, Z, RedX, RedY;
+ if (!sd_match(Red0, m_AllOf(m_OneUse(m_UnaryOp(RedOpc, m_Value(X))),
+ m_Value(RedX))) ||
+ !sd_match(Chain, m_OneUse(m_c_BinOp(
+ Opc,
+ m_AllOf(m_OneUse(m_UnaryOp(RedOpc, m_Value(Y))),
+ m_Value(RedY)),
+ m_Value(Z)))) ||
+ sd_match(Z, m_UnaryOp(RedOpc, m_Value())) ||
+ X.getValueType() != Y.getValueType() ||
+ !hasOperation(Opc, X.getValueType()) ||
+ !TLI.shouldReassociateReduction(RedOpc, VT))
+ return SDValue();
+ if ((Opc == ISD::FADD || Opc == ISD::FMUL) &&
+ (!Chain->getFlags().hasAllowReassociation() ||
+ !RedX->getFlags().hasAllowReassociation() ||
+ !RedY->getFlags().hasAllowReassociation()))
+ return SDValue();
+ SelectionDAG::FlagInserter FlagsInserter(
+ DAG, Flags & Chain->getFlags() & RedX->getFlags() & RedY->getFlags());
+ SDValue Sum = DAG.getNode(Opc, DL, X.getValueType(), X, Y);
+ SDValue Red = DAG.getNode(RedOpc, DL, VT, Sum);
+ return DAG.getNode(Opc, DL, VT, Red, Z);
+ };
+ if (SDValue V = FoldReductionChain(N0, N1))
+ return V;
+ if (SDValue V = FoldReductionChain(N1, N0))
+ return V;
+
return SDValue();
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 37f75be08493f..e5ce36afb7b2f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -16633,52 +16633,6 @@ static SDValue combineAddMulh(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(RISCVISD::MULHSU, DL, VT, X, Mulh.getOperand(1));
}
-// Reassociate a scalar accumulator chain of vector reductions so the
-// reductions become adjacent and can be merged:
-// add(vecreduce_add(X), add(vecreduce_add(Y), Z))
-// -> add(vecreduce_add(add(X, Y)), Z)
-// The generic DAGCombiner only folds add(vecreduce(x), vecreduce(y)) ->
-// vecreduce(add(x, y)) when both operands of a single add are reductions
-// (DAGCombiner::reassociateReduction). SLP often produces a left-leaning
-// scalar chain add(reduce_i, acc) where the cascade breaks, leaving one
-// reduction (and, with Zvdot4a8i, one vredsum) per term. Applied to fixpoint
-// by the combiner worklist, this collapses the whole chain into
-// add(vecreduce_add(BigSum), acc), after which the generic fold and the
-// vdot4a accumulator chaining produce a single reduction.
-static SDValue combineReduceAddChain(SDNode *N, SelectionDAG &DAG,
- const RISCVSubtarget &Subtarget) {
- using namespace SDPatternMatch;
- EVT VT = N->getValueType(0);
- // Only scalar integer adds; vector reductions feed a scalar result.
- if (!Subtarget.hasVInstructions() || !VT.isScalarInteger())
- return SDValue();
-
- // Match add(vecreduce_add(X), Tail) where Tail is add(vecreduce_add(Y), Z).
- SDValue X, Y, Z;
- if (!sd_match(N, m_Add(m_OneUse(m_UnaryOp(ISD::VECREDUCE_ADD, m_Value(X))),
- m_OneUse(m_Add(m_OneUse(m_UnaryOp(ISD::VECREDUCE_ADD,
- m_Value(Y))),
- m_Value(Z))))))
- return SDValue();
-
- // The two reductions must be over the same vector type to merge them.
- EVT SrcVT = X.getValueType();
- if (SrcVT != Y.getValueType())
- return SDValue();
-
- // The add on the source vector type must be available.
- const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- if (!TLI.isOperationLegalOrCustom(ISD::ADD, SrcVT))
- return SDValue();
-
- // Reassociating the wrapping reduction sum invalidates the per-step nsw/nuw
- // facts, so build the new nodes without flags.
- SDLoc DL(N);
- SDValue Sum = DAG.getNode(ISD::ADD, DL, SrcVT, X, Y);
- SDValue Red = DAG.getNode(ISD::VECREDUCE_ADD, DL, VT, Sum);
- return DAG.getNode(ISD::ADD, DL, VT, Red, Z);
-}
-
static SDValue performADDCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const RISCVSubtarget &Subtarget) {
@@ -16693,8 +16647,6 @@ static SDValue performADDCombine(SDNode *N,
if (SDValue V = combineShlAddIAdd(N, DAG, Subtarget))
return V;
}
- if (SDValue V = combineReduceAddChain(N, DAG, Subtarget))
- return V;
if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
return V;
if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index 7ef38401bf76e..995bf28060e8d 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -4421,50 +4421,51 @@ define i32 @full(ptr %p1, i32 noundef %s1, ptr %p2, i32 noundef %s2) {
; CHECK-SD-BASE-NEXT: // kill: def $w1 killed $w1 def $x1
; CHECK-SD-BASE-NEXT: sxtw x8, w3
; CHECK-SD-BASE-NEXT: sxtw x9, w1
-; CHECK-SD-BASE-NEXT: ldr d0, [x0]
-; CHECK-SD-BASE-NEXT: ldr d1, [x2]
+; CHECK-SD-BASE-NEXT: ldr d3, [x0]
+; CHECK-SD-BASE-NEXT: ldr d4, [x2]
; CHECK-SD-BASE-NEXT: add x10, x0, x9
; CHECK-SD-BASE-NEXT: add x11, x2, x8
+; CHECK-SD-BASE-NEXT: add x12, x10, x9
+; CHECK-SD-BASE-NEXT: add x13, x11, x8
+; CHECK-SD-BASE-NEXT: add x14, x12, x9
+; CHECK-SD-BASE-NEXT: add x15, x13, x8
+; CHECK-SD-BASE-NEXT: ldr d5, [x13]
+; CHECK-SD-BASE-NEXT: add x16, x14, x9
+; CHECK-SD-BASE-NEXT: add x17, x15, x8
+; CHECK-SD-BASE-NEXT: add x18, x16, x9
+; CHECK-SD-BASE-NEXT: add x1, x17, x8
+; CHECK-SD-BASE-NEXT: add x3, x18, x9
+; CHECK-SD-BASE-NEXT: add x4, x1, x8
+; CHECK-SD-BASE-NEXT: ldr d0, [x3]
+; CHECK-SD-BASE-NEXT: ldr d1, [x4]
+; CHECK-SD-BASE-NEXT: ldr d2, [x4, x8]
; CHECK-SD-BASE-NEXT: uabdl v0.8h, v0.8b, v1.8b
-; CHECK-SD-BASE-NEXT: ldr d1, [x10]
-; CHECK-SD-BASE-NEXT: ldr d2, [x11]
-; CHECK-SD-BASE-NEXT: add x10, x10, x9
-; CHECK-SD-BASE-NEXT: add x11, x11, x8
+; CHECK-SD-BASE-NEXT: ldr d1, [x3, x9]
; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b
-; CHECK-SD-BASE-NEXT: ldr d2, [x11]
-; CHECK-SD-BASE-NEXT: add x11, x11, x8
+; CHECK-SD-BASE-NEXT: ldr d2, [x1]
; CHECK-SD-BASE-NEXT: uaddlp v0.4s, v0.8h
; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h
-; CHECK-SD-BASE-NEXT: ldr d1, [x10]
-; CHECK-SD-BASE-NEXT: add x10, x10, x9
+; CHECK-SD-BASE-NEXT: ldr d1, [x18]
; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b
-; CHECK-SD-BASE-NEXT: ldr d2, [x11]
-; CHECK-SD-BASE-NEXT: add x11, x11, x8
+; CHECK-SD-BASE-NEXT: ldr d2, [x17]
; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h
-; CHECK-SD-BASE-NEXT: ldr d1, [x10]
-; CHECK-SD-BASE-NEXT: add x10, x10, x9
+; CHECK-SD-BASE-NEXT: ldr d1, [x16]
; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b
-; CHECK-SD-BASE-NEXT: ldr d2, [x11]
-; CHECK-SD-BASE-NEXT: add x11, x11, x8
+; CHECK-SD-BASE-NEXT: ldr d2, [x15]
; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h
-; CHECK-SD-BASE-NEXT: ldr d1, [x10]
-; CHECK-SD-BASE-NEXT: add x10, x10, x9
+; CHECK-SD-BASE-NEXT: ldr d1, [x14]
; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b
-; CHECK-SD-BASE-NEXT: ldr d2, [x11]
-; CHECK-SD-BASE-NEXT: add x11, x11, x8
+; CHECK-SD-BASE-NEXT: uabdl v2.8h, v3.8b, v4.8b
+; CHECK-SD-BASE-NEXT: ldr d3, [x11]
+; CHECK-SD-BASE-NEXT: ldr d4, [x12]
+; CHECK-SD-BASE-NEXT: uaddlp v2.4s, v2.8h
; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h
; CHECK-SD-BASE-NEXT: ldr d1, [x10]
-; CHECK-SD-BASE-NEXT: add x10, x10, x9
-; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b
-; CHECK-SD-BASE-NEXT: ldr d2, [x11]
-; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h
-; CHECK-SD-BASE-NEXT: ldr d1, [x10]
-; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b
-; CHECK-SD-BASE-NEXT: ldr d2, [x11, x8]
-; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h
-; CHECK-SD-BASE-NEXT: ldr d1, [x10, x9]
-; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b
-; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h
+; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v3.8b
+; CHECK-SD-BASE-NEXT: uabdl v3.8h, v4.8b, v5.8b
+; CHECK-SD-BASE-NEXT: uadalp v2.4s, v1.8h
+; CHECK-SD-BASE-NEXT: uadalp v0.4s, v3.8h
+; CHECK-SD-BASE-NEXT: add v0.4s, v2.4s, v0.4s
; CHECK-SD-BASE-NEXT: addv s0, v0.4s
; CHECK-SD-BASE-NEXT: fmov w0, s0
; CHECK-SD-BASE-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
index 037417f044a8e..6cbaf32549b58 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
@@ -639,15 +639,23 @@ define float @fadd_reduct_reassoc_v4f32(<4 x float> %a, <4 x float> %b) {
}
define float @fadd_reduct_reassoc_v4f32_init(float %i, <4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fadd_reduct_reassoc_v4f32_init:
-; CHECK: // %bb.0:
-; CHECK-NEXT: faddp v1.4s, v1.4s, v1.4s
-; CHECK-NEXT: faddp v2.4s, v2.4s, v2.4s
-; CHECK-NEXT: faddp s1, v1.2s
-; CHECK-NEXT: fadd s0, s0, s1
-; CHECK-NEXT: faddp s1, v2.2s
-; CHECK-NEXT: fadd s0, s0, s1
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: fadd_reduct_reassoc_v4f32_init:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: fadd v1.4s, v2.4s, v1.4s
+; CHECK-SD-NEXT: faddp v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT: faddp s1, v1.2s
+; CHECK-SD-NEXT: fadd s0, s1, s0
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: fadd_reduct_reassoc_v4f32_init:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: faddp v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT: faddp v2.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT: faddp s1, v1.2s
+; CHECK-GI-NEXT: fadd s0, s0, s1
+; CHECK-GI-NEXT: faddp s1, v2.2s
+; CHECK-GI-NEXT: fadd s0, s0, s1
+; CHECK-GI-NEXT: ret
%r1 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float %i, <4 x float> %a)
%r2 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %b)
%r = fadd fast float %r1, %r2
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
index 8e7316c0d4bb9..578568a6eeff1 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmul.ll
@@ -416,13 +416,10 @@ define float @fmul_reduct_reassoc_v4f32(<4 x float> %a, <4 x float> %b) {
define float @fmul_reduct_reassoc_v4f32_init(float %i, <4 x float> %a, <4 x float> %b) {
; CHECK-SD-LABEL: fmul_reduct_reassoc_v4f32_init:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: mov d3, v1.d[1]
-; CHECK-SD-NEXT: fmul v1.2s, v1.2s, v3.2s
-; CHECK-SD-NEXT: mov d3, v2.d[1]
+; CHECK-SD-NEXT: fmul v1.4s, v2.4s, v1.4s
+; CHECK-SD-NEXT: mov d2, v1.d[1]
+; CHECK-SD-NEXT: fmul v1.2s, v1.2s, v2.2s
; CHECK-SD-NEXT: fmul v1.2s, v1.2s, v1.s[1]
-; CHECK-SD-NEXT: fmul v2.2s, v2.2s, v3.2s
-; CHECK-SD-NEXT: fmul s0, s0, s1
-; CHECK-SD-NEXT: fmul v1.2s, v2.2s, v2.s[1]
; CHECK-SD-NEXT: fmul s0, s0, s1
; CHECK-SD-NEXT: ret
;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
index 85b090ee356f2..e2c85e58cd34a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sad.ll
@@ -141,41 +141,37 @@ entry:
define signext i32 @sad_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stridea, i32 signext %strideb) {
; CHECK-LABEL: sad_2block_16xi8_as_i32:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: add a4, a0, a2
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT: vle8.v v8, (a4)
-; CHECK-NEXT: add a5, a1, a3
-; CHECK-NEXT: vle8.v v9, (a5)
-; CHECK-NEXT: vminu.vv v10, v8, v9
+; CHECK-NEXT: vle8.v v8, (a0)
+; CHECK-NEXT: vle8.v v9, (a1)
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vle8.v v10, (a0)
+; CHECK-NEXT: add a1, a1, a3
+; CHECK-NEXT: vle8.v v11, (a1)
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vle8.v v12, (a0)
+; CHECK-NEXT: vminu.vv v13, v8, v9
; CHECK-NEXT: vmaxu.vv v8, v8, v9
-; CHECK-NEXT: add a4, a4, a2
-; CHECK-NEXT: vle8.v v9, (a4)
-; CHECK-NEXT: add a2, a4, a2
-; CHECK-NEXT: vsub.vv v8, v8, v10
-; CHECK-NEXT: vle8.v v10, (a2)
-; CHECK-NEXT: add a5, a5, a3
-; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT: vzext.vf2 v12, v8
-; CHECK-NEXT: vle8.v v8, (a5)
-; CHECK-NEXT: add a3, a5, a3
-; CHECK-NEXT: vle8.v v11, (a3)
-; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT: vminu.vv v14, v9, v8
-; CHECK-NEXT: vmaxu.vv v8, v9, v8
+; CHECK-NEXT: add a1, a1, a3
; CHECK-NEXT: vminu.vv v9, v10, v11
+; CHECK-NEXT: vle8.v v14, (a1)
; CHECK-NEXT: vmaxu.vv v10, v10, v11
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vsub.vv v8, v8, v13
; CHECK-NEXT: vle8.v v11, (a0)
-; CHECK-NEXT: vsub.vv v8, v8, v14
-; CHECK-NEXT: vle8.v v14, (a1)
; CHECK-NEXT: vsub.vv v9, v10, v9
-; CHECK-NEXT: vminu.vv v10, v11, v14
-; CHECK-NEXT: vmaxu.vv v11, v11, v14
-; CHECK-NEXT: vwaddu.vv v14, v9, v8
-; CHECK-NEXT: vsub.vv v16, v11, v10
+; CHECK-NEXT: add a1, a1, a3
+; CHECK-NEXT: vle8.v v10, (a1)
+; CHECK-NEXT: vminu.vv v13, v12, v14
+; CHECK-NEXT: vmaxu.vv v12, v12, v14
+; CHECK-NEXT: vminu.vv v14, v11, v10
+; CHECK-NEXT: vmaxu.vv v10, v11, v10
+; CHECK-NEXT: vsub.vv v11, v12, v13
+; CHECK-NEXT: vsub.vv v10, v10, v14
+; CHECK-NEXT: vwaddu.vv v12, v9, v8
+; CHECK-NEXT: vwaddu.vv v14, v10, v11
; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; CHECK-NEXT: vwaddu.vv v8, v14, v12
-; CHECK-NEXT: vzext.vf2 v12, v16
-; CHECK-NEXT: vwaddu.wv v8, v8, v12
; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT: vmv.s.x v12, zero
; CHECK-NEXT: vredsum.vs v8, v8, v12
@@ -184,35 +180,33 @@ define signext i32 @sad_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stridea
;
; ZVABD-LABEL: sad_2block_16xi8_as_i32:
; ZVABD: # %bb.0: # %entry
-; ZVABD-NEXT: add a4, a0, a2
; ZVABD-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; ZVABD-NEXT: vle8.v v8, (a4)
-; ZVABD-NEXT: add a5, a1, a3
-; ZVABD-NEXT: vle8.v v9, (a5)
+; ZVABD-NEXT: vle8.v v8, (a0)
+; ZVABD-NEXT: vle8.v v9, (a1)
+; ZVABD-NEXT: add a0, a0, a2
+; ZVABD-NEXT: add a4, a0, a2
; ZVABD-NEXT: vabdu.vv v8, v8, v9
-; ZVABD-NEXT: add a4, a4, a2
; ZVABD-NEXT: vle8.v v9, (a4)
-; ZVABD-NEXT: add a5, a5, a3
+; ZVABD-NEXT: add a1, a1, a3
+; ZVABD-NEXT: add a5, a1, a3
+; ZVABD-NEXT: vle8.v v10, (a5)
; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vzext.vf2 v12, v8
-; ZVABD-NEXT: vle8.v v8, (a5)
+; ZVABD-NEXT: vle8.v v8, (a0)
; ZVABD-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; ZVABD-NEXT: vabdu.vv v8, v9, v8
+; ZVABD-NEXT: vabdu.vv v9, v9, v10
+; ZVABD-NEXT: vle8.v v10, (a1)
; ZVABD-NEXT: add a2, a4, a2
+; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
+; ZVABD-NEXT: vzext.vf2 v14, v9
; ZVABD-NEXT: vle8.v v9, (a2)
; ZVABD-NEXT: add a3, a5, a3
-; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; ZVABD-NEXT: vzext.vf2 v14, v8
-; ZVABD-NEXT: vle8.v v8, (a3)
+; ZVABD-NEXT: vle8.v v11, (a3)
; ZVABD-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; ZVABD-NEXT: vwabdau.vv v14, v9, v8
+; ZVABD-NEXT: vwabdau.vv v12, v8, v10
+; ZVABD-NEXT: vwabdau.vv v14, v9, v11
; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vwaddu.vv v8, v14, v12
-; ZVABD-NEXT: vle8.v v14, (a1)
-; ZVABD-NEXT: vzext.vf2 v12, v14
-; ZVABD-NEXT: vle8.v v16, (a0)
-; ZVABD-NEXT: vzext.vf2 v14, v16
-; ZVABD-NEXT: vwabdau.vv v8, v14, v12
; ZVABD-NEXT: vsetvli zero, zero, e32, m4, ta, ma
; ZVABD-NEXT: vmv.s.x v12, zero
; ZVABD-NEXT: vredsum.vs v8, v8, v12
@@ -264,41 +258,37 @@ entry:
define signext i32 @sadu_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stridea, i32 signext %strideb) {
; CHECK-LABEL: sadu_2block_16xi8_as_i32:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: add a4, a0, a2
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT: vle8.v v8, (a4)
-; CHECK-NEXT: add a5, a1, a3
-; CHECK-NEXT: vle8.v v9, (a5)
-; CHECK-NEXT: vmin.vv v10, v8, v9
+; CHECK-NEXT: vle8.v v8, (a0)
+; CHECK-NEXT: vle8.v v9, (a1)
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vle8.v v10, (a0)
+; CHECK-NEXT: add a1, a1, a3
+; CHECK-NEXT: vle8.v v11, (a1)
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vle8.v v12, (a0)
+; CHECK-NEXT: vmin.vv v13, v8, v9
; CHECK-NEXT: vmax.vv v8, v8, v9
-; CHECK-NEXT: add a4, a4, a2
-; CHECK-NEXT: vle8.v v9, (a4)
-; CHECK-NEXT: add a2, a4, a2
-; CHECK-NEXT: vsub.vv v8, v8, v10
-; CHECK-NEXT: vle8.v v10, (a2)
-; CHECK-NEXT: add a5, a5, a3
-; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT: vzext.vf2 v12, v8
-; CHECK-NEXT: vle8.v v8, (a5)
-; CHECK-NEXT: add a3, a5, a3
-; CHECK-NEXT: vle8.v v11, (a3)
-; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT: vmin.vv v14, v9, v8
-; CHECK-NEXT: vmax.vv v8, v9, v8
+; CHECK-NEXT: add a1, a1, a3
; CHECK-NEXT: vmin.vv v9, v10, v11
+; CHECK-NEXT: vle8.v v14, (a1)
; CHECK-NEXT: vmax.vv v10, v10, v11
+; CHECK-NEXT: add a0, a0, a2
+; CHECK-NEXT: vsub.vv v8, v8, v13
; CHECK-NEXT: vle8.v v11, (a0)
-; CHECK-NEXT: vsub.vv v8, v8, v14
-; CHECK-NEXT: vle8.v v14, (a1)
; CHECK-NEXT: vsub.vv v9, v10, v9
-; CHECK-NEXT: vmin.vv v10, v11, v14
-; CHECK-NEXT: vmax.vv v11, v11, v14
-; CHECK-NEXT: vwaddu.vv v14, v9, v8
-; CHECK-NEXT: vsub.vv v16, v11, v10
+; CHECK-NEXT: add a1, a1, a3
+; CHECK-NEXT: vle8.v v10, (a1)
+; CHECK-NEXT: vmin.vv v13, v12, v14
+; CHECK-NEXT: vmax.vv v12, v12, v14
+; CHECK-NEXT: vmin.vv v14, v11, v10
+; CHECK-NEXT: vmax.vv v10, v11, v10
+; CHECK-NEXT: vsub.vv v11, v12, v13
+; CHECK-NEXT: vsub.vv v10, v10, v14
+; CHECK-NEXT: vwaddu.vv v12, v9, v8
+; CHECK-NEXT: vwaddu.vv v14, v10, v11
; CHECK-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; CHECK-NEXT: vwaddu.vv v8, v14, v12
-; CHECK-NEXT: vzext.vf2 v12, v16
-; CHECK-NEXT: vwaddu.wv v8, v8, v12
; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT: vmv.s.x v12, zero
; CHECK-NEXT: vredsum.vs v8, v8, v12
@@ -307,35 +297,33 @@ define signext i32 @sadu_2block_16xi8_as_i32(ptr %a, ptr %b, i32 signext %stride
;
; ZVABD-LABEL: sadu_2block_16xi8_as_i32:
; ZVABD: # %bb.0: # %entry
-; ZVABD-NEXT: add a4, a0, a2
; ZVABD-NEXT: vsetivli zero, 16, e8, m1, ta, ma
-; ZVABD-NEXT: vle8.v v8, (a4)
-; ZVABD-NEXT: add a5, a1, a3
-; ZVABD-NEXT: vle8.v v9, (a5)
+; ZVABD-NEXT: vle8.v v8, (a0)
+; ZVABD-NEXT: vle8.v v9, (a1)
+; ZVABD-NEXT: add a0, a0, a2
+; ZVABD-NEXT: add a4, a0, a2
; ZVABD-NEXT: vabd.vv v8, v8, v9
-; ZVABD-NEXT: add a4, a4, a2
; ZVABD-NEXT: vle8.v v9, (a4)
-; ZVABD-NEXT: add a5, a5, a3
+; ZVABD-NEXT: add a1, a1, a3
+; ZVABD-NEXT: add a5, a1, a3
+; ZVABD-NEXT: vle8.v v10, (a5)
; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vzext.vf2 v12, v8
-; ZVABD-NEXT: vle8.v v8, (a5)
+; ZVABD-NEXT: vle8.v v8, (a0)
; ZVABD-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; ZVABD-NEXT: vabd.vv v8, v9, v8
+; ZVABD-NEXT: vabd.vv v9, v9, v10
+; ZVABD-NEXT: vle8.v v10, (a1)
; ZVABD-NEXT: add a2, a4, a2
+; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
+; ZVABD-NEXT: vzext.vf2 v14, v9
; ZVABD-NEXT: vle8.v v9, (a2)
; ZVABD-NEXT: add a3, a5, a3
-; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; ZVABD-NEXT: vzext.vf2 v14, v8
-; ZVABD-NEXT: vle8.v v8, (a3)
+; ZVABD-NEXT: vle8.v v11, (a3)
; ZVABD-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; ZVABD-NEXT: vwabda.vv v14, v9, v8
+; ZVABD-NEXT: vwabda.vv v12, v8, v10
+; ZVABD-NEXT: vwabda.vv v14, v9, v11
; ZVABD-NEXT: vsetvli zero, zero, e16, m2, ta, ma
; ZVABD-NEXT: vwaddu.vv v8, v14, v12
-; ZVABD-NEXT: vle8.v v14, (a1)
-; ZVABD-NEXT: vsext.vf2 v12, v14
-; ZVABD-NEXT: vle8.v v16, (a0)
-; ZVABD-NEXT: vsext.vf2 v14, v16
-; ZVABD-NEXT: vwabda.vv v8, v14, v12
; ZVABD-NEXT: vsetvli zero, zero, e32, m4, ta, ma
; ZVABD-NEXT: vmv.s.x v12, zero
; ZVABD-NEXT: vredsum.vs v8, v8, v12
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
index e49c9a7e6136e..21591478cebc2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zvdot4a8i.ll
@@ -2011,20 +2011,16 @@ define i32 @vdot4au_vv_chain4(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x
; NODOT: # %bb.0: # %entry
; NODOT-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; NODOT-NEXT: vwmulu.vv v16, v14, v15
-; NODOT-NEXT: vwmulu.vv v18, v12, v13
+; NODOT-NEXT: vwmulu.vv v14, v10, v11
+; NODOT-NEXT: vwmulu.vv v18, v8, v9
+; NODOT-NEXT: vwmulu.vv v20, v12, v13
; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vwaddu.vv v12, v16, v18
-; NODOT-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; NODOT-NEXT: vwmulu.vv v16, v10, v11
-; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vwaddu.wv v12, v12, v16
-; NODOT-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; NODOT-NEXT: vwmulu.vv v10, v8, v9
-; NODOT-NEXT: vsetvli zero, zero, e16, m2, ta, ma
-; NODOT-NEXT: vwaddu.wv v12, v12, v10
+; NODOT-NEXT: vwaddu.vv v8, v14, v18
+; NODOT-NEXT: vwaddu.vv v12, v16, v20
; NODOT-NEXT: vsetvli zero, zero, e32, m4, ta, ma
-; NODOT-NEXT: vmv.s.x v8, zero
-; NODOT-NEXT: vredsum.vs v8, v12, v8
+; NODOT-NEXT: vadd.vv v8, v12, v8
+; NODOT-NEXT: vmv.s.x v12, zero
+; NODOT-NEXT: vredsum.vs v8, v8, v12
; NODOT-NEXT: vmv.x.s a0, v8
; NODOT-NEXT: ret
;
@@ -2032,10 +2028,10 @@ define i32 @vdot4au_vv_chain4(<16 x i8> %a0, <16 x i8> %b0, <16 x i8> %a1, <16 x
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; DOT-NEXT: vmv.v.i v16, 0
-; DOT-NEXT: vdot4au.vv v16, v14, v15
-; DOT-NEXT: vdot4au.vv v16, v12, v13
; DOT-NEXT: vdot4au.vv v16, v10, v11
; DOT-NEXT: vdot4au.vv v16, v8, v9
+; DOT-NEXT: vdot4au.vv v16, v14, v15
+; DOT-NEXT: vdot4au.vv v16, v12, v13
; DOT-NEXT: vmv.s.x v8, zero
; DOT-NEXT: vredsum.vs v8, v16, v8
; DOT-NEXT: vmv.x.s a0, v8
diff --git a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
index 88dd44b6165f6..148c28eec5067 100644
--- a/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/zvdot4a8i-sdnode.ll
@@ -1106,20 +1106,18 @@ define i32 @vdot4au_vv_chain4(<vscale x 16 x i8> %a0, <vscale x 16 x i8> %b0, <v
; NODOT: # %bb.0: # %entry
; NODOT-NEXT: vsetvli a0, zero, e8, m2, ta, ma
; NODOT-NEXT: vwmulu.vv v24, v20, v22
-; NODOT-NEXT: vwmulu.vv v28, v16, v18
-; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
-; NODOT-NEXT: vwaddu.vv v16, v24, v28
-; NODOT-NEXT: vsetvli zero, zero, e8, m2, ta, ma
-; NODOT-NEXT: vwmulu.vv v24, v12, v14
+; NODOT-NEXT: vwmulu.vv v20, v8, v10
+; NODOT-NEXT: vwmulu.vv v28, v12, v14
; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
-; NODOT-NEXT: vwaddu.wv v16, v16, v24
+; NODOT-NEXT: vwaddu.vv v8, v28, v20
; NODOT-NEXT: vsetvli zero, zero, e8, m2, ta, ma
-; NODOT-NEXT: vwmulu.vv v12, v8, v10
+; NODOT-NEXT: vwmulu.vv v28, v16, v18
; NODOT-NEXT: vsetvli zero, zero, e16, m4, ta, ma
-; NODOT-NEXT: vwaddu.wv v16, v16, v12
+; NODOT-NEXT: vwaddu.vv v16, v24, v28
; NODOT-NEXT: vsetvli zero, zero, e32, m8, ta, ma
-; NODOT-NEXT: vmv.s.x v8, zero
-; NODOT-NEXT: vredsum.vs v8, v16, v8
+; NODOT-NEXT: vadd.vv v8, v16, v8
+; NODOT-NEXT: vmv.s.x v16, zero
+; NODOT-NEXT: vredsum.vs v8, v8, v16
; NODOT-NEXT: vmv.x.s a0, v8
; NODOT-NEXT: ret
;
@@ -1127,10 +1125,10 @@ define i32 @vdot4au_vv_chain4(<vscale x 16 x i8> %a0, <vscale x 16 x i8> %b0, <v
; DOT: # %bb.0: # %entry
; DOT-NEXT: vsetvli a0, zero, e32, m2, ta, ma
; DOT-NEXT: vmv.v.i v24, 0
-; DOT-NEXT: vdot4au.vv v24, v20, v22
-; DOT-NEXT: vdot4au.vv v24, v16, v18
; DOT-NEXT: vdot4au.vv v24, v12, v14
; DOT-NEXT: vdot4au.vv v24, v8, v10
+; DOT-NEXT: vdot4au.vv v24, v20, v22
+; DOT-NEXT: vdot4au.vv v24, v16, v18
; DOT-NEXT: vmv.s.x v8, zero
; DOT-NEXT: vredsum.vs v8, v24, v8
; DOT-NEXT: vmv.x.s a0, v8
More information about the llvm-commits
mailing list