[llvm] [RISCV] Reduce the LMUL for a vrgather operation if legal (PR #125768)
Philip Reames via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 5 09:11:28 PST 2025
https://github.com/preames updated https://github.com/llvm/llvm-project/pull/125768
>From f2c0c9f195510b956da7ad716c14e8c589fe0d60 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Tue, 4 Feb 2025 13:15:23 -0800
Subject: [PATCH 1/5] [RISCV] Reduce the LMUL for a vrgather operation if legal
If we're lowering a shuffle to a vrgather (or vcompress), and we know
that a prefix of the operation can be done while producing the same
(defined) lanes, do the operation with a narrower LMUL.
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 25 ++
.../rvv/fixed-vectors-shuffle-reverse.ll | 340 ++++++++++--------
2 files changed, 209 insertions(+), 156 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 7c3b58389da28ee..a6cca57943de1f0 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -5619,6 +5619,31 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
}
}
+ // If only a prefix of the source elements influence a prefix of the
+ // destination elements, try to see if we can reduce the required LMUL
+ unsigned MinVLen = Subtarget.getRealMinVLen();
+ unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
+ if (NumElts > MinVLMAX) {
+ unsigned MaxIdx = 0;
+ for (auto [I, M] : enumerate(Mask)) {
+ if (M == -1)
+ continue;
+ MaxIdx = std::max(std::max((unsigned)I,(unsigned)M), MaxIdx);
+ }
+ unsigned NewNumElts = NumElts;
+ while (MaxIdx < NewNumElts / 2 && NewNumElts != MinVLMAX)
+ NewNumElts /= 2;
+ if (NewNumElts != NumElts) {
+ MVT NewVT = MVT::getVectorVT(VT.getVectorElementType(), NewNumElts);
+ SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
+ V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewVT, V1, ZeroIdx);
+ SDValue Res = DAG.getVectorShuffle(NewVT, DL, V1, DAG.getUNDEF(NewVT),
+ Mask.take_front(NewNumElts));
+ return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
+ Res, ZeroIdx);
+ }
+ }
+
// Before hitting generic lowering fallbacks, try to widen the mask
// to a wider SEW.
if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index 5fd7e47507f71e4..69c101c79afbab6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -874,27 +874,30 @@ define <16 x i8> @reverse_v16i8_2(<8 x i8> %a, <8 x i8> %b) {
define <32 x i8> @reverse_v32i8_2(<16 x i8> %a, <16 x i8> %b) {
; CHECK-LABEL: reverse_v32i8_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma
-; CHECK-NEXT: vmv1r.v v10, v9
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v12
+; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT: vid.v v10
; CHECK-NEXT: addi a1, a0, -1
-; CHECK-NEXT: vrsub.vx v12, v12, a1
+; CHECK-NEXT: vrsub.vx v10, v10, a1
; CHECK-NEXT: lui a1, 16
; CHECK-NEXT: addi a1, a1, -1
; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma
-; CHECK-NEXT: vrgatherei16.vv v15, v8, v12
-; CHECK-NEXT: vrgatherei16.vv v14, v9, v12
+; CHECK-NEXT: vrgatherei16.vv v15, v8, v10
+; CHECK-NEXT: vrgatherei16.vv v14, v12, v10
; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma
; CHECK-NEXT: vmv.s.x v0, a1
; CHECK-NEXT: li a1, 32
-; CHECK-NEXT: slli a0, a0, 1
-; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, mu
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: slli a0, a0, 1
+; CHECK-NEXT: vrsub.vi v8, v8, 15
; CHECK-NEXT: addi a0, a0, -32
-; CHECK-NEXT: vrsub.vi v12, v8, 15
-; CHECK-NEXT: vslidedown.vx v8, v14, a0
-; CHECK-NEXT: vrgather.vv v8, v10, v12, v0.t
+; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT: vslidedown.vx v10, v14, a0
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vrgather.vv v12, v9, v8
+; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v10, v12, v0
; CHECK-NEXT: ret
%res = shufflevector <16 x i8> %a, <16 x i8> %b, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <32 x i8> %res
@@ -943,23 +946,26 @@ define <8 x i16> @reverse_v8i16_2(<4 x i16> %a, <4 x i16> %b) {
define <16 x i16> @reverse_v16i16_2(<8 x i16> %a, <8 x i16> %b) {
; CHECK-LABEL: reverse_v16i16_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT: vmv1r.v v10, v9
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v9
+; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT: vid.v v10
; CHECK-NEXT: srli a1, a0, 1
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v9, v9, a1
-; CHECK-NEXT: vrgather.vv v13, v8, v9
-; CHECK-NEXT: vrgather.vv v12, v11, v9
-; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, mu
+; CHECK-NEXT: vrsub.vx v10, v10, a1
+; CHECK-NEXT: vrgather.vv v13, v8, v10
+; CHECK-NEXT: vrgather.vv v12, v11, v10
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vid.v v8
; CHECK-NEXT: li a1, 255
; CHECK-NEXT: addi a0, a0, -16
-; CHECK-NEXT: vrsub.vi v14, v8, 7
+; CHECK-NEXT: vrsub.vi v8, v8, 7
; CHECK-NEXT: vmv.s.x v0, a1
-; CHECK-NEXT: vslidedown.vx v8, v12, a0
-; CHECK-NEXT: vrgather.vv v8, v10, v14, v0.t
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT: vslidedown.vx v10, v12, a0
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT: vrgather.vv v12, v9, v8
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v10, v12, v0
; CHECK-NEXT: ret
%res = shufflevector <8 x i16> %a, <8 x i16> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <16 x i16> %res
@@ -968,30 +974,33 @@ define <16 x i16> @reverse_v16i16_2(<8 x i16> %a, <8 x i16> %b) {
define <32 x i16> @reverse_v32i16_2(<16 x i16> %a, <16 x i16> %b) {
; CHECK-LABEL: reverse_v32i16_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT: vmv2r.v v12, v10
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v10
-; CHECK-NEXT: lui a1, 16
-; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT: vmv.s.x v0, a1
+; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT: vid.v v12
; CHECK-NEXT: srli a1, a0, 1
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT: vrsub.vx v10, v10, a1
+; CHECK-NEXT: vrsub.vx v13, v12, a1
+; CHECK-NEXT: vrgather.vv v15, v8, v13
+; CHECK-NEXT: vrgather.vv v14, v9, v13
+; CHECK-NEXT: vrgather.vv v9, v10, v13
+; CHECK-NEXT: vrgather.vv v8, v11, v13
; CHECK-NEXT: li a1, 32
-; CHECK-NEXT: slli a0, a0, 1
-; CHECK-NEXT: vrgather.vv v19, v8, v10
-; CHECK-NEXT: vrgather.vv v18, v9, v10
-; CHECK-NEXT: vrgather.vv v16, v11, v10
-; CHECK-NEXT: vsetvli zero, a1, e16, m4, ta, mu
-; CHECK-NEXT: vid.v v8
-; CHECK-NEXT: addi a0, a0, -32
-; CHECK-NEXT: vrsub.vi v20, v8, 15
-; CHECK-NEXT: vmv1r.v v17, v16
-; CHECK-NEXT: vslidedown.vx v8, v16, a0
-; CHECK-NEXT: vrgather.vv v8, v12, v20, v0.t
+; CHECK-NEXT: lui a2, 16
+; CHECK-NEXT: addi a2, a2, -1
+; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT: vmv.s.x v0, a2
+; CHECK-NEXT: slli a2, a0, 1
+; CHECK-NEXT: addi a0, a0, -16
+; CHECK-NEXT: vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT: vrgather.vv v12, v10, v13
+; CHECK-NEXT: addi a2, a2, -32
+; CHECK-NEXT: vmv.v.v v13, v12
+; CHECK-NEXT: vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT: vslidedown.vx v12, v12, a2
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT: vslidedown.vx v8, v8, a0
+; CHECK-NEXT: vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0
; CHECK-NEXT: ret
%res = shufflevector <16 x i16> %a, <16 x i16> %b, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <32 x i16> %res
@@ -1024,24 +1033,26 @@ define <4 x i32> @reverse_v4i32_2(<2 x i32> %a, < 2 x i32> %b) {
define <8 x i32> @reverse_v8i32_2(<4 x i32> %a, <4 x i32> %b) {
; CHECK-LABEL: reverse_v8i32_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT: vmv1r.v v10, v9
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v9
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v10
; CHECK-NEXT: srli a1, a0, 2
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v9, v9, a1
-; CHECK-NEXT: vrgather.vv v13, v8, v9
-; CHECK-NEXT: vrgather.vv v12, v11, v9
-; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT: vrsub.vx v10, v10, a1
+; CHECK-NEXT: vrgather.vv v13, v8, v10
+; CHECK-NEXT: vrgather.vv v12, v11, v10
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vid.v v8
; CHECK-NEXT: vmv.v.i v0, 15
; CHECK-NEXT: srli a0, a0, 1
-; CHECK-NEXT: vrsub.vi v14, v8, 3
+; CHECK-NEXT: vrsub.vi v8, v8, 3
; CHECK-NEXT: addi a0, a0, -8
-; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT: vslidedown.vx v8, v12, a0
-; CHECK-NEXT: vrgatherei16.vv v8, v10, v14, v0.t
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT: vslidedown.vx v10, v12, a0
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vrgather.vv v12, v9, v8
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v10, v12, v0
; CHECK-NEXT: ret
%res = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <8 x i32> %res
@@ -1050,26 +1061,29 @@ define <8 x i32> @reverse_v8i32_2(<4 x i32> %a, <4 x i32> %b) {
define <16 x i32> @reverse_v16i32_2(<8 x i32> %a, <8 x i32> %b) {
; CHECK-LABEL: reverse_v16i32_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT: vmv2r.v v12, v10
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v10
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v12
; CHECK-NEXT: srli a1, a0, 2
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v14, v10, a1
-; CHECK-NEXT: vrgather.vv v11, v8, v14
-; CHECK-NEXT: vrgather.vv v10, v9, v14
-; CHECK-NEXT: vrgather.vv v8, v9, v14
-; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT: vid.v v14
+; CHECK-NEXT: vrsub.vx v13, v12, a1
+; CHECK-NEXT: vrgather.vv v15, v8, v13
+; CHECK-NEXT: vrgather.vv v14, v9, v13
+; CHECK-NEXT: vrgather.vv v9, v10, v13
+; CHECK-NEXT: vrgather.vv v8, v11, v13
; CHECK-NEXT: li a1, 255
-; CHECK-NEXT: addi a0, a0, -16
-; CHECK-NEXT: vrsub.vi v16, v14, 7
; CHECK-NEXT: vmv.s.x v0, a1
-; CHECK-NEXT: vmv1r.v v9, v8
-; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT: addi a1, a0, -16
+; CHECK-NEXT: srli a0, a0, 1
+; CHECK-NEXT: vrgather.vv v12, v10, v13
+; CHECK-NEXT: addi a0, a0, -8
+; CHECK-NEXT: vmv.v.v v13, v12
+; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT: vslidedown.vx v12, v12, a1
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vslidedown.vx v8, v8, a0
-; CHECK-NEXT: vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0
; CHECK-NEXT: ret
%res = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <16 x i32> %res
@@ -1078,32 +1092,36 @@ define <16 x i32> @reverse_v16i32_2(<8 x i32> %a, <8 x i32> %b) {
define <32 x i32> @reverse_v32i32_2(<16 x i32> %a, <16 x i32> %b) {
; CHECK-LABEL: reverse_v32i32_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT: vmv4r.v v16, v12
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v12
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v16
; CHECK-NEXT: srli a1, a0, 2
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v20, v12, a1
-; CHECK-NEXT: vrgather.vv v15, v8, v20
-; CHECK-NEXT: vrgather.vv v14, v9, v20
-; CHECK-NEXT: vrgather.vv v13, v10, v20
-; CHECK-NEXT: vrgather.vv v12, v11, v20
-; CHECK-NEXT: lui a1, 16
-; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vmv.s.x v0, a1
+; CHECK-NEXT: vrsub.vx v17, v16, a1
+; CHECK-NEXT: vrgather.vv v23, v8, v17
+; CHECK-NEXT: vrgather.vv v22, v9, v17
+; CHECK-NEXT: vrgather.vv v21, v10, v17
+; CHECK-NEXT: vrgather.vv v20, v11, v17
+; CHECK-NEXT: vrgather.vv v11, v12, v17
+; CHECK-NEXT: vrgather.vv v10, v13, v17
+; CHECK-NEXT: vrgather.vv v9, v14, v17
+; CHECK-NEXT: vrgather.vv v8, v15, v17
; CHECK-NEXT: li a1, 32
-; CHECK-NEXT: slli a0, a0, 1
-; CHECK-NEXT: vrgather.vv v8, v9, v20
-; CHECK-NEXT: vsetvli zero, a1, e16, m4, ta, ma
-; CHECK-NEXT: vid.v v20
-; CHECK-NEXT: addi a0, a0, -32
-; CHECK-NEXT: vmv1r.v v9, v8
-; CHECK-NEXT: vrsub.vi v24, v20, 15
-; CHECK-NEXT: vmv2r.v v10, v8
-; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT: lui a2, 16
+; CHECK-NEXT: addi a2, a2, -1
+; CHECK-NEXT: vmv.s.x v0, a2
+; CHECK-NEXT: slli a2, a0, 1
+; CHECK-NEXT: vrgather.vv v16, v12, v17
+; CHECK-NEXT: addi a0, a0, -16
+; CHECK-NEXT: vmv.v.v v17, v16
+; CHECK-NEXT: addi a2, a2, -32
+; CHECK-NEXT: vmv2r.v v18, v16
+; CHECK-NEXT: vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-NEXT: vslidedown.vx v16, v16, a2
+; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma
; CHECK-NEXT: vslidedown.vx v8, v8, a0
-; CHECK-NEXT: vrgatherei16.vv v8, v16, v24, v0.t
+; CHECK-NEXT: vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v16, v8, v0
; CHECK-NEXT: ret
%res = shufflevector <16 x i32> %a, <16 x i32> %b, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <32 x i32> %res
@@ -1127,28 +1145,29 @@ define <4 x i64> @reverse_v4i64_2(<2 x i64> %a, < 2 x i64> %b) {
define <8 x i64> @reverse_v8i64_2(<4 x i64> %a, <4 x i64> %b) {
; CHECK-LABEL: reverse_v8i64_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT: vmv2r.v v12, v10
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v10
+; CHECK-NEXT: vsetvli a1, zero, e64, m1, ta, ma
+; CHECK-NEXT: vid.v v12
; CHECK-NEXT: srli a1, a0, 3
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v14, v10, a1
-; CHECK-NEXT: vrgather.vv v11, v8, v14
-; CHECK-NEXT: vrgather.vv v10, v9, v14
-; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT: vid.v v15
-; CHECK-NEXT: vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT: vrgather.vv v8, v9, v14
+; CHECK-NEXT: vrsub.vx v13, v12, a1
+; CHECK-NEXT: vrgather.vv v15, v8, v13
+; CHECK-NEXT: vrgather.vv v14, v9, v13
+; CHECK-NEXT: vrgather.vv v9, v10, v13
+; CHECK-NEXT: vrgather.vv v8, v11, v13
; CHECK-NEXT: vmv.v.i v0, 15
-; CHECK-NEXT: srli a0, a0, 1
-; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT: vrsub.vi v16, v15, 3
-; CHECK-NEXT: addi a0, a0, -8
-; CHECK-NEXT: vmv1r.v v9, v8
-; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT: srli a1, a0, 1
+; CHECK-NEXT: srli a0, a0, 2
+; CHECK-NEXT: addi a1, a1, -8
+; CHECK-NEXT: vrgather.vv v12, v10, v13
+; CHECK-NEXT: addi a0, a0, -4
+; CHECK-NEXT: vmv.v.v v13, v12
+; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT: vslidedown.vx v12, v12, a1
+; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; CHECK-NEXT: vslidedown.vx v8, v8, a0
-; CHECK-NEXT: vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0
; CHECK-NEXT: ret
%res = shufflevector <4 x i64> %a, <4 x i64> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <8 x i64> %res
@@ -1197,23 +1216,26 @@ define <8 x half> @reverse_v8f16_2(<4 x half> %a, <4 x half> %b) {
define <16 x half> @reverse_v16f16_2(<8 x half> %a, <8 x half> %b) {
; CHECK-LABEL: reverse_v16f16_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT: vmv1r.v v10, v9
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v9
+; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT: vid.v v10
; CHECK-NEXT: srli a1, a0, 1
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v9, v9, a1
-; CHECK-NEXT: vrgather.vv v13, v8, v9
-; CHECK-NEXT: vrgather.vv v12, v11, v9
-; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, mu
+; CHECK-NEXT: vrsub.vx v10, v10, a1
+; CHECK-NEXT: vrgather.vv v13, v8, v10
+; CHECK-NEXT: vrgather.vv v12, v11, v10
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
; CHECK-NEXT: vid.v v8
; CHECK-NEXT: li a1, 255
; CHECK-NEXT: addi a0, a0, -16
-; CHECK-NEXT: vrsub.vi v14, v8, 7
+; CHECK-NEXT: vrsub.vi v8, v8, 7
; CHECK-NEXT: vmv.s.x v0, a1
-; CHECK-NEXT: vslidedown.vx v8, v12, a0
-; CHECK-NEXT: vrgather.vv v8, v10, v14, v0.t
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT: vslidedown.vx v10, v12, a0
+; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT: vrgather.vv v12, v9, v8
+; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v10, v12, v0
; CHECK-NEXT: ret
%res = shufflevector <8 x half> %a, <8 x half> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <16 x half> %res
@@ -1269,24 +1291,26 @@ define <4 x float> @reverse_v4f32_2(<2 x float> %a, <2 x float> %b) {
define <8 x float> @reverse_v8f32_2(<4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: reverse_v8f32_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT: vmv1r.v v10, v9
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v9
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v10
; CHECK-NEXT: srli a1, a0, 2
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v9, v9, a1
-; CHECK-NEXT: vrgather.vv v13, v8, v9
-; CHECK-NEXT: vrgather.vv v12, v11, v9
-; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT: vrsub.vx v10, v10, a1
+; CHECK-NEXT: vrgather.vv v13, v8, v10
+; CHECK-NEXT: vrgather.vv v12, v11, v10
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vid.v v8
; CHECK-NEXT: vmv.v.i v0, 15
; CHECK-NEXT: srli a0, a0, 1
-; CHECK-NEXT: vrsub.vi v14, v8, 3
+; CHECK-NEXT: vrsub.vi v8, v8, 3
; CHECK-NEXT: addi a0, a0, -8
-; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT: vslidedown.vx v8, v12, a0
-; CHECK-NEXT: vrgatherei16.vv v8, v10, v14, v0.t
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT: vslidedown.vx v10, v12, a0
+; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT: vrgather.vv v12, v9, v8
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v10, v12, v0
; CHECK-NEXT: ret
%res = shufflevector <4 x float> %a, <4 x float> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <8 x float> %res
@@ -1295,26 +1319,29 @@ define <8 x float> @reverse_v8f32_2(<4 x float> %a, <4 x float> %b) {
define <16 x float> @reverse_v16f32_2(<8 x float> %a, <8 x float> %b) {
; CHECK-LABEL: reverse_v16f32_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma
-; CHECK-NEXT: vmv2r.v v12, v10
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v10
+; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT: vid.v v12
; CHECK-NEXT: srli a1, a0, 2
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v14, v10, a1
-; CHECK-NEXT: vrgather.vv v11, v8, v14
-; CHECK-NEXT: vrgather.vv v10, v9, v14
-; CHECK-NEXT: vrgather.vv v8, v9, v14
-; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT: vid.v v14
+; CHECK-NEXT: vrsub.vx v13, v12, a1
+; CHECK-NEXT: vrgather.vv v15, v8, v13
+; CHECK-NEXT: vrgather.vv v14, v9, v13
+; CHECK-NEXT: vrgather.vv v9, v10, v13
+; CHECK-NEXT: vrgather.vv v8, v11, v13
; CHECK-NEXT: li a1, 255
-; CHECK-NEXT: addi a0, a0, -16
-; CHECK-NEXT: vrsub.vi v16, v14, 7
; CHECK-NEXT: vmv.s.x v0, a1
-; CHECK-NEXT: vmv1r.v v9, v8
-; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT: addi a1, a0, -16
+; CHECK-NEXT: srli a0, a0, 1
+; CHECK-NEXT: vrgather.vv v12, v10, v13
+; CHECK-NEXT: addi a0, a0, -8
+; CHECK-NEXT: vmv.v.v v13, v12
+; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT: vslidedown.vx v12, v12, a1
+; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vslidedown.vx v8, v8, a0
-; CHECK-NEXT: vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0
; CHECK-NEXT: ret
%res = shufflevector <8 x float> %a, <8 x float> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <16 x float> %res
@@ -1338,28 +1365,29 @@ define <4 x double> @reverse_v4f64_2(<2 x double> %a, < 2 x double> %b) {
define <8 x double> @reverse_v8f64_2(<4 x double> %a, <4 x double> %b) {
; CHECK-LABEL: reverse_v8f64_2:
; CHECK: # %bb.0:
-; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT: vmv2r.v v12, v10
; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: vid.v v10
+; CHECK-NEXT: vsetvli a1, zero, e64, m1, ta, ma
+; CHECK-NEXT: vid.v v12
; CHECK-NEXT: srli a1, a0, 3
; CHECK-NEXT: addi a1, a1, -1
-; CHECK-NEXT: vrsub.vx v14, v10, a1
-; CHECK-NEXT: vrgather.vv v11, v8, v14
-; CHECK-NEXT: vrgather.vv v10, v9, v14
-; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT: vid.v v15
-; CHECK-NEXT: vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT: vrgather.vv v8, v9, v14
+; CHECK-NEXT: vrsub.vx v13, v12, a1
+; CHECK-NEXT: vrgather.vv v15, v8, v13
+; CHECK-NEXT: vrgather.vv v14, v9, v13
+; CHECK-NEXT: vrgather.vv v9, v10, v13
+; CHECK-NEXT: vrgather.vv v8, v11, v13
; CHECK-NEXT: vmv.v.i v0, 15
-; CHECK-NEXT: srli a0, a0, 1
-; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT: vrsub.vi v16, v15, 3
-; CHECK-NEXT: addi a0, a0, -8
-; CHECK-NEXT: vmv1r.v v9, v8
-; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT: srli a1, a0, 1
+; CHECK-NEXT: srli a0, a0, 2
+; CHECK-NEXT: addi a1, a1, -8
+; CHECK-NEXT: vrgather.vv v12, v10, v13
+; CHECK-NEXT: addi a0, a0, -4
+; CHECK-NEXT: vmv.v.v v13, v12
+; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT: vslidedown.vx v12, v12, a1
+; CHECK-NEXT: vsetivli zero, 4, e64, m2, ta, ma
; CHECK-NEXT: vslidedown.vx v8, v8, a0
-; CHECK-NEXT: vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT: vsetivli zero, 8, e64, m4, ta, ma
+; CHECK-NEXT: vmerge.vvm v8, v12, v8, v0
; CHECK-NEXT: ret
%res = shufflevector <4 x double> %a, <4 x double> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
ret <8 x double> %res
>From 86af4deea6c931bcf65bb14c000dc4e155c97440 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Tue, 4 Feb 2025 15:21:12 -0800
Subject: [PATCH 2/5] Missing test update
---
.../rvv/fixed-vectors-shufflevector-vnsrl.ll | 20 ++++++++-----------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
index 847ef9a7b36017c..3c28e978842b99a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shufflevector-vnsrl.ll
@@ -525,13 +525,10 @@ define void @vnsrl_0_i32_single_src_m8(ptr %in, ptr %out) {
; V-NEXT: li a2, 64
; V-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; V-NEXT: vle32.v v8, (a0)
-; V-NEXT: lui a0, 341
-; V-NEXT: addiw a0, a0, 1365
-; V-NEXT: vsetivli zero, 1, e64, m1, ta, ma
-; V-NEXT: vmv.s.x v16, a0
+; V-NEXT: vsetivli zero, 16, e32, m2, ta, ma
+; V-NEXT: vnsrl.wi v16, v8, 0
; V-NEXT: vsetvli zero, a2, e32, m8, ta, ma
-; V-NEXT: vcompress.vm v24, v8, v16
-; V-NEXT: vse32.v v24, (a1)
+; V-NEXT: vse32.v v16, (a1)
; V-NEXT: ret
;
; ZVE32F-LABEL: vnsrl_0_i32_single_src_m8:
@@ -539,15 +536,14 @@ define void @vnsrl_0_i32_single_src_m8(ptr %in, ptr %out) {
; ZVE32F-NEXT: li a2, 64
; ZVE32F-NEXT: vsetvli zero, a2, e32, m8, ta, ma
; ZVE32F-NEXT: vle32.v v8, (a0)
-; ZVE32F-NEXT: vsetivli zero, 2, e32, m1, ta, ma
-; ZVE32F-NEXT: vmv.v.i v16, 0
; ZVE32F-NEXT: lui a0, 341
; ZVE32F-NEXT: addi a0, a0, 1365
-; ZVE32F-NEXT: vsetvli zero, zero, e32, m1, tu, ma
-; ZVE32F-NEXT: vmv.s.x v16, a0
+; ZVE32F-NEXT: vmv.s.x v12, a0
+; ZVE32F-NEXT: li a0, 32
+; ZVE32F-NEXT: vsetvli zero, a0, e32, m4, ta, ma
+; ZVE32F-NEXT: vcompress.vm v16, v8, v12
; ZVE32F-NEXT: vsetvli zero, a2, e32, m8, ta, ma
-; ZVE32F-NEXT: vcompress.vm v24, v8, v16
-; ZVE32F-NEXT: vse32.v v24, (a1)
+; ZVE32F-NEXT: vse32.v v16, (a1)
; ZVE32F-NEXT: ret
entry:
%0 = load <64 x i32>, ptr %in, align 4
>From 74bf146f789581fc76c35ae96ad192fa25970450 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Tue, 4 Feb 2025 15:26:58 -0800
Subject: [PATCH 3/5] clang-format
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index a6cca57943de1f0..535b94c7d2b29cd 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -5628,7 +5628,7 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
for (auto [I, M] : enumerate(Mask)) {
if (M == -1)
continue;
- MaxIdx = std::max(std::max((unsigned)I,(unsigned)M), MaxIdx);
+ MaxIdx = std::max(std::max((unsigned)I, (unsigned)M), MaxIdx);
}
unsigned NewNumElts = NumElts;
while (MaxIdx < NewNumElts / 2 && NewNumElts != MinVLMAX)
@@ -5639,8 +5639,8 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewVT, V1, ZeroIdx);
SDValue Res = DAG.getVectorShuffle(NewVT, DL, V1, DAG.getUNDEF(NewVT),
Mask.take_front(NewNumElts));
- return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
- Res, ZeroIdx);
+ return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Res,
+ ZeroIdx);
}
}
>From 17d3c812de66f8ee7f4b6f583701f018d7a34ddf Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Wed, 5 Feb 2025 08:22:00 -0800
Subject: [PATCH 4/5] Address review comment
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 535b94c7d2b29cd..579b96315dff3f6 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -5630,9 +5630,8 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
continue;
MaxIdx = std::max(std::max((unsigned)I, (unsigned)M), MaxIdx);
}
- unsigned NewNumElts = NumElts;
- while (MaxIdx < NewNumElts / 2 && NewNumElts != MinVLMAX)
- NewNumElts /= 2;
+ unsigned NewNumElts =
+ std::max((uint64_t)MinVLMAX, PowerOf2Ceil(MaxIdx + 1));
if (NewNumElts != NumElts) {
MVT NewVT = MVT::getVectorVT(VT.getVectorElementType(), NewNumElts);
SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
>From 5b4716a27a2ab8f6417a0d7370fe29f1b705f38b Mon Sep 17 00:00:00 2001
From: Philip Reames <preames at rivosinc.com>
Date: Wed, 5 Feb 2025 09:09:35 -0800
Subject: [PATCH 5/5] Remove now stale comment
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index d249d779132f83d..9a6a0de9b565ec4 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -5731,9 +5731,7 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
SDValue Gather;
// If we have a locally repeating mask, then we can reuse the first register
// in the index register group for all registers within the source register
- // group. TODO: This generalizes to m2, and m4. Also, this is currently
- // picking up cases with a fully undef tail which could be more directly
- // handled with fewer redundant vrgathers
+ // group. TODO: This generalizes to m2, and m4.
const MVT M1VT = getLMUL1VT(ContainerVT);
auto VLMAX = RISCVTargetLowering::computeVLMAXBounds(M1VT, Subtarget).first;
if (ContainerVT.bitsGT(M1VT) && isLocalRepeatingShuffle(Mask, VLMAX)) {
More information about the llvm-commits
mailing list