[llvm] [RISCV] Sink vp.splat operands of VP intrinsic. (PR #133245)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 14 03:27:47 PDT 2025
https://github.com/NexMing updated https://github.com/llvm/llvm-project/pull/133245
>From fe9907438c645b943211b238e88789826fa7c8f7 Mon Sep 17 00:00:00 2001
From: yanming <ming.yan at terapines.com>
Date: Thu, 27 Mar 2025 11:00:23 +0800
Subject: [PATCH 1/2] Add testcase.
---
.../CodeGen/RISCV/rvv/sink-splat-operands.ll | 67 +++++++++++++++++++
1 file changed, 67 insertions(+)
diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
index 9b794538c404e..0f09d5a35c98c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
@@ -5890,3 +5890,70 @@ vector.body: ; preds = %vector.body, %entry
for.cond.cleanup: ; preds = %vector.body
ret void
}
+
+define void @sink_vp_splat(ptr nocapture %out, ptr nocapture %in) {
+; CHECK-LABEL: sink_vp_splat:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: li a2, 0
+; CHECK-NEXT: li a3, 1024
+; CHECK-NEXT: lui a4, 1
+; CHECK-NEXT: .LBB129_1: # %vector.body
+; CHECK-NEXT: # =>This Loop Header: Depth=1
+; CHECK-NEXT: # Child Loop BB129_2 Depth 2
+; CHECK-NEXT: vsetvli a5, a3, e32, m4, ta, ma
+; CHECK-NEXT: slli a6, a2, 2
+; CHECK-NEXT: vmv.v.i v8, 0
+; CHECK-NEXT: vmv.v.i v12, 3
+; CHECK-NEXT: add a7, a1, a6
+; CHECK-NEXT: li t0, 1024
+; CHECK-NEXT: .LBB129_2: # %for.body424
+; CHECK-NEXT: # Parent Loop BB129_1 Depth=1
+; CHECK-NEXT: # => This Inner Loop Header: Depth=2
+; CHECK-NEXT: vle32.v v16, (a7)
+; CHECK-NEXT: addi t0, t0, -1
+; CHECK-NEXT: vmacc.vv v8, v16, v12
+; CHECK-NEXT: add a7, a7, a4
+; CHECK-NEXT: bnez t0, .LBB129_2
+; CHECK-NEXT: # %bb.3: # %vector.latch
+; CHECK-NEXT: # in Loop: Header=BB129_1 Depth=1
+; CHECK-NEXT: add a6, a0, a6
+; CHECK-NEXT: sub a3, a3, a5
+; CHECK-NEXT: vse32.v v8, (a6)
+; CHECK-NEXT: add a2, a2, a5
+; CHECK-NEXT: bnez a3, .LBB129_1
+; CHECK-NEXT: # %bb.4: # %for.cond.cleanup
+; CHECK-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body: ; preds = %vector.latch, %entry
+ %scalar.ind = phi i64 [ 0, %entry ], [ %next.ind, %vector.latch ]
+ %trip.count = phi i64 [ 1024, %entry ], [ %remaining.trip.count, %vector.latch ]
+ %evl = tail call i32 @llvm.experimental.get.vector.length.i64(i64 %trip.count, i32 8, i1 true)
+ %vp.splat1 = tail call <vscale x 8 x i32> @llvm.experimental.vp.splat.nxv8i32(i32 0, <vscale x 8 x i1> splat(i1 true), i32 %evl)
+ %vp.splat2 = tail call <vscale x 8 x i32> @llvm.experimental.vp.splat.nxv8i32(i32 3, <vscale x 8 x i1> splat(i1 true), i32 %evl)
+ %evl.cast = zext i32 %evl to i64
+ br label %for.body424
+
+for.body424: ; preds = %for.body424, %vector.body
+ %scalar.phi = phi i64 [ 0, %vector.body ], [ %indvars.iv.next27, %for.body424 ]
+ %vector.phi = phi <vscale x 8 x i32> [ %vp.splat1, %vector.body ], [ %vp.binary26, %for.body424 ]
+ %arrayidx625 = getelementptr inbounds [1024 x i32], ptr %in, i64 %scalar.phi, i64 %scalar.ind
+ %widen.load = tail call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %arrayidx625, <vscale x 8 x i1> splat (i1 true), i32 %evl)
+ %vp.binary = tail call <vscale x 8 x i32> @llvm.vp.mul.nxv8i32(<vscale x 8 x i32> %widen.load, <vscale x 8 x i32> %vp.splat2, <vscale x 8 x i1> splat (i1 true), i32 %evl)
+ %vp.binary26 = tail call <vscale x 8 x i32> @llvm.vp.add.nxv8i32(<vscale x 8 x i32> %vector.phi, <vscale x 8 x i32> %vp.binary, <vscale x 8 x i1> splat (i1 true), i32 %evl)
+ %indvars.iv.next27 = add nuw nsw i64 %scalar.phi, 1
+ %exitcond.not28 = icmp eq i64 %indvars.iv.next27, 1024
+ br i1 %exitcond.not28, label %vector.latch, label %for.body424
+
+vector.latch: ; preds = %for.body424
+ %arrayidx830 = getelementptr inbounds i32, ptr %out, i64 %scalar.ind
+ tail call void @llvm.vp.store.nxv8i32.p0(<vscale x 8 x i32> %vp.binary26, ptr %arrayidx830, <vscale x 8 x i1> splat (i1 true), i32 %evl)
+ %remaining.trip.count = sub nuw i64 %trip.count, %evl.cast
+ %next.ind = add i64 %scalar.ind, %evl.cast
+ %6 = icmp eq i64 %remaining.trip.count, 0
+ br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %vector.latch
+ ret void
+}
>From d6011d8401f4f1bd224182d9c8926f747fd4883e Mon Sep 17 00:00:00 2001
From: yanming <ming.yan at terapines.com>
Date: Mon, 14 Apr 2025 18:24:09 +0800
Subject: [PATCH 2/2] [RISCV] Sink vp.splat operands of VP intrinsic.
---
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 46 ++++++++++---------
.../CodeGen/RISCV/rvv/sink-splat-operands.ll | 30 ++++++------
2 files changed, 40 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 9b91de36a688a..e21b888ebb0c3 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -2837,17 +2837,19 @@ bool RISCVTTIImpl::isProfitableToSinkOperands(
if (!ST->sinkSplatOperands())
return false;
- for (auto OpIdx : enumerate(I->operands())) {
- if (!canSplatOperand(I, OpIdx.index()))
- continue;
-
- Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
+ for (auto &U : I->operands()) {
+ auto *Op = dyn_cast<Instruction>(U.get());
// Make sure we are not already sinking this operand
if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
continue;
- // We are looking for a splat that can be sunk.
- if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
+ // We are looking for a splat/vp.splat that can be sunk.
+ // FIXME: Should we care about the poison value of inactive elements in
+ // vp.splat?
+ bool IsVPSplat = match(Op, m_Intrinsic<Intrinsic::experimental_vp_splat>(
+ m_Value(), m_Value(), m_Value()));
+ if (!IsVPSplat &&
+ !match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
m_Undef(), m_ZeroMask())))
continue;
@@ -2855,21 +2857,23 @@ bool RISCVTTIImpl::isProfitableToSinkOperands(
if (cast<VectorType>(Op->getType())->getElementType()->isIntegerTy(1))
continue;
- // All uses of the shuffle should be sunk to avoid duplicating it across gpr
- // and vector registers
- for (Use &U : Op->uses()) {
- Instruction *Insn = cast<Instruction>(U.getUser());
- if (!canSplatOperand(Insn, U.getOperandNo()))
- return false;
- }
+ // All uses of the splat/vp.splat should be sunk to avoid duplicating it
+ // across gpr and vector registers
+ if (any_of(Op->uses(), [this](Use &U) {
+ return !canSplatOperand(cast<Instruction>(U.getUser()),
+ U.getOperandNo());
+ }))
+ continue;
- Use *InsertEltUse = &Op->getOperandUse(0);
- // Sink any fpexts since they might be used in a widening fp pattern.
- auto *InsertElt = cast<InsertElementInst>(InsertEltUse);
- if (isa<FPExtInst>(InsertElt->getOperand(1)))
- Ops.push_back(&InsertElt->getOperandUse(1));
- Ops.push_back(InsertEltUse);
- Ops.push_back(&OpIdx.value());
+ if (!IsVPSplat) {
+ Use *InsertEltUse = &Op->getOperandUse(0);
+ // Sink any fpexts since they might be used in a widening fp pattern.
+ auto *InsertElt = cast<InsertElementInst>(InsertEltUse);
+ if (isa<FPExtInst>(InsertElt->getOperand(1)))
+ Ops.push_back(&InsertElt->getOperandUse(1));
+ Ops.push_back(InsertEltUse);
+ }
+ Ops.push_back(&U);
}
return true;
}
diff --git a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
index 0f09d5a35c98c..7402a3513e17d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/sink-splat-operands.ll
@@ -5896,30 +5896,30 @@ define void @sink_vp_splat(ptr nocapture %out, ptr nocapture %in) {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: li a2, 0
; CHECK-NEXT: li a3, 1024
-; CHECK-NEXT: lui a4, 1
+; CHECK-NEXT: li a4, 3
+; CHECK-NEXT: lui a5, 1
; CHECK-NEXT: .LBB129_1: # %vector.body
; CHECK-NEXT: # =>This Loop Header: Depth=1
; CHECK-NEXT: # Child Loop BB129_2 Depth 2
-; CHECK-NEXT: vsetvli a5, a3, e32, m4, ta, ma
-; CHECK-NEXT: slli a6, a2, 2
+; CHECK-NEXT: vsetvli a6, a3, e32, m4, ta, ma
+; CHECK-NEXT: slli a7, a2, 2
; CHECK-NEXT: vmv.v.i v8, 0
-; CHECK-NEXT: vmv.v.i v12, 3
-; CHECK-NEXT: add a7, a1, a6
-; CHECK-NEXT: li t0, 1024
+; CHECK-NEXT: add t0, a1, a7
+; CHECK-NEXT: li t1, 1024
; CHECK-NEXT: .LBB129_2: # %for.body424
; CHECK-NEXT: # Parent Loop BB129_1 Depth=1
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vle32.v v16, (a7)
-; CHECK-NEXT: addi t0, t0, -1
-; CHECK-NEXT: vmacc.vv v8, v16, v12
-; CHECK-NEXT: add a7, a7, a4
-; CHECK-NEXT: bnez t0, .LBB129_2
+; CHECK-NEXT: vle32.v v12, (t0)
+; CHECK-NEXT: addi t1, t1, -1
+; CHECK-NEXT: vmacc.vx v8, a4, v12
+; CHECK-NEXT: add t0, t0, a5
+; CHECK-NEXT: bnez t1, .LBB129_2
; CHECK-NEXT: # %bb.3: # %vector.latch
; CHECK-NEXT: # in Loop: Header=BB129_1 Depth=1
-; CHECK-NEXT: add a6, a0, a6
-; CHECK-NEXT: sub a3, a3, a5
-; CHECK-NEXT: vse32.v v8, (a6)
-; CHECK-NEXT: add a2, a2, a5
+; CHECK-NEXT: add a7, a0, a7
+; CHECK-NEXT: sub a3, a3, a6
+; CHECK-NEXT: vse32.v v8, (a7)
+; CHECK-NEXT: add a2, a2, a6
; CHECK-NEXT: bnez a3, .LBB129_1
; CHECK-NEXT: # %bb.4: # %for.cond.cleanup
; CHECK-NEXT: ret
More information about the llvm-commits
mailing list