[llvm] [VPlan] Use VPWidenIntrinsicRecipe to support binary and unary operations with EVL-vectorization (PR #114205)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Nov 17 23:09:54 PST 2024
https://github.com/LiqinWeng updated https://github.com/llvm/llvm-project/pull/114205
>From 7f56a9c0c27aff5b7fe6c841b8d1121ad41c8853 Mon Sep 17 00:00:00 2001
From: LiqinWeng <liqin.weng at spacemit.com>
Date: Wed, 30 Oct 2024 18:20:24 +0800
Subject: [PATCH 1/4] [VPlan] Use VPWidenIntrinsicRecipe to support binary and
unary operations with EVL-vectorization
---
llvm/include/llvm/IR/VectorBuilder.h | 4 +-
llvm/lib/IR/VectorBuilder.cpp | 9 +-
llvm/lib/Transforms/Utils/LoopUtils.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 79 ++----------
.../Transforms/Vectorize/VPlanAnalysis.cpp | 5 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 119 +++++++++---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 1 -
.../Transforms/Vectorize/VPlanVerifier.cpp | 4 -
.../RISCV/vplan-vp-intrinsics.ll | 2 +-
.../RISCV/vplan-vp-select-intrinsics.ll | 4 +-
11 files changed, 91 insertions(+), 149 deletions(-)
diff --git a/llvm/include/llvm/IR/VectorBuilder.h b/llvm/include/llvm/IR/VectorBuilder.h
index b0277c2b52595e..830163984e37ba 100644
--- a/llvm/include/llvm/IR/VectorBuilder.h
+++ b/llvm/include/llvm/IR/VectorBuilder.h
@@ -99,11 +99,11 @@ class VectorBuilder {
const Twine &Name = Twine());
/// Emit a VP reduction intrinsic call for recurrence kind.
- /// \param RdxID The intrinsic ID of llvm.vector.reduce.*
+ /// \param ID The intrinsic ID of call Intrinsic
/// \param ValTy The type of operand which the reduction operation is
/// performed.
/// \param VecOpArray The operand list.
- Value *createSimpleReduction(Intrinsic::ID RdxID, Type *ValTy,
+ Value *createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy,
ArrayRef<Value *> VecOpArray,
const Twine &Name = Twine());
};
diff --git a/llvm/lib/IR/VectorBuilder.cpp b/llvm/lib/IR/VectorBuilder.cpp
index 737f49b1334d76..d629a2fb6af7b3 100644
--- a/llvm/lib/IR/VectorBuilder.cpp
+++ b/llvm/lib/IR/VectorBuilder.cpp
@@ -60,13 +60,12 @@ Value *VectorBuilder::createVectorInstruction(unsigned Opcode, Type *ReturnTy,
return createVectorInstructionImpl(VPID, ReturnTy, InstOpArray, Name);
}
-Value *VectorBuilder::createSimpleReduction(Intrinsic::ID RdxID,
- Type *ValTy,
+Value *VectorBuilder::createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy,
ArrayRef<Value *> InstOpArray,
const Twine &Name) {
- auto VPID = VPIntrinsic::getForIntrinsic(RdxID);
- assert(VPReductionIntrinsic::isVPReduction(VPID) &&
- "No VPIntrinsic for this reduction");
+ auto VPID = VPIntrinsic::getForIntrinsic(ID);
+ assert(VPIntrinsic::isVPIntrinsic(VPID) &&
+ "No VPIntrinsic for this Intrinsic");
return createVectorInstructionImpl(VPID, ValTy, InstOpArray, Name);
}
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 70047273c3b9af..2dac2d43f7f3a3 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1300,7 +1300,7 @@ Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src,
Type *SrcEltTy = SrcTy->getElementType();
Value *Iden = getRecurrenceIdentity(Kind, SrcEltTy, Desc.getFastMathFlags());
Value *Ops[] = {Iden, Src};
- return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
+ return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops);
}
Value *llvm::createReduction(IRBuilderBase &B,
@@ -1343,7 +1343,7 @@ Value *llvm::createOrderedReduction(VectorBuilder &VBuilder,
Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd);
auto *SrcTy = cast<VectorType>(Src->getType());
Value *Ops[] = {Start, Src};
- return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
+ return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops);
}
void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index abfe97b4ab55b6..2b2cec1055b7a6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -867,7 +867,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
case VPRecipeBase::VPWidenGEPSC:
case VPRecipeBase::VPWidenIntrinsicSC:
case VPRecipeBase::VPWidenSC:
- case VPRecipeBase::VPWidenEVLSC:
case VPRecipeBase::VPWidenSelectSC:
case VPRecipeBase::VPBlendSC:
case VPRecipeBase::VPPredInstPHISC:
@@ -1058,7 +1057,6 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe {
static inline bool classof(const VPRecipeBase *R) {
return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
R->getVPDefID() == VPRecipeBase::VPWidenSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenEVLSC ||
R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
@@ -1438,16 +1436,11 @@ class VPIRInstruction : public VPRecipeBase {
class VPWidenRecipe : public VPRecipeWithIRFlags {
unsigned Opcode;
-protected:
- template <typename IterT>
- VPWidenRecipe(unsigned VPDefOpcode, Instruction &I,
- iterator_range<IterT> Operands)
- : VPRecipeWithIRFlags(VPDefOpcode, Operands, I), Opcode(I.getOpcode()) {}
-
public:
template <typename IterT>
VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands)
- : VPWidenRecipe(VPDef::VPWidenSC, I, Operands) {}
+ : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I),
+ Opcode(I.getOpcode()) {}
~VPWidenRecipe() override = default;
@@ -1457,15 +1450,7 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
return R;
}
- static inline bool classof(const VPRecipeBase *R) {
- return R->getVPDefID() == VPRecipeBase::VPWidenSC ||
- R->getVPDefID() == VPRecipeBase::VPWidenEVLSC;
- }
-
- static inline bool classof(const VPUser *U) {
- auto *R = dyn_cast<VPRecipeBase>(U);
- return R && classof(R);
- }
+ VP_CLASSOF_IMPL(VPDef::VPWidenSC)
/// Produce a widened instruction using the opcode and operands of the recipe,
/// processing State.VF elements.
@@ -1484,54 +1469,6 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
#endif
};
-/// A recipe for widening operations with vector-predication intrinsics with
-/// explicit vector length (EVL).
-class VPWidenEVLRecipe : public VPWidenRecipe {
- using VPRecipeWithIRFlags::transferFlags;
-
-public:
- template <typename IterT>
- VPWidenEVLRecipe(Instruction &I, iterator_range<IterT> Operands, VPValue &EVL)
- : VPWidenRecipe(VPDef::VPWidenEVLSC, I, Operands) {
- addOperand(&EVL);
- }
- VPWidenEVLRecipe(VPWidenRecipe &W, VPValue &EVL)
- : VPWidenEVLRecipe(*W.getUnderlyingInstr(), W.operands(), EVL) {
- transferFlags(W);
- }
-
- ~VPWidenEVLRecipe() override = default;
-
- VPWidenRecipe *clone() override final {
- llvm_unreachable("VPWidenEVLRecipe cannot be cloned");
- return nullptr;
- }
-
- VP_CLASSOF_IMPL(VPDef::VPWidenEVLSC);
-
- VPValue *getEVL() { return getOperand(getNumOperands() - 1); }
- const VPValue *getEVL() const { return getOperand(getNumOperands() - 1); }
-
- /// Produce a vp-intrinsic using the opcode and operands of the recipe,
- /// processing EVL elements.
- void execute(VPTransformState &State) override final;
-
- /// Returns true if the recipe only uses the first lane of operand \p Op.
- bool onlyFirstLaneUsed(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- // EVL in that recipe is always the last operand, thus any use before means
- // the VPValue should be vectorized.
- return getEVL() == Op;
- }
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override final;
-#endif
-};
-
/// VPWidenCastRecipe is a recipe to create vector cast instructions.
class VPWidenCastRecipe : public VPRecipeWithIRFlags {
/// Cast instruction opcode.
@@ -1657,6 +1594,16 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
+ template <typename IterT>
+ VPWidenIntrinsicRecipe(Instruction &I, Intrinsic::ID VectorIntrinsicID,
+ iterator_range<IterT> Operands, Type *Ty,
+ DebugLoc DL = {})
+ : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, Operands, I),
+ VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
+ MayReadFromMemory(I.mayReadFromMemory()),
+ MayWriteToMemory(I.mayWriteToMemory()),
+ MayHaveSideEffects(I.mayHaveSideEffects()) {}
+
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
DebugLoc DL = {})
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 8b8ab6be99b0d5..f50a1286316a08 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -267,9 +267,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
[this](const VPRecipeBase *R) {
return inferScalarType(R->getOperand(0));
})
- .Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPWidenEVLRecipe,
- VPReplicateRecipe, VPWidenCallRecipe, VPWidenMemoryRecipe,
- VPWidenSelectRecipe>(
+ .Case<VPBlendRecipe, VPInstruction, VPWidenRecipe, VPReplicateRecipe,
+ VPWidenCallRecipe, VPWidenMemoryRecipe, VPWidenSelectRecipe>(
[this](const auto *R) { return inferScalarTypeForRecipe(R); })
.Case<VPWidenIntrinsicRecipe>([](const VPWidenIntrinsicRecipe *R) {
return R->getResultType();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ef2ca9af7268d1..eb6374e3ab6edb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -98,7 +98,6 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPWidenLoadSC:
case VPWidenPHISC:
case VPWidenSC:
- case VPWidenEVLSC:
case VPWidenSelectSC: {
const Instruction *I =
dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
@@ -142,7 +141,6 @@ bool VPRecipeBase::mayReadFromMemory() const {
case VPWidenIntOrFpInductionSC:
case VPWidenPHISC:
case VPWidenSC:
- case VPWidenEVLSC:
case VPWidenSelectSC: {
const Instruction *I =
dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
@@ -183,7 +181,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
case VPWidenPHISC:
case VPWidenPointerInductionSC:
case VPWidenSC:
- case VPWidenEVLSC:
case VPWidenSelectSC: {
const Instruction *I =
dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
@@ -957,24 +954,53 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
Args.push_back(Arg);
}
- // Use vector version of the intrinsic.
- Module *M = State.Builder.GetInsertBlock()->getModule();
- Function *VectorF =
- Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
- assert(VectorF && "Can't retrieve vector intrinsic.");
+ if (VPIntrinsic::isVPIntrinsic(VectorIntrinsicID) &&
+ VectorIntrinsicID != Intrinsic::vp_select) {
+ VectorBuilder VBuilder(State.Builder);
+ Value *Mask =
+ State.Builder.CreateVectorSplat(State.VF, State.Builder.getTrue());
+ VBuilder.setMask(Mask).setEVL(Args.back());
+ // Remove EVL from Args
+ Args.pop_back();
+
+ if (VectorIntrinsicID == Intrinsic::vp_icmp ||
+ VectorIntrinsicID == Intrinsic::vp_fcmp) {
+ auto &Ctx = State.Builder.getContext();
+ Value *Pred = MetadataAsValue::get(
+ Ctx, MDString::get(Ctx, CmpInst::getPredicateName(getPredicate())));
+ Args.push_back(Pred);
+ }
- auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
- SmallVector<OperandBundleDef, 1> OpBundles;
- if (CI)
- CI->getOperandBundlesAsDefs(OpBundles);
+ Value *VPInst = VBuilder.createSimpleIntrinsic(
+ VectorIntrinsicID, TysForDecl[0], Args, "vp.call");
- CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+ if (isa<FPMathOperator>(VPInst))
+ setFlags(cast<Instruction>(VPInst));
- setFlags(V);
+ if (!VPInst->getType()->isVoidTy())
+ State.set(this, VPInst);
+ State.addMetadata(VPInst,
+ dyn_cast_or_null<Instruction>(getUnderlyingValue()));
+ } else {
+ // Use vector version of the intrinsic.
+ Module *M = State.Builder.GetInsertBlock()->getModule();
+ Function *VectorF =
+ Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
+ assert(VectorF && "Can't retrieve vector intrinsic.");
- if (!V->getType()->isVoidTy())
- State.set(this, V);
- State.addMetadata(V, CI);
+ auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
+ SmallVector<OperandBundleDef, 1> OpBundles;
+ if (CI)
+ CI->getOperandBundlesAsDefs(OpBundles);
+
+ CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+
+ setFlags(V);
+
+ if (!V->getType()->isVoidTy())
+ State.set(this, V);
+ State.addMetadata(V, CI);
+ }
}
InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
@@ -1006,6 +1032,20 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
ParamTys.push_back(
ToVectorTy(Ctx.Types.inferScalarType(getOperand(I)), VF));
+ // TODO: Implment in cost model
+ if (std::optional<unsigned> FOp =
+ VPIntrinsic::getFunctionalOpcodeForVP(VectorIntrinsicID)) {
+ if (FOp == Instruction::FNeg) {
+ // Instruction *CtxI =
+ dyn_cast_or_null<Instruction>(getUnderlyingValue());
+ Type *VectorTy = ToVectorTy(getResultType(), VF);
+ return Ctx.TTI.getArithmeticInstrCost(
+ FOp.value(), VectorTy, CostKind,
+ {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+ {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None});
+ }
+ }
+
// TODO: Rework TTI interface to avoid reliance on underlying IntrinsicInst.
FastMathFlags FMF = hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
IntrinsicCostAttributes CostAttrs(
@@ -1417,42 +1457,6 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
}
}
-void VPWidenEVLRecipe::execute(VPTransformState &State) {
- unsigned Opcode = getOpcode();
- // TODO: Support other opcodes
- if (!Instruction::isBinaryOp(Opcode) && !Instruction::isUnaryOp(Opcode))
- llvm_unreachable("Unsupported opcode in VPWidenEVLRecipe::execute");
-
- State.setDebugLocFrom(getDebugLoc());
-
- assert(State.get(getOperand(0))->getType()->isVectorTy() &&
- "VPWidenEVLRecipe should not be used for scalars");
-
- VPValue *EVL = getEVL();
- Value *EVLArg = State.get(EVL, /*NeedsScalar=*/true);
- IRBuilderBase &BuilderIR = State.Builder;
- VectorBuilder Builder(BuilderIR);
- Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue());
-
- SmallVector<Value *, 4> Ops;
- for (unsigned I = 0, E = getNumOperands() - 1; I < E; ++I) {
- VPValue *VPOp = getOperand(I);
- Ops.push_back(State.get(VPOp));
- }
-
- Builder.setMask(Mask).setEVL(EVLArg);
- Value *VPInst =
- Builder.createVectorInstruction(Opcode, Ops[0]->getType(), Ops, "vp.op");
- // Currently vp-intrinsics only accept FMF flags.
- // TODO: Enable other flags when support is added.
- if (isa<FPMathOperator>(VPInst))
- setFlags(cast<Instruction>(VPInst));
-
- State.set(this, VPInst);
- State.addMetadata(VPInst,
- dyn_cast_or_null<Instruction>(getUnderlyingValue()));
-}
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
@@ -1462,15 +1466,6 @@ void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
printFlags(O);
printOperands(O, SlotTracker);
}
-
-void VPWidenEVLRecipe::print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "WIDEN ";
- printAsOperand(O, SlotTracker);
- O << " = vp." << Instruction::getOpcodeName(getOpcode());
- printFlags(O);
- printOperands(O, SlotTracker);
-}
#endif
void VPWidenCastRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index c1b9d6ede51090..2779ce9cbefc72 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1465,10 +1465,17 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
})
.Case<VPWidenRecipe>([&](VPWidenRecipe *W) -> VPRecipeBase * {
unsigned Opcode = W->getOpcode();
+ // TODO: Support other opcodes
if (!Instruction::isBinaryOp(Opcode) &&
!Instruction::isUnaryOp(Opcode))
return nullptr;
- return new VPWidenEVLRecipe(*W, EVL);
+ auto *I = cast<Instruction>(W->getUnderlyingInstr());
+ SmallVector<VPValue *> Ops(W->operands());
+ Ops.push_back(&EVL);
+ Intrinsic::ID VPID = VPIntrinsic::getForOpcode(W->getOpcode());
+ return new VPWidenIntrinsicRecipe(
+ *I, VPID, make_range(Ops.begin(), Ops.end()), I->getType(),
+ I->getDebugLoc());
})
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
VPValue *NewMask = GetNewMask(Red->getCondOp());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 691b0d40823cfb..e35bd66cdaab82 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -344,7 +344,6 @@ class VPDef {
VPWidenStoreEVLSC,
VPWidenStoreSC,
VPWidenSC,
- VPWidenEVLSC,
VPWidenSelectSC,
VPBlendSC,
VPHistogramSC,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 8bdb3133243582..f4764c11af9315 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -146,10 +146,6 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
.Case<VPWidenLoadEVLRecipe>([&](const VPWidenLoadEVLRecipe *L) {
return VerifyEVLUse(*L, 1);
})
- .Case<VPWidenEVLRecipe>([&](const VPWidenEVLRecipe *W) {
- return VerifyEVLUse(
- *W, Instruction::isUnaryOp(W->getOpcode()) ? 1 : 2);
- })
.Case<VPReductionEVLRecipe>([&](const VPReductionEVLRecipe *R) {
return VerifyEVLUse(*R, 2);
})
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
index 706b6f88829848..f1f7dbca0b10b8 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -32,7 +32,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
-; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = vp.add nsw ir<[[LD2]]>, ir<[[LD1]]>, vp<[[EVL]]>
+; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call nsw llvm.vp.add(ir<[[LD2]]>, ir<[[LD1]]>, vp<[[EVL]]>)
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
index 6d6cfb5e9d18ed..f174f9d007ef55 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -28,9 +28,9 @@
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
; IF-EVL-NEXT: WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]>
- ; IF-EVL-NEXT: WIDEN ir<[[SUB:%.+]]> = vp.sub ir<0>, ir<[[LD2]]>, vp<[[EVL]]>
+ ; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SUB:%.+]]> = call llvm.vp.sub(ir<0>, ir<[[LD2]]>, vp<[[EVL]]>)
; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%1>, ir<%2>, vp<[[EVL]]>)
- ; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = vp.add vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]>
+ ; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call llvm.vp.add(vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]>)
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]>
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
>From 177bee58ac175d587975f177cc502b8ead13c118 Mon Sep 17 00:00:00 2001
From: LiqinWeng <liqin.weng at spacemit.com>
Date: Wed, 30 Oct 2024 19:19:23 +0800
Subject: [PATCH 2/4] Remove some unused code && implement the vp_fneg cost in
TTI
---
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 2 ++
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 22 -------------------
2 files changed, 2 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 4edaeef354a59d..5928e4b32b5c14 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1118,6 +1118,8 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
case Intrinsic::vp_udiv:
case Intrinsic::vp_urem:
case Intrinsic::vp_xor:
+ // TODO: add new patch for it.
+ case Intrinsic::vp_fneg:
// vp float arithmetic ops.
case Intrinsic::vp_fadd:
case Intrinsic::vp_fsub:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index eb6374e3ab6edb..e276efdcd53047 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -963,14 +963,6 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
// Remove EVL from Args
Args.pop_back();
- if (VectorIntrinsicID == Intrinsic::vp_icmp ||
- VectorIntrinsicID == Intrinsic::vp_fcmp) {
- auto &Ctx = State.Builder.getContext();
- Value *Pred = MetadataAsValue::get(
- Ctx, MDString::get(Ctx, CmpInst::getPredicateName(getPredicate())));
- Args.push_back(Pred);
- }
-
Value *VPInst = VBuilder.createSimpleIntrinsic(
VectorIntrinsicID, TysForDecl[0], Args, "vp.call");
@@ -1032,20 +1024,6 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
ParamTys.push_back(
ToVectorTy(Ctx.Types.inferScalarType(getOperand(I)), VF));
- // TODO: Implment in cost model
- if (std::optional<unsigned> FOp =
- VPIntrinsic::getFunctionalOpcodeForVP(VectorIntrinsicID)) {
- if (FOp == Instruction::FNeg) {
- // Instruction *CtxI =
- dyn_cast_or_null<Instruction>(getUnderlyingValue());
- Type *VectorTy = ToVectorTy(getResultType(), VF);
- return Ctx.TTI.getArithmeticInstrCost(
- FOp.value(), VectorTy, CostKind,
- {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
- {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None});
- }
- }
-
// TODO: Rework TTI interface to avoid reliance on underlying IntrinsicInst.
FastMathFlags FMF = hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
IntrinsicCostAttributes CostAttrs(
>From 9f1312793650488b63135463a15aca081352aab9 Mon Sep 17 00:00:00 2001
From: "Liqin.Weng" <liqin.weng at spacemit.com>
Date: Wed, 6 Nov 2024 16:11:46 +0800
Subject: [PATCH 3/4] 1. add mask Op in VPlanTransform 2. remove the
vpintrinsic's implment in execute
---
llvm/include/llvm/IR/VectorBuilder.h | 4 +-
llvm/lib/IR/VectorBuilder.cpp | 9 +--
.../Target/RISCV/RISCVTargetTransformInfo.cpp | 2 -
llvm/lib/Transforms/Utils/LoopUtils.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 12 +---
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 56 +++++++------------
.../Transforms/Vectorize/VPlanTransforms.cpp | 15 +++--
.../RISCV/vplan-vp-intrinsics.ll | 2 +-
.../RISCV/vplan-vp-select-intrinsics.ll | 4 +-
9 files changed, 45 insertions(+), 63 deletions(-)
diff --git a/llvm/include/llvm/IR/VectorBuilder.h b/llvm/include/llvm/IR/VectorBuilder.h
index 830163984e37ba..b0277c2b52595e 100644
--- a/llvm/include/llvm/IR/VectorBuilder.h
+++ b/llvm/include/llvm/IR/VectorBuilder.h
@@ -99,11 +99,11 @@ class VectorBuilder {
const Twine &Name = Twine());
/// Emit a VP reduction intrinsic call for recurrence kind.
- /// \param ID The intrinsic ID of call Intrinsic
+ /// \param RdxID The intrinsic ID of llvm.vector.reduce.*
/// \param ValTy The type of operand which the reduction operation is
/// performed.
/// \param VecOpArray The operand list.
- Value *createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy,
+ Value *createSimpleReduction(Intrinsic::ID RdxID, Type *ValTy,
ArrayRef<Value *> VecOpArray,
const Twine &Name = Twine());
};
diff --git a/llvm/lib/IR/VectorBuilder.cpp b/llvm/lib/IR/VectorBuilder.cpp
index d629a2fb6af7b3..737f49b1334d76 100644
--- a/llvm/lib/IR/VectorBuilder.cpp
+++ b/llvm/lib/IR/VectorBuilder.cpp
@@ -60,12 +60,13 @@ Value *VectorBuilder::createVectorInstruction(unsigned Opcode, Type *ReturnTy,
return createVectorInstructionImpl(VPID, ReturnTy, InstOpArray, Name);
}
-Value *VectorBuilder::createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy,
+Value *VectorBuilder::createSimpleReduction(Intrinsic::ID RdxID,
+ Type *ValTy,
ArrayRef<Value *> InstOpArray,
const Twine &Name) {
- auto VPID = VPIntrinsic::getForIntrinsic(ID);
- assert(VPIntrinsic::isVPIntrinsic(VPID) &&
- "No VPIntrinsic for this Intrinsic");
+ auto VPID = VPIntrinsic::getForIntrinsic(RdxID);
+ assert(VPReductionIntrinsic::isVPReduction(VPID) &&
+ "No VPIntrinsic for this reduction");
return createVectorInstructionImpl(VPID, ValTy, InstOpArray, Name);
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 5928e4b32b5c14..4edaeef354a59d 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1118,8 +1118,6 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
case Intrinsic::vp_udiv:
case Intrinsic::vp_urem:
case Intrinsic::vp_xor:
- // TODO: add new patch for it.
- case Intrinsic::vp_fneg:
// vp float arithmetic ops.
case Intrinsic::vp_fadd:
case Intrinsic::vp_fsub:
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 2dac2d43f7f3a3..70047273c3b9af 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1300,7 +1300,7 @@ Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src,
Type *SrcEltTy = SrcTy->getElementType();
Value *Iden = getRecurrenceIdentity(Kind, SrcEltTy, Desc.getFastMathFlags());
Value *Ops[] = {Iden, Src};
- return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops);
+ return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
}
Value *llvm::createReduction(IRBuilderBase &B,
@@ -1343,7 +1343,7 @@ Value *llvm::createOrderedReduction(VectorBuilder &VBuilder,
Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd);
auto *SrcTy = cast<VectorType>(Src->getType());
Value *Ops[] = {Start, Src};
- return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops);
+ return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
}
void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 2b2cec1055b7a6..1c8f015d6c9353 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1585,18 +1585,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
bool MayHaveSideEffects;
public:
- VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
- ArrayRef<VPValue *> CallArguments, Type *Ty,
- DebugLoc DL = {})
- : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
- VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
- MayReadFromMemory(CI.mayReadFromMemory()),
- MayWriteToMemory(CI.mayWriteToMemory()),
- MayHaveSideEffects(CI.mayHaveSideEffects()) {}
-
- template <typename IterT>
VPWidenIntrinsicRecipe(Instruction &I, Intrinsic::ID VectorIntrinsicID,
- iterator_range<IterT> Operands, Type *Ty,
+ ArrayRef<VPValue *> Operands, Type *Ty,
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, Operands, I),
VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e276efdcd53047..82dcfe2182eade 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -954,45 +954,31 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
Args.push_back(Arg);
}
- if (VPIntrinsic::isVPIntrinsic(VectorIntrinsicID) &&
- VectorIntrinsicID != Intrinsic::vp_select) {
- VectorBuilder VBuilder(State.Builder);
- Value *Mask =
- State.Builder.CreateVectorSplat(State.VF, State.Builder.getTrue());
- VBuilder.setMask(Mask).setEVL(Args.back());
- // Remove EVL from Args
- Args.pop_back();
-
- Value *VPInst = VBuilder.createSimpleIntrinsic(
- VectorIntrinsicID, TysForDecl[0], Args, "vp.call");
-
- if (isa<FPMathOperator>(VPInst))
- setFlags(cast<Instruction>(VPInst));
-
- if (!VPInst->getType()->isVoidTy())
- State.set(this, VPInst);
- State.addMetadata(VPInst,
- dyn_cast_or_null<Instruction>(getUnderlyingValue()));
- } else {
- // Use vector version of the intrinsic.
- Module *M = State.Builder.GetInsertBlock()->getModule();
- Function *VectorF =
- Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
- assert(VectorF && "Can't retrieve vector intrinsic.");
-
- auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
- SmallVector<OperandBundleDef, 1> OpBundles;
- if (CI)
- CI->getOperandBundlesAsDefs(OpBundles);
+ // Use vector version of the intrinsic.
+ Module *M = State.Builder.GetInsertBlock()->getModule();
+ bool IsVPIntrinsic = VPIntrinsic::isVPIntrinsic(VectorIntrinsicID);
+ Function *VectorF =
+ Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
+ assert(VectorF && "Can't retrieve vector intrinsic.");
- CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+ SmallVector<OperandBundleDef, 1> OpBundles;
+ if (!IsVPIntrinsic) {
+ if (auto *CI = cast_or_null<CallInst>(getUnderlyingValue()))
+ CI->getOperandBundlesAsDefs(OpBundles);
+ }
- setFlags(V);
+ Value *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
- if (!V->getType()->isVoidTy())
- State.set(this, V);
- State.addMetadata(V, CI);
+ if (IsVPIntrinsic) {
+ if (isa<FPMathOperator>(V))
+ setFlags(cast<Instruction>(V));
+ } else {
+ setFlags(cast<Instruction>(V));
}
+
+ if (!V->getType()->isVoidTy())
+ State.set(this, V);
+ State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
}
InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2779ce9cbefc72..d26e3e0f45eb31 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1469,13 +1469,20 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
if (!Instruction::isBinaryOp(Opcode) &&
!Instruction::isUnaryOp(Opcode))
return nullptr;
+
auto *I = cast<Instruction>(W->getUnderlyingInstr());
+ Intrinsic::ID VPID = VPIntrinsic::getForOpcode(Opcode);
SmallVector<VPValue *> Ops(W->operands());
+ assert(VPIntrinsic::getMaskParamPos(VPID) &&
+ VPIntrinsic::getVectorLengthParamPos(VPID) &&
+ "Expected VP intrinsic");
+ VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(
+ IntegerType::getInt1Ty(I->getContext())));
+ Ops.push_back(Mask);
Ops.push_back(&EVL);
- Intrinsic::ID VPID = VPIntrinsic::getForOpcode(W->getOpcode());
- return new VPWidenIntrinsicRecipe(
- *I, VPID, make_range(Ops.begin(), Ops.end()), I->getType(),
- I->getDebugLoc());
+ return new VPWidenIntrinsicRecipe(*I, VPID, Ops,
+ TypeInfo.inferScalarType(W),
+ W->getDebugLoc());
})
.Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
VPValue *NewMask = GetNewMask(Red->getCondOp());
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
index f1f7dbca0b10b8..77ff66c2812a92 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -32,7 +32,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
-; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call nsw llvm.vp.add(ir<[[LD2]]>, ir<[[LD1]]>, vp<[[EVL]]>)
+; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call nsw llvm.vp.add(ir<[[LD2]]>, ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
index f174f9d007ef55..f5cfb5fc11d232 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -28,9 +28,9 @@
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
; IF-EVL-NEXT: WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]>
- ; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SUB:%.+]]> = call llvm.vp.sub(ir<0>, ir<[[LD2]]>, vp<[[EVL]]>)
+ ; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[SUB:%.+]]> = call llvm.vp.sub(ir<0>, ir<[[LD2]]>, ir<true>, vp<[[EVL]]>)
; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%1>, ir<%2>, vp<[[EVL]]>)
- ; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call llvm.vp.add(vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]>)
+ ; IF-EVL-NEXT: WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call llvm.vp.add(vp<[[SELECT]]>, ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]>
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
>From 4ead4d417414350be63e1e9bc2a880fd7b88af85 Mon Sep 17 00:00:00 2001
From: LiqinWeng <liqin.weng at spacemit.com>
Date: Mon, 18 Nov 2024 15:08:34 +0800
Subject: [PATCH 4/4] fix the comments 1. clang-format 2. fix the failed test
---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 3 ++-
.../LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 82dcfe2182eade..ee92dbd75e7597 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -971,7 +971,8 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
if (IsVPIntrinsic) {
if (isa<FPMathOperator>(V))
- setFlags(cast<Instruction>(V));
+ // Currently vp-intrinsics only accept FMF flags.
+ setFlags(cast<Instruction>(V));
} else {
setFlags(cast<Instruction>(V));
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index 414f39d557044a..c61a61af36ccd0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -43,7 +43,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
; IF-EVL-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
; IF-EVL-OUTLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
-; IF-EVL-OUTLOOP-NEXT: WIDEN ir<[[ADD:%.+]]> = vp.add ir<[[LD1]]>, ir<[[RDX_PHI]]>, vp<[[EVL]]>
+; IF-EVL-OUTLOOP-NEXT: WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call nsw llvm.vp.add(ir<[[LD1]]>, ir<[[RDX_PHI]]>, ir<true>, vp<[[EVL]]>)
; IF-EVL-OUTLOOP-NEXT: WIDEN-INTRINSIC vp<[[RDX_SELECT]]> = call llvm.vp.merge(ir<true>, ir<[[ADD]]>, ir<[[RDX_PHI]]>, vp<[[EVL]]>)
; IF-EVL-OUTLOOP-NEXT: SCALAR-CAST vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
More information about the llvm-commits
mailing list