[llvm] 511726c - [LV] Move getStepVector out of ILV (NFC).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Dec 26 12:17:49 PST 2021
Author: Florian Hahn
Date: 2021-12-26T21:17:26+01:00
New Revision: 511726c64d3b6cca66f7c54d457d586aa3129f67
URL: https://github.com/llvm/llvm-project/commit/511726c64d3b6cca66f7c54d457d586aa3129f67
DIFF: https://github.com/llvm/llvm-project/commit/511726c64d3b6cca66f7c54d457d586aa3129f67.diff
LOG: [LV] Move getStepVector out of ILV (NFC).
First step to split up induction handling and move it outside ILV.
Used in D116123 and following.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 4747f34fcc625..83f9e3f58993b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -606,14 +606,6 @@ class InnerLoopVectorizer {
/// represented as.
void truncateToMinimalBitwidths(VPTransformState &State);
- /// This function adds
- /// (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
- /// to each vector element of Val. The sequence starts at StartIndex.
- /// \p Opcode is relevant for FP induction variable.
- virtual Value *
- getStepVector(Value *Val, Value *StartIdx, Value *Step,
- Instruction::BinaryOps Opcode = Instruction::BinaryOpsEnd);
-
/// Compute scalar induction steps. \p ScalarIV is the scalar induction
/// variable on which to base the steps, \p Step is the size of the step, and
/// \p EntryVal is the value from the original loop that maps to the steps.
@@ -856,9 +848,6 @@ class InnerLoopUnroller : public InnerLoopVectorizer {
private:
Value *getBroadcastInstrs(Value *V) override;
- Value *getStepVector(
- Value *Val, Value *StartIdx, Value *Step,
- Instruction::BinaryOps Opcode = Instruction::BinaryOpsEnd) override;
Value *reverseVector(Value *Vec) override;
};
@@ -2335,6 +2324,72 @@ Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
return Shuf;
}
+/// This function adds
+/// (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
+/// to each vector element of Val. The sequence starts at StartIndex.
+/// \p Opcode is relevant for FP induction variable.
+static Value *getStepVector(Value *Val, Value *StartIdx, Value *Step,
+ Instruction::BinaryOps BinOp, ElementCount VF,
+ IRBuilder<> &Builder) {
+ if (VF.isScalar()) {
+ // When unrolling and the VF is 1, we only need to add a simple scalar.
+ Type *Ty = Val->getType();
+ assert(!Ty->isVectorTy() && "Val must be a scalar");
+
+ if (Ty->isFloatingPointTy()) {
+ // Floating-point operations inherit FMF via the builder's flags.
+ Value *MulOp = Builder.CreateFMul(StartIdx, Step);
+ return Builder.CreateBinOp(BinOp, Val, MulOp);
+ }
+ return Builder.CreateAdd(Val, Builder.CreateMul(StartIdx, Step),
+ "induction");
+ }
+
+ // Create and check the types.
+ auto *ValVTy = cast<VectorType>(Val->getType());
+ ElementCount VLen = ValVTy->getElementCount();
+
+ Type *STy = Val->getType()->getScalarType();
+ assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
+ "Induction Step must be an integer or FP");
+ assert(Step->getType() == STy && "Step has wrong type");
+
+ SmallVector<Constant *, 8> Indices;
+
+ // Create a vector of consecutive numbers from zero to VF.
+ VectorType *InitVecValVTy = ValVTy;
+ Type *InitVecValSTy = STy;
+ if (STy->isFloatingPointTy()) {
+ InitVecValSTy =
+ IntegerType::get(STy->getContext(), STy->getScalarSizeInBits());
+ InitVecValVTy = VectorType::get(InitVecValSTy, VLen);
+ }
+ Value *InitVec = Builder.CreateStepVector(InitVecValVTy);
+
+ // Splat the StartIdx
+ Value *StartIdxSplat = Builder.CreateVectorSplat(VLen, StartIdx);
+
+ if (STy->isIntegerTy()) {
+ InitVec = Builder.CreateAdd(InitVec, StartIdxSplat);
+ Step = Builder.CreateVectorSplat(VLen, Step);
+ assert(Step->getType() == Val->getType() && "Invalid step vec");
+ // FIXME: The newly created binary instructions should contain nsw/nuw
+ // flags, which can be found from the original scalar operations.
+ Step = Builder.CreateMul(InitVec, Step);
+ return Builder.CreateAdd(Val, Step, "induction");
+ }
+
+ // Floating point induction.
+ assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
+ "Binary Opcode should be specified for FP induction");
+ InitVec = Builder.CreateUIToFP(InitVec, ValVTy);
+ InitVec = Builder.CreateFAdd(InitVec, StartIdxSplat);
+
+ Step = Builder.CreateVectorSplat(VLen, Step);
+ Value *MulOp = Builder.CreateFMul(InitVec, Step);
+ return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
+}
+
void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
const InductionDescriptor &II, Value *Step, Value *Start,
Instruction *EntryVal, VPValue *Def, VPTransformState &State) {
@@ -2355,8 +2410,8 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
Value *Zero = getSignedIntOrFpConstant(Start->getType(), 0);
Value *SplatStart = Builder.CreateVectorSplat(State.VF, Start);
- Value *SteppedStart =
- getStepVector(SplatStart, Zero, Step, II.getInductionOpcode());
+ Value *SteppedStart = getStepVector(
+ SplatStart, Zero, Step, II.getInductionOpcode(), State.VF, State.Builder);
// We create vector phi nodes for both integer and floating-point induction
// variables. Here, we determine the kind of arithmetic we will perform.
@@ -2502,7 +2557,8 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV,
StartIdx = getRuntimeVF(Builder, Step->getType(), State.VF * Part);
Value *EntryPart =
- getStepVector(Broadcasted, StartIdx, Step, ID.getInductionOpcode());
+ getStepVector(Broadcasted, StartIdx, Step, ID.getInductionOpcode(),
+ State.VF, State.Builder);
State.set(Def, EntryPart, Part);
if (Trunc)
addMetadata(EntryPart, Trunc);
@@ -2554,54 +2610,6 @@ void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV,
buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, State);
}
-Value *InnerLoopVectorizer::getStepVector(Value *Val, Value *StartIdx,
- Value *Step,
- Instruction::BinaryOps BinOp) {
- // Create and check the types.
- auto *ValVTy = cast<VectorType>(Val->getType());
- ElementCount VLen = ValVTy->getElementCount();
-
- Type *STy = Val->getType()->getScalarType();
- assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
- "Induction Step must be an integer or FP");
- assert(Step->getType() == STy && "Step has wrong type");
-
- SmallVector<Constant *, 8> Indices;
-
- // Create a vector of consecutive numbers from zero to VF.
- VectorType *InitVecValVTy = ValVTy;
- Type *InitVecValSTy = STy;
- if (STy->isFloatingPointTy()) {
- InitVecValSTy =
- IntegerType::get(STy->getContext(), STy->getScalarSizeInBits());
- InitVecValVTy = VectorType::get(InitVecValSTy, VLen);
- }
- Value *InitVec = Builder.CreateStepVector(InitVecValVTy);
-
- // Splat the StartIdx
- Value *StartIdxSplat = Builder.CreateVectorSplat(VLen, StartIdx);
-
- if (STy->isIntegerTy()) {
- InitVec = Builder.CreateAdd(InitVec, StartIdxSplat);
- Step = Builder.CreateVectorSplat(VLen, Step);
- assert(Step->getType() == Val->getType() && "Invalid step vec");
- // FIXME: The newly created binary instructions should contain nsw/nuw flags,
- // which can be found from the original scalar operations.
- Step = Builder.CreateMul(InitVec, Step);
- return Builder.CreateAdd(Val, Step, "induction");
- }
-
- // Floating point induction.
- assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
- "Binary Opcode should be specified for FP induction");
- InitVec = Builder.CreateUIToFP(InitVec, ValVTy);
- InitVec = Builder.CreateFAdd(InitVec, StartIdxSplat);
-
- Step = Builder.CreateVectorSplat(VLen, Step);
- Value *MulOp = Builder.CreateFMul(InitVec, Step);
- return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
-}
-
void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
Instruction *EntryVal,
const InductionDescriptor &ID,
@@ -8036,21 +8044,6 @@ Value *InnerLoopUnroller::reverseVector(Value *Vec) { return Vec; }
Value *InnerLoopUnroller::getBroadcastInstrs(Value *V) { return V; }
-Value *InnerLoopUnroller::getStepVector(Value *Val, Value *StartIdx,
- Value *Step,
- Instruction::BinaryOps BinOp) {
- // When unrolling and the VF is 1, we only need to add a simple scalar.
- Type *Ty = Val->getType();
- assert(!Ty->isVectorTy() && "Val must be a scalar");
-
- if (Ty->isFloatingPointTy()) {
- // Floating-point operations inherit FMF via the builder's flags.
- Value *MulOp = Builder.CreateFMul(StartIdx, Step);
- return Builder.CreateBinOp(BinOp, Val, MulOp);
- }
- return Builder.CreateAdd(Val, Builder.CreateMul(StartIdx, Step), "induction");
-}
-
static void AddRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
More information about the llvm-commits
mailing list