[PATCH] D92282: [VPlan] Handle scalarized values in VPTransformState.

[Florian Hahn via Phabricator via llvm-commits]

fhahn created this revision.
fhahn added reviewers: gilr, rengolin, Ayal.
Herald added subscribers: psnobl, rogfer01, bollu, hiraditya.
Herald added a project: LLVM.
fhahn requested review of this revision.
Herald added a subscriber: vkmr.

This patch adds plumbing to handle scalarized values directly in
VPTransformState.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D92282

Files:
  llvm/lib/Transforms/Vectorize/VPlan.cpp
  llvm/lib/Transforms/Vectorize/VPlan.h


Index: llvm/lib/Transforms/Vectorize/VPlan.h
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.h
+++ llvm/lib/Transforms/Vectorize/VPlan.h
@@ -270,6 +270,9 @@
     typedef SmallVector<Value *, 2> PerPartValuesTy;
 
     DenseMap<VPValue *, PerPartValuesTy> PerPartOutput;
+
+    typedef SmallVector<SmallVector<Value *, 4>, 2> ScalarsPerPartValuesTy;
+    DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars;
   } Data;
 
   /// Get the generated Value for a given VPValue and a given Part. Note that
@@ -288,6 +291,21 @@
   /// Get the generated Value for a given VPValue and given Part and Lane.
   Value *get(VPValue *Def, VPIteration Instance);
 
+  bool hasVectorValue(VPValue *Def, unsigned Part) {
+    auto I = Data.PerPartOutput.find(Def);
+    return I != Data.PerPartOutput.end() && Part < I->second.size() &&
+           I->second[Part];
+  }
+
+  bool hasScalarValue(VPValue *Def, VPIteration Instance) {
+    auto I = Data.PerPartScalars.find(Def);
+    if (I == Data.PerPartScalars.end())
+      return false;
+    return Instance.Part < I->second.size() &&
+           Instance.Lane < I->second[Instance.Part].size() &&
+           I->second[Instance.Part][Instance.Lane];
+  }
+
   /// Set the generated Value for a given VPValue and a given Part.
   void set(VPValue *Def, Value *V, unsigned Part) {
     if (!Data.PerPartOutput.count(Def)) {
@@ -298,6 +316,19 @@
   }
   void set(VPValue *Def, Value *IRDef, Value *V, unsigned Part);
 
+  void set(VPValue *Def, Value *IRDef, Value *V, const VPIteration &Instance);
+
+  void set(VPValue *Def, Value *V, const VPIteration &Instance) {
+    auto Iter = Data.PerPartScalars.insert({Def, {}});
+    auto &PerPartVec = Iter.first->second;
+    while (PerPartVec.size() <= Instance.Part)
+      PerPartVec.emplace_back();
+    auto &Scalars = PerPartVec[Instance.Part];
+    while (Scalars.size() <= Instance.Lane)
+      Scalars.push_back(nullptr);
+    Scalars[Instance.Lane] = V;
+  }
+
   /// Hold state information used when constructing the CFG of the output IR,
   /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
   struct CFGState {
Index: llvm/lib/Transforms/Vectorize/VPlan.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -217,25 +217,30 @@
 }
 
 Value *VPTransformState::get(VPValue *Def, VPIteration Instance) {
+  if (auto *C = Def->getOutOfScopeIRValue())
+    return C;
+
   // For uniform definitions, all lanes produce the same value, so we can always
   // return the first lane.
   if (auto *ReplicateR = dyn_cast<VPReplicateRecipe>(Def))
     if (ReplicateR->isUniform())
       Instance.Lane = 0;
-  // If the Def is managed directly by VPTransformState, extract the lane from
-  // the relevant part. Note that currently only VPInstructions and external
-  // defs are managed by VPTransformState. Other Defs are still created by ILV
-  // and managed in its ValueMap. For those this method currently just
-  // delegates the call to ILV below.
-  if (Data.PerPartOutput.count(Def)) {
+
+  if (hasScalarValue(Def, Instance))
+    return Data.PerPartScalars[Def][Instance.Part][Instance.Lane];
+
+  if (hasVectorValue(Def, Instance.Part)) {
+    assert(Data.PerPartOutput.count(Def));
     auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
     if (!VecPart->getType()->isVectorTy()) {
       assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar");
       return VecPart;
     }
     // TODO: Cache created scalar values.
-    return Builder.CreateExtractElement(VecPart,
-                                        Builder.getInt32(Instance.Lane));
+    auto *Extract =
+        Builder.CreateExtractElement(VecPart, Builder.getInt32(Instance.Lane));
+    // set(Def, Extract, Instance);
+    return Extract;
   }
 
   return Callback.getOrCreateScalarValue(VPValue2Value[Def], Instance);


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D92282.308225.patch
Type: text/x-patch
Size: 3996 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20201129/8957c08c/attachment.bin>