[PATCH] Extend LoopVectorizationLegality::isConsecutivePtr to handle multiple level GEPs

[Wei Mi]

Hi qcolombet, hfinkel, nadav,

No matter whether gep merging takes effect or not, it is better for the analysis not to depend on having only one level GEP, just as DecomposeGEPExpression does right now.

The patch extends consecutive analysis in LoopVectorizer pass to handle multiple level GEPs. This is a following patch for http://reviews.llvm.org/D9865.

I also tried other way to solve the problem more generally by generating a temporarily merged GEP everytime when analyzing a GEP and removing it after the analysis, but it failed. A lot of existing analysis requires GEP to be a valid inst inserted in the function. We need to insert the temporarily combined GEP into the original BB, do the analysis, then delete it -- making a dangling GEP insn just for the analysis doesn't work. But it makes the IR during the analysis messy this way. Another way is to make the combined GEP kind of meta data just for analysis, but I am not sure how much effort it will cost because the meta data needs to be updated from time to time.

REPOSITORY
  rL LLVM

http://reviews.llvm.org/D10281

Files:
  lib/Transforms/Vectorize/LoopVectorize.cpp

Index: lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- lib/Transforms/Vectorize/LoopVectorize.cpp
+++ lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1556,25 +1556,61 @@
   if (!Gep)
     return 0;
 
-  unsigned NumOperands = Gep->getNumOperands();
-  Value *GpPtr = Gep->getPointerOperand();
-  // If this GEP value is a consecutive pointer induction variable and all of
-  // the indices are constant then we know it is consecutive. We can
-  Phi = dyn_cast<PHINode>(GpPtr);
-  if (Phi && Inductions.count(Phi)) {
+  const DataLayout &DL = Gep->getModule()->getDataLayout();
+  unsigned GEPAllocSize = DL.getTypeAllocSize(
+      cast<PointerType>(Gep->getType()->getScalarType())->getElementType());
 
-    // Make sure that the pointer does not point to structs.
-    PointerType *GepPtrType = cast<PointerType>(GpPtr->getType());
-    if (GepPtrType->getElementType()->isAggregateType())
-      return 0;
+  unsigned NumOperands;
+  while (Gep) {
+    NumOperands = Gep->getNumOperands();
+    Value *GpPtr = Gep->getPointerOperand();
+    Phi = dyn_cast<PHINode>(GpPtr);
+    if (Phi && Inductions.count(Phi)) {
+      // If this GEP value is a consecutive pointer induction variable and
+      // all of the indices are constant then we know it is consecutive.
+
+      // Make sure that the pointer does not point to structs.
+      PointerType *GepPtrType = cast<PointerType>(GpPtr->getType());
+
+      // Make sure that all of the index operands are loop invariant.
+      for (unsigned i = 1; i < NumOperands; ++i)
+        if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop))
+          return 0;
 
-    // Make sure that all of the index operands are loop invariant.
-    for (unsigned i = 1; i < NumOperands; ++i)
-      if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop))
-        return 0;
+      if (GepPtrType->getElementType()->isAggregateType()) {
+        if ((Gep = dyn_cast<GetElementPtrInst>(GpPtr)))
+          continue;
+        else
+          return 0;
+      }
+      InductionInfo II = Inductions[Phi];
+      return II.getConsecutiveDirection();
+    } else {
+      // If the pointer operand of the GEP is a SCEVAddRecExpr, and all the
+      // other operand is 0, and the pointer operand is another
+      // GetElementPtrInst, recursively find the induction variable in the
+      // pointer operand.
+      const SCEV *PtrScev = SE->getSCEV(GpPtr);
+      if (dyn_cast<SCEVAddRecExpr>(PtrScev)) {
+        for (unsigned i = 1; i < NumOperands; ++i)
+          if (!match(Gep->getOperand(i), m_Zero()))
+            return 0;
+
+        Gep = dyn_cast<GetElementPtrInst>(GpPtr);
+        if (!Gep)
+          return 0;
+
+        unsigned NewAllocSize = DL.getTypeAllocSize(
+            cast<PointerType>(Gep->getType()->getScalarType())
+                ->getElementType());
+        if (GEPAllocSize != NewAllocSize)
+          return 0;
 
-    InductionInfo II = Inductions[Phi];
-    return II.getConsecutiveDirection();
+        continue;
+      } else {
+        break;
+      }
+    }
   }
 
   unsigned InductionOperand = getGEPInductionOperand(Gep);
@@ -1696,8 +1732,14 @@
   VectorParts &Entry = WidenMap.get(Instr);
 
   // Handle consecutive loads/stores.
+  Value *GepPtr;
+  Instruction *GepPtrInst;
   GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
-  if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
+  if (Gep && (GepPtr = Gep->getPointerOperand()) &&
+      (GepPtrInst = dyn_cast<Instruction>(GepPtr)) &&
+      !SE->isLoopInvariant(SE->getSCEV(GepPtrInst), OrigLoop)) {
+    // The case Gep->getPointerOperand() is an induction variable
+    // or a SCEVAddRecExpr.
     setDebugLocFromInst(Builder, Gep);
     Value *PtrOperand = Gep->getPointerOperand();
     Value *FirstBasePtr = getVectorValue(PtrOperand)[0];

EMAIL PREFERENCES
  http://reviews.llvm.org/settings/panel/emailpreferences/
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D10281.27216.patch
Type: text/x-patch
Size: 3899 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20150605/7844f0f1/attachment.bin>