[PATCH] D132750: [SLP]Fix PR57322: vectorize constant float stores.

[Alexey Bataev via Phabricator via llvm-commits]

ABataev created this revision.
ABataev added reviewers: RKSimon, vdmitrie, vzakhari.
Herald added subscribers: vporpo, hiraditya.
Herald added a project: All.
ABataev requested review of this revision.
Herald added a subscriber: pcwang-thead.
Herald added a project: LLVM.

Stores for constant floats must be vectorized, improve analysis in SLP
vectorizer for stores.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D132750

Files:
  llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
  llvm/test/Transforms/SLPVectorizer/X86/stores_constant_float.ll


Index: llvm/test/Transforms/SLPVectorizer/X86/stores_constant_float.ll
===================================================================
--- llvm/test/Transforms/SLPVectorizer/X86/stores_constant_float.ll
+++ llvm/test/Transforms/SLPVectorizer/X86/stores_constant_float.ll
@@ -5,9 +5,7 @@
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = alloca { double, double }, align 8
-; CHECK-NEXT:    [[C_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[C]], i64 0, i32 1
-; CHECK-NEXT:    store double 0.000000e+00, ptr [[C]], align 8
-; CHECK-NEXT:    store double 1.000000e+00, ptr [[C_IMAGP]], align 8
+; CHECK-NEXT:    store <2 x double> <double 0.000000e+00, double 1.000000e+00>, ptr [[C]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
Index: llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2125,8 +2125,9 @@
 
   /// Return information about the vector formed for the specified index
   /// of a vector of (the same) instruction.
+  /// \param WithFloat - If true, check for float constants.
   TargetTransformInfo::OperandValueInfo
-  getOperandInfo(ArrayRef<Value *> VL, unsigned OpIdx);
+  getOperandInfo(ArrayRef<Value *> VL, unsigned OpIdx, bool WithFloat = false);
 
   /// \returns the cost of the vectorizable entry.
   InstructionCost getEntryCost(const TreeEntry *E,
@@ -5814,8 +5815,8 @@
   return I->getOpcode() == AltOp->getOpcode();
 }
 
-TTI::OperandValueInfo BoUpSLP::getOperandInfo(ArrayRef<Value *> VL, unsigned OpIdx) {
-
+TTI::OperandValueInfo BoUpSLP::getOperandInfo(ArrayRef<Value *> VL,
+                                              unsigned OpIdx, bool WithFloat) {
   TTI::OperandValueKind VK = TTI::OK_UniformConstantValue;
   TTI::OperandValueProperties VP = TTI::OP_PowerOf2;
 
@@ -5825,19 +5826,25 @@
   // to OK_AnyValue. If all operands are constants but not the same,
   // then set the operand kind to OK_NonUniformConstantValue.
   ConstantInt *CInt0 = nullptr;
-  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
-    const Instruction *I = cast<Instruction>(VL[i]);
-    assert(I->getOpcode() == cast<Instruction>(VL[0])->getOpcode());
-    ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(OpIdx));
-    if (!CInt) {
+  for (auto [I, V] : enumerate(VL)) {
+    const auto *Inst = cast<Instruction>(V);
+    assert(Inst->getOpcode() == cast<Instruction>(VL[0])->getOpcode() &&
+           "Expected same opcode");
+    auto *CInt = dyn_cast<ConstantInt>(Inst->getOperand(OpIdx));
+    auto *CFlt = dyn_cast<ConstantFP>(Inst->getOperand(OpIdx));
+    if (!CInt && (!WithFloat || !CFlt)) {
       VK = TTI::OK_AnyValue;
       VP = TTI::OP_None;
       break;
     }
-    if (VP == TTI::OP_PowerOf2 &&
-        !CInt->getValue().isPowerOf2())
+    if (CFlt) {
+      VK = TTI::OK_NonUniformConstantValue;
+      VP = TTI::OP_None;
+      continue;
+    }
+    if (VP == TTI::OP_PowerOf2 && !CInt->getValue().isPowerOf2())
       VP = TTI::OP_None;
-    if (i == 0) {
+    if (I == 0) {
       CInt0 = CInt;
       continue;
     }
@@ -6500,19 +6507,12 @@
       auto *SI =
           cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
       Align Alignment = SI->getAlign();
-      TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(SI->getOperand(0));
+      TTI::OperandValueInfo OpInfo = getOperandInfo(VL, 0, /*WithFloat=*/true);
       InstructionCost ScalarEltCost = TTI->getMemoryOpCost(
           Instruction::Store, ScalarTy, Alignment, 0, CostKind, OpInfo, VL0);
       InstructionCost ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
       TTI::OperandValueKind OpVK = TTI::OK_AnyValue;
-      if (all_of(E->Scalars,
-                 [](Value *V) {
-                   return isConstant(cast<Instruction>(V)->getOperand(0));
-                 }) &&
-          any_of(E->Scalars, [](Value *V) {
-            Value *Op = cast<Instruction>(V)->getOperand(0);
-            return !isa<UndefValue>(Op) && !cast<Constant>(Op)->isZeroValue();
-          }))
+      if (OpInfo.isConstant())
         OpVK = TTI::OK_NonUniformConstantValue;
       InstructionCost VecStCost = TTI->getMemoryOpCost(
           Instruction::Store, VecTy, Alignment, 0, CostKind,


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D132750.455956.patch
Type: text/x-patch
Size: 4347 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20220826/ff26be6b/attachment.bin>