[llvm] r261140 - Create masked gather and scatter intrinsics in Loop Vectorizer.
Elena Demikhovsky via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 17 11:23:06 PST 2016
Author: delena
Date: Wed Feb 17 13:23:04 2016
New Revision: 261140
URL: http://llvm.org/viewvc/llvm-project?rev=261140&view=rev
Log:
Create masked gather and scatter intrinsics in Loop Vectorizer.
Loop vectorizer now knows to vectorize GEP and create masked gather and scatter intrinsics for random memory access.
The feature is enabled on AVX-512 target.
Differential Revision: http://reviews.llvm.org/D15690
Added:
llvm/trunk/test/Transforms/LoopVectorize/X86/gather_scatter.ll
Modified:
llvm/trunk/include/llvm/IR/IRBuilder.h
llvm/trunk/lib/IR/IRBuilder.cpp
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/trunk/test/Transforms/LoopVectorize/X86/masked_load_store.ll
Modified: llvm/trunk/include/llvm/IR/IRBuilder.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/IR/IRBuilder.h?rev=261140&r1=261139&r2=261140&view=diff
==============================================================================
--- llvm/trunk/include/llvm/IR/IRBuilder.h (original)
+++ llvm/trunk/include/llvm/IR/IRBuilder.h Wed Feb 17 13:23:04 2016
@@ -436,6 +436,14 @@ public:
CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align,
Value *Mask);
+ /// \brief Create a call to Masked Gather intrinsic
+ CallInst *CreateMaskedGather(Value *Ptrs, unsigned Align, Value *Mask = 0,
+ Value *PassThru = 0, const Twine& Name = "");
+
+ /// \brief Create a call to Masked Scatter intrinsic
+ CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, unsigned Align,
+ Value *Mask = 0);
+
/// \brief Create an assume intrinsic call that allows the optimizer to
/// assume that the provided condition will be true.
CallInst *CreateAssumption(Value *Cond);
Modified: llvm/trunk/lib/IR/IRBuilder.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/IR/IRBuilder.cpp?rev=261140&r1=261139&r2=261140&view=diff
==============================================================================
--- llvm/trunk/lib/IR/IRBuilder.cpp (original)
+++ llvm/trunk/lib/IR/IRBuilder.cpp Wed Feb 17 13:23:04 2016
@@ -201,18 +201,17 @@ CallInst *IRBuilderBase::CreateAssumptio
return createCallHelper(FnAssume, Ops, this);
}
-/// Create a call to a Masked Load intrinsic.
-/// Ptr - the base pointer for the load
-/// Align - alignment of the source location
-/// Mask - an vector of booleans which indicates what vector lanes should
-/// be accessed in memory
-/// PassThru - a pass-through value that is used to fill the masked-off lanes
-/// of the result
-/// Name - name of the result variable
+/// \brief Create a call to a Masked Load intrinsic.
+/// \p Ptr - base pointer for the load
+/// \p Align - alignment of the source location
+/// \p Mask - vector of booleans which indicates what vector lanes should
+/// be accessed in memory
+/// \p PassThru - pass-through value that is used to fill the masked-off lanes
+/// of the result
+/// \p Name - name of the result variable
CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
Value *Mask, Value *PassThru,
const Twine &Name) {
- assert(Ptr->getType()->isPointerTy() && "Ptr must be of pointer type");
// DataTy is the overloaded type
Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType();
assert(DataTy->isVectorTy() && "Ptr should point to a vector");
@@ -222,12 +221,12 @@ CallInst *IRBuilderBase::CreateMaskedLoa
return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, DataTy, Name);
}
-/// Create a call to a Masked Store intrinsic.
-/// Val - the data to be stored,
-/// Ptr - the base pointer for the store
-/// Align - alignment of the destination location
-/// Mask - an vector of booleans which indicates what vector lanes should
-/// be accessed in memory
+/// \brief Create a call to a Masked Store intrinsic.
+/// \p Val - data to be stored,
+/// \p Ptr - base pointer for the store
+/// \p Align - alignment of the destination location
+/// \p Mask - vector of booleans which indicates what vector lanes should
+/// be accessed in memory
CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
unsigned Align, Value *Mask) {
Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
@@ -247,6 +246,62 @@ CallInst *IRBuilderBase::CreateMaskedInt
return createCallHelper(TheFn, Ops, this, Name);
}
+/// \brief Create a call to a Masked Gather intrinsic.
+/// \p Ptrs - vector of pointers for loading
+/// \p Align - alignment for one element
+/// \p Mask - vector of booleans which indicates what vector lanes should
+/// be accessed in memory
+/// \p PassThru - pass-through value that is used to fill the masked-off lanes
+/// of the result
+/// \p Name - name of the result variable
+CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
+ Value *Mask, Value *PassThru,
+ const Twine& Name) {
+ auto PtrsTy = cast<VectorType>(Ptrs->getType());
+ auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
+ unsigned NumElts = PtrsTy->getVectorNumElements();
+ Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
+
+ if (!Mask)
+ Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
+ NumElts));
+
+ Value * Ops[] = {Ptrs, getInt32(Align), Mask, UndefValue::get(DataTy)};
+
+ // We specify only one type when we create this intrinsic. Types of other
+ // arguments are derived from this type.
+ return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, DataTy, Name);
+}
+
+/// \brief Create a call to a Masked Scatter intrinsic.
+/// \p Data - data to be stored,
+/// \p Ptrs - the vector of pointers, where the \p Data elements should be
+/// stored
+/// \p Align - alignment for one element
+/// \p Mask - vector of booleans which indicates what vector lanes should
+/// be accessed in memory
+CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
+ unsigned Align, Value *Mask) {
+ auto PtrsTy = cast<VectorType>(Ptrs->getType());
+ auto DataTy = cast<VectorType>(Data->getType());
+
+ auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
+ unsigned NumElts = PtrsTy->getVectorNumElements();
+
+ assert(NumElts == DataTy->getVectorNumElements() &&
+ PtrTy->getElementType() == DataTy->getElementType() &&
+ "Incompatible pointer and data types");
+
+ if (!Mask)
+ Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
+ NumElts));
+ Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
+
+ // We specify only one type when we create this intrinsic. Types of other
+ // arguments are derived from this type.
+ return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, DataTy);
+}
+
template <typename T0, typename T1, typename T2, typename T3>
static std::vector<Value *>
getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=261140&r1=261139&r2=261140&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Wed Feb 17 13:23:04 2016
@@ -1282,6 +1282,17 @@ public:
bool isLegalMaskedLoad(Type *DataType, Value *Ptr) {
return isConsecutivePtr(Ptr) && TTI->isLegalMaskedLoad(DataType);
}
+ /// Returns true if the target machine supports masked scatter operation
+ /// for the given \p DataType.
+ bool isLegalMaskedScatter(Type *DataType) {
+ return TTI->isLegalMaskedScatter(DataType);
+ }
+ /// Returns true if the target machine supports masked gather operation
+ /// for the given \p DataType.
+ bool isLegalMaskedGather(Type *DataType) {
+ return TTI->isLegalMaskedGather(DataType);
+ }
+
/// Returns true if vector representation of the instruction \p I
/// requires mask.
bool isMaskRequired(const Instruction* I) {
@@ -2379,70 +2390,107 @@ void InnerLoopVectorizer::vectorizeMemor
if (ScalarAllocatedSize != VectorElementSize)
return scalarizeInstruction(Instr);
- // If the pointer is loop invariant or if it is non-consecutive,
- // scalarize the load.
+ // If the pointer is loop invariant scalarize the load.
+ if (LI && Legal->isUniform(Ptr))
+ return scalarizeInstruction(Instr);
+
+ // If the pointer is non-consecutive and gather/scatter is not supported
+ // scalarize the instruction.
int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);
bool Reverse = ConsecutiveStride < 0;
- bool UniformLoad = LI && Legal->isUniform(Ptr);
- if (!ConsecutiveStride || UniformLoad)
+ bool CreateGatherScatter = !ConsecutiveStride &&
+ ((LI && Legal->isLegalMaskedGather(ScalarDataTy)) ||
+ (SI && Legal->isLegalMaskedScatter(ScalarDataTy)));
+
+ if (!ConsecutiveStride && !CreateGatherScatter)
return scalarizeInstruction(Instr);
Constant *Zero = Builder.getInt32(0);
VectorParts &Entry = WidenMap.get(Instr);
+ VectorParts VectorGep;
// Handle consecutive loads/stores.
GetElementPtrInst *Gep = getGEPInstruction(Ptr);
- if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
- setDebugLocFromInst(Builder, Gep);
- Value *PtrOperand = Gep->getPointerOperand();
- Value *FirstBasePtr = getVectorValue(PtrOperand)[0];
- FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero);
-
- // Create the new GEP with the new induction variable.
- GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
- Gep2->setOperand(0, FirstBasePtr);
- Gep2->setName("gep.indvar.base");
- Ptr = Builder.Insert(Gep2);
- } else if (Gep) {
- setDebugLocFromInst(Builder, Gep);
- assert(PSE.getSE()->isLoopInvariant(PSE.getSCEV(Gep->getPointerOperand()),
- OrigLoop) &&
- "Base ptr must be invariant");
-
- // The last index does not have to be the induction. It can be
- // consecutive and be a function of the index. For example A[I+1];
- unsigned NumOperands = Gep->getNumOperands();
- unsigned InductionOperand = getGEPInductionOperand(Gep);
- // Create the new GEP with the new induction variable.
- GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
-
- for (unsigned i = 0; i < NumOperands; ++i) {
- Value *GepOperand = Gep->getOperand(i);
- Instruction *GepOperandInst = dyn_cast<Instruction>(GepOperand);
-
- // Update last index or loop invariant instruction anchored in loop.
- if (i == InductionOperand ||
- (GepOperandInst && OrigLoop->contains(GepOperandInst))) {
- assert((i == InductionOperand ||
- PSE.getSE()->isLoopInvariant(PSE.getSCEV(GepOperandInst),
- OrigLoop)) &&
- "Must be last index or loop invariant");
-
- VectorParts &GEPParts = getVectorValue(GepOperand);
- Value *Index = GEPParts[0];
- Index = Builder.CreateExtractElement(Index, Zero);
- Gep2->setOperand(i, Index);
- Gep2->setName("gep.indvar.idx");
+ if (ConsecutiveStride) {
+ if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
+ setDebugLocFromInst(Builder, Gep);
+ Value *PtrOperand = Gep->getPointerOperand();
+ Value *FirstBasePtr = getVectorValue(PtrOperand)[0];
+ FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero);
+
+ // Create the new GEP with the new induction variable.
+ GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
+ Gep2->setOperand(0, FirstBasePtr);
+ Gep2->setName("gep.indvar.base");
+ Ptr = Builder.Insert(Gep2);
+ } else if (Gep) {
+ setDebugLocFromInst(Builder, Gep);
+ assert(PSE.getSE()->isLoopInvariant(PSE.getSCEV(Gep->getPointerOperand()),
+ OrigLoop) &&
+ "Base ptr must be invariant");
+ // The last index does not have to be the induction. It can be
+ // consecutive and be a function of the index. For example A[I+1];
+ unsigned NumOperands = Gep->getNumOperands();
+ unsigned InductionOperand = getGEPInductionOperand(Gep);
+ // Create the new GEP with the new induction variable.
+ GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
+
+ for (unsigned i = 0; i < NumOperands; ++i) {
+ Value *GepOperand = Gep->getOperand(i);
+ Instruction *GepOperandInst = dyn_cast<Instruction>(GepOperand);
+
+ // Update last index or loop invariant instruction anchored in loop.
+ if (i == InductionOperand ||
+ (GepOperandInst && OrigLoop->contains(GepOperandInst))) {
+ assert((i == InductionOperand ||
+ PSE.getSE()->isLoopInvariant(PSE.getSCEV(GepOperandInst),
+ OrigLoop)) &&
+ "Must be last index or loop invariant");
+
+ VectorParts &GEPParts = getVectorValue(GepOperand);
+ Value *Index = GEPParts[0];
+ Index = Builder.CreateExtractElement(Index, Zero);
+ Gep2->setOperand(i, Index);
+ Gep2->setName("gep.indvar.idx");
+ }
+ }
+ Ptr = Builder.Insert(Gep2);
+ } else { // No GEP
+ // Use the induction element ptr.
+ assert(isa<PHINode>(Ptr) && "Invalid induction ptr");
+ setDebugLocFromInst(Builder, Ptr);
+ VectorParts &PtrVal = getVectorValue(Ptr);
+ Ptr = Builder.CreateExtractElement(PtrVal[0], Zero);
}
+ } else {
+ // At this point we should vector version of GEP for Gather or Scatter
+ assert(CreateGatherScatter && "The instruction should be scalarized");
+ if (Gep) {
+ SmallVector<VectorParts, 4> OpsV;
+ // Vectorizing GEP, across UF parts, we want to keep each loop-invariant
+ // base or index of GEP scalar
+ for (Value *Op : Gep->operands()) {
+ if (PSE.getSE()->isLoopInvariant(PSE.getSCEV(Op), OrigLoop))
+ OpsV.push_back(VectorParts(UF, Op));
+ else
+ OpsV.push_back(getVectorValue(Op));
+ }
+
+ for (unsigned Part = 0; Part < UF; ++Part) {
+ SmallVector<Value*, 4> Ops;
+ Value *GEPBasePtr = OpsV[0][Part];
+ for (unsigned i = 1; i < Gep->getNumOperands(); i++)
+ Ops.push_back(OpsV[i][Part]);
+ Value *NewGep = Builder.CreateGEP(nullptr, GEPBasePtr, Ops,
+ "VectorGep");
+ assert(NewGep->getType()->isVectorTy() && "Expected vector GEP");
+ NewGep = Builder.CreateBitCast(NewGep,
+ VectorType::get(Ptr->getType(), VF));
+ VectorGep.push_back(NewGep);
+ }
+ } else
+ VectorGep = getVectorValue(Ptr);
}
- Ptr = Builder.Insert(Gep2);
- } else {
- // Use the induction element ptr.
- assert(isa<PHINode>(Ptr) && "Invalid induction ptr");
- setDebugLocFromInst(Builder, Ptr);
- VectorParts &PtrVal = getVectorValue(Ptr);
- Ptr = Builder.CreateExtractElement(PtrVal[0], Zero);
- }
VectorParts Mask = createBlockInMask(Instr->getParent());
// Handle Stores:
@@ -2455,30 +2503,37 @@ void InnerLoopVectorizer::vectorizeMemor
VectorParts StoredVal = getVectorValue(SI->getValueOperand());
for (unsigned Part = 0; Part < UF; ++Part) {
- // Calculate the pointer for the specific unroll-part.
- Value *PartPtr =
+ Instruction *NewSI = nullptr;
+ if (CreateGatherScatter) {
+ Value *MaskPart = Legal->isMaskRequired(SI) ? Mask[Part] : nullptr;
+ NewSI = Builder.CreateMaskedScatter(StoredVal[Part], VectorGep[Part],
+ Alignment, MaskPart);
+ } else {
+ // Calculate the pointer for the specific unroll-part.
+ Value *PartPtr =
Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(Part * VF));
- if (Reverse) {
- // If we store to reverse consecutive memory locations, then we need
- // to reverse the order of elements in the stored value.
- StoredVal[Part] = reverseVector(StoredVal[Part]);
- // If the address is consecutive but reversed, then the
- // wide store needs to start at the last vector element.
- PartPtr = Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(-Part * VF));
- PartPtr = Builder.CreateGEP(nullptr, PartPtr, Builder.getInt32(1 - VF));
- Mask[Part] = reverseVector(Mask[Part]);
- }
+ if (Reverse) {
+ // If we store to reverse consecutive memory locations, then we need
+ // to reverse the order of elements in the stored value.
+ StoredVal[Part] = reverseVector(StoredVal[Part]);
+ // If the address is consecutive but reversed, then the
+ // wide store needs to start at the last vector element.
+ PartPtr = Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(-Part * VF));
+ PartPtr = Builder.CreateGEP(nullptr, PartPtr, Builder.getInt32(1 - VF));
+ Mask[Part] = reverseVector(Mask[Part]);
+ }
- Value *VecPtr = Builder.CreateBitCast(PartPtr,
- DataTy->getPointerTo(AddressSpace));
+ Value *VecPtr = Builder.CreateBitCast(PartPtr,
+ DataTy->getPointerTo(AddressSpace));
- Instruction *NewSI;
- if (Legal->isMaskRequired(SI))
- NewSI = Builder.CreateMaskedStore(StoredVal[Part], VecPtr, Alignment,
- Mask[Part]);
- else
- NewSI = Builder.CreateAlignedStore(StoredVal[Part], VecPtr, Alignment);
+ if (Legal->isMaskRequired(SI))
+ NewSI = Builder.CreateMaskedStore(StoredVal[Part], VecPtr, Alignment,
+ Mask[Part]);
+ else
+ NewSI = Builder.CreateAlignedStore(StoredVal[Part], VecPtr,
+ Alignment);
+ }
propagateMetadata(NewSI, SI);
}
return;
@@ -2488,29 +2543,36 @@ void InnerLoopVectorizer::vectorizeMemor
assert(LI && "Must have a load instruction");
setDebugLocFromInst(Builder, LI);
for (unsigned Part = 0; Part < UF; ++Part) {
- // Calculate the pointer for the specific unroll-part.
- Value *PartPtr =
+ Instruction* NewLI;
+ if (CreateGatherScatter) {
+ Value *MaskPart = Legal->isMaskRequired(LI) ? Mask[Part] : nullptr;
+ NewLI = Builder.CreateMaskedGather(VectorGep[Part], Alignment,
+ MaskPart, 0, "wide.masked.gather");
+ Entry[Part] = NewLI;
+ } else {
+ // Calculate the pointer for the specific unroll-part.
+ Value *PartPtr =
Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(Part * VF));
- if (Reverse) {
- // If the address is consecutive but reversed, then the
- // wide load needs to start at the last vector element.
- PartPtr = Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(-Part * VF));
- PartPtr = Builder.CreateGEP(nullptr, PartPtr, Builder.getInt32(1 - VF));
- Mask[Part] = reverseVector(Mask[Part]);
- }
+ if (Reverse) {
+ // If the address is consecutive but reversed, then the
+ // wide load needs to start at the last vector element.
+ PartPtr = Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(-Part * VF));
+ PartPtr = Builder.CreateGEP(nullptr, PartPtr, Builder.getInt32(1 - VF));
+ Mask[Part] = reverseVector(Mask[Part]);
+ }
- Instruction* NewLI;
- Value *VecPtr = Builder.CreateBitCast(PartPtr,
- DataTy->getPointerTo(AddressSpace));
- if (Legal->isMaskRequired(LI))
- NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part],
- UndefValue::get(DataTy),
- "wide.masked.load");
- else
- NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load");
+ Value *VecPtr = Builder.CreateBitCast(PartPtr,
+ DataTy->getPointerTo(AddressSpace));
+ if (Legal->isMaskRequired(LI))
+ NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part],
+ UndefValue::get(DataTy),
+ "wide.masked.load");
+ else
+ NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load");
+ Entry[Part] = Reverse ? reverseVector(NewLI) : NewLI;
+ }
propagateMetadata(NewLI, LI);
- Entry[Part] = Reverse ? reverseVector(NewLI) : NewLI;
}
}
@@ -4520,7 +4582,8 @@ bool LoopVectorizationLegality::blockCan
if (!LI)
return false;
if (!SafePtrs.count(LI->getPointerOperand())) {
- if (isLegalMaskedLoad(LI->getType(), LI->getPointerOperand())) {
+ if (isLegalMaskedLoad(LI->getType(), LI->getPointerOperand()) ||
+ isLegalMaskedGather(LI->getType())) {
MaskedOp.insert(LI);
continue;
}
@@ -4545,7 +4608,8 @@ bool LoopVectorizationLegality::blockCan
// scalarize the block.
bool isLegalMaskedOp =
isLegalMaskedStore(SI->getValueOperand()->getType(),
- SI->getPointerOperand());
+ SI->getPointerOperand()) ||
+ isLegalMaskedScatter(SI->getValueOperand()->getType());
if (isLegalMaskedOp) {
--NumPredStores;
MaskedOp.insert(SI);
@@ -5281,6 +5345,19 @@ unsigned LoopVectorizationCostModel::exp
return Cost;
}
+/// \brief Check if the load/store instruction \p I may be translated into
+/// gather/scatter during vectorization.
+///
+/// Pointer \p Ptr specifies address in memory for the given scalar memory
+/// instruction. We need it to retrieve data type.
+/// Using gather/scatter is possible when it is supported by target.
+static bool isGatherOrScatterLegal(Instruction *I, Value *Ptr,
+ LoopVectorizationLegality *Legal) {
+ Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType();
+ return (isa<LoadInst>(I) && Legal->isLegalMaskedGather(DataTy)) ||
+ (isa<StoreInst>(I) && Legal->isLegalMaskedScatter(DataTy));
+}
+
/// \brief Check whether the address computation for a non-consecutive memory
/// access looks like an unlikely candidate for being merged into the indexing
/// mode.
@@ -5500,11 +5577,15 @@ LoopVectorizationCostModel::getInstructi
// Scalarized loads/stores.
int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);
+ bool UseGatherOrScatter = (ConsecutiveStride == 0) &&
+ isGatherOrScatterLegal(I, Ptr, Legal);
+
bool Reverse = ConsecutiveStride < 0;
const DataLayout &DL = I->getModule()->getDataLayout();
unsigned ScalarAllocatedSize = DL.getTypeAllocSize(ValTy);
unsigned VectorElementSize = DL.getTypeStoreSize(VectorTy) / VF;
- if (!ConsecutiveStride || ScalarAllocatedSize != VectorElementSize) {
+ if ((!ConsecutiveStride && !UseGatherOrScatter) ||
+ ScalarAllocatedSize != VectorElementSize) {
bool IsComplexComputation =
isLikelyComplexAddressComputation(Ptr, Legal, SE, TheLoop);
unsigned Cost = 0;
@@ -5528,8 +5609,15 @@ LoopVectorizationCostModel::getInstructi
return Cost;
}
- // Wide load/stores.
unsigned Cost = TTI.getAddressComputationCost(VectorTy);
+ if (UseGatherOrScatter) {
+ assert(ConsecutiveStride == 0 &&
+ "Gather/Scatter are not used for consecutive stride");
+ return Cost +
+ TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,
+ Legal->isMaskRequired(I), Alignment);
+ }
+ // Wide load/stores.
if (Legal->isMaskRequired(I))
Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment,
AS);
Added: llvm/trunk/test/Transforms/LoopVectorize/X86/gather_scatter.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/X86/gather_scatter.ll?rev=261140&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/LoopVectorize/X86/gather_scatter.ll (added)
+++ llvm/trunk/test/Transforms/LoopVectorize/X86/gather_scatter.ll Wed Feb 17 13:23:04 2016
@@ -0,0 +1,236 @@
+; RUN: opt < %s -O3 -mcpu=knl -S | FileCheck %s -check-prefix=AVX512
+
+;AVX1-NOT: llvm.masked
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc_linux"
+
+; The source code:
+;
+;void foo1(float * __restrict__ in, float * __restrict__ out, int * __restrict__ trigger, int * __restrict__ index) {
+;
+; for (int i=0; i < SIZE; ++i) {
+; if (trigger[i] > 0) {
+; out[i] = in[index[i]] + (float) 0.5;
+; }
+; }
+;}
+
+;AVX512-LABEL: @foo1
+;AVX512: llvm.masked.load.v8i32
+;AVX512: llvm.masked.gather.v8f32
+;AVX512: llvm.masked.store.v8f32
+;AVX512: ret void
+
+; Function Attrs: nounwind uwtable
+define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger, i32* noalias %index) {
+entry:
+ %in.addr = alloca float*, align 8
+ %out.addr = alloca float*, align 8
+ %trigger.addr = alloca i32*, align 8
+ %index.addr = alloca i32*, align 8
+ %i = alloca i32, align 4
+ store float* %in, float** %in.addr, align 8
+ store float* %out, float** %out.addr, align 8
+ store i32* %trigger, i32** %trigger.addr, align 8
+ store i32* %index, i32** %index.addr, align 8
+ store i32 0, i32* %i, align 4
+ br label %for.cond
+
+for.cond: ; preds = %for.inc, %entry
+ %0 = load i32, i32* %i, align 4
+ %cmp = icmp slt i32 %0, 4096
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %1 = load i32, i32* %i, align 4
+ %idxprom = sext i32 %1 to i64
+ %2 = load i32*, i32** %trigger.addr, align 8
+ %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+ %3 = load i32, i32* %arrayidx, align 4
+ %cmp1 = icmp sgt i32 %3, 0
+ br i1 %cmp1, label %if.then, label %if.end
+
+if.then: ; preds = %for.body
+ %4 = load i32, i32* %i, align 4
+ %idxprom2 = sext i32 %4 to i64
+ %5 = load i32*, i32** %index.addr, align 8
+ %arrayidx3 = getelementptr inbounds i32, i32* %5, i64 %idxprom2
+ %6 = load i32, i32* %arrayidx3, align 4
+ %idxprom4 = sext i32 %6 to i64
+ %7 = load float*, float** %in.addr, align 8
+ %arrayidx5 = getelementptr inbounds float, float* %7, i64 %idxprom4
+ %8 = load float, float* %arrayidx5, align 4
+ %add = fadd float %8, 5.000000e-01
+ %9 = load i32, i32* %i, align 4
+ %idxprom6 = sext i32 %9 to i64
+ %10 = load float*, float** %out.addr, align 8
+ %arrayidx7 = getelementptr inbounds float, float* %10, i64 %idxprom6
+ store float %add, float* %arrayidx7, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %for.body
+ br label %for.inc
+
+for.inc: ; preds = %if.end
+ %11 = load i32, i32* %i, align 4
+ %inc = add nsw i32 %11, 1
+ store i32 %inc, i32* %i, align 4
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret void
+}
+
+; The source code
+;void foo2 (In * __restrict__ in, float * __restrict__ out, int * __restrict__ trigger) {
+;
+; for (int i=0; i<SIZE; ++i) {
+; if (trigger[i] > 0) {
+; out[i] = in[i].b + (float) 0.5;
+; }
+; }
+;}
+
+%struct.In = type { float, float }
+
+;AVX512-LABEL: @foo2
+;AVX512: getelementptr %struct.In, %struct.In* %in, <16 x i64> %induction, i32 1
+;AVX512: llvm.masked.gather.v16f32
+;AVX512: llvm.masked.store.v16f32
+;AVX512: ret void
+define void @foo2(%struct.In* noalias %in, float* noalias %out, i32* noalias %trigger, i32* noalias %index) #0 {
+entry:
+ %in.addr = alloca %struct.In*, align 8
+ %out.addr = alloca float*, align 8
+ %trigger.addr = alloca i32*, align 8
+ %index.addr = alloca i32*, align 8
+ %i = alloca i32, align 4
+ store %struct.In* %in, %struct.In** %in.addr, align 8
+ store float* %out, float** %out.addr, align 8
+ store i32* %trigger, i32** %trigger.addr, align 8
+ store i32* %index, i32** %index.addr, align 8
+ store i32 0, i32* %i, align 4
+ br label %for.cond
+
+for.cond: ; preds = %for.inc, %entry
+ %0 = load i32, i32* %i, align 4
+ %cmp = icmp slt i32 %0, 4096
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %1 = load i32, i32* %i, align 4
+ %idxprom = sext i32 %1 to i64
+ %2 = load i32*, i32** %trigger.addr, align 8
+ %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+ %3 = load i32, i32* %arrayidx, align 4
+ %cmp1 = icmp sgt i32 %3, 0
+ br i1 %cmp1, label %if.then, label %if.end
+
+if.then: ; preds = %for.body
+ %4 = load i32, i32* %i, align 4
+ %idxprom2 = sext i32 %4 to i64
+ %5 = load %struct.In*, %struct.In** %in.addr, align 8
+ %arrayidx3 = getelementptr inbounds %struct.In, %struct.In* %5, i64 %idxprom2
+ %b = getelementptr inbounds %struct.In, %struct.In* %arrayidx3, i32 0, i32 1
+ %6 = load float, float* %b, align 4
+ %add = fadd float %6, 5.000000e-01
+ %7 = load i32, i32* %i, align 4
+ %idxprom4 = sext i32 %7 to i64
+ %8 = load float*, float** %out.addr, align 8
+ %arrayidx5 = getelementptr inbounds float, float* %8, i64 %idxprom4
+ store float %add, float* %arrayidx5, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %for.body
+ br label %for.inc
+
+for.inc: ; preds = %if.end
+ %9 = load i32, i32* %i, align 4
+ %inc = add nsw i32 %9, 1
+ store i32 %inc, i32* %i, align 4
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret void
+}
+
+; The source code
+;struct Out {
+; float a;
+; float b;
+;};
+;void foo3 (In * __restrict__ in, Out * __restrict__ out, int * __restrict__ trigger) {
+;
+; for (int i=0; i<SIZE; ++i) {
+; if (trigger[i] > 0) {
+; out[i].b = in[i].b + (float) 0.5;
+; }
+; }
+;}
+
+;AVX512-LABEL: @foo3
+;AVX512: getelementptr %struct.In, %struct.In* %in, <16 x i64> %induction, i32 1
+;AVX512: llvm.masked.gather.v16f32
+;AVX512: fadd <16 x float>
+;AVX512: getelementptr %struct.Out, %struct.Out* %out, <16 x i64> %induction, i32 1
+;AVX512: llvm.masked.scatter.v16f32
+;AVX512: ret void
+
+%struct.Out = type { float, float }
+
+define void @foo3(%struct.In* noalias %in, %struct.Out* noalias %out, i32* noalias %trigger) {
+entry:
+ %in.addr = alloca %struct.In*, align 8
+ %out.addr = alloca %struct.Out*, align 8
+ %trigger.addr = alloca i32*, align 8
+ %i = alloca i32, align 4
+ store %struct.In* %in, %struct.In** %in.addr, align 8
+ store %struct.Out* %out, %struct.Out** %out.addr, align 8
+ store i32* %trigger, i32** %trigger.addr, align 8
+ store i32 0, i32* %i, align 4
+ br label %for.cond
+
+for.cond: ; preds = %for.inc, %entry
+ %0 = load i32, i32* %i, align 4
+ %cmp = icmp slt i32 %0, 4096
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body: ; preds = %for.cond
+ %1 = load i32, i32* %i, align 4
+ %idxprom = sext i32 %1 to i64
+ %2 = load i32*, i32** %trigger.addr, align 8
+ %arrayidx = getelementptr inbounds i32, i32* %2, i64 %idxprom
+ %3 = load i32, i32* %arrayidx, align 4
+ %cmp1 = icmp sgt i32 %3, 0
+ br i1 %cmp1, label %if.then, label %if.end
+
+if.then: ; preds = %for.body
+ %4 = load i32, i32* %i, align 4
+ %idxprom2 = sext i32 %4 to i64
+ %5 = load %struct.In*, %struct.In** %in.addr, align 8
+ %arrayidx3 = getelementptr inbounds %struct.In, %struct.In* %5, i64 %idxprom2
+ %b = getelementptr inbounds %struct.In, %struct.In* %arrayidx3, i32 0, i32 1
+ %6 = load float, float* %b, align 4
+ %add = fadd float %6, 5.000000e-01
+ %7 = load i32, i32* %i, align 4
+ %idxprom4 = sext i32 %7 to i64
+ %8 = load %struct.Out*, %struct.Out** %out.addr, align 8
+ %arrayidx5 = getelementptr inbounds %struct.Out, %struct.Out* %8, i64 %idxprom4
+ %b6 = getelementptr inbounds %struct.Out, %struct.Out* %arrayidx5, i32 0, i32 1
+ store float %add, float* %b6, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %for.body
+ br label %for.inc
+
+for.inc: ; preds = %if.end
+ %9 = load i32, i32* %i, align 4
+ %inc = add nsw i32 %9, 1
+ store i32 %inc, i32* %i, align 4
+ br label %for.cond
+
+for.end: ; preds = %for.cond
+ ret void
+}
+declare void @llvm.masked.scatter.v16f32(<16 x float>, <16 x float*>, i32, <16 x i1>)
Modified: llvm/trunk/test/Transforms/LoopVectorize/X86/masked_load_store.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/X86/masked_load_store.ll?rev=261140&r1=261139&r2=261140&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/LoopVectorize/X86/masked_load_store.ll (original)
+++ llvm/trunk/test/Transforms/LoopVectorize/X86/masked_load_store.ll Wed Feb 17 13:23:04 2016
@@ -278,7 +278,8 @@ for.end:
;AVX: ret void
;AVX512-LABEL: @foo4
-;AVX512-NOT: llvm.masked
+;AVX512-NOT: llvm.masked.load
+;AVX512: llvm.masked.gather
;AVX512: ret void
; Function Attrs: nounwind uwtable
More information about the llvm-commits
mailing list