[llvm] 7cd1cfd - [NFC][TTI] Add Alignment for isLegalMasked[Gather/Scatter]
Anna Welker via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 18 01:15:00 PST 2019
Author: Anna Welker
Date: 2019-12-18T09:14:39Z
New Revision: 7cd1cfdd6b6c89f74af69f3513b5856a5c837317
URL: https://github.com/llvm/llvm-project/commit/7cd1cfdd6b6c89f74af69f3513b5856a5c837317
DIFF: https://github.com/llvm/llvm-project/commit/7cd1cfdd6b6c89f74af69f3513b5856a5c837317.diff
LOG: [NFC][TTI] Add Alignment for isLegalMasked[Gather/Scatter]
Add an extra parameter so alignment can be taken under
consideration in gather/scatter legalization.
Differential Revision: https://reviews.llvm.org/D71610
Added:
Modified:
llvm/include/llvm/Analysis/TargetTransformInfo.h
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
llvm/lib/Analysis/TargetTransformInfo.cpp
llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
llvm/lib/Target/ARM/ARMTargetTransformInfo.h
llvm/lib/Target/X86/X86TargetTransformInfo.cpp
llvm/lib/Target/X86/X86TargetTransformInfo.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 26bfe2692ff9..5382d76813a7 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -597,9 +597,9 @@ class TargetTransformInfo {
bool isLegalNTLoad(Type *DataType, Align Alignment) const;
/// Return true if the target supports masked scatter.
- bool isLegalMaskedScatter(Type *DataType) const;
+ bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) const;
/// Return true if the target supports masked gather.
- bool isLegalMaskedGather(Type *DataType) const;
+ bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) const;
/// Return true if the target supports masked compress store.
bool isLegalMaskedCompressStore(Type *DataType) const;
@@ -1237,8 +1237,8 @@ class TargetTransformInfo::Concept {
virtual bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) = 0;
virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
- virtual bool isLegalMaskedScatter(Type *DataType) = 0;
- virtual bool isLegalMaskedGather(Type *DataType) = 0;
+ virtual bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) = 0;
+ virtual bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) = 0;
virtual bool isLegalMaskedCompressStore(Type *DataType) = 0;
virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0;
virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0;
@@ -1537,11 +1537,11 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
bool isLegalNTLoad(Type *DataType, Align Alignment) override {
return Impl.isLegalNTLoad(DataType, Alignment);
}
- bool isLegalMaskedScatter(Type *DataType) override {
- return Impl.isLegalMaskedScatter(DataType);
+ bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) override {
+ return Impl.isLegalMaskedScatter(DataType, Alignment);
}
- bool isLegalMaskedGather(Type *DataType) override {
- return Impl.isLegalMaskedGather(DataType);
+ bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) override {
+ return Impl.isLegalMaskedGather(DataType, Alignment);
}
bool isLegalMaskedCompressStore(Type *DataType) override {
return Impl.isLegalMaskedCompressStore(DataType);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 45bff97c3f29..ac0609e29270 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -272,9 +272,13 @@ class TargetTransformInfoImplBase {
return Alignment >= DataSize && isPowerOf2_32(DataSize);
}
- bool isLegalMaskedScatter(Type *DataType) { return false; }
+ bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) {
+ return false;
+ }
- bool isLegalMaskedGather(Type *DataType) { return false; }
+ bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) {
+ return false;
+ }
bool isLegalMaskedCompressStore(Type *DataType) { return false; }
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 0813c0ba88fe..f2c63f789d89 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -315,12 +315,14 @@ bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
return TTIImpl->isLegalNTLoad(DataType, Alignment);
}
-bool TargetTransformInfo::isLegalMaskedGather(Type *DataType) const {
- return TTIImpl->isLegalMaskedGather(DataType);
+bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
+ MaybeAlign Alignment) const {
+ return TTIImpl->isLegalMaskedGather(DataType, Alignment);
}
-bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType) const {
- return TTIImpl->isLegalMaskedScatter(DataType);
+bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
+ MaybeAlign Alignment) const {
+ return TTIImpl->isLegalMaskedScatter(DataType, Alignment);
}
bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const {
diff --git a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
index a64e49e17df5..ee72de67d875 100644
--- a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
@@ -849,21 +849,20 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
bool &ModifiedDT) {
IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
if (II) {
+ unsigned Alignment;
switch (II->getIntrinsicID()) {
default:
break;
case Intrinsic::masked_load: {
// Scalarize unsupported vector masked load
- unsigned Alignment =
- cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
+ Alignment = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
if (TTI->isLegalMaskedLoad(CI->getType(), MaybeAlign(Alignment)))
return false;
scalarizeMaskedLoad(CI, ModifiedDT);
return true;
}
case Intrinsic::masked_store: {
- unsigned Alignment =
- cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
+ Alignment = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
if (TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType(),
MaybeAlign(Alignment)))
return false;
@@ -871,12 +870,15 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
return true;
}
case Intrinsic::masked_gather:
- if (TTI->isLegalMaskedGather(CI->getType()))
+ Alignment = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
+ if (TTI->isLegalMaskedGather(CI->getType(), MaybeAlign(Alignment)))
return false;
scalarizeMaskedGather(CI, ModifiedDT);
return true;
case Intrinsic::masked_scatter:
- if (TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType()))
+ Alignment = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
+ if (TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType(),
+ MaybeAlign(Alignment)))
return false;
scalarizeMaskedScatter(CI, ModifiedDT);
return true;
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 76eb82ef0e5f..f1273b4b8b37 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -159,6 +159,10 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
return isLegalMaskedLoad(DataTy, Alignment);
}
+ bool isLegalMaskedGather(Type *Ty, MaybeAlign Alignment) { return false; }
+
+ bool isLegalMaskedScatter(Type *Ty, MaybeAlign Alignment) { return false; }
+
int getMemcpyCost(const Instruction *I);
int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp);
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index d6bae86b92d3..0c31c31f6ff3 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -3296,8 +3296,10 @@ int X86TTIImpl::getGatherScatterOpCost(unsigned Opcode, Type *SrcVTy,
unsigned AddressSpace = PtrTy->getAddressSpace();
bool Scalarize = false;
- if ((Opcode == Instruction::Load && !isLegalMaskedGather(SrcVTy)) ||
- (Opcode == Instruction::Store && !isLegalMaskedScatter(SrcVTy)))
+ if ((Opcode == Instruction::Load &&
+ !isLegalMaskedGather(SrcVTy, MaybeAlign(Alignment))) ||
+ (Opcode == Instruction::Store &&
+ !isLegalMaskedScatter(SrcVTy, MaybeAlign(Alignment))))
Scalarize = true;
// Gather / Scatter for vector 2 is not profitable on KNL / SKX
// Vector-4 of gather/scatter instruction does not exist on KNL.
@@ -3420,7 +3422,7 @@ bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy) {
return isLegalMaskedExpandLoad(DataTy);
}
-bool X86TTIImpl::isLegalMaskedGather(Type *DataTy) {
+bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, MaybeAlign Alignment) {
// Some CPUs have better gather performance than others.
// TODO: Remove the explicit ST->hasAVX512()?, That would mean we would only
// enable gather with a -march.
@@ -3458,11 +3460,11 @@ bool X86TTIImpl::isLegalMaskedGather(Type *DataTy) {
return IntWidth == 32 || IntWidth == 64;
}
-bool X86TTIImpl::isLegalMaskedScatter(Type *DataType) {
+bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) {
// AVX2 doesn't support scatter
if (!ST->hasAVX512())
return false;
- return isLegalMaskedGather(DataType);
+ return isLegalMaskedGather(DataType, Alignment);
}
bool X86TTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 8a8735e1052f..b9c2dbd78058 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -189,8 +189,8 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment);
bool isLegalNTLoad(Type *DataType, Align Alignment);
bool isLegalNTStore(Type *DataType, Align Alignment);
- bool isLegalMaskedGather(Type *DataType);
- bool isLegalMaskedScatter(Type *DataType);
+ bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment);
+ bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment);
bool isLegalMaskedExpandLoad(Type *DataType);
bool isLegalMaskedCompressStore(Type *DataType);
bool hasDivRemOp(Type *DataType, bool IsSigned);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index be4bb933dd02..f98682a86e9e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1210,14 +1210,14 @@ class LoopVectorizationCostModel {
/// Returns true if the target machine supports masked scatter operation
/// for the given \p DataType.
- bool isLegalMaskedScatter(Type *DataType) {
- return TTI.isLegalMaskedScatter(DataType);
+ bool isLegalMaskedScatter(Type *DataType, MaybeAlign Alignment) {
+ return TTI.isLegalMaskedScatter(DataType, Alignment);
}
/// Returns true if the target machine supports masked gather operation
/// for the given \p DataType.
- bool isLegalMaskedGather(Type *DataType) {
- return TTI.isLegalMaskedGather(DataType);
+ bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) {
+ return TTI.isLegalMaskedGather(DataType, Alignment);
}
/// Returns true if the target machine can represent \p V as a masked gather
@@ -1228,7 +1228,9 @@ class LoopVectorizationCostModel {
if (!LI && !SI)
return false;
auto *Ty = getMemInstValueType(V);
- return (LI && isLegalMaskedGather(Ty)) || (SI && isLegalMaskedScatter(Ty));
+ MaybeAlign Align = getLoadStoreAlignment(V);
+ return (LI && isLegalMaskedGather(Ty, Align)) ||
+ (SI && isLegalMaskedScatter(Ty, Align));
}
/// Returns true if \p I is an instruction that will be scalarized with
@@ -4579,9 +4581,10 @@ bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, unsigne
return WideningDecision == CM_Scalarize;
}
const MaybeAlign Alignment = getLoadStoreAlignment(I);
- return isa<LoadInst>(I) ?
- !(isLegalMaskedLoad(Ty, Ptr, Alignment) || isLegalMaskedGather(Ty))
- : !(isLegalMaskedStore(Ty, Ptr, Alignment) || isLegalMaskedScatter(Ty));
+ return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
+ isLegalMaskedGather(Ty, Alignment))
+ : !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
+ isLegalMaskedScatter(Ty, Alignment));
}
case Instruction::UDiv:
case Instruction::SDiv:
More information about the llvm-commits
mailing list