[llvm] r236613 - [X86] Disable loop unrolling in loop vectorization pass when VF is 1.
Wei Mi
wmi at google.com
Wed May 6 10:12:26 PDT 2015
Author: wmi
Date: Wed May 6 12:12:25 2015
New Revision: 236613
URL: http://llvm.org/viewvc/llvm-project?rev=236613&view=rev
Log:
[X86] Disable loop unrolling in loop vectorization pass when VF is 1.
The patch disabled unrolling in loop vectorization pass when VF==1 on x86 architecture,
by setting MaxInterleaveFactor to 1. Unrolling in loop vectorization pass may introduce
the cost of overflow check, memory boundary check and extra prologue/epilogue code when
regular unroller will unroll the loop another time. Disable it when VF==1 remove the
unnecessary cost on x86. The same can be done for other platforms after verifying
interleaving/memory bound checking to be not perf critical on those platforms.
Differential Revision: http://reviews.llvm.org/D9515
Added:
llvm/trunk/test/Transforms/LoopVectorize/unroll.ll
Modified:
llvm/trunk/include/llvm/Analysis/TargetTransformInfo.h
llvm/trunk/include/llvm/Analysis/TargetTransformInfoImpl.h
llvm/trunk/include/llvm/CodeGen/BasicTTIImpl.h
llvm/trunk/lib/Analysis/TargetTransformInfo.cpp
llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.h
llvm/trunk/lib/Target/ARM/ARMTargetTransformInfo.h
llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.h
llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.cpp
llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.h
llvm/trunk/lib/Target/X86/X86TargetTransformInfo.cpp
llvm/trunk/lib/Target/X86/X86TargetTransformInfo.h
llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/trunk/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
Modified: llvm/trunk/include/llvm/Analysis/TargetTransformInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Analysis/TargetTransformInfo.h?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Analysis/TargetTransformInfo.h (original)
+++ llvm/trunk/include/llvm/Analysis/TargetTransformInfo.h Wed May 6 12:12:25 2015
@@ -403,7 +403,7 @@ public:
/// \return The maximum interleave factor that any transform should try to
/// perform for this target. This number depends on the level of parallelism
/// and the number of execution units in the CPU.
- unsigned getMaxInterleaveFactor() const;
+ unsigned getMaxInterleaveFactor(unsigned VF) const;
/// \return The expected cost of arithmetic ops, such as mul, xor, fsub, etc.
unsigned
@@ -562,7 +562,7 @@ public:
const APInt &Imm, Type *Ty) = 0;
virtual unsigned getNumberOfRegisters(bool Vector) = 0;
virtual unsigned getRegisterBitWidth(bool Vector) = 0;
- virtual unsigned getMaxInterleaveFactor() = 0;
+ virtual unsigned getMaxInterleaveFactor(unsigned VF) = 0;
virtual unsigned
getArithmeticInstrCost(unsigned Opcode, Type *Ty, OperandValueKind Opd1Info,
OperandValueKind Opd2Info,
@@ -703,8 +703,8 @@ public:
unsigned getRegisterBitWidth(bool Vector) override {
return Impl.getRegisterBitWidth(Vector);
}
- unsigned getMaxInterleaveFactor() override {
- return Impl.getMaxInterleaveFactor();
+ unsigned getMaxInterleaveFactor(unsigned VF) override {
+ return Impl.getMaxInterleaveFactor(VF);
}
unsigned
getArithmeticInstrCost(unsigned Opcode, Type *Ty, OperandValueKind Opd1Info,
Modified: llvm/trunk/include/llvm/Analysis/TargetTransformInfoImpl.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Analysis/TargetTransformInfoImpl.h?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Analysis/TargetTransformInfoImpl.h (original)
+++ llvm/trunk/include/llvm/Analysis/TargetTransformInfoImpl.h Wed May 6 12:12:25 2015
@@ -263,7 +263,7 @@ public:
unsigned getRegisterBitWidth(bool Vector) { return 32; }
- unsigned getMaxInterleaveFactor() { return 1; }
+ unsigned getMaxInterleaveFactor(unsigned VF) { return 1; }
unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty,
TTI::OperandValueKind Opd1Info,
Modified: llvm/trunk/include/llvm/CodeGen/BasicTTIImpl.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/CodeGen/BasicTTIImpl.h?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/include/llvm/CodeGen/BasicTTIImpl.h (original)
+++ llvm/trunk/include/llvm/CodeGen/BasicTTIImpl.h Wed May 6 12:12:25 2015
@@ -285,7 +285,7 @@ public:
unsigned getRegisterBitWidth(bool Vector) { return 32; }
- unsigned getMaxInterleaveFactor() { return 1; }
+ unsigned getMaxInterleaveFactor(unsigned VF) { return 1; }
unsigned getArithmeticInstrCost(
unsigned Opcode, Type *Ty,
Modified: llvm/trunk/lib/Analysis/TargetTransformInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Analysis/TargetTransformInfo.cpp?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Analysis/TargetTransformInfo.cpp (original)
+++ llvm/trunk/lib/Analysis/TargetTransformInfo.cpp Wed May 6 12:12:25 2015
@@ -186,8 +186,8 @@ unsigned TargetTransformInfo::getRegiste
return TTIImpl->getRegisterBitWidth(Vector);
}
-unsigned TargetTransformInfo::getMaxInterleaveFactor() const {
- return TTIImpl->getMaxInterleaveFactor();
+unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const {
+ return TTIImpl->getMaxInterleaveFactor(VF);
}
unsigned TargetTransformInfo::getArithmeticInstrCost(
Modified: llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.cpp?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.cpp Wed May 6 12:12:25 2015
@@ -419,7 +419,7 @@ unsigned AArch64TTIImpl::getCostOfKeepin
return Cost;
}
-unsigned AArch64TTIImpl::getMaxInterleaveFactor() {
+unsigned AArch64TTIImpl::getMaxInterleaveFactor(unsigned VF) {
if (ST->isCortexA57())
return 4;
return 2;
Modified: llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.h?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.h (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64TargetTransformInfo.h Wed May 6 12:12:25 2015
@@ -110,7 +110,7 @@ public:
return 64;
}
- unsigned getMaxInterleaveFactor();
+ unsigned getMaxInterleaveFactor(unsigned VF);
unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src);
Modified: llvm/trunk/lib/Target/ARM/ARMTargetTransformInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/ARM/ARMTargetTransformInfo.h?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/ARM/ARMTargetTransformInfo.h (original)
+++ llvm/trunk/lib/Target/ARM/ARMTargetTransformInfo.h Wed May 6 12:12:25 2015
@@ -96,7 +96,7 @@ public:
return 32;
}
- unsigned getMaxInterleaveFactor() {
+ unsigned getMaxInterleaveFactor(unsigned VF) {
// These are out of order CPUs:
if (ST->isCortexA15() || ST->isSwift())
return 2;
Modified: llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.cpp?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.cpp (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.cpp Wed May 6 12:12:25 2015
@@ -215,7 +215,7 @@ unsigned PPCTTIImpl::getRegisterBitWidth
}
-unsigned PPCTTIImpl::getMaxInterleaveFactor() {
+unsigned PPCTTIImpl::getMaxInterleaveFactor(unsigned VF) {
unsigned Directive = ST->getDarwinDirective();
// The 440 has no SIMD support, but floating-point instructions
// have a 5-cycle latency, so unroll by 5x for latency hiding.
Modified: llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.h?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.h (original)
+++ llvm/trunk/lib/Target/PowerPC/PPCTargetTransformInfo.h Wed May 6 12:12:25 2015
@@ -81,7 +81,7 @@ public:
bool enableAggressiveInterleaving(bool LoopHasReductions);
unsigned getNumberOfRegisters(bool Vector);
unsigned getRegisterBitWidth(bool Vector);
- unsigned getMaxInterleaveFactor();
+ unsigned getMaxInterleaveFactor(unsigned VF);
unsigned getArithmeticInstrCost(
unsigned Opcode, Type *Ty,
TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
Modified: llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.cpp?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.cpp (original)
+++ llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.cpp Wed May 6 12:12:25 2015
@@ -76,7 +76,7 @@ unsigned AMDGPUTTIImpl::getNumberOfRegis
unsigned AMDGPUTTIImpl::getRegisterBitWidth(bool) { return 32; }
-unsigned AMDGPUTTIImpl::getMaxInterleaveFactor() {
+unsigned AMDGPUTTIImpl::getMaxInterleaveFactor(unsigned VF) {
// Semi-arbitrary large amount.
return 64;
}
Modified: llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.h?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.h (original)
+++ llvm/trunk/lib/Target/R600/AMDGPUTargetTransformInfo.h Wed May 6 12:12:25 2015
@@ -70,7 +70,7 @@ public:
unsigned getNumberOfRegisters(bool Vector);
unsigned getRegisterBitWidth(bool Vector);
- unsigned getMaxInterleaveFactor();
+ unsigned getMaxInterleaveFactor(unsigned VF);
};
} // end namespace llvm
Modified: llvm/trunk/lib/Target/X86/X86TargetTransformInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86TargetTransformInfo.cpp?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86TargetTransformInfo.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86TargetTransformInfo.cpp Wed May 6 12:12:25 2015
@@ -66,7 +66,13 @@ unsigned X86TTIImpl::getRegisterBitWidth
}
-unsigned X86TTIImpl::getMaxInterleaveFactor() {
+unsigned X86TTIImpl::getMaxInterleaveFactor(unsigned VF) {
+ // If the loop will not be vectorized, don't interleave the loop.
+ // Let regular unroll to unroll the loop, which saves the overflow
+ // check and memory check cost.
+ if (VF == 1)
+ return 1;
+
if (ST->isAtom())
return 1;
Modified: llvm/trunk/lib/Target/X86/X86TargetTransformInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86TargetTransformInfo.h?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86TargetTransformInfo.h (original)
+++ llvm/trunk/lib/Target/X86/X86TargetTransformInfo.h Wed May 6 12:12:25 2015
@@ -72,7 +72,7 @@ public:
unsigned getNumberOfRegisters(bool Vector);
unsigned getRegisterBitWidth(bool Vector);
- unsigned getMaxInterleaveFactor();
+ unsigned getMaxInterleaveFactor(unsigned VF);
unsigned getArithmeticInstrCost(
unsigned Opcode, Type *Ty,
TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
Modified: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp (original)
+++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp Wed May 6 12:12:25 2015
@@ -4160,7 +4160,7 @@ LoopVectorizationCostModel::selectUnroll
std::max(1U, (R.MaxLocalUsers - 1)));
// Clamp the unroll factor ranges to reasonable factors.
- unsigned MaxInterleaveSize = TTI.getMaxInterleaveFactor();
+ unsigned MaxInterleaveSize = TTI.getMaxInterleaveFactor(VF);
// Check if the user has overridden the unroll max.
if (VF == 1) {
Modified: llvm/trunk/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll?rev=236613&r1=236612&r2=236613&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll (original)
+++ llvm/trunk/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll Wed May 6 12:12:25 2015
@@ -47,9 +47,11 @@ define i32 @foo(i32* nocapture %A) nounw
; CHECK-VECTOR: store <4 x i32>
; CHECK-VECTOR: ret
;
+; For x86, loop unroll in loop vectorizer is disabled when VF==1.
+;
; CHECK-SCALAR-LABEL: @bar(
; CHECK-SCALAR: store i32
-; CHECK-SCALAR: store i32
+; CHECK-SCALAR-NOT: store i32
; CHECK-SCALAR: ret
define i32 @bar(i32* nocapture %A, i32 %n) nounwind uwtable ssp {
%1 = icmp sgt i32 %n, 0
Added: llvm/trunk/test/Transforms/LoopVectorize/unroll.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/LoopVectorize/unroll.ll?rev=236613&view=auto
==============================================================================
--- llvm/trunk/test/Transforms/LoopVectorize/unroll.ll (added)
+++ llvm/trunk/test/Transforms/LoopVectorize/unroll.ll Wed May 6 12:12:25 2015
@@ -0,0 +1,37 @@
+; This test makes sure that loop will not be unrolled in vectorization if VF computed
+; equals to 1.
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+; Make sure there are no geps being merged.
+; CHECK-LABEL: @foo(
+; CHECK: getelementptr
+; CHECK-NOT: getelementptr
+
+ at N = common global i32 0, align 4
+ at a = common global [1000 x i32] zeroinitializer, align 16
+
+define void @foo() #0 {
+entry:
+ %0 = load i32, i32* @N, align 4
+ %cmp5 = icmp sgt i32 %0, 0
+ br i1 %cmp5, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph: ; preds = %entry
+ %conv = sext i32 %0 to i64
+ br label %for.body
+
+for.body: ; preds = %for.body.lr.ph, %for.body
+ %i.06 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+ %mul = mul nuw nsw i64 %i.06, 7
+ %arrayidx = getelementptr inbounds [1000 x i32], [1000 x i32]* @a, i64 0, i64 %mul
+ store i32 3, i32* %arrayidx, align 4
+ %inc = add nuw nsw i64 %i.06, 1
+ %cmp = icmp slt i64 %inc, %conv
+ br i1 %cmp, label %for.body, label %for.end.loopexit
+
+for.end.loopexit: ; preds = %for.body
+ br label %for.end
+
+for.end: ; preds = %for.end.loopexit, %entry
+ ret void
+}
More information about the llvm-commits
mailing list