[clang] e3a4701 - [clang][CodeGen] Lower Likelihood attributes to @llvm.expect intrin instead of branch weights
Roman Lebedev via cfe-commits
cfe-commits at lists.llvm.org
Sun Mar 21 12:51:02 PDT 2021
Author: Roman Lebedev
Date: 2021-03-21T22:50:21+03:00
New Revision: e3a470162738871bba982416748ae5f5e3572947
URL: https://github.com/llvm/llvm-project/commit/e3a470162738871bba982416748ae5f5e3572947
DIFF: https://github.com/llvm/llvm-project/commit/e3a470162738871bba982416748ae5f5e3572947.diff
LOG: [clang][CodeGen] Lower Likelihood attributes to @llvm.expect intrin instead of branch weights
08196e0b2e1f8aaa8a854585335c17ba479114df exposed LowerExpectIntrinsic's
internal implementation detail in the form of
LikelyBranchWeight/UnlikelyBranchWeight options to the outside.
While this isn't incorrect from the results viewpoint,
this is suboptimal from the layering viewpoint,
and causes confusion - should transforms also use those weights,
or should they use something else, D98898?
So go back to status quo by making LikelyBranchWeight/UnlikelyBranchWeight
internal again, and fixing all the code that used it directly,
which currently is only clang codegen, thankfully,
to emit proper @llvm.expect intrinsics instead.
Added:
Modified:
clang/lib/CodeGen/CGStmt.cpp
clang/lib/CodeGen/CodeGenFunction.cpp
clang/lib/CodeGen/CodeGenFunction.h
clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
Removed:
################################################################################
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 38f3aa941415e..fb719efb1a35d 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -821,8 +821,11 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
if (ConditionScope.requiresCleanups())
ExitBlock = createBasicBlock("while.exit");
- llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
- S.getCond(), getProfileCount(S.getBody()), S.getBody());
+ llvm::MDNode *Weights =
+ createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+ if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+ BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+ BoolCondVal, Stmt::getLikelihood(S.getBody()));
Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights);
if (ExitBlock != LoopExit.getBlock()) {
@@ -1008,8 +1011,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
// C99 6.8.5p2/p4: The first substatement is executed if the expression
// compares unequal to 0. The condition must be a scalar type.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
- llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
- S.getCond(), getProfileCount(S.getBody()), S.getBody());
+ llvm::MDNode *Weights =
+ createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+ if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+ BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+ BoolCondVal, Stmt::getLikelihood(S.getBody()));
if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal))
if (C->isOne())
@@ -1094,8 +1100,11 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
// The body is executed if the expression, contextually converted
// to bool, is true.
llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
- llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
- S.getCond(), getProfileCount(S.getBody()), S.getBody());
+ llvm::MDNode *Weights =
+ createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+ if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+ BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+ BoolCondVal, Stmt::getLikelihood(S.getBody()));
Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
if (ExitBlock != LoopExit.getBlock()) {
@@ -1369,7 +1378,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
// this case.
(*SwitchWeights)[0] += ThisCount;
} else if (SwitchLikelihood)
- Weights = createBranchWeights(LH);
+ Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index a00ae74fa165f..1c538262737dd 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1764,31 +1764,39 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
return;
}
- // If the branch has a condition wrapped by __builtin_unpredictable,
- // create metadata that specifies that the branch is unpredictable.
- // Don't bother if not optimizing because that metadata would not be used.
- llvm::MDNode *Unpredictable = nullptr;
- auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
- if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
- auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
- if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
- llvm::MDBuilder MDHelper(getLLVMContext());
- Unpredictable = MDHelper.createUnpredictable();
- }
- }
-
- llvm::MDNode *Weights = createBranchWeights(LH);
- if (!Weights) {
- uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
- Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
- }
-
// Emit the code with the fully general case.
llvm::Value *CondV;
{
ApplyDebugLocation DL(*this, Cond);
CondV = EvaluateExprAsBool(Cond);
}
+
+ llvm::MDNode *Weights = nullptr;
+ llvm::MDNode *Unpredictable = nullptr;
+
+ // If optimizing, lower unpredictability/probability knowledge about cond.
+ if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
+ // If the branch has a condition wrapped by __builtin_unpredictable,
+ // create metadata that specifies that the branch is unpredictable.
+ if (auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts())) {
+ auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
+ if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
+ llvm::MDBuilder MDHelper(getLLVMContext());
+ Unpredictable = MDHelper.createUnpredictable();
+ }
+ }
+
+ // If there is a Likelihood knowledge for the cond, lower it.
+ llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
+ if (CondV != NewCondV)
+ CondV = NewCondV;
+ else {
+ // Otherwise, lower profile counts.
+ uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
+ Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
+ }
+ }
+
Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
}
@@ -2632,35 +2640,26 @@ llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
return llvm::DebugLoc();
}
-static Optional<std::pair<uint32_t, uint32_t>>
-getLikelihoodWeights(Stmt::Likelihood LH) {
+llvm::Value *
+CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
+ Stmt::Likelihood LH) {
switch (LH) {
- case Stmt::LH_Unlikely:
- return std::pair<uint32_t, uint32_t>(llvm::UnlikelyBranchWeight,
- llvm::LikelyBranchWeight);
case Stmt::LH_None:
- return None;
+ return Cond;
case Stmt::LH_Likely:
- return std::pair<uint32_t, uint32_t>(llvm::LikelyBranchWeight,
- llvm::UnlikelyBranchWeight);
+ case Stmt::LH_Unlikely:
+ // Don't generate llvm.expect on -O0 as the backend won't use it for
+ // anything.
+ if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+ return Cond;
+ llvm::Type *CondTy = Cond->getType();
+ assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
+ llvm::Function *FnExpect =
+ CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
+ llvm::Value *ExpectedValueOfCond =
+ llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
+ return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
+ Cond->getName() + ".expval");
}
llvm_unreachable("Unknown Likelihood");
}
-
-llvm::MDNode *CodeGenFunction::createBranchWeights(Stmt::Likelihood LH) const {
- Optional<std::pair<uint32_t, uint32_t>> LHW = getLikelihoodWeights(LH);
- if (!LHW)
- return nullptr;
-
- llvm::MDBuilder MDHelper(CGM.getLLVMContext());
- return MDHelper.createBranchWeights(LHW->first, LHW->second);
-}
-
-llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop(
- const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const {
- llvm::MDNode *Weights = createProfileWeightsForLoop(Cond, LoopCount);
- if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
- Weights = createBranchWeights(Stmt::getLikelihood(Body));
-
- return Weights;
-}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 4bca21a51f9c7..11de9166335b4 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -1445,8 +1445,9 @@ class CodeGenFunction : public CodeGenTypeCache {
};
OpenMPCancelExitStack OMPCancelStack;
- /// Calculate branch weights for the likelihood attribute
- llvm::MDNode *createBranchWeights(Stmt::Likelihood LH) const;
+ /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
+ llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
+ Stmt::Likelihood LH);
CodeGenPGO PGO;
@@ -1457,13 +1458,6 @@ class CodeGenFunction : public CodeGenTypeCache {
llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
uint64_t LoopCount) const;
- /// Calculate the branch weight for PGO data or the likelihood attribute.
- /// The function tries to get the weight of \ref createProfileWeightsForLoop.
- /// If that fails it gets the weight of \ref createBranchWeights.
- llvm::MDNode *createProfileOrBranchWeightsForLoop(const Stmt *Cond,
- uint64_t LoopCount,
- const Stmt *Body) const;
-
public:
/// Increment the profiler's counter for the given statement by \p StepV.
/// If \p StepV is null, the default increment is 1.
diff --git a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
index ec2ee37f97c07..27e8a4999c328 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
@@ -1,60 +1,263 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu -verify
// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s
+// CHECK-LABEL: @_Z2wli(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT: br label [[WHILE_COND:%.*]]
+// CHECK: while.cond:
+// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+// CHECK-NEXT: [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 true)
+// CHECK-NEXT: br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+// CHECK: while.body:
+// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[WHILE_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK: while.end:
+// CHECK-NEXT: ret void
+//
void wl(int e){
- // CHECK-LABEL: define{{.*}}wl
- // CHECK: br {{.*}} !prof !6
while(e) [[likely]] ++e;
}
+// CHECK-LABEL: @_Z2wui(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[WHILE_COND:%.*]]
+// CHECK: while.cond:
+// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+// CHECK-NEXT: [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 false)
+// CHECK-NEXT: br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+// CHECK: while.body:
+// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[WHILE_COND]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK: while.end:
+// CHECK-NEXT: ret void
+//
void wu(int e){
- // CHECK-LABEL: define{{.*}}wu
- // CHECK: br {{.*}} !prof !10
while(e) [[unlikely]] ++e;
}
+// CHECK-LABEL: @_Z15w_branch_elidedj(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[WHILE_BODY:%.*]]
+// CHECK: while.body:
+// CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[INC:%.*]] = add i32 [[TMP0]], 1
+// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[WHILE_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+//
void w_branch_elided(unsigned e){
- // CHECK-LABEL: define{{.*}}w_branch_elided
- // CHECK-NOT: br {{.*}} !prof
// expected-warning at +2 {{attribute 'likely' has no effect when annotating an infinite loop}}
// expected-note at +1 {{annotating the infinite loop here}}
while(1) [[likely]] ++e;
}
+// CHECK-LABEL: @_Z2flj(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4:[0-9]+]]
+// CHECK-NEXT: store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[FOR_COND:%.*]]
+// CHECK: for.cond:
+// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
+// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
+// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK: for.cond.cleanup:
+// CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]]
+// CHECK-NEXT: br label [[FOR_END:%.*]]
+// CHECK: for.body:
+// CHECK-NEXT: br label [[FOR_INC:%.*]]
+// CHECK: for.inc:
+// CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[INC:%.*]] = add i32 [[TMP4]], 1
+// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK: for.end:
+// CHECK-NEXT: ret void
+//
void fl(unsigned e)
{
- // CHECK-LABEL: define{{.*}}fl
- // CHECK: br {{.*}} !prof !6
for(int i = 0; i != e; ++e) [[likely]];
}
+// CHECK-LABEL: @_Z2fui(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT: store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT: store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[FOR_COND:%.*]]
+// CHECK: for.cond:
+// CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
+// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
+// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK: for.cond.cleanup:
+// CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]]
+// CHECK-NEXT: br label [[FOR_END:%.*]]
+// CHECK: for.body:
+// CHECK-NEXT: br label [[FOR_INC:%.*]]
+// CHECK: for.inc:
+// CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP4]], 1
+// CHECK-NEXT: store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK: for.end:
+// CHECK-NEXT: ret void
+//
void fu(int e)
{
- // CHECK-LABEL: define{{.*}}fu
- // CHECK: br {{.*}} !prof !10
for(int i = 0; i != e; ++e) [[unlikely]];
}
+// CHECK-LABEL: @_Z15f_branch_elidedv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: br label [[FOR_COND:%.*]]
+// CHECK: for.cond:
+// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]]
+//
void f_branch_elided()
{
- // CHECK-LABEL: define{{.*}}f_branch_elided
- // CHECK-NOT: br {{.*}} !prof
for(;;) [[likely]];
}
+// CHECK-LABEL: @_Z3frlOA4_i(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT: [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT: [[__BEGIN1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT: [[__END1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT: store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14:![0-9]+]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0
+// CHECK-NEXT: store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0
+// CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4
+// CHECK-NEXT: store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: br label [[FOR_COND:%.*]]
+// CHECK: for.cond:
+// CHECK-NEXT: [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]]
+// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
+// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK: for.cond.cleanup:
+// CHECK-NEXT: [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]]
+// CHECK-NEXT: br label [[FOR_END:%.*]]
+// CHECK: for.body:
+// CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]]
+// CHECK-NEXT: br label [[FOR_INC:%.*]]
+// CHECK: for.inc:
+// CHECK-NEXT: [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1
+// CHECK-NEXT: store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
+// CHECK: for.end:
+// CHECK-NEXT: ret void
+//
void frl(int (&&e) [4])
{
- // CHECK-LABEL: define{{.*}}frl
- // CHECK: br {{.*}} !prof !6
for(int i : e) [[likely]];
}
+// CHECK-LABEL: @_Z3fruOA4_i(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT: [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT: [[__BEGIN1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT: [[__END1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT: store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0
+// CHECK-NEXT: store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0
+// CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4
+// CHECK-NEXT: store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: br label [[FOR_COND:%.*]]
+// CHECK: for.cond:
+// CHECK-NEXT: [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]]
+// CHECK-NEXT: [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
+// CHECK-NEXT: br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK: for.cond.cleanup:
+// CHECK-NEXT: [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]]
+// CHECK-NEXT: br label [[FOR_END:%.*]]
+// CHECK: for.body:
+// CHECK-NEXT: [[TMP11:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]]
+// CHECK-NEXT: [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[TMP14:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]]
+// CHECK-NEXT: br label [[FOR_INC:%.*]]
+// CHECK: for.inc:
+// CHECK-NEXT: [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1
+// CHECK-NEXT: store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK: for.end:
+// CHECK-NEXT: ret void
+//
void fru(int (&&e) [4])
{
- // CHECK-LABEL: define{{.*}}fru
- // CHECK: br {{.*}} !prof !10
for(int i : e) [[unlikely]];
}
-
-// CHECK: !6 = !{!"branch_weights", i32 2000, i32 1}
-// CHECK: !10 = !{!"branch_weights", i32 1, i32 2000}
diff --git a/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp b/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
index 5fb7a67a7d9e8..05729eb5a789e 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
@@ -1,59 +1,111 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
// RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s
extern volatile int i;
+// CHECK-LABEL: @_Z8OneCaseLv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT: ], !prof !6
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void OneCaseL() {
- // CHECK-LABEL: define{{.*}}OneCaseL
- // CHECK: switch
- // CHECK: {{.*}} !prof !6
switch (i) {
[[likely]] case 1: break;
}
}
+// CHECK-LABEL: @_Z8OneCaseUv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT: ], !prof !7
+// CHECK: sw.bb:
+// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void OneCaseU() {
- // CHECK-LABEL: define{{.*}}OneCaseU
- // CHECK: switch
- // CHECK: {{.*}} !prof !7
switch (i) {
[[unlikely]] case 1: ++i; break;
}
}
+// CHECK-LABEL: @_Z10TwoCasesLNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT: ], !prof !8
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void TwoCasesLN() {
- // CHECK-LABEL: define{{.*}}TwoCasesLN
- // CHECK: switch
- // CHECK: {{.*}} !prof !8
switch (i) {
[[likely]] case 1: break;
case 2: break;
}
}
+// CHECK-LABEL: @_Z10TwoCasesUNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT: ], !prof !9
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void TwoCasesUN() {
- // CHECK-LABEL: define{{.*}}TwoCasesUN
- // CHECK: switch
- // CHECK: {{.*}} !prof !9
switch (i) {
[[unlikely]] case 1: break;
case 2: break;
}
}
+// CHECK-LABEL: @_Z10TwoCasesLUv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT: ], !prof !10
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void TwoCasesLU() {
- // CHECK-LABEL: define{{.*}}TwoCasesLU
- // CHECK: switch
- // CHECK: {{.*}} !prof !10
switch (i) {
[[likely]] case 1: break;
[[unlikely]] case 2: break;
}
}
+// CHECK-LABEL: @_Z20CasesFallthroughNNLNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT: i32 2, label [[SW_BB]]
+// CHECK-NEXT: i32 3, label [[SW_BB1:%.*]]
+// CHECK-NEXT: i32 4, label [[SW_BB1]]
+// CHECK-NEXT: ], !prof !11
+// CHECK: sw.bb:
+// CHECK-NEXT: br label [[SW_BB1]]
+// CHECK: sw.bb1:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void CasesFallthroughNNLN() {
- // CHECK-LABEL: define{{.*}}CasesFallthroughNNLN
- // CHECK: switch
- // CHECK: {{.*}} !prof !11
switch (i) {
case 1:
case 2:
@@ -62,10 +114,23 @@ void CasesFallthroughNNLN() {
}
}
+// CHECK-LABEL: @_Z20CasesFallthroughNNUNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT: i32 2, label [[SW_BB]]
+// CHECK-NEXT: i32 3, label [[SW_BB1:%.*]]
+// CHECK-NEXT: i32 4, label [[SW_BB1]]
+// CHECK-NEXT: ], !prof !12
+// CHECK: sw.bb:
+// CHECK-NEXT: br label [[SW_BB1]]
+// CHECK: sw.bb1:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void CasesFallthroughNNUN() {
- // CHECK-LABEL: define{{.*}}CasesFallthroughNNUN
- // CHECK: switch
- // CHECK: {{.*}} !prof !12
switch (i) {
case 1:
case 2:
@@ -74,10 +139,32 @@ void CasesFallthroughNNUN() {
}
}
+// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallLNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT: i32 2, label [[SW_BB]]
+// CHECK-NEXT: i32 3, label [[SW_BB]]
+// CHECK-NEXT: i32 4, label [[SW_BB]]
+// CHECK-NEXT: i32 5, label [[SW_BB]]
+// CHECK-NEXT: i32 102, label [[SW_BB1:%.*]]
+// CHECK-NEXT: i32 103, label [[SW_BB2:%.*]]
+// CHECK-NEXT: i32 104, label [[SW_BB2]]
+// CHECK-NEXT: ], !prof !13
+// CHECK: sw.bb:
+// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[SW_BB1]]
+// CHECK: sw.bb1:
+// CHECK-NEXT: br label [[SW_BB2]]
+// CHECK: sw.bb2:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void CasesFallthroughRangeSmallLN() {
- // CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallLN
- // CHECK: switch
- // CHECK: {{.*}} !prof !13
switch (i) {
case 1 ... 5: ++i;
case 102:
@@ -86,10 +173,32 @@ void CasesFallthroughRangeSmallLN() {
}
}
+// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallUNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT: i32 2, label [[SW_BB]]
+// CHECK-NEXT: i32 3, label [[SW_BB]]
+// CHECK-NEXT: i32 4, label [[SW_BB]]
+// CHECK-NEXT: i32 5, label [[SW_BB]]
+// CHECK-NEXT: i32 102, label [[SW_BB1:%.*]]
+// CHECK-NEXT: i32 103, label [[SW_BB2:%.*]]
+// CHECK-NEXT: i32 104, label [[SW_BB2]]
+// CHECK-NEXT: ], !prof !14
+// CHECK: sw.bb:
+// CHECK-NEXT: [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT: store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: br label [[SW_BB1]]
+// CHECK: sw.bb1:
+// CHECK-NEXT: br label [[SW_BB2]]
+// CHECK: sw.bb2:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void CasesFallthroughRangeSmallUN() {
- // CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallUN
- // CHECK: switch
- // CHECK: {{.*}} !prof !14
switch (i) {
case 1 ... 5: ++i;
case 102:
@@ -98,12 +207,26 @@ void CasesFallthroughRangeSmallUN() {
}
}
+// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeLLNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [
+// CHECK-NEXT: i32 1003, label [[SW_BB1:%.*]]
+// CHECK-NEXT: i32 104, label [[SW_BB1]]
+// CHECK-NEXT: ], !prof !8
+// CHECK: sw.bb:
+// CHECK-NEXT: br label [[SW_BB1]]
+// CHECK: sw.bb1:
+// CHECK-NEXT: br label [[SW_EPILOG:%.*]]
+// CHECK: sw.caserange:
+// CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 0
+// CHECK-NEXT: [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64
+// CHECK-NEXT: [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 true)
+// CHECK-NEXT: br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void CasesFallthroughRangeLargeLLN() {
- // CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeLLN
- // CHECK: switch
- // CHECK: {{.*}} !prof !8
- // CHECK: caserange
- // CHECK: br{{.*}} !prof !15
switch (i) {
[[likely]] case 0 ... 64:
[[likely]] case 1003:
@@ -111,12 +234,26 @@ void CasesFallthroughRangeLargeLLN() {
}
}
+// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeUUNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [
+// CHECK-NEXT: i32 1003, label [[SW_BB1:%.*]]
+// CHECK-NEXT: i32 104, label [[SW_BB1]]
+// CHECK-NEXT: ], !prof !9
+// CHECK: sw.bb:
+// CHECK-NEXT: br label [[SW_BB1]]
+// CHECK: sw.bb1:
+// CHECK-NEXT: br label [[SW_EPILOG:%.*]]
+// CHECK: sw.caserange:
+// CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 0
+// CHECK-NEXT: [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64
+// CHECK-NEXT: [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 false)
+// CHECK-NEXT: br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void CasesFallthroughRangeLargeUUN() {
- // CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeUUN
- // CHECK: switch
- // CHECK: {{.*}} !prof !9
- // CHECK: caserange
- // CHECK: br{{.*}} !prof !16
switch (i) {
[[unlikely]] case 0 ... 64:
[[unlikely]] case 1003:
@@ -124,30 +261,55 @@ void CasesFallthroughRangeLargeUUN() {
}
}
+// CHECK-LABEL: @_Z15OneCaseDefaultLv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT: ], !prof !15
+// CHECK: sw.default:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void OneCaseDefaultL() {
- // CHECK-LABEL: define{{.*}}OneCaseDefaultL
- // CHECK: switch
- // CHECK: {{.*}} !prof !17
switch (i) {
case 1: break;
[[likely]] default: break;
}
}
+// CHECK-LABEL: @_Z15OneCaseDefaultUv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT: ], !prof !16
+// CHECK: sw.default:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void OneCaseDefaultU() {
- // CHECK-LABEL: define{{.*}}OneCaseDefaultU
- // CHECK: switch
- // CHECK: {{.*}} !prof !18
switch (i) {
case 1: break;
[[unlikely]] default: break;
}
}
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNLv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT: ], !prof !17
+// CHECK: sw.default:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void TwoCasesDefaultLNL() {
- // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNL
- // CHECK: switch
- // CHECK: {{.*}} !prof !19
switch (i) {
[[likely]] case 1: break;
case 2: break;
@@ -155,10 +317,19 @@ void TwoCasesDefaultLNL() {
}
}
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNNv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT: ], !prof !8
+// CHECK: sw.default:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void TwoCasesDefaultLNN() {
- // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNN
- // CHECK: switch
- // CHECK: {{.*}} !prof !8
switch (i) {
[[likely]] case 1: break;
case 2: break;
@@ -166,29 +337,22 @@ void TwoCasesDefaultLNN() {
}
}
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNUv(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT: switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT: i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT: i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT: ], !prof !18
+// CHECK: sw.default:
+// CHECK-NEXT: br label [[SW_EPILOG]]
+// CHECK: sw.epilog:
+// CHECK-NEXT: ret void
+//
void TwoCasesDefaultLNU() {
- // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNU
- // CHECK: switch
- // CHECK: {{.*}} !prof !20
switch (i) {
[[likely]] case 1: break;
case 2: break;
[[unlikely]] default: break;
}
}
-
-// CHECK: !6 = !{!"branch_weights", i32 357913942, i32 715827883}
-// CHECK: !7 = !{!"branch_weights", i32 536870912, i32 1}
-// CHECK: !8 = !{!"branch_weights", i32 238609295, i32 715827883, i32 238609295}
-// CHECK: !9 = !{!"branch_weights", i32 357913942, i32 1, i32 357913942}
-// CHECK: !10 = !{!"branch_weights", i32 357913942, i32 715827883, i32 1}
-// CHECK: !11 = !{!"branch_weights", i32 143165577, i32 143165577, i32 143165577, i32 715827883, i32 143165577}
-// CHECK: !12 = !{!"branch_weights", i32 214748365, i32 214748365, i32 214748365, i32 1, i32 214748365}
-// CHECK: !13 = !{!"branch_weights", i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 715827883, i32 79536432}
-// CHECK: !14 = !{!"branch_weights", i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 1, i32 119304648}
-// CHECK: !15 = !{!"branch_weights", i32 2000, i32 1}
-// CHECK: !16 = !{!"branch_weights", i32 1, i32 2000}
-// CHECK: !17 = !{!"branch_weights", i32 715827883, i32 357913942}
-// CHECK: !18 = !{!"branch_weights", i32 1, i32 536870912}
-// CHECK: !19 = !{!"branch_weights", i32 536870912, i32 536870912, i32 268435456}
-// CHECK: !20 = !{!"branch_weights", i32 1, i32 715827883, i32 357913942}
diff --git a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
index 22b2e649e4d48..4e47ff70d5574 100644
--- a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
+++ b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
@@ -17,7 +17,6 @@
#include "llvm/IR/Function.h"
#include "llvm/IR/PassManager.h"
-#include "llvm/Support/CommandLine.h"
namespace llvm {
@@ -32,8 +31,6 @@ struct LowerExpectIntrinsicPass : PassInfoMixin<LowerExpectIntrinsicPass> {
PreservedAnalyses run(Function &F, FunctionAnalysisManager &);
};
-extern cl::opt<uint32_t> LikelyBranchWeight;
-extern cl::opt<uint32_t> UnlikelyBranchWeight;
}
#endif
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index da13075dfee26..7f3549aaaabf9 100644
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -24,6 +24,7 @@
#include "llvm/IR/Metadata.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Scalar.h"
@@ -41,15 +42,15 @@ STATISTIC(ExpectIntrinsicsHandled,
// only be used in extreme cases, we could make this ratio higher. As it stands,
// programmers may be using __builtin_expect() / llvm.expect to annotate that a
// branch is likely or unlikely to be taken.
-//
-// There is a known dependency on this ratio in CodeGenPrepare when transforming
-// 'select' instructions. It may be worthwhile to hoist these values to some
-// shared space, so they can be used directly by other passes.
-cl::opt<uint32_t> llvm::LikelyBranchWeight(
+// WARNING: these values are internal implementation detail of the pass.
+// They should not be exposed to the outside of the pass, front-end codegen
+// should emit @llvm.expect intrinsics instead of using these weights directly.
+// Transforms should use TargetLowering getPredictableBranchThreshold() hook.
+static cl::opt<uint32_t> LikelyBranchWeight(
"likely-branch-weight", cl::Hidden, cl::init(2000),
cl::desc("Weight of the branch likely to be taken (default = 2000)"));
-cl::opt<uint32_t> llvm::UnlikelyBranchWeight(
+static cl::opt<uint32_t> UnlikelyBranchWeight(
"unlikely-branch-weight", cl::Hidden, cl::init(1),
cl::desc("Weight of the branch unlikely to be taken (default = 1)"));
More information about the cfe-commits
mailing list