[llvm] 9740a4b - [SelectOpt] Preserve Profile Information (#200680)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 13:48:19 PDT 2026
Author: Aiden Grossman
Date: 2026-06-01T20:48:14Z
New Revision: 9740a4bb88463581a4fd6c1362b9d1d14c995cb3
URL: https://github.com/llvm/llvm-project/commit/9740a4bb88463581a4fd6c1362b9d1d14c995cb3
DIFF: https://github.com/llvm/llvm-project/commit/9740a4bb88463581a4fd6c1362b9d1d14c995cb3.diff
LOG: [SelectOpt] Preserve Profile Information (#200680)
If at least one of the SelectLike instructions in the group has profile
metadata, we can propagate it given they all share the same condition.
Added:
Modified:
llvm/lib/CodeGen/SelectOptimize.cpp
llvm/test/CodeGen/AArch64/selectopt-cast.ll
llvm/utils/profcheck-xfail.txt
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectOptimize.cpp b/llvm/lib/CodeGen/SelectOptimize.cpp
index 9d1e113cd2fb9..399adf4467d8a 100644
--- a/llvm/lib/CodeGen/SelectOptimize.cpp
+++ b/llvm/lib/CodeGen/SelectOptimize.cpp
@@ -60,6 +60,10 @@ STATISTIC(NumSelectConvertedLoop,
"Number of select groups converted due to loop-level analysis");
STATISTIC(NumSelectsConverted, "Number of selects converted");
+namespace llvm {
+extern cl::opt<bool> ProfcheckDisableMetadataFixes;
+}
+
static cl::opt<unsigned> ColdOperandThreshold(
"cold-operand-threshold",
cl::desc("Maximum frequency of path for an operand to be considered cold."),
@@ -560,6 +564,8 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
SmallVector<std::stack<Instruction *>, 2> TrueSlices, FalseSlices;
typedef std::stack<Instruction *>::size_type StackSizeType;
StackSizeType maxTrueSliceLen = 0, maxFalseSliceLen = 0;
+ Instruction *SelectWithProfile = nullptr;
+ bool SelectWithProfileIsInverted = false;
for (SelectLike &SI : ASI.Selects) {
if (!isa<SelectInst>(SI.getI()))
continue;
@@ -579,6 +585,16 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
FalseSlices.push_back(FalseSlice);
}
}
+ // Also see if the select has profile data that we can propagate later
+ // to the conditional branch.
+ Value *SelectCondition = cast<SelectInst>(SI.getI())->getCondition();
+ if (hasProfMD(*SI.getI()) && ASI.Condition == SelectCondition) {
+ SelectWithProfile = SI.getI();
+ } else if (hasProfMD(*SI.getI()) &&
+ match(SelectCondition, m_Not(m_Value(ASI.Condition)))) {
+ SelectWithProfile = SI.getI();
+ SelectWithProfileIsInverted = true;
+ }
}
// In the case of multiple select instructions in the same group, the order
// of non-dependent instructions (instructions of
diff erent dependence
@@ -749,7 +765,14 @@ void SelectOptimizeImpl::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
PN->setDebugLoc(SI.getI()->getDebugLoc());
++NumSelectsConverted;
}
- IB.CreateCondBr(CondFr, TT, FT, SI.getI());
+ Instruction *CondBr = IB.CreateCondBr(CondFr, TT, FT, SI.getI());
+ if (!ProfcheckDisableMetadataFixes && SelectWithProfile) {
+ CondBr->copyMetadata(*SelectWithProfile, {llvm::LLVMContext::MD_prof});
+ if (SelectWithProfileIsInverted)
+ CondBr->swapProfMetadata();
+ } else {
+ setExplicitlyUnknownBranchWeightsIfProfiled(*CondBr, DEBUG_TYPE);
+ }
// Remove the old select instructions, now that they are not longer used.
for (SelectLike &SI : ASI.Selects)
diff --git a/llvm/test/CodeGen/AArch64/selectopt-cast.ll b/llvm/test/CodeGen/AArch64/selectopt-cast.ll
index e363cb369286e..2ea9f94e8e4bc 100644
--- a/llvm/test/CodeGen/AArch64/selectopt-cast.ll
+++ b/llvm/test/CodeGen/AArch64/selectopt-cast.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals smart
; RUN: opt -select-optimize -mtriple=arm64-apple-macosx -S %s | FileCheck %s
; RUN: opt -passes='require<profile-summary>,function(select-optimize)' -mtriple=arm64-apple-macosx -S %s | FileCheck %s
@@ -287,7 +287,7 @@ define void @test_add_sext_not_and_regular_select(ptr %dst, ptr %src, i64 %j.sta
; CHECK-NEXT: [[NOT_CMP3:%.*]] = xor i1 [[CMP3]], true
; CHECK-NEXT: [[DEC:%.*]] = sext i1 [[NOT_CMP3]] to i64
; CHECK-NEXT: [[CMP3_FROZEN:%.*]] = freeze i1 [[CMP3]]
-; CHECK-NEXT: br i1 [[CMP3_FROZEN]], label [[SELECT_END1]], label [[SELECT_FALSE_SINK:%.*]]
+; CHECK-NEXT: br i1 [[CMP3_FROZEN]], label [[SELECT_END1]], label [[SELECT_FALSE_SINK:%.*]], !prof [[PROF0:![0-9]+]]
; CHECK: select.false.sink:
; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[J]], -1
; CHECK-NEXT: br label [[SELECT_END1]]
@@ -317,7 +317,64 @@ loop:
%not.cmp3 = xor i1 %cmp3, true
%dec = sext i1 %not.cmp3 to i64
%j.next = add nsw i64 %j, %dec
- %sink = select i1 %cmp3, ptr %l.i, ptr %l.j
+ %sink = select i1 %cmp3, ptr %l.i, ptr %l.j, !prof !0
+ %gep.dst = getelementptr inbounds ptr, ptr %dst, i64 %iv
+ store ptr %sink, ptr %gep.dst, align 8
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv, %j.start
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define void @test_add_sext_not_and_regular_select_inverted(ptr %dst, ptr %src, i64 %j.start, i64 %p, i64 %i.start) {
+; CHECK-LABEL: @test_add_sext_not_and_regular_select_inverted(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[SELECT_END:%.*]] ]
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[J_START:%.*]], [[ENTRY]] ], [ [[J_NEXT:%.*]], [[SELECT_END]] ]
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_START:%.*]], [[ENTRY]] ], [ [[J_NEXT]], [[SELECT_END]] ]
+; CHECK-NEXT: [[GEP_I:%.*]] = getelementptr inbounds ptr, ptr [[SRC:%.*]], i64 [[I]]
+; CHECK-NEXT: [[L_I:%.*]] = load ptr, ptr [[GEP_I]], align 8
+; CHECK-NEXT: [[GEP_J:%.*]] = getelementptr inbounds ptr, ptr [[SRC]], i64 [[J]]
+; CHECK-NEXT: [[L_J:%.*]] = load ptr, ptr [[GEP_J]], align 8
+; CHECK-NEXT: [[CMP3:%.*]] = icmp ult ptr [[L_I]], [[L_J]]
+; CHECK-NEXT: [[NOT_CMP3:%.*]] = xor i1 [[CMP3]], true
+; CHECK-NEXT: [[DEC:%.*]] = sext i1 [[NOT_CMP3]] to i64
+; CHECK-NEXT: [[CMP3_FROZEN:%.*]] = freeze i1 [[CMP3]]
+; CHECK-NEXT: br i1 [[CMP3_FROZEN]], label [[SELECT_END]], label [[SELECT_FALSE_SINK:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK: select.false.sink:
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[J]], -1
+; CHECK-NEXT: br label [[SELECT_END]]
+; CHECK: select.end:
+; CHECK-NEXT: [[J_NEXT]] = phi i64 [ [[J]], [[LOOP]] ], [ [[TMP0]], [[SELECT_FALSE_SINK]] ]
+; CHECK-NEXT: [[SINK:%.*]] = phi ptr [ [[L_J]], [[LOOP]] ], [ [[L_I]], [[SELECT_FALSE_SINK]] ]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds ptr, ptr [[DST:%.*]], i64 [[IV]]
+; CHECK-NEXT: store ptr [[SINK]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[J_START]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %j = phi i64 [ %j.start, %entry ], [ %j.next, %loop ]
+ %i = phi i64 [ %i.start, %entry ], [ %j.next, %loop ]
+ %gep.i = getelementptr inbounds ptr, ptr %src, i64 %i
+ %l.i = load ptr, ptr %gep.i, align 8
+ %gep.j = getelementptr inbounds ptr, ptr %src, i64 %j
+ %l.j = load ptr, ptr %gep.j, align 8
+ %cmp3 = icmp ult ptr %l.i, %l.j
+ %not.cmp3 = xor i1 %cmp3, true
+ %dec = sext i1 %not.cmp3 to i64
+ %j.next = add nsw i64 %j, %dec
+ %sink = select i1 %not.cmp3, ptr %l.i, ptr %l.j, !prof !0
%gep.dst = getelementptr inbounds ptr, ptr %dst, i64 %iv
store ptr %sink, ptr %gep.dst, align 8
%iv.next = add i64 %iv, 1
@@ -938,3 +995,10 @@ loop:
exit:
ret void
}
+
+!0 = !{!"branch_weights", i32 1000, i32 1}
+
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 1000, i32 1}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1000}
+;.
diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt
index e229a0c2d348f..688ecaec888f0 100644
--- a/llvm/utils/profcheck-xfail.txt
+++ b/llvm/utils/profcheck-xfail.txt
@@ -1,5 +1,3 @@
-CodeGen/AArch64/selectopt-cast.ll
-CodeGen/AArch64/selectopt.ll
CodeGen/ARM/sjljeh-swifterror.ll
CodeGen/WinEH/wineh-comdat.ll
CodeGen/WinEH/wineh-scope-statenumbering.ll
More information about the llvm-commits
mailing list