[llvm] [MemProf] Support cold-only and min hint when optimizing existing hot/cold new (PR #218803)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 15:53:42 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Teresa Johnson (teresajohnson)
<details>
<summary>Changes</summary>
Change -optimize-existing-hot-cold-new to an enum option supporting:
- none (default): Do not optimize existing hot/cold new calls
- cold: Only optimize existing hot/cold new calls if determined to be cold
- always: Always optimize existing hot/cold new calls
Add a new option -min-existing-hot-cold-new-hint (off by default) that,
when optimizing an existing hot/cold operator new call, takes the minimum
of the compiler hint and the existing hint. If the existing hint is a
compile-time constant, the minimum is computed at compile time; otherwise,
an llvm.umin intrinsic is emitted.
---
Patch is 29.23 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/218803.diff
4 Files Affected:
- (modified) llvm/include/llvm/Transforms/Utils/BuildLibCalls.h (+6-7)
- (modified) llvm/lib/Transforms/Utils/BuildLibCalls.cpp (+12-14)
- (modified) llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp (+97-45)
- (modified) llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll (+59-5)
``````````diff
diff --git a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
index f53e7d056ca9f..67b4ff985db63 100644
--- a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
+++ b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
@@ -284,27 +284,26 @@ namespace llvm {
/// Emit a call to the hot/cold operator new function.
LLVM_ABI Value *emitHotColdNew(Value *Num, IRBuilderBase &B,
const TargetLibraryInfo *TLI, LibFunc NewFunc,
- uint8_t HotCold);
+ Value *HotCold);
LLVM_ABI Value *emitHotColdNewNoThrow(Value *Num, Value *NoThrow,
IRBuilderBase &B,
const TargetLibraryInfo *TLI,
- LibFunc NewFunc, uint8_t HotCold);
+ LibFunc NewFunc, Value *HotCold);
LLVM_ABI Value *emitHotColdNewAligned(Value *Num, Value *Align,
IRBuilderBase &B,
const TargetLibraryInfo *TLI,
- LibFunc NewFunc, uint8_t HotCold);
+ LibFunc NewFunc, Value *HotCold);
LLVM_ABI Value *emitHotColdNewAlignedNoThrow(Value *Num, Value *Align,
Value *NoThrow, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
- LibFunc NewFunc,
- uint8_t HotCold);
+ LibFunc NewFunc, Value *HotCold);
LLVM_ABI Value *emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
- LibFunc NewFunc, uint8_t HotCold);
+ LibFunc NewFunc, Value *HotCold);
LLVM_ABI Value *
emitHotColdSizeReturningNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
- LibFunc NewFunc, uint8_t HotCold);
+ LibFunc NewFunc, Value *HotCold);
}
#endif
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index 0afcc9df40070..e9ef993e71ec1 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -2160,7 +2160,7 @@ Value *llvm::emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
LibFunc SizeFeedbackNewFunc,
- uint8_t HotCold) {
+ Value *HotCold) {
Module *M = B.GetInsertBlock()->getModule();
if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
return nullptr;
@@ -2173,7 +2173,7 @@ Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
FunctionCallee Func =
M->getOrInsertFunction(Name, SizedPtrT, Num->getType(), B.getInt8Ty());
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
- CallInst *CI = B.CreateCall(Func, {Num, B.getInt8(HotCold)}, "sized_ptr");
+ CallInst *CI = B.CreateCall(Func, {Num, HotCold}, "sized_ptr");
if (const Function *F = dyn_cast<Function>(Func.getCallee()))
CI->setCallingConv(F->getCallingConv());
@@ -2185,7 +2185,7 @@ Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
IRBuilderBase &B,
const TargetLibraryInfo *TLI,
LibFunc SizeFeedbackNewFunc,
- uint8_t HotCold) {
+ Value *HotCold) {
Module *M = B.GetInsertBlock()->getModule();
if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
return nullptr;
@@ -2198,8 +2198,7 @@ Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
FunctionCallee Func = M->getOrInsertFunction(Name, SizedPtrT, Num->getType(),
Align->getType(), B.getInt8Ty());
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
- CallInst *CI =
- B.CreateCall(Func, {Num, Align, B.getInt8(HotCold)}, "sized_ptr");
+ CallInst *CI = B.CreateCall(Func, {Num, Align, HotCold}, "sized_ptr");
if (const Function *F = dyn_cast<Function>(Func.getCallee()))
CI->setCallingConv(F->getCallingConv());
@@ -2209,7 +2208,7 @@ Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
const TargetLibraryInfo *TLI, LibFunc NewFunc,
- uint8_t HotCold) {
+ Value *HotCold) {
Module *M = B.GetInsertBlock()->getModule();
if (!isLibFuncEmittable(M, TLI, NewFunc))
return nullptr;
@@ -2218,7 +2217,7 @@ Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
FunctionCallee Func =
M->getOrInsertFunction(Name, B.getPtrTy(), Num->getType(), B.getInt8Ty());
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
- CallInst *CI = B.CreateCall(Func, {Num, B.getInt8(HotCold)}, Name);
+ CallInst *CI = B.CreateCall(Func, {Num, HotCold}, Name);
if (const Function *F =
dyn_cast<Function>(Func.getCallee()->stripPointerCasts()))
@@ -2229,7 +2228,7 @@ Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
Value *llvm::emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
- LibFunc NewFunc, uint8_t HotCold) {
+ LibFunc NewFunc, Value *HotCold) {
Module *M = B.GetInsertBlock()->getModule();
if (!isLibFuncEmittable(M, TLI, NewFunc))
return nullptr;
@@ -2238,7 +2237,7 @@ Value *llvm::emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B,
FunctionCallee Func = M->getOrInsertFunction(
Name, B.getPtrTy(), Num->getType(), NoThrow->getType(), B.getInt8Ty());
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
- CallInst *CI = B.CreateCall(Func, {Num, NoThrow, B.getInt8(HotCold)}, Name);
+ CallInst *CI = B.CreateCall(Func, {Num, NoThrow, HotCold}, Name);
if (const Function *F =
dyn_cast<Function>(Func.getCallee()->stripPointerCasts()))
@@ -2249,7 +2248,7 @@ Value *llvm::emitHotColdNewNoThrow(Value *Num, Value *NoThrow, IRBuilderBase &B,
Value *llvm::emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
- LibFunc NewFunc, uint8_t HotCold) {
+ LibFunc NewFunc, Value *HotCold) {
Module *M = B.GetInsertBlock()->getModule();
if (!isLibFuncEmittable(M, TLI, NewFunc))
return nullptr;
@@ -2258,7 +2257,7 @@ Value *llvm::emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
FunctionCallee Func = M->getOrInsertFunction(
Name, B.getPtrTy(), Num->getType(), Align->getType(), B.getInt8Ty());
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
- CallInst *CI = B.CreateCall(Func, {Num, Align, B.getInt8(HotCold)}, Name);
+ CallInst *CI = B.CreateCall(Func, {Num, Align, HotCold}, Name);
if (const Function *F =
dyn_cast<Function>(Func.getCallee()->stripPointerCasts()))
@@ -2270,7 +2269,7 @@ Value *llvm::emitHotColdNewAligned(Value *Num, Value *Align, IRBuilderBase &B,
Value *llvm::emitHotColdNewAlignedNoThrow(Value *Num, Value *Align,
Value *NoThrow, IRBuilderBase &B,
const TargetLibraryInfo *TLI,
- LibFunc NewFunc, uint8_t HotCold) {
+ LibFunc NewFunc, Value *HotCold) {
Module *M = B.GetInsertBlock()->getModule();
if (!isLibFuncEmittable(M, TLI, NewFunc))
return nullptr;
@@ -2280,8 +2279,7 @@ Value *llvm::emitHotColdNewAlignedNoThrow(Value *Num, Value *Align,
Name, B.getPtrTy(), Num->getType(), Align->getType(), NoThrow->getType(),
B.getInt8Ty());
inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
- CallInst *CI =
- B.CreateCall(Func, {Num, Align, NoThrow, B.getInt8(HotCold)}, Name);
+ CallInst *CI = B.CreateCall(Func, {Num, Align, NoThrow, HotCold}, Name);
if (const Function *F =
dyn_cast<Function>(Func.getCallee()->stripPointerCasts()))
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 44a3cbca85d0c..0275d01fed134 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -57,13 +57,36 @@ static cl::opt<bool>
static cl::opt<bool>
OptimizeHotColdNew("optimize-hot-cold-new", cl::Hidden, cl::init(false),
cl::desc("Enable hot/cold operator new library calls"));
-static cl::opt<bool> OptimizeExistingHotColdNew(
- "optimize-existing-hot-cold-new", cl::Hidden, cl::init(false),
+enum class OptimizeExistingHotColdNewKind {
+ None,
+ Cold,
+ Always,
+};
+static cl::opt<OptimizeExistingHotColdNewKind> OptimizeExistingHotColdNew(
+ "optimize-existing-hot-cold-new", cl::Hidden,
cl::desc(
- "Enable optimization of existing hot/cold operator new library calls"));
+ "Enable optimization of existing hot/cold operator new library calls"),
+ cl::values(
+ clEnumValN(
+ OptimizeExistingHotColdNewKind::None, "none",
+ "Do not optimize existing hot/cold operator new library calls"),
+ clEnumValN(OptimizeExistingHotColdNewKind::Cold, "cold",
+ "Only optimize existing hot/cold operator new library calls "
+ "if determined to be cold"),
+ clEnumValN(
+ OptimizeExistingHotColdNewKind::Always, "always",
+ "Always optimize existing hot/cold operator new library calls"),
+ clEnumValN(
+ OptimizeExistingHotColdNewKind::Always, "",
+ "Always optimize existing hot/cold operator new library calls")),
+ cl::init(OptimizeExistingHotColdNewKind::None), cl::ValueOptional);
static cl::opt<bool> OptimizeNoBuiltinHotColdNew(
"optimize-nobuiltin-hot-cold-new-new", cl::Hidden, cl::init(false),
cl::desc("Enable transformation of nobuiltin operator new library calls"));
+static cl::opt<bool> MinExistingHotColdNewHint(
+ "min-existing-hot-cold-new-hint", cl::Hidden, cl::init(false),
+ cl::desc("Take the minimum of compiler hint and existing hint when "
+ "optimizing existing hot/cold operator new library calls"));
namespace {
@@ -1777,7 +1800,7 @@ Value *LibCallSimplifier::maybeOptimizeNoBuiltinOperatorNew(CallInst *CI,
case LibFunc_size_returning_new_aligned_hot_cold:
// If the nobuiltin call already passes a hot_cold_t parameter, allow update
// of that parameter when enabled.
- if (!OptimizeExistingHotColdNew)
+ if (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::None)
return nullptr;
break;
default:
@@ -1796,10 +1819,12 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
return nullptr;
uint8_t HotCold;
- if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "cold")
+ bool IsCold = false;
+ if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "cold") {
HotCold = ColdNewHintValue;
- else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() ==
- "notcold")
+ IsCold = true;
+ } else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() ==
+ "notcold")
HotCold = NotColdNewHintValue;
else if (CI->getAttributes().getFnAttr("memprof").getValueAsString() == "hot")
HotCold = HotNewHintValue;
@@ -1809,6 +1834,24 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
else
return nullptr;
+ bool ShouldOptimizeExisting =
+ OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Always ||
+ (OptimizeExistingHotColdNew == OptimizeExistingHotColdNewKind::Cold &&
+ IsCold);
+
+ Value *HotColdVal = B.getInt8(HotCold);
+ auto getHotColdHintForExisting = [&](uint8_t HotCold) -> Value * {
+ if (!MinExistingHotColdNewHint)
+ return HotColdVal;
+ Value *ExistingHint = CI->getArgOperand(CI->arg_size() - 1);
+ if (auto *CIHint = dyn_cast<ConstantInt>(ExistingHint))
+ return B.getInt8(
+ std::min(HotCold, static_cast<uint8_t>(CIHint->getZExtValue())));
+ if (ExistingHint->getType() != B.getInt8Ty())
+ ExistingHint = B.CreateTruncOrBitCast(ExistingHint, B.getInt8Ty());
+ return B.CreateBinaryIntrinsic(Intrinsic::umin, ExistingHint, HotColdVal);
+ };
+
// For calls that already pass a hot/cold hint, only update the hint if
// directed by OptimizeExistingHotColdNew. For other calls to new, add a hint
// if cold or hot, and leave as-is for default handling if "notcold" aka warm.
@@ -1819,112 +1862,121 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
Value *NewCall = nullptr;
switch (Func) {
case LibFunc_Znwm12__hot_cold_t:
- if (OptimizeExistingHotColdNew)
+ if (ShouldOptimizeExisting)
NewCall = emitHotColdNew(CI->getArgOperand(0), B, TLI,
- LibFunc_Znwm12__hot_cold_t, HotCold);
+ LibFunc_Znwm12__hot_cold_t,
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_Znwm:
NewCall = emitHotColdNew(CI->getArgOperand(0), B, TLI,
- LibFunc_Znwm12__hot_cold_t, HotCold);
+ LibFunc_Znwm12__hot_cold_t, HotColdVal);
break;
case LibFunc_Znam12__hot_cold_t:
- if (OptimizeExistingHotColdNew)
+ if (ShouldOptimizeExisting)
NewCall = emitHotColdNew(CI->getArgOperand(0), B, TLI,
- LibFunc_Znam12__hot_cold_t, HotCold);
+ LibFunc_Znam12__hot_cold_t,
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_Znam:
NewCall = emitHotColdNew(CI->getArgOperand(0), B, TLI,
- LibFunc_Znam12__hot_cold_t, HotCold);
+ LibFunc_Znam12__hot_cold_t, HotColdVal);
break;
case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
- if (OptimizeExistingHotColdNew)
- NewCall = emitHotColdNewNoThrow(
- CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
- LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
+ if (ShouldOptimizeExisting)
+ NewCall =
+ emitHotColdNewNoThrow(CI->getArgOperand(0), CI->getArgOperand(1), B,
+ TLI, LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t,
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_ZnwmRKSt9nothrow_t:
NewCall = emitHotColdNewNoThrow(
CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
- LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotCold);
+ LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t, HotColdVal);
break;
case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
- if (OptimizeExistingHotColdNew)
- NewCall = emitHotColdNewNoThrow(
- CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
- LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
+ if (ShouldOptimizeExisting)
+ NewCall =
+ emitHotColdNewNoThrow(CI->getArgOperand(0), CI->getArgOperand(1), B,
+ TLI, LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t,
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_ZnamRKSt9nothrow_t:
NewCall = emitHotColdNewNoThrow(
CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
- LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotCold);
+ LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t, HotColdVal);
break;
case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:
- if (OptimizeExistingHotColdNew)
- NewCall = emitHotColdNewAligned(
- CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
- LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
+ if (ShouldOptimizeExisting)
+ NewCall =
+ emitHotColdNewAligned(CI->getArgOperand(0), CI->getArgOperand(1), B,
+ TLI, LibFunc_ZnwmSt11align_val_t12__hot_cold_t,
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_ZnwmSt11align_val_t:
NewCall = emitHotColdNewAligned(
CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
- LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotCold);
+ LibFunc_ZnwmSt11align_val_t12__hot_cold_t, HotColdVal);
break;
case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
- if (OptimizeExistingHotColdNew)
- NewCall = emitHotColdNewAligned(
- CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
- LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
+ if (ShouldOptimizeExisting)
+ NewCall =
+ emitHotColdNewAligned(CI->getArgOperand(0), CI->getArgOperand(1), B,
+ TLI, LibFunc_ZnamSt11align_val_t12__hot_cold_t,
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_ZnamSt11align_val_t:
NewCall = emitHotColdNewAligned(
CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
- LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotCold);
+ LibFunc_ZnamSt11align_val_t12__hot_cold_t, HotColdVal);
break;
case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
- if (OptimizeExistingHotColdNew)
+ if (ShouldOptimizeExisting)
NewCall = emitHotColdNewAlignedNoThrow(
CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
- HotCold);
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:
NewCall = emitHotColdNewAlignedNoThrow(
CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
- TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
+ TLI, LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
+ HotColdVal);
break;
case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
- if (OptimizeExistingHotColdNew)
+ if (ShouldOptimizeExisting)
NewCall = emitHotColdNewAlignedNoThrow(
CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
- HotCold);
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
NewCall = emitHotColdNewAlignedNoThrow(
CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2), B,
- TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t, HotCold);
+ TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
+ HotColdVal);
break;
case LibFunc_size_returning_new:
NewCall = emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
LibFunc_size_returning_new_hot_cold,
- HotCold);
+ HotColdVal);
break;
case LibFunc_size_returning_new_hot_cold:
- if (OptimizeExistingHotColdNew)
+ if (ShouldOptimizeExisting)
NewCall = emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
LibFunc_size_returning_new_hot_cold,
- HotCold);
+ getHotColdHintForExisting(HotCold));
break;
case LibFunc_size_returning_new_aligned...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/218803
More information about the llvm-commits
mailing list