[libcxx-commits] [libcxx] 8618bb0 - [libc++] Add the --ignore-under option to compare-benchmarks

[Louis Dionne via libcxx-commits]

Author: Louis Dionne
Date: 2025-12-17T12:59:15-05:00
New Revision: 8618bb0e4cb0746152cb797a9a5ba12a2a1a8892

URL: https://github.com/llvm/llvm-project/commit/8618bb0e4cb0746152cb797a9a5ba12a2a1a8892
DIFF: https://github.com/llvm/llvm-project/commit/8618bb0e4cb0746152cb797a9a5ba12a2a1a8892.diff

LOG: [libc++] Add the --ignore-under option to compare-benchmarks

This allows excluding benchmark results that are very susceptible to
noise due to their extremely small absolute value. For example, two
benchmarks measured at 1.5 nanoseconds are likely to be just within
the noise of each other, but failure to account for that may lead to
seemingly large swings when looking at data.

Added: 
    

Modified: 
    libcxx/utils/compare-benchmarks

Removed: 
    


################################################################################
diff  --git a/libcxx/utils/compare-benchmarks b/libcxx/utils/compare-benchmarks
index 19ab453132fd9..b9489073568d0 100755
--- a/libcxx/utils/compare-benchmarks
+++ b/libcxx/utils/compare-benchmarks
@@ -137,6 +137,10 @@ def main(argv):
     parser.add_argument('--filter', type=str, required=False,
         help='An optional regular expression used to filter the benchmarks included in the comparison. '
              'Only benchmarks whose names match the regular expression will be included.')
+    parser.add_argument('--ignore-under', type=float, required=False,
+        help='Ignore benchmarks whose value (in absolute terms) is less than the provided float for all '
+             'the data sets being compared. This allows ignoring benchmarks that are likely to contain '
+             'a significant amount of noise.')
     parser.add_argument('--sort', type=str, required=False, default='benchmark',
                         choices=['benchmark', 'baseline', 'candidate', 'percent_
diff '],
         help='Optional sorting criteria for displaying results. By default, results are displayed in '
@@ -213,7 +217,8 @@ def main(argv):
 
     # Parse the raw LNT data and store each input in a dataframe
     lnt_inputs = [parse_lnt(file.readlines()) for file in args.files]
-    inputs = [pandas.DataFrame(lnt).rename(columns={args.metric: f'{args.metric}_{i}'}) for (i, lnt) in enumerate(lnt_inputs)]
+    series = [f'{args.metric}_{i}' for (i, _) in enumerate(lnt_inputs)]
+    inputs = [pandas.DataFrame(lnt).rename(columns={args.metric: s}) for (s, lnt) in zip(series, lnt_inputs)]
 
     # Join the inputs into a single dataframe
     data = functools.reduce(lambda a, b: a.merge(b, how='outer', on='benchmark'), inputs)
@@ -227,6 +232,9 @@ def main(argv):
         keeplist = [b for b in data['benchmark'] if re.search(args.filter, b) is not None]
         data = data[data['benchmark'].isin(keeplist)]
 
+    if args.ignore_under is not None:
+        data = data[~(data[series] < args.ignore_under).all(axis=1)]
+
     # Sort the data by the appropriate criteria
     if args.sort == 'benchmark':
         data = data.sort_values(by='benchmark')
@@ -245,13 +253,12 @@ def main(argv):
     elif args.format == 'kpi':
         if args.discard_benchmarks_introduced_after is not None:
             index = args.series_names.index(args.discard_benchmarks_introduced_after)
-            series_to_filter = [f'{args.metric}_{i}' for i in range(index+1, len(lnt_inputs))]
-            for candidate in series_to_filter:
+            for candidate in series[index+1:]:
                 first_candidate = f'{args.metric}_1'
                 data = data[~(data[first_candidate].isna() & data[candidate].notna())]
         produce_kpis(data, noise=args.noise_threshold,
                            extrema=args.top_performer_threshold,
-                           series=[f'{args.metric}_{i}' for i in range(len(lnt_inputs))],
+                           series=series,
                            series_names=args.series_names,
                            meta_candidate=args.meta_candidate,
                            title=args.subtitle)