[libc-commits] [libc] da5a4f1 - [libc][automemcpy] Introduce geomean of scores as a tie breaker

[Guillaume Chatelet via libc-commits]

Author: Guillaume Chatelet
Date: 2022-02-17T13:37:05Z
New Revision: da5a4f16e84b6f0bbfe12e5b743951081502f771

URL: https://github.com/llvm/llvm-project/commit/da5a4f16e84b6f0bbfe12e5b743951081502f771
DIFF: https://github.com/llvm/llvm-project/commit/da5a4f16e84b6f0bbfe12e5b743951081502f771.diff

LOG: [libc][automemcpy] Introduce geomean of scores as a tie breaker

Differential Revision: https://reviews.llvm.org/D120040

Added: 
    

Modified: 
    libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h
    libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp
    libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp
    libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp

Removed: 
    


################################################################################
diff  --git a/libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h b/libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h
index 9b861c625061..2991df0aceba 100644
--- a/libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h
+++ b/libc/benchmarks/automemcpy/include/automemcpy/ResultAnalyzer.h
@@ -60,7 +60,8 @@ struct PerDistributionData {
 struct FunctionData {
   FunctionId Id;
   StringMap<PerDistributionData> PerDistributionData;
-  GradeHistogram GradeHisto = {};           // GradeEnum indexed array
+  double ScoresGeoMean;           // Geomean of scores for each distribution.
+  GradeHistogram GradeHisto = {}; // GradeEnum indexed array
   Grade::GradeEnum FinalGrade = Grade::BAD; // Overall grade for this function
 };
 

diff  --git a/libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp b/libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp
index ed9cd1f286c2..6bfde0d2cb4b 100644
--- a/libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp
+++ b/libc/benchmarks/automemcpy/lib/ResultAnalyzer.cpp
@@ -168,14 +168,17 @@ void fillScores(MutableArrayRef<FunctionData> Functions) {
 }
 
 void castVotes(MutableArrayRef<FunctionData> Functions) {
-  for (FunctionData &Function : Functions)
+  for (FunctionData &Function : Functions) {
+    Function.ScoresGeoMean = 1.0;
     for (const auto &Pair : Function.PerDistributionData) {
       const StringRef Distribution = Pair.getKey();
       const double Score = Pair.getValue().Score;
+      Function.ScoresGeoMean *= Score;
       const auto G = Grade::judge(Score);
       ++(Function.GradeHisto[G]);
       Function.PerDistributionData[Distribution].Grade = G;
     }
+  }
 
   for (FunctionData &Function : Functions) {
     const auto &GradeHisto = Function.GradeHisto;

diff  --git a/libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp b/libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp
index 4a6caec469e5..422bc575b6b7 100644
--- a/libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp
+++ b/libc/benchmarks/automemcpy/lib/ResultAnalyzerMain.cpp
@@ -141,18 +141,16 @@ int Main(int argc, char **argv) {
   fillScores(Functions);
   castVotes(Functions);
 
-  // TODO: Implement tie breaking algorithm.
+  // Present data by function type, Grade and Geomean of scores.
   std::sort(Functions.begin(), Functions.end(),
             [](const FunctionData &A, const FunctionData &B) {
-              return A.FinalGrade < B.FinalGrade;
+              const auto Less = [](const FunctionData &FD) {
+                return std::make_tuple(FD.Id.Type, FD.FinalGrade,
+                                       -FD.ScoresGeoMean);
+              };
+              return Less(A) < Less(B);
             });
 
-  // Present data by function type.
-  std::stable_sort(Functions.begin(), Functions.end(),
-                   [](const FunctionData &A, const FunctionData &B) {
-                     return A.Id.Type < B.Id.Type;
-                   });
-
   // Print result.
   for (const FunctionData &Function : Functions) {
     outs() << formatv("{0,-10}", Grade::getString(Function.FinalGrade));

diff  --git a/libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp b/libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp
index 56f7bbf3d5f8..10d0f98272b4 100644
--- a/libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp
+++ b/libc/benchmarks/automemcpy/unittests/ResultAnalyzerTest.cpp
@@ -139,27 +139,35 @@ TEST(AutomemcpyJsonResultsAnalyzer, castVotes) {
   EXPECT_THAT(Data[1].Id, Foo2);
   EXPECT_THAT(Data[2].Id, Foo3);
 
+  const auto GetDistData = [&Data](size_t Index, StringRef Name) {
+    return Data[Index].PerDistributionData.lookup(Name);
+  };
+
   // Distribution A
   // Throughput is 0, 1 and 7, so normalized scores are 0, 1/7 and 1.
-  EXPECT_NEAR(Data[0].PerDistributionData.lookup("A").Score, 0, kAbsErr);
-  EXPECT_NEAR(Data[1].PerDistributionData.lookup("A").Score, 1. / 7, kAbsErr);
-  EXPECT_NEAR(Data[2].PerDistributionData.lookup("A").Score, 1, kAbsErr);
+  EXPECT_THAT(GetDistData(0, "A").Score, DoubleNear(0, kAbsErr));
+  EXPECT_THAT(GetDistData(1, "A").Score, DoubleNear(1. / 7, kAbsErr));
+  EXPECT_THAT(GetDistData(2, "A").Score, DoubleNear(1, kAbsErr));
   // which are turned into grades BAD,  MEDIOCRE and EXCELLENT.
-  EXPECT_THAT(Data[0].PerDistributionData.lookup("A").Grade, Grade::BAD);
-  EXPECT_THAT(Data[1].PerDistributionData.lookup("A").Grade, Grade::MEDIOCRE);
-  EXPECT_THAT(Data[2].PerDistributionData.lookup("A").Grade, Grade::EXCELLENT);
+  EXPECT_THAT(GetDistData(0, "A").Grade, Grade::BAD);
+  EXPECT_THAT(GetDistData(1, "A").Grade, Grade::MEDIOCRE);
+  EXPECT_THAT(GetDistData(2, "A").Grade, Grade::EXCELLENT);
 
   // Distribution B
   // Throughput is 30, 100 and 100, so normalized scores are 0, 1 and 1.
-  EXPECT_NEAR(Data[0].PerDistributionData.lookup("B").Score, 0, kAbsErr);
-  EXPECT_NEAR(Data[1].PerDistributionData.lookup("B").Score, 1, kAbsErr);
-  EXPECT_NEAR(Data[2].PerDistributionData.lookup("B").Score, 1, kAbsErr);
+  EXPECT_THAT(GetDistData(0, "B").Score, DoubleNear(0, kAbsErr));
+  EXPECT_THAT(GetDistData(1, "B").Score, DoubleNear(1, kAbsErr));
+  EXPECT_THAT(GetDistData(2, "B").Score, DoubleNear(1, kAbsErr));
   // which are turned into grades BAD, EXCELLENT and EXCELLENT.
-  EXPECT_THAT(Data[0].PerDistributionData.lookup("B").Grade, Grade::BAD);
-  EXPECT_THAT(Data[1].PerDistributionData.lookup("B").Grade, Grade::EXCELLENT);
-  EXPECT_THAT(Data[2].PerDistributionData.lookup("B").Grade, Grade::EXCELLENT);
+  EXPECT_THAT(GetDistData(0, "B").Grade, Grade::BAD);
+  EXPECT_THAT(GetDistData(1, "B").Grade, Grade::EXCELLENT);
+  EXPECT_THAT(GetDistData(2, "B").Grade, Grade::EXCELLENT);
 
   // Now looking from the functions point of view.
+  EXPECT_THAT(Data[0].ScoresGeoMean, DoubleNear(0, kAbsErr));
+  EXPECT_THAT(Data[1].ScoresGeoMean, DoubleNear(1. * (1. / 7), kAbsErr));
+  EXPECT_THAT(Data[2].ScoresGeoMean, DoubleNear(1, kAbsErr));
+
   // Note the array is indexed by GradeEnum values (EXCELLENT=0 / BAD = 6)
   EXPECT_THAT(Data[0].GradeHisto, ElementsAre(0, 0, 0, 0, 0, 0, 2));
   EXPECT_THAT(Data[1].GradeHisto, ElementsAre(1, 0, 0, 0, 0, 1, 0));