[llvm] [llvm] Add a tool to check mustache compliance against the public spec (PR #142813)
Paul Kirth via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 11 08:24:14 PDT 2025
https://github.com/ilovepi updated https://github.com/llvm/llvm-project/pull/142813
>From dab7e62e4c7005047c46df4d3adf8470813a6f7a Mon Sep 17 00:00:00 2001
From: Paul Kirth <pk1574 at gmail.com>
Date: Sat, 31 May 2025 13:30:32 -0700
Subject: [PATCH] [llvm] Add a tool to check mustache compliance against the
public spec
This is a cli tool to that tests the conformance of LLVM's mustache
implementation against the public Mustache spec, hosted at
https://github.com/mustache/spec. This is a revised version of the
patches in #111487.
Co-authored-by: Peter Chou <peter.chou at mail.utoronto.ca>
---
llvm/CMakeLists.txt | 1 +
llvm/docs/CommandGuide/index.rst | 1 +
.../CommandGuide/llvm-test-mustache-spec.rst | 23 ++
.../llvm-test-mustache-spec/CMakeLists.txt | 5 +
.../llvm-test-mustache-spec.cpp | 268 ++++++++++++++++++
5 files changed, 298 insertions(+)
create mode 100644 llvm/docs/CommandGuide/llvm-test-mustache-spec.rst
create mode 100644 llvm/utils/llvm-test-mustache-spec/CMakeLists.txt
create mode 100644 llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 206f009b45f59..cfb67472aa71e 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -1313,6 +1313,7 @@ if( LLVM_INCLUDE_UTILS )
add_subdirectory(utils/yaml-bench)
add_subdirectory(utils/split-file)
add_subdirectory(utils/mlgo-utils)
+ add_subdirectory(utils/llvm-test-mustache-spec)
if( LLVM_INCLUDE_TESTS )
set(LLVM_SUBPROJECT_TITLE "Third-Party/Google Test")
add_subdirectory(${LLVM_THIRD_PARTY_DIR}/unittest ${CMAKE_CURRENT_BINARY_DIR}/third-party/unittest)
diff --git a/llvm/docs/CommandGuide/index.rst b/llvm/docs/CommandGuide/index.rst
index 643951eca2a26..88fc1fd326b76 100644
--- a/llvm/docs/CommandGuide/index.rst
+++ b/llvm/docs/CommandGuide/index.rst
@@ -87,6 +87,7 @@ Developer Tools
llvm-exegesis
llvm-ifs
llvm-locstats
+ llvm-test-mustache-spec
llvm-pdbutil
llvm-profgen
llvm-tli-checker
diff --git a/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst b/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst
new file mode 100644
index 0000000000000..31b29e6829080
--- /dev/null
+++ b/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst
@@ -0,0 +1,23 @@
+llvm-test-mustache-spec - LLVM tool to test Mustache Compliance Library
+=======================================================================
+
+llvm-test-mustache-spec tests the mustache spec conformance of the LLVM
+mustache library. The spec can be found here: https://github.com/mustache/spec
+
+To test against the spec, simply download the spec and pass the test JSON files
+to the driver. Each spec file should have a list of tests for compliance with
+the spec. These are loaded as test cases, and rendered with our Mustache
+implementation, which is then compared against the expected output from the
+spec.
+
+The current implementation only supports non-optional parts of the spec, so
+we do not expect any of the dynamic-names, inheritance, or lambda tests to
+pass. Additionally, Triple Mustache is not supported. Unsupported tests are
+marked as XFail and are removed from the XFail list as they are fixed.
+
+ $ llvm-test-mustache-spec path/to/specs/*.json
+
+.. program:: llvm-test-mustache-spec
+
+Outputs the number of test failures and successes in each of the test files.
+
diff --git a/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt b/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt
new file mode 100644
index 0000000000000..dc1aa73371ffc
--- /dev/null
+++ b/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt
@@ -0,0 +1,5 @@
+add_llvm_utility(llvm-test-mustache-spec
+ llvm-test-mustache-spec.cpp
+)
+
+target_link_libraries(llvm-test-mustache-spec PRIVATE LLVMSupport)
diff --git a/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp b/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp
new file mode 100644
index 0000000000000..28ed1b876672d
--- /dev/null
+++ b/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp
@@ -0,0 +1,268 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Simple drivers to test the mustache spec found at:
+// https://github.com/mustache/spec
+//
+// It is used to verify that the current implementation conforms to the spec.
+// Simply download the spec and pass the test JSON files to the driver. Each
+// spec file should have a list of tests for compliance with the spec. These
+// are loaded as test cases, and rendered with our Mustache implementation,
+// which is then compared against the expected output from the spec.
+//
+// The current implementation only supports non-optional parts of the spec, so
+// we do not expect any of the dynamic-names, inheritance, or lambda tests to
+// pass. Additionally, Triple Mustache is not supported. Unsupported tests are
+// marked as XFail and are removed from the XFail list as they are fixed.
+//
+// Usage:
+// llvm-test-mustache-spec path/to/test/file.json path/to/test/file2.json ...
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Mustache.h"
+#include "llvm/Support/Path.h"
+#include <string>
+
+using namespace llvm;
+using namespace llvm::json;
+using namespace llvm::mustache;
+
+#define DEBUG_TYPE "llvm-test-mustache-spec"
+
+static cl::OptionCategory Cat("llvm-test-mustache-spec Options");
+
+static cl::list<std::string>
+ InputFiles(cl::Positional, cl::desc("<input files>"), cl::OneOrMore);
+
+static cl::opt<bool> ReportErrors("report-errors",
+ cl::desc("Report errors in spec tests"),
+ cl::cat(Cat));
+
+static ExitOnError ExitOnErr;
+
+static int NumXFail = 0;
+static int NumSuccess = 0;
+
+static const StringMap<StringSet<>> XFailTestNames = {{
+ {"delimiters.json",
+ {
+ "Pair Behavior",
+ "Special Characters",
+ "Sections",
+ "Inverted Sections",
+ "Partial Inheritence",
+ "Post-Partial Behavior",
+ "Standalone Tag",
+ "Indented Standalone Tag",
+ "Standalone Line Endings",
+ "Standalone Without Previous Line",
+ "Standalone Without Newline",
+ }},
+ {"~dynamic-names.json",
+ {
+ "Basic Behavior - Partial",
+ "Basic Behavior - Name Resolution",
+ "Context",
+ "Dotted Names",
+ "Dotted Names - Failed Lookup",
+ "Dotted names - Context Stacking",
+ "Dotted names - Context Stacking Under Repetition",
+ "Dotted names - Context Stacking Failed Lookup",
+ "Recursion",
+ "Surrounding Whitespace",
+ "Inline Indentation",
+ "Standalone Line Endings",
+ "Standalone Without Previous Line",
+ "Standalone Without Newline",
+ "Standalone Indentation",
+ "Padding Whitespace",
+ }},
+ {"~inheritance.json",
+ {
+ "Default",
+ "Variable",
+ "Triple Mustache",
+ "Sections",
+ "Negative Sections",
+ "Mustache Injection",
+ "Inherit",
+ "Overridden content",
+ "Data does not override block default",
+ "Two overridden parents",
+ "Override parent with newlines",
+ "Inherit indentation",
+ "Only one override",
+ "Parent template",
+ "Recursion",
+ "Multi-level inheritance, no sub child",
+ "Text inside parent",
+ "Text inside parent",
+ "Block scope",
+ "Standalone parent",
+ "Standalone block",
+ "Block reindentation",
+ "Intrinsic indentation",
+ "Nested block reindentation",
+
+ }},
+ {"~lambdas.json",
+ {
+ "Interpolation",
+ "Interpolation - Expansion",
+ "Interpolation - Alternate Delimiters",
+ "Interpolation - Multiple Calls",
+ "Escaping",
+ "Section",
+ "Section - Expansion",
+ "Section - Alternate Delimiters",
+ "Section - Multiple Calls",
+
+ }},
+ {"interpolation.json",
+ {
+ "Triple Mustache",
+ "Triple Mustache Integer Interpolation",
+ "Triple Mustache Decimal Interpolation",
+ "Triple Mustache Null Interpolation",
+ "Triple Mustache Context Miss Interpolation",
+ "Dotted Names - Triple Mustache Interpolation",
+ "Implicit Iterators - Triple Mustache",
+ "Triple Mustache - Surrounding Whitespace",
+ "Triple Mustache - Standalone",
+ "Triple Mustache With Padding",
+ }},
+ {"partials.json", {"Standalone Indentation"}},
+ {"sections.json", {"Implicit Iterator - Triple mustache"}},
+}};
+
+struct TestData {
+ static Expected<TestData> createTestData(json::Object *TestCase,
+ StringRef InputFile) {
+ // If any of the needed elements are missing, we cannot continue.
+ // NOTE: partials are optional in the test schema.
+ if (!TestCase || !TestCase->getString("template") ||
+ !TestCase->getString("expected") || !TestCase->getString("name") ||
+ !TestCase->get("data"))
+ return createStringError(
+ llvm::inconvertibleErrorCode(),
+ "invalid JSON schema in test file: " + InputFile + "\n");
+
+ return TestData{TestCase->getString("template").value(),
+ TestCase->getString("expected").value(),
+ TestCase->getString("name").value(), TestCase->get("data"),
+ TestCase->get("partials")};
+ }
+
+ TestData() = default;
+
+ StringRef TemplateStr;
+ StringRef ExpectedStr;
+ StringRef Name;
+ Value *Data;
+ Value *Partials;
+};
+
+static void reportTestFailure(const TestData &TD, StringRef ActualStr,
+ bool IsXFail) {
+ LLVM_DEBUG(dbgs() << "Template: " << TD.TemplateStr << "\n");
+ if (TD.Partials) {
+ LLVM_DEBUG(dbgs() << "Partial: ");
+ LLVM_DEBUG(TD.Partials->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
+ }
+ LLVM_DEBUG(dbgs() << "JSON Data: ");
+ LLVM_DEBUG(TD.Data->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\n");
+ outs() << formatv("Test {}: {}\n", (IsXFail ? "XFailed" : "Failed"), TD.Name);
+ if (ReportErrors) {
+ outs() << " Expected: \'" << TD.ExpectedStr << "\'\n"
+ << " Actual: \'" << ActualStr << "\'\n"
+ << " ====================\n";
+ }
+}
+
+static void registerPartials(Value *Partials, Template &T) {
+ if (!Partials)
+ return;
+ for (const auto &[Partial, Str] : *Partials->getAsObject())
+ T.registerPartial(Partial.str(), Str.getAsString()->str());
+}
+
+static json::Value readJsonFromFile(StringRef &InputFile) {
+ std::unique_ptr<MemoryBuffer> Buffer =
+ ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFile)));
+ return ExitOnErr(parse(Buffer->getBuffer()));
+}
+
+static bool isTestXFail(StringRef FileName, StringRef TestName) {
+ auto P = llvm::sys::path::filename(FileName);
+ auto It = XFailTestNames.find(P);
+ return It != XFailTestNames.end() && It->second.contains(TestName);
+}
+
+static bool evaluateTest(StringRef &InputFile, TestData &TestData,
+ std::string &ActualStr) {
+ bool IsXFail = isTestXFail(InputFile, TestData.Name);
+ bool Matches = TestData.ExpectedStr == ActualStr;
+ if ((Matches && IsXFail) || (!Matches && !IsXFail)) {
+ reportTestFailure(TestData, ActualStr, IsXFail);
+ return false;
+ }
+ IsXFail ? NumXFail++ : NumSuccess++;
+ return true;
+}
+
+static void runTest(StringRef InputFile) {
+ NumXFail = 0;
+ NumSuccess = 0;
+ outs() << "Running Tests: " << InputFile << "\n";
+ json::Value Json = readJsonFromFile(InputFile);
+
+ json::Object *Obj = Json.getAsObject();
+ Array *TestArray = Obj->getArray("tests");
+ // Even though we parsed the JSON, it can have a bad format, so check it.
+ if (!TestArray)
+ ExitOnErr(createStringError(
+ llvm::inconvertibleErrorCode(),
+ "invalid JSON schema in test file: " + InputFile + "\n"));
+
+ const size_t Total = TestArray->size();
+
+ for (Value V : *TestArray) {
+ auto TestData =
+ ExitOnErr(TestData::createTestData(V.getAsObject(), InputFile));
+ Template T(TestData.TemplateStr);
+ registerPartials(TestData.Partials, T);
+
+ std::string ActualStr;
+ raw_string_ostream OS(ActualStr);
+ T.render(*TestData.Data, OS);
+ evaluateTest(InputFile, TestData, ActualStr);
+ }
+
+ const int NumFailed = Total - NumSuccess - NumXFail;
+ outs() << formatv("===Results===\n"
+ " Suceeded: {}\n"
+ " Expectedly Failed: {}\n"
+ " Failed: {}\n"
+ " Total: {}\n",
+ NumSuccess, NumXFail, NumFailed, Total);
+}
+
+int main(int argc, char **argv) {
+ ExitOnErr.setBanner(std::string(argv[0]) + " error: ");
+ cl::ParseCommandLineOptions(argc, argv);
+ for (const auto &FileName : InputFiles)
+ runTest(FileName);
+ return 0;
+}
More information about the llvm-commits
mailing list