[clang-tools-extra] [llvm] [llvm] add support for mustache templating language (PR #105893)

[Paul Kirth via cfe-commits]

================
@@ -0,0 +1,463 @@
+//===-- Mustache.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Mustache.h"
+#include "llvm/Support/Error.h"
+
+using namespace llvm;
+using namespace llvm::json;
+using namespace llvm::mustache;
+
+SmallString<128> escapeString(StringRef Input,
+                              DenseMap<char, StringRef> &Escape) {
+  SmallString<128> EscapedString("");
+  for (char C : Input) {
+    if (Escape.find(C) != Escape.end())
+      EscapedString += Escape[C];
+    else
+      EscapedString += C;
+  }
+  return EscapedString;
+}
+
+Accessor split(StringRef Str, char Delimiter) {
+  Accessor Tokens;
+  if (Str == ".") {
+    Tokens.emplace_back(Str);
+    return Tokens;
+  }
+  StringRef Ref(Str);
+  while (!Ref.empty()) {
+    StringRef Part;
+    std::tie(Part, Ref) = Ref.split(Delimiter);
+    Tokens.emplace_back(Part.trim());
+  }
+  return Tokens;
+}
+
+Token::Token(StringRef RawBody, StringRef InnerBody, char Identifier)
+    : RawBody(RawBody), TokenBody(InnerBody) {
+  TokenType = getTokenType(Identifier);
+  if (TokenType == Type::Comment)
+    return;
+
+  StringRef AccessorStr =
+      TokenType == Type::Variable ? InnerBody : InnerBody.substr(1);
+
+  Accessor = split(AccessorStr.trim(), '.');
+}
+
+Token::Token(StringRef Str)
+    : TokenType(Type::Text), RawBody(Str), Accessor({}), TokenBody(Str) {}
+
+Token::Type Token::getTokenType(char Identifier) {
+  switch (Identifier) {
+  case '#':
+    return Type::SectionOpen;
+  case '/':
+    return Type::SectionClose;
+  case '^':
+    return Type::InvertSectionOpen;
+  case '!':
+    return Type::Comment;
+  case '>':
+    return Type::Partial;
+  case '&':
+    return Type::UnescapeVariable;
+  default:
+    return Type::Variable;
+  }
+}
+
+// Function to check if there's no meaningful text behind
+bool noTextBehind(size_t Idx, const SmallVector<Token, 0> &Tokens) {
+  if (Idx == 0 || Tokens[Idx - 1].getType() != Token::Type::Text)
+    return false;
+  const Token &PrevToken = Tokens[Idx - 1];
+  StringRef TokenBody = PrevToken.getRawBody().rtrim(" \t\v\t");
+  return TokenBody.ends_with("\n") || TokenBody.ends_with("\r\n") ||
+         (TokenBody.empty() && Idx == 1);
+}
+// Function to check if there's no meaningful text ahead
+bool noTextAhead(size_t Idx, const SmallVector<Token, 0> &Tokens) {
+  if (Idx >= Tokens.size() - 1 ||
+      Tokens[Idx + 1].getType() != Token::Type::Text)
+    return false;
+
+  const Token &NextToken = Tokens[Idx + 1];
+  StringRef TokenBody = NextToken.getRawBody().ltrim(" ");
+  return TokenBody.starts_with("\r\n") || TokenBody.starts_with("\n");
+}
+
+// Simple tokenizer that splits the template into tokens
+// the mustache spec allows {{{ }}} to unescape variables
+// but we don't support that here unescape variable
+// is represented only by {{& variable}}
+SmallVector<Token, 0> tokenize(StringRef Template) {
+  SmallVector<Token, 0> Tokens;
+  StringRef Open("{{");
+  StringRef Close("}}");
+  std::size_t Start = 0;
+  std::size_t DelimiterStart = Template.find(Open);
+  if (DelimiterStart == StringRef::npos) {
+    Tokens.emplace_back(Template);
+    return Tokens;
+  }
+  while (DelimiterStart != StringRef::npos) {
+    if (DelimiterStart != Start) {
+      Tokens.emplace_back(Template.substr(Start, DelimiterStart - Start));
+    }
+
+    size_t DelimiterEnd = Template.find(Close, DelimiterStart);
+    if (DelimiterEnd == StringRef::npos) {
+      break;
+    }
+
+    // Extract the Interpolated variable without {{ and }}
+    size_t InterpolatedStart = DelimiterStart + Open.size();
+    size_t InterpolatedEnd = DelimiterEnd - DelimiterStart - Close.size();
+    SmallString<128> Interpolated =
+        Template.substr(InterpolatedStart, InterpolatedEnd);
+    SmallString<128> RawBody;
+    RawBody += Open;
+    RawBody += Interpolated;
+    RawBody += Close;
+
+    Tokens.emplace_back(RawBody, Interpolated, Interpolated[0]);
+    Start = DelimiterEnd + Close.size();
+    DelimiterStart = Template.find(Open, Start);
+  }
+
+  if (Start < Template.size())
+    Tokens.emplace_back(Template.substr(Start));
+
+  // fix up white spaces for
+  // open sections/inverted sections/close section/comment
+  // This loop attempts to find standalone tokens and tries to trim out
+  // the whitespace around them
+  // for example:
+  // if you have the template string
+  //  "Line 1\n {{#section}} \n Line 2 \n {{/section}} \n Line 3"
+  // The output would be
+  //  "Line 1\n Line 2\n Line 3"
+  size_t LastIdx = Tokens.size() - 1;
+  for (size_t Idx = 0, End = Tokens.size(); Idx < End; ++Idx) {
+    Token::Type CurrentType = Tokens[Idx].getType();
+    // Check if token type requires cleanup
+    bool RequiresCleanUp = (CurrentType == Token::Type::SectionOpen ||
+                            CurrentType == Token::Type::InvertSectionOpen ||
+                            CurrentType == Token::Type::SectionClose ||
+                            CurrentType == Token::Type::Comment ||
+                            CurrentType == Token::Type::Partial);
+
+    if (!RequiresCleanUp)
+      continue;
+
+    bool NoTextBehind = noTextBehind(Idx, Tokens);
+    bool NoTextAhead = noTextAhead(Idx, Tokens);
+
+    // Adjust next token body if no text ahead
+    if ((NoTextBehind && NoTextAhead) || (NoTextAhead && Idx == 0)) {
+      Token &NextToken = Tokens[Idx + 1];
+      StringRef NextTokenBody = NextToken.getTokenBody();
+      if (NextTokenBody.starts_with("\r\n")) {
+        NextToken.setTokenBody(NextTokenBody.substr(2));
+      } else if (NextTokenBody.starts_with("\n")) {
+        NextToken.setTokenBody(NextTokenBody.substr(1));
+      }
+    }
+    // Adjust previous token body if no text behind
+    if ((NoTextBehind && NoTextAhead) || (NoTextBehind && Idx == LastIdx)) {
+      Token &PrevToken = Tokens[Idx - 1];
+      StringRef PrevTokenBody = PrevToken.getTokenBody();
+      PrevToken.setTokenBody(PrevTokenBody.rtrim(" \t\v\t"));
+    }
+  }
+  return Tokens;
+}
+
+class Parser {
+public:
+  Parser(StringRef TemplateStr, BumpPtrAllocator &Allocator)
+      : Allocator(Allocator), TemplateStr(TemplateStr) {}
+
+  ASTNode *parse();
+
+private:
+  void parseMustache(ASTNode *Parent);
+
+  BumpPtrAllocator &Allocator;
+  SmallVector<Token, 0> Tokens;
+  size_t CurrentPtr;
+  StringRef TemplateStr;
+};
+
+ASTNode *Parser::parse() {
+  Tokens = tokenize(TemplateStr);
+  CurrentPtr = 0;
+  void *Root = Allocator.Allocate(sizeof(ASTNode), alignof(ASTNode));
+  ASTNode *RootNode = new (Root) ASTNode();
+  parseMustache(RootNode);
+  return RootNode;
+}
+
+void Parser::parseMustache(ASTNode *Parent) {
+
+  while (CurrentPtr < Tokens.size()) {
+    Token CurrentToken = Tokens[CurrentPtr];
+    CurrentPtr++;
+    Accessor A = CurrentToken.getAccessor();
+    ASTNode *CurrentNode;
+    void *Node = Allocator.Allocate(sizeof(ASTNode), alignof(ASTNode));
+
+    switch (CurrentToken.getType()) {
+    case Token::Type::Text: {
+      CurrentNode = new (Node) ASTNode(CurrentToken.getTokenBody(), Parent);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::Variable: {
+      CurrentNode = new (Node) ASTNode(ASTNode::Variable, A, Parent);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::UnescapeVariable: {
+      CurrentNode = new (Node) ASTNode(ASTNode::UnescapeVariable, A, Parent);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::Partial: {
+      CurrentNode = new (Node) ASTNode(ASTNode::Partial, A, Parent);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::SectionOpen: {
+      CurrentNode = new (Node) ASTNode(ASTNode::Section, A, Parent);
+      size_t Start = CurrentPtr;
+      parseMustache(CurrentNode);
+      size_t End = CurrentPtr;
+      SmallString<128> RawBody;
+      if (Start + 1 < End - 1) {
+        for (std::size_t I = Start + 1; I < End - 1; I++)
+          RawBody += Tokens[I].getRawBody();
+      } else if (Start + 1 == End - 1) {
+        RawBody = Tokens[Start].getRawBody();
+      }
+      CurrentNode->setRawBody(RawBody);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::InvertSectionOpen: {
+      CurrentNode = new (Node) ASTNode(ASTNode::InvertSection, A, Parent);
+      size_t Start = CurrentPtr;
+      parseMustache(CurrentNode);
+      size_t End = CurrentPtr;
+      SmallString<128> RawBody;
+      if (Start + 1 < End - 1) {
+        for (size_t Idx = Start + 1; Idx < End - 1; Idx++)
+          RawBody += Tokens[Idx].getRawBody();
+      } else if (Start + 1 == End - 1) {
+        RawBody = Tokens[Start].getRawBody();
+      }
+      CurrentNode->setRawBody(RawBody);
+      Parent->addChild(CurrentNode);
+      break;
+    }
+    case Token::Type::SectionClose:
+      return;
+    default:
+      break;
+    }
+  }
+}
+
+SmallString<128> Template::render(Value Data) {
+  BumpPtrAllocator LocalAllocator;
+  return Tree->render(Data, LocalAllocator, Partials, Lambdas, SectionLambdas,
+                      Escapes);
+}
+
+void Template::registerPartial(StringRef Name, StringRef Partial) {
+  Parser P = Parser(Partial, Allocator);
+  ASTNode *PartialTree = P.parse();
+  Partials[Name] = PartialTree;
+}
+
+void Template::registerLambda(StringRef Name, Lambda L) { Lambdas[Name] = L; }
+
+void Template::registerLambda(StringRef Name, SectionLambda L) {
+  SectionLambdas[Name] = L;
+}
+
+void Template::registerEscape(DenseMap<char, StringRef> E) { Escapes = E; }
+
+Template::Template(StringRef TemplateStr) {
+  Parser P = Parser(TemplateStr, Allocator);
+  Tree = P.parse();
+  // the default behaviour is to escape html entities
+  DenseMap<char, StringRef> HtmlEntities = {{'&', "&"},
+                                            {'<', "<"},
+                                            {'>', ">"},
+                                            {'"', """},
+                                            {'\'', "'"}};
+  registerEscape(HtmlEntities);
+}
+
+SmallString<128> printJson(Value &Data) {
----------------
ilovepi wrote:

I'm not totally sure it makes sense for a `printJson` function to return a string, instead of writing to an `Ostream`. Maybe rename to `JsonToString` or `toJsonString`?

https://github.com/llvm/llvm-project/pull/105893