[clang-tools-extra] [llvm] [llvm] add support for mustache templating language (PR #105893)

[Paul Kirth via cfe-commits]

================
@@ -0,0 +1,731 @@
+//===-- Mustache.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Mustache.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+#include <sstream>
+
+using namespace llvm;
+using namespace llvm::json;
+
+namespace llvm {
+namespace mustache {
+namespace {
+
+class Token {
+public:
+  enum class Type {
+    Text,
+    Variable,
+    Partial,
+    SectionOpen,
+    SectionClose,
+    InvertSectionOpen,
+    UnescapeVariable,
+    Comment,
+  };
+
+  Token(std::string Str);
+
+  Token(std::string RawBody, std::string TokenBody, char Identifier);
+
+  StringRef getTokenBody() const { return TokenBody; };
+
+  StringRef getRawBody() const { return RawBody; };
+
+  void setTokenBody(std::string NewBody) { TokenBody = std::move(NewBody); };
+
+  Accessor getAccessor() const { return Accessor; };
+
+  Type getType() const { return TokenType; };
+
+  void setIndentation(size_t NewIndentation) { Indentation = NewIndentation; };
+
+  size_t getIndentation() const { return Indentation; };
+
+  static Type getTokenType(char Identifier);
+
+private:
+  Type TokenType;
+  // RawBody is the original string that was tokenized.
+  std::string RawBody;
+  // TokenBody is the original string with the identifier removed.
+  std::string TokenBody;
+  Accessor Accessor;
+  size_t Indentation;
+};
+
+class ASTNode {
+public:
+  enum Type {
+    Root,
+    Text,
+    Partial,
+    Variable,
+    UnescapeVariable,
+    Section,
+    InvertSection,
+  };
+
+  ASTNode() : T(Type::Root), ParentContext(nullptr) {};
+
+  ASTNode(StringRef Body, ASTNode *Parent)
+      : T(Type::Text), Body(Body), Parent(Parent), ParentContext(nullptr) {};
+
+  // Constructor for Section/InvertSection/Variable/UnescapeVariable.
+  ASTNode(Type T, Accessor Accessor, ASTNode *Parent)
+      : T(T), Parent(Parent), Children({}), Accessor(Accessor),
+        ParentContext(nullptr) {};
+
+  void addChild(ASTNode *Child) { Children.emplace_back(Child); };
+
+  void setRawBody(std::string NewBody) { RawBody = std::move(NewBody); };
+
+  void setIndentation(size_t NewIndentation) { Indentation = NewIndentation; };
+
+  void render(const llvm::json::Value &Data, llvm::raw_ostream &OS);
+
+  void setUpNode(llvm::BumpPtrAllocator &Alloc, StringMap<ASTNode *> &Partials,
+                 StringMap<Lambda> &Lambdas,
+                 StringMap<SectionLambda> &SectionLambdas,
+                 DenseMap<char, std::string> &Escapes);
+
+private:
+  void renderLambdas(const llvm::json::Value &Contexts, llvm::raw_ostream &OS,
+                     Lambda &L);
+
+  void renderSectionLambdas(const llvm::json::Value &Contexts,
+                            llvm::raw_ostream &OS, SectionLambda &L);
+
+  void renderPartial(const llvm::json::Value &Contexts, llvm::raw_ostream &OS,
+                     ASTNode *Partial);
+
+  void renderChild(const llvm::json::Value &Context, llvm::raw_ostream &OS);
+
+  const llvm::json::Value *findContext();
+
+  llvm::BumpPtrAllocator *Allocator;
+  StringMap<ASTNode *> *Partials;
+  StringMap<Lambda> *Lambdas;
+  StringMap<SectionLambda> *SectionLambdas;
+  DenseMap<char, std::string> *Escapes;
+  Type T;
+  size_t Indentation = 0;
+  std::string RawBody;
+  std::string Body;
+  ASTNode *Parent;
+  // TODO: switch implementation to SmallVector<T>
+  std::vector<ASTNode *> Children;
+  const Accessor Accessor;
+  const llvm::json::Value *ParentContext;
+};
+
+// Custom stream to escape strings.
+class EscapeStringStream : public raw_ostream {
+public:
+  explicit EscapeStringStream(llvm::raw_ostream &WrappedStream,
+                              DenseMap<char, std::string> &Escape)
+      : Escape(Escape), WrappedStream(WrappedStream) {
+    SetUnbuffered();
+  }
+
+protected:
+  void write_impl(const char *Ptr, size_t Size) override {
+    llvm::StringRef Data(Ptr, Size);
+    for (char C : Data) {
+      auto It = Escape.find(C);
+      if (It != Escape.end())
+        WrappedStream << It->getSecond();
+      else
+        WrappedStream << C;
+    }
+  }
+
+  uint64_t current_pos() const override { return WrappedStream.tell(); }
+
+private:
+  DenseMap<char, std::string> &Escape;
+  llvm::raw_ostream &WrappedStream;
+};
+
+// Custom stream to add indentation used to for rendering partials.
+class AddIndentationStringStream : public raw_ostream {
+public:
+  explicit AddIndentationStringStream(llvm::raw_ostream &WrappedStream,
+                                      size_t Indentation)
+      : Indentation(Indentation), WrappedStream(WrappedStream) {
+    SetUnbuffered();
+  }
+
+protected:
+  void write_impl(const char *Ptr, size_t Size) override {
+    llvm::StringRef Data(Ptr, Size);
+    SmallString<0> Indent;
+    Indent.resize(Indentation, ' ');
+    for (char C : Data) {
+      WrappedStream << C;
+      if (C == '\n')
+        WrappedStream << Indent;
+    }
+  }
+
+  uint64_t current_pos() const override { return WrappedStream.tell(); }
+
+private:
+  size_t Indentation;
+  llvm::raw_ostream &WrappedStream;
+};
+
+Accessor splitMustacheString(StringRef Str) {
+  // We split the mustache string into an accessor.
+  // For example:
+  //    "a.b.c" would be split into {"a", "b", "c"}
+  // We make an exception for a single dot which
+  // refers to the current context.
+  Accessor Tokens;
+  if (Str == ".") {
+    Tokens.emplace_back(Str);
+    return Tokens;
+  }
+  while (!Str.empty()) {
+    StringRef Part;
+    std::tie(Part, Str) = Str.split(".");
+    Tokens.emplace_back(Part.trim());
+  }
+  return Tokens;
+}
+
+Token::Token(std::string RawBody, std::string TokenBody, char Identifier)
+    : RawBody(std::move(RawBody)), TokenBody(std::move(TokenBody)),
+      Indentation(0) {
+  TokenType = getTokenType(Identifier);
+  if (TokenType == Type::Comment)
+    return;
+
+  std::string AccessorStr =
+      TokenType == Type::Variable ? this->TokenBody : this->TokenBody.substr(1);
+
+  Accessor = splitMustacheString(StringRef(AccessorStr).trim());
+}
+
+Token::Token(std::string Str)
+    : TokenType(Type::Text), RawBody(std::move(Str)), Accessor({}),
+      TokenBody(RawBody), Indentation(0) {}
+
+Token::Type Token::getTokenType(char Identifier) {
+  switch (Identifier) {
+  case '#':
+    return Type::SectionOpen;
+  case '/':
+    return Type::SectionClose;
+  case '^':
+    return Type::InvertSectionOpen;
+  case '!':
+    return Type::Comment;
+  case '>':
+    return Type::Partial;
+  case '&':
+    return Type::UnescapeVariable;
+  default:
+    return Type::Variable;
+  }
+}
+
+// Function to check if there is meaningful text behind.
+// We determine if a token has meaningful text behind
+// if the right of previous token contains anything that is
+// not a newline.
+// For example:
+//  "Stuff {{#Section}}" (returns true)
+//   vs
+//  "{{#Section}} \n" (returns false)
+// We make an exception for when previous token is empty
+// and the current token is the second token.
+// For example:
+//  "{{#Section}}"
+bool hasTextBehind(size_t Idx, const ArrayRef<Token> &Tokens) {
+  if (Idx == 0)
+    return true;
+
+  int PrevIdx = Idx - 1;
+  if (Tokens[PrevIdx].getType() != Token::Type::Text)
+    return true;
+
+  const Token &PrevToken = Tokens[PrevIdx];
+  StringRef TokenBody = PrevToken.getRawBody().rtrim(" \t\v");
+  return !TokenBody.ends_with("\n") && !(TokenBody.empty() && Idx == 1);
+}
+
+// Function to check if there's no meaningful text ahead.
+// We determine if a token has text ahead if the left of previous
+// token does not start with a newline.
+bool hasTextAhead(size_t Idx, const ArrayRef<Token> &Tokens) {
+  if (Idx >= Tokens.size() - 1)
+    return true;
+
+  int NextIdx = Idx + 1;
+  if (Tokens[NextIdx].getType() != Token::Type::Text)
+    return true;
+
+  const Token &NextToken = Tokens[NextIdx];
+  StringRef TokenBody = NextToken.getRawBody().ltrim(" ");
+  return !TokenBody.starts_with("\r\n") && !TokenBody.starts_with("\n");
+}
+
+bool requiresCleanUp(Token::Type T) {
+  // We must clean up all the tokens that could contain child nodes.
+  return T == Token::Type::SectionOpen || T == Token::Type::InvertSectionOpen ||
+         T == Token::Type::SectionClose || T == Token::Type::Comment ||
+         T == Token::Type::Partial;
+}
+
+// Adjust next token body if there is no text ahead.
+// For example:
+// The template string
+//  "{{! Comment }} \nLine 2"
+// would be considered as no text ahead and should be rendered as
+//  " Line 2"
+void stripTokenAhead(SmallVectorImpl<Token> &Tokens, size_t Idx) {
+  Token &NextToken = Tokens[Idx + 1];
+  StringRef NextTokenBody = NextToken.getTokenBody();
+  // cut off the leading newline which could be \n or \r\n
+  if (NextTokenBody.starts_with("\r\n"))
+    NextToken.setTokenBody(NextTokenBody.substr(2).str());
+  else if (NextTokenBody.starts_with("\n"))
+    NextToken.setTokenBody(NextTokenBody.substr(1).str());
+}
+
+// Adjust previous token body if there no text behind.
+// For example:
+//  The template string
+//  " \t{{#section}}A{{/section}}"
+// would be considered as having no text ahead and would be render as
+//  "A"
+// The exception for this is partial tag which requires us to
+// keep track of the indentation once it's rendered.
+void stripTokenBefore(SmallVectorImpl<Token> &Tokens, size_t Idx,
+                      Token &CurrentToken, Token::Type CurrentType) {
+  Token &PrevToken = Tokens[Idx - 1];
+  StringRef PrevTokenBody = PrevToken.getTokenBody();
+  StringRef Unindented = PrevTokenBody.rtrim(" \t\v");
+  size_t Indentation = PrevTokenBody.size() - Unindented.size();
+  if (CurrentType != Token::Type::Partial)
+    PrevToken.setTokenBody(Unindented.str());
+  CurrentToken.setIndentation(Indentation);
+}
+
+// Simple tokenizer that splits the template into tokens.
+// The mustache spec allows {{{ }}} to unescape variables,
+// but we don't support that here. An unescape variable
+// is represented only by {{& variable}}.
+SmallVector<Token> tokenize(StringRef Template) {
+  SmallVector<Token> Tokens;
+  std::string Open("{{");
+  std::string Close("}}");
----------------
ilovepi wrote:

These should probably be `StringRef`, but  if not then `const` at the very least.

https://github.com/llvm/llvm-project/pull/105893