Add KaTeX rendering to Markdown. (#20571)

This PR adds mathematical rendering with KaTeX.

The first step is to add a Goldmark extension that detects the latex
(and tex) mathematics delimiters.

The second step to make this extension only run if math support is
enabled.

The second step is to then add KaTeX CSS and JS to the head which will
load after the dom is rendered.

Fix #3445

Signed-off-by: Andrew Thornton <[email protected]>

Signed-off-by: Andrew Thornton <[email protected]>
Co-authored-by: silverwind <[email protected]>
Co-authored-by: Lunny Xiao <[email protected]>

Author: 3 people

custom/conf/app.example.ini

@@ -1262,6 +1262,9 @@ ROUTER = console
 ;; List of file extensions that should be rendered/edited as Markdown
 ;; Separate the extensions with a comma. To render files without any extension as markdown, just put a comma
 ;FILE_EXTENSIONS = .md,.markdown,.mdown,.mkd
+;;
+;; Enables math inline and block detection
+;ENABLE_MATH = true
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -236,6 +236,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `CUSTOM_URL_SCHEMES`: Use a comma separated list (ftp,git,svn) to indicate additional
   URL hyperlinks to be rendered in Markdown. URLs beginning in http and https are
   always displayed
+- `ENABLE_MATH`: **true**: Enables detection of `\(...\)`, `\[...\]`, `$...$` and `$$...$$` blocks as math blocks.
 
 ## Server (`server`)
 

docs/content/doc/advanced/external-renderers.en-us.md

@@ -74,12 +74,13 @@ RENDER_COMMAND = "timeout 30s pandoc +RTS -M512M -RTS -f rst"
 IS_INPUT_FILE = false
 ```
 
-If your external markup relies on additional classes and attributes on the generated HTML elements, you might need to enable custom sanitizer policies. Gitea uses the [`bluemonday`](https://godoc.org/github.com/microcosm-cc/bluemonday) package as our HTML sanitizier. The example below will support [KaTeX](https://katex.org/) output from [`pandoc`](https://pandoc.org/).
+If your external markup relies on additional classes and attributes on the generated HTML elements, you might need to enable custom sanitizer policies. Gitea uses the [`bluemonday`](https://godoc.org/github.com/microcosm-cc/bluemonday) package as our HTML sanitizer. The example below could be used to support server-side [KaTeX](https://katex.org/) rendering output from [`pandoc`](https://pandoc.org/).
 
 ```ini
 [markup.sanitizer.TeX]
 ; Pandoc renders TeX segments as <span>s with the "math" class, optionally
 ; with "inline" or "display" classes depending on context.
+; - note this is different from the built-in math support in our markdown parser which uses <code>
 ELEMENT = span
 ALLOW_ATTR = class
 REGEXP = ^\s*((math(\s+|$)|inline(\s+|$)|display(\s+|$)))+

docs/content/doc/features/comparison.en-us.md

@@ -53,6 +53,8 @@ _Symbols used in table:_
 | WebAuthn (2FA)                      | ✓                                                  | ✘    | ✓         | ✓         | ✓         | ✓              | ?            |
 | Built-in CI/CD                      | ✘                                                  | ✘    | ✓         | ✓         | ✓         | ✘              | ✘            |
 | Subgroups: groups within groups     | [✘](https://github.com/go-gitea/gitea/issues/1872) | ✘    | ✘         | ✓         | ✓         | ✘              | ✓            |
+| Mermaid diagrams in Markdown        | ✓                                                  | ✘    | ✓         | ✓         | ✓         | ✘              | ✘            |
+| Math syntax in Markdown             | ✓                                                  | ✘    | ✓         | ✓         | ✓         | ✘              | ✘            |
 
 ## Code management
 

docs/content/page/index.en-us.md

@@ -131,7 +131,8 @@ You can try it out using [the online demo](https://try.gitea.io/).
   - Environment variables
   - Command line options
 - Multi-language support ([21 languages](https://github.com/go-gitea/gitea/tree/main/options/locale))
-- [Mermaid](https://mermaidjs.github.io/) Diagram support
+- [Mermaid](https://mermaidjs.github.io/) diagrams in Markdown
+- Math syntax in Markdown
 - Mail service
   - Notifications
   - Registration confirmation

go.mod

@@ -103,6 +103,7 @@ require (
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 	gopkg.in/ini.v1 v1.67.0
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	mvdan.cc/xurls/v2 v2.4.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
 	xorm.io/builder v0.3.11
@@ -290,7 +291,6 @@ require (
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )
 

modules/markup/markdown/convertyaml.go

@@ -0,0 +1,84 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package markdown
+
+import (
+	"github.com/yuin/goldmark/ast"
+	east "github.com/yuin/goldmark/extension/ast"
+	"gopkg.in/yaml.v3"
+)
+
+func nodeToTable(meta *yaml.Node) ast.Node {
+	for {
+		if meta == nil {
+			return nil
+		}
+		switch meta.Kind {
+		case yaml.DocumentNode:
+			meta = meta.Content[0]
+			continue
+		default:
+		}
+		break
+	}
+	switch meta.Kind {
+	case yaml.MappingNode:
+		return mappingNodeToTable(meta)
+	case yaml.SequenceNode:
+		return sequenceNodeToTable(meta)
+	default:
+		return ast.NewString([]byte(meta.Value))
+	}
+}
+
+func mappingNodeToTable(meta *yaml.Node) ast.Node {
+	table := east.NewTable()
+	alignments := []east.Alignment{}
+	for i := 0; i < len(meta.Content); i += 2 {
+		alignments = append(alignments, east.AlignNone)
+	}
+
+	headerRow := east.NewTableRow(alignments)
+	valueRow := east.NewTableRow(alignments)
+	for i := 0; i < len(meta.Content); i += 2 {
+		cell := east.NewTableCell()
+
+		cell.AppendChild(cell, nodeToTable(meta.Content[i]))
+		headerRow.AppendChild(headerRow, cell)
+
+		if i+1 < len(meta.Content) {
+			cell = east.NewTableCell()
+			cell.AppendChild(cell, nodeToTable(meta.Content[i+1]))
+			valueRow.AppendChild(valueRow, cell)
+		}
+	}
+
+	table.AppendChild(table, east.NewTableHeader(headerRow))
+	table.AppendChild(table, valueRow)
+	return table
+}
+
+func sequenceNodeToTable(meta *yaml.Node) ast.Node {
+	table := east.NewTable()
+	alignments := []east.Alignment{east.AlignNone}
+	for _, item := range meta.Content {
+		row := east.NewTableRow(alignments)
+		cell := east.NewTableCell()
+		cell.AppendChild(cell, nodeToTable(item))
+		row.AppendChild(row, cell)
+		table.AppendChild(table, row)
+	}
+	return table
+}
+
+func nodeToDetails(meta *yaml.Node, icon string) ast.Node {
+	details := NewDetails()
+	summary := NewSummary()
+	summary.AppendChild(summary, NewIcon(icon))
+	details.AppendChild(details, summary)
+	details.AppendChild(details, nodeToTable(meta))
+
+	return details
+}

modules/markup/markdown/goldmark.go

@@ -15,7 +15,6 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	giteautil "code.gitea.io/gitea/modules/util"
 
-	meta "github.com/yuin/goldmark-meta"
 	"github.com/yuin/goldmark/ast"
 	east "github.com/yuin/goldmark/extension/ast"
 	"github.com/yuin/goldmark/parser"
@@ -32,20 +31,12 @@ type ASTTransformer struct{}
 
 // Transform transforms the given AST tree.
 func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) {
-	metaData := meta.GetItems(pc)
 	firstChild := node.FirstChild()
 	createTOC := false
 	ctx := pc.Get(renderContextKey).(*markup.RenderContext)
-	rc := &RenderConfig{
-		Meta: "table",
-		Icon: "table",
-		Lang: "",
-	}
-
-	if metaData != nil {
-		rc.ToRenderConfig(metaData)
-
-		metaNode := rc.toMetaNode(metaData)
+	rc := pc.Get(renderConfigKey).(*RenderConfig)
+	if rc.yamlNode != nil {
+		metaNode := rc.toMetaNode()
 		if metaNode != nil {
 			node.InsertBefore(node, firstChild, metaNode)
 		}

modules/markup/markdown/markdown.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/common"
+	"code.gitea.io/gitea/modules/markup/markdown/math"
 	"code.gitea.io/gitea/modules/setting"
 	giteautil "code.gitea.io/gitea/modules/util"
 
@@ -38,6 +39,7 @@ var (
 	isWikiKey        = parser.NewContextKey()
 	renderMetasKey   = parser.NewContextKey()
 	renderContextKey = parser.NewContextKey()
+	renderConfigKey  = parser.NewContextKey()
 )
 
 type limitWriter struct {
@@ -98,7 +100,7 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer)
 							languageStr := string(language)
 
 							preClasses := []string{"code-block"}
-							if languageStr == "mermaid" {
+							if languageStr == "mermaid" || languageStr == "math" {
 								preClasses = append(preClasses, "is-loading")
 							}
 
@@ -120,6 +122,9 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer)
 						}
 					}),
 				),
+				math.NewExtension(
+					math.Enabled(setting.Markdown.EnableMath),
+				),
 				meta.Meta,
 			),
 			goldmark.WithParserOptions(
@@ -167,7 +172,18 @@ func actualRender(ctx *markup.RenderContext, input io.Reader, output io.Writer)
 		log.Error("Unable to ReadAll: %v", err)
 		return err
 	}
-	if err := converter.Convert(giteautil.NormalizeEOL(buf), lw, parser.WithContext(pc)); err != nil {
+	buf = giteautil.NormalizeEOL(buf)
+
+	rc := &RenderConfig{
+		Meta: "table",
+		Icon: "table",
+		Lang: "",
+	}
+	buf, _ = ExtractMetadataBytes(buf, rc)
+
+	pc.Set(renderConfigKey, rc)
+
+	if err := converter.Convert(buf, lw, parser.WithContext(pc)); err != nil {
 		log.Error("Unable to render: %v", err)
 		return err
 	}

modules/markup/markdown/math/block_node.go

@@ -0,0 +1,42 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package math
+
+import "github.com/yuin/goldmark/ast"
+
+// Block represents a display math block e.g. $$...$$ or \[...\]
+type Block struct {
+	ast.BaseBlock
+	Dollars bool
+	Indent  int
+	Closed  bool
+}
+
+// KindBlock is the node kind for math blocks
+var KindBlock = ast.NewNodeKind("MathBlock")
+
+// NewBlock creates a new math Block
+func NewBlock(dollars bool, indent int) *Block {
+	return &Block{
+		Dollars: dollars,
+		Indent:  indent,
+	}
+}
+
+// Dump dumps the block to a string
+func (n *Block) Dump(source []byte, level int) {
+	m := map[string]string{}
+	ast.DumpHelper(n, source, level, m, nil)
+}
+
+// Kind returns KindBlock for math Blocks
+func (n *Block) Kind() ast.NodeKind {
+	return KindBlock
+}
+
+// IsRaw returns true as this block should not be processed further
+func (n *Block) IsRaw() bool {
+	return true
+}