Share HTML template renderers and create a watcher framework (#20218)

The recovery, API, Web and package frameworks all create their own HTML
Renderers. This increases the memory requirements of Gitea
unnecessarily with duplicate templates being kept in memory.

Further the reloading framework in dev mode for these involves locking
and recompiling all of the templates on each load. This will potentially
hide concurrency issues and it is inefficient.

This PR stores the templates renderer in the context and stores this
context in the NormalRoutes, it then creates a fsnotify.Watcher
framework to watch files.

The watching framework is then extended to the mailer templates which
were previously not being reloaded in dev.

Then the locales are simplified to a similar structure.

Fix #20210 
Fix #20211
Fix #20217

Signed-off-by: Andrew Thornton <art27@cantab.net>

modules/templates/base.go

@@ -5,15 +5,16 @@
 package templates
 
 import (
+	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
-
-	"github.com/unrolled/render"
 )
 
 // Vars represents variables to be render in golang templates
@@ -47,8 +48,16 @@ func BaseVars() Vars {
 	}
 }
 
-func getDirAssetNames(dir string) []string {
+func getDirTemplateAssetNames(dir string) []string {
+	return getDirAssetNames(dir, false)
+}
+
+func getDirAssetNames(dir string, mailer bool) []string {
 	var tmpls []string
+
+	if mailer {
+		dir += filepath.Join(dir, "mail")
+	}
 	f, err := os.Stat(dir)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -67,8 +76,13 @@ func getDirAssetNames(dir string) []string {
 		log.Warn("Failed to read %s templates dir. %v", dir, err)
 		return tmpls
 	}
+
+	prefix := "templates/"
+	if mailer {
+		prefix += "mail/"
+	}
 	for _, filePath := range files {
-		if strings.HasPrefix(filePath, "mail/") {
+		if !mailer && strings.HasPrefix(filePath, "mail/") {
 			continue
 		}
 
@@ -76,20 +90,39 @@ func getDirAssetNames(dir string) []string {
 			continue
 		}
 
-		tmpls = append(tmpls, "templates/"+filePath)
+		tmpls = append(tmpls, prefix+filePath)
 	}
 	return tmpls
 }
 
-// HTMLRenderer returns a render.
-func HTMLRenderer() *render.Render {
-	return render.New(render.Options{
-		Extensions:                []string{".tmpl"},
-		Directory:                 "templates",
-		Funcs:                     NewFuncMap(),
-		Asset:                     GetAsset,
-		AssetNames:                GetAssetNames,
-		IsDevelopment:             !setting.IsProd,
-		DisableHTTPErrorRendering: true,
-	})
+func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error {
+	mailRoot := filepath.Join(root, "mail")
+	if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		name := path[len(root):]
+		if len(name) > 0 && name[0] == '/' {
+			name = name[1:]
+		}
+		if err != nil {
+			if os.IsNotExist(err) {
+				return callback(path, name, d, err)
+			}
+			return err
+		}
+		if skipMail && path == mailRoot && d.IsDir() {
+			return fs.SkipDir
+		}
+		if util.CommonSkip(d.Name()) {
+			if d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() {
+			return callback(path, name, d, err)
+		}
+		return nil
+	}); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("unable to get files for template assets in %s: %w", root, err)
+	}
+	return nil
 }

modules/templates/dynamic.go

@@ -8,15 +8,12 @@ package templates
 
 import (
 	"html/template"
+	"io/fs"
 	"os"
-	"path"
 	"path/filepath"
-	"strings"
 	texttmpl "text/template"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/util"
 )
 
 var (
@@ -36,77 +33,42 @@ func GetAsset(name string) ([]byte, error) {
 	return os.ReadFile(filepath.Join(setting.StaticRootPath, name))
 }
 
-// GetAssetNames returns assets list
-func GetAssetNames() []string {
-	tmpls := getDirAssetNames(filepath.Join(setting.CustomPath, "templates"))
-	tmpls2 := getDirAssetNames(filepath.Join(setting.StaticRootPath, "templates"))
+// walkTemplateFiles calls a callback for each template asset
+func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// GetTemplateAssetNames returns list of template names
+func GetTemplateAssetNames() []string {
+	tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates"))
+	tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates"))
 	return append(tmpls, tmpls2...)
 }
 
-// Mailer provides the templates required for sending notification mails.
-func Mailer() (*texttmpl.Template, *template.Template) {
-	for _, funcs := range NewTextFuncMap() {
-		subjectTemplates.Funcs(funcs)
+func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
+		return err
 	}
-	for _, funcs := range NewFuncMap() {
-		bodyTemplates.Funcs(funcs)
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
+		return err
 	}
-
-	staticDir := path.Join(setting.StaticRootPath, "templates", "mail")
-
-	isDir, err := util.IsDir(staticDir)
-	if err != nil {
-		log.Warn("Unable to check if templates dir %s is a directory. Error: %v", staticDir, err)
-	}
-	if isDir {
-		files, err := util.StatDir(staticDir)
-
-		if err != nil {
-			log.Warn("Failed to read %s templates dir. %v", staticDir, err)
-		} else {
-			for _, filePath := range files {
-				if !strings.HasSuffix(filePath, ".tmpl") {
-					continue
-				}
-
-				content, err := os.ReadFile(path.Join(staticDir, filePath))
-				if err != nil {
-					log.Warn("Failed to read static %s template. %v", filePath, err)
-					continue
-				}
-
-				buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, strings.TrimSuffix(filePath, ".tmpl"), content)
-			}
-		}
-	}
-
-	customDir := path.Join(setting.CustomPath, "templates", "mail")
-
-	isDir, err = util.IsDir(customDir)
-	if err != nil {
-		log.Warn("Unable to check if templates dir %s is a directory. Error: %v", customDir, err)
-	}
-	if isDir {
-		files, err := util.StatDir(customDir)
-
-		if err != nil {
-			log.Warn("Failed to read %s templates dir. %v", customDir, err)
-		} else {
-			for _, filePath := range files {
-				if !strings.HasSuffix(filePath, ".tmpl") {
-					continue
-				}
-
-				content, err := os.ReadFile(path.Join(customDir, filePath))
-				if err != nil {
-					log.Warn("Failed to read custom %s template. %v", filePath, err)
-					continue
-				}
-
-				buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, strings.TrimSuffix(filePath, ".tmpl"), content)
-			}
-		}
-	}
-
-	return subjectTemplates, bodyTemplates
+	return nil
+}
+
+// BuiltinAsset will read the provided asset from the embedded assets
+// (This always returns os.ErrNotExist)
+func BuiltinAsset(name string) ([]byte, error) {
+	return nil, os.ErrNotExist
+}
+
+// BuiltinAssetNames returns the names of the embedded assets
+// (This always returns nil)
+func BuiltinAssetNames() []string {
+	return nil
 }

modules/templates/htmlrenderer.go

@@ -0,0 +1,52 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package templates
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/watcher"
+
+	"github.com/unrolled/render"
+)
+
+var rendererKey interface{} = "templatesHtmlRendereer"
+
+// HTMLRenderer returns the current html renderer for the context or creates and stores one within the context for future use
+func HTMLRenderer(ctx context.Context) (context.Context, *render.Render) {
+	rendererInterface := ctx.Value(rendererKey)
+	if rendererInterface != nil {
+		renderer, ok := rendererInterface.(*render.Render)
+		if ok {
+			return ctx, renderer
+		}
+	}
+
+	rendererType := "static"
+	if !setting.IsProd {
+		rendererType = "auto-reloading"
+	}
+	log.Log(1, log.DEBUG, "Creating "+rendererType+" HTML Renderer")
+
+	renderer := render.New(render.Options{
+		Extensions:                []string{".tmpl"},
+		Directory:                 "templates",
+		Funcs:                     NewFuncMap(),
+		Asset:                     GetAsset,
+		AssetNames:                GetTemplateAssetNames,
+		UseMutexLock:              !setting.IsProd,
+		IsDevelopment:             false,
+		DisableHTTPErrorRendering: true,
+	})
+	if !setting.IsProd {
+		watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{
+			PathsCallback:   walkTemplateFiles,
+			BetweenCallback: renderer.CompileTemplates,
+		})
+	}
+	return context.WithValue(ctx, rendererKey, renderer), renderer
+}

modules/templates/mailer.go

@@ -0,0 +1,92 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package templates
+
+import (
+	"context"
+	"html/template"
+	"io/fs"
+	"os"
+	"strings"
+	texttmpl "text/template"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/watcher"
+)
+
+// Mailer provides the templates required for sending notification mails.
+func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
+	for _, funcs := range NewTextFuncMap() {
+		subjectTemplates.Funcs(funcs)
+	}
+	for _, funcs := range NewFuncMap() {
+		bodyTemplates.Funcs(funcs)
+	}
+
+	refreshTemplates := func() {
+		for _, assetPath := range BuiltinAssetNames() {
+			if !strings.HasPrefix(assetPath, "mail/") {
+				continue
+			}
+
+			if !strings.HasSuffix(assetPath, ".tmpl") {
+				continue
+			}
+
+			content, err := BuiltinAsset(assetPath)
+			if err != nil {
+				log.Warn("Failed to read embedded %s template. %v", assetPath, err)
+				continue
+			}
+
+			assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
+
+			log.Trace("Adding built-in mailer template for %s", assetName)
+			buildSubjectBodyTemplate(subjectTemplates,
+				bodyTemplates,
+				assetName,
+				content)
+		}
+
+		if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				return nil
+			}
+
+			content, err := os.ReadFile(path)
+			if err != nil {
+				log.Warn("Failed to read custom %s template. %v", path, err)
+				return nil
+			}
+
+			assetName := strings.TrimSuffix(name, ".tmpl")
+			log.Trace("Adding mailer template for %s from %q", assetName, path)
+			buildSubjectBodyTemplate(subjectTemplates,
+				bodyTemplates,
+				assetName,
+				content)
+			return nil
+		}); err != nil && !os.IsNotExist(err) {
+			log.Warn("Error whilst walking mailer templates directories. %v", err)
+		}
+	}
+
+	refreshTemplates()
+
+	if !setting.IsProd {
+		// Now subjectTemplates and bodyTemplates are both synchronized
+		// thus it is safe to call refresh from a different goroutine
+		watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{
+			PathsCallback:   walkMailerTemplates,
+			BetweenCallback: refreshTemplates,
+		})
+	}
+
+	return subjectTemplates, bodyTemplates
+}

modules/templates/static.go

@@ -9,6 +9,7 @@ package templates
 import (
 	"html/template"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -16,10 +17,8 @@ import (
 	texttmpl "text/template"
 	"time"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
-	"code.gitea.io/gitea/modules/util"
 )
 
 var (
@@ -40,95 +39,42 @@ func GetAsset(name string) ([]byte, error) {
 	} else if err == nil {
 		return bs, nil
 	}
-	return Asset(strings.TrimPrefix(name, "templates/"))
+	return BuiltinAsset(strings.TrimPrefix(name, "templates/"))
 }
 
-// GetAssetNames only for chi
-func GetAssetNames() []string {
+// GetFiles calls a callback for each template asset
+func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// GetTemplateAssetNames only for chi
+func GetTemplateAssetNames() []string {
 	realFS := Assets.(vfsgen۰FS)
 	tmpls := make([]string, 0, len(realFS))
 	for k := range realFS {
+		if strings.HasPrefix(k, "/mail/") {
+			continue
+		}
 		tmpls = append(tmpls, "templates/"+k[1:])
 	}
 
 	customDir := path.Join(setting.CustomPath, "templates")
-	customTmpls := getDirAssetNames(customDir)
+	customTmpls := getDirTemplateAssetNames(customDir)
 	return append(tmpls, customTmpls...)
 }
 
-// Mailer provides the templates required for sending notification mails.
-func Mailer() (*texttmpl.Template, *template.Template) {
-	for _, funcs := range NewTextFuncMap() {
-		subjectTemplates.Funcs(funcs)
+func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
+		return err
 	}
-	for _, funcs := range NewFuncMap() {
-		bodyTemplates.Funcs(funcs)
-	}
-
-	for _, assetPath := range AssetNames() {
-		if !strings.HasPrefix(assetPath, "mail/") {
-			continue
-		}
-
-		if !strings.HasSuffix(assetPath, ".tmpl") {
-			continue
-		}
-
-		content, err := Asset(assetPath)
-		if err != nil {
-			log.Warn("Failed to read embedded %s template. %v", assetPath, err)
-			continue
-		}
-
-		buildSubjectBodyTemplate(subjectTemplates,
-			bodyTemplates,
-			strings.TrimPrefix(
-				strings.TrimSuffix(
-					assetPath,
-					".tmpl",
-				),
-				"mail/",
-			),
-			content)
-	}
-
-	customDir := path.Join(setting.CustomPath, "templates", "mail")
-	isDir, err := util.IsDir(customDir)
-	if err != nil {
-		log.Warn("Failed to check if custom directory %s is a directory. %v", err)
-	}
-	if isDir {
-		files, err := util.StatDir(customDir)
-
-		if err != nil {
-			log.Warn("Failed to read %s templates dir. %v", customDir, err)
-		} else {
-			for _, filePath := range files {
-				if !strings.HasSuffix(filePath, ".tmpl") {
-					continue
-				}
-
-				content, err := os.ReadFile(path.Join(customDir, filePath))
-				if err != nil {
-					log.Warn("Failed to read custom %s template. %v", filePath, err)
-					continue
-				}
-
-				buildSubjectBodyTemplate(subjectTemplates,
-					bodyTemplates,
-					strings.TrimSuffix(
-						filePath,
-						".tmpl",
-					),
-					content)
-			}
-		}
-	}
-
-	return subjectTemplates, bodyTemplates
+	return nil
 }
 
-func Asset(name string) ([]byte, error) {
+// BuiltinAsset reads the provided asset from the builtin embedded assets
+func BuiltinAsset(name string) ([]byte, error) {
 	f, err := Assets.Open("/" + name)
 	if err != nil {
 		return nil, err
@@ -137,7 +83,8 @@ func Asset(name string) ([]byte, error) {
 	return io.ReadAll(f)
 }
 
-func AssetNames() []string {
+// BuiltinAssetNames returns the names of the built-in embedded assets
+func BuiltinAssetNames() []string {
 	realFS := Assets.(vfsgen۰FS)
 	results := make([]string, 0, len(realFS))
 	for k := range realFS {
@@ -146,7 +93,8 @@ func AssetNames() []string {
 	return results
 }
 
-func AssetIsDir(name string) (bool, error) {
+// BuiltinAssetIsDir returns if a provided asset is a directory
+func BuiltinAssetIsDir(name string) (bool, error) {
 	if f, err := Assets.Open("/" + name); err != nil {
 		return false, err
 	} else {