Share HTML template renderers and create a watcher framework (#20218)

The recovery, API, Web and package frameworks all create their own HTML
Renderers. This increases the memory requirements of Gitea
unnecessarily with duplicate templates being kept in memory.

Further the reloading framework in dev mode for these involves locking
and recompiling all of the templates on each load. This will potentially
hide concurrency issues and it is inefficient.

This PR stores the templates renderer in the context and stores this
context in the NormalRoutes, it then creates a fsnotify.Watcher
framework to watch files.

The watching framework is then extended to the mailer templates which
were previously not being reloaded in dev.

Then the locales are simplified to a similar structure.

Fix #20210 
Fix #20211
Fix #20217

Signed-off-by: Andrew Thornton <art27@cantab.net>

modules/translation/translation.go

@@ -5,15 +5,16 @@
 package translation
 
 import (
-	"path"
+	"context"
 	"sort"
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/translation/i18n"
-	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/modules/watcher"
 
 	"golang.org/x/text/language"
 )
@@ -31,6 +32,7 @@ type LangType struct {
 }
 
 var (
+	lock          *sync.RWMutex
 	matcher       language.Matcher
 	allLangs      []*LangType
 	allLangMap    map[string]*LangType
@@ -43,57 +45,52 @@ func AllLangs() []*LangType {
 }
 
 // InitLocales loads the locales
-func InitLocales() {
-	i18n.ResetDefaultLocales(setting.IsProd)
-	localeNames, err := options.Dir("locale")
-	if err != nil {
-		log.Fatal("Failed to list locale files: %v", err)
+func InitLocales(ctx context.Context) {
+	if lock != nil {
+		lock.Lock()
+		defer lock.Unlock()
+	} else if !setting.IsProd && lock == nil {
+		lock = &sync.RWMutex{}
 	}
 
-	localFiles := make(map[string]interface{}, len(localeNames))
-	for _, name := range localeNames {
-		if options.IsDynamic() {
-			// Try to check if CustomPath has the file, otherwise fallback to StaticRootPath
-			value := path.Join(setting.CustomPath, "options/locale", name)
+	refreshLocales := func() {
+		i18n.ResetDefaultLocales()
+		localeNames, err := options.Dir("locale")
+		if err != nil {
+			log.Fatal("Failed to list locale files: %v", err)
+		}
 
-			isFile, err := util.IsFile(value)
-			if err != nil {
-				log.Fatal("Failed to load %s locale file. %v", name, err)
-			}
-
-			if isFile {
-				localFiles[name] = value
-			} else {
-				localFiles[name] = path.Join(setting.StaticRootPath, "options/locale", name)
-			}
-		} else {
+		localFiles := make(map[string]interface{}, len(localeNames))
+		for _, name := range localeNames {
 			localFiles[name], err = options.Locale(name)
 			if err != nil {
 				log.Fatal("Failed to load %s locale file. %v", name, err)
 			}
 		}
-	}
 
-	supportedTags = make([]language.Tag, len(setting.Langs))
-	for i, lang := range setting.Langs {
-		supportedTags[i] = language.Raw.Make(lang)
-	}
+		supportedTags = make([]language.Tag, len(setting.Langs))
+		for i, lang := range setting.Langs {
+			supportedTags[i] = language.Raw.Make(lang)
+		}
 
-	matcher = language.NewMatcher(supportedTags)
-	for i := range setting.Names {
-		key := "locale_" + setting.Langs[i] + ".ini"
+		matcher = language.NewMatcher(supportedTags)
+		for i := range setting.Names {
+			key := "locale_" + setting.Langs[i] + ".ini"
 
-		if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil {
-			log.Error("Failed to set messages to %s: %v", setting.Langs[i], err)
+			if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil {
+				log.Error("Failed to set messages to %s: %v", setting.Langs[i], err)
+			}
+		}
+		if len(setting.Langs) != 0 {
+			defaultLangName := setting.Langs[0]
+			if defaultLangName != "en-US" {
+				log.Info("Use the first locale (%s) in LANGS setting option as default", defaultLangName)
+			}
+			i18n.DefaultLocales.SetDefaultLang(defaultLangName)
 		}
 	}
-	if len(setting.Langs) != 0 {
-		defaultLangName := setting.Langs[0]
-		if defaultLangName != "en-US" {
-			log.Info("Use the first locale (%s) in LANGS setting option as default", defaultLangName)
-		}
-		i18n.DefaultLocales.SetDefaultLang(defaultLangName)
-	}
+
+	refreshLocales()
 
 	langs, descs := i18n.DefaultLocales.ListLangNameDesc()
 	allLangs = make([]*LangType, 0, len(langs))
@@ -108,6 +105,17 @@ func InitLocales() {
 	sort.Slice(allLangs, func(i, j int) bool {
 		return strings.ToLower(allLangs[i].Name) < strings.ToLower(allLangs[j].Name)
 	})
+
+	if !setting.IsProd {
+		watcher.CreateWatcher(ctx, "Locales", &watcher.CreateWatcherOpts{
+			PathsCallback: options.WalkLocales,
+			BetweenCallback: func() {
+				lock.Lock()
+				defer lock.Unlock()
+				refreshLocales()
+			},
+		})
+	}
 }
 
 // Match matches accept languages
@@ -118,16 +126,24 @@ func Match(tags ...language.Tag) language.Tag {
 
 // locale represents the information of localization.
 type locale struct {
+	i18n.Locale
 	Lang, LangName string // these fields are used directly in templates: .i18n.Lang
 }
 
 // NewLocale return a locale
 func NewLocale(lang string) Locale {
+	if lock != nil {
+		lock.RLock()
+		defer lock.RUnlock()
+	}
+
 	langName := "unknown"
 	if l, ok := allLangMap[lang]; ok {
 		langName = l.Name
 	}
+	i18nLocale, _ := i18n.GetLocale(lang)
 	return &locale{
+		Locale:   i18nLocale,
 		Lang:     lang,
 		LangName: langName,
 	}
@@ -137,11 +153,6 @@ func (l *locale) Language() string {
 	return l.Lang
 }
 
-// Tr translates content to target language.
-func (l *locale) Tr(format string, args ...interface{}) string {
-	return i18n.Tr(l.Lang, format, args...)
-}
-
 // Language specific rules for translating plural texts
 var trNLangRules = map[string]func(int64) int{
 	// the default rule is "en-US" if a language isn't listed here