tdsds.com/gitea
1
1
Fork 0
mirror of https://github.com/go-gitea/gitea synced 2025-07-22 18:28:37 +00:00
Code Releases Activity

Share HTML template renderers and create a watcher framework (#20218)

Browse Source 
The recovery, API, Web and package frameworks all create their own HTML
Renderers. This increases the memory requirements of Gitea
unnecessarily with duplicate templates being kept in memory.

Further the reloading framework in dev mode for these involves locking
and recompiling all of the templates on each load. This will potentially
hide concurrency issues and it is inefficient.

This PR stores the templates renderer in the context and stores this
context in the NormalRoutes, it then creates a fsnotify.Watcher
framework to watch files.

The watching framework is then extended to the mailer templates which
were previously not being reloaded in dev.

Then the locales are simplified to a similar structure.

Fix #20210 
Fix #20211
Fix #20217

Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
zeripath
2022-08-28 10:43:25 +01:00
committed by GitHub
parent c21d6511a8
commit bb0ff77e46
43 changed files with 902 additions and 618 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
modules/translation/i18n/i18n.go
View File

@@ -5,297 +5,48 @@
package i18n
import (
"errors"
"fmt"
"os"
"reflect"
"sync"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"gopkg.in/ini.v1"
"io"
)
var (
ErrLocaleAlreadyExist = errors.New("lang already exists")
var DefaultLocales = NewLocaleStore()
DefaultLocales = NewLocaleStore(true)
)
type locale struct {
// This mutex will be set if we have live-reload enabled (e.g. dev mode)
reloadMu *sync.RWMutex
store *LocaleStore
langName string
idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
sourceFileName string
sourceFileInfo os.FileInfo
lastReloadCheckTime time.Time
type Locale interface {
// Tr translates a given key and arguments for a language
Tr(trKey string, trArgs ...interface{}) string
// Has reports if a locale has a translation for a given key
Has(trKey string) bool
}
type LocaleStore struct {
// This mutex will be set if we have live-reload enabled (e.g. dev mode)
reloadMu *sync.RWMutex
// LocaleStore provides the functions common to all locale stores
type LocaleStore interface {
io.Closer
langNames []string
langDescs []string
localeMap map[string]*locale
// this needs to be locked when live-reloading
trKeyToIdxMap map[string]int
defaultLang string
}
func NewLocaleStore(isProd bool) *LocaleStore {
store := &LocaleStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
if !isProd {
store.reloadMu = &sync.RWMutex{}
}
return store
}
// AddLocaleByIni adds locale by ini into the store
// if source is a string, then the file is loaded. In dev mode, this file will be checked for live-reloading
// if source is a []byte, then the content is used
// Note: this is not concurrent safe
func (store *LocaleStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
if _, ok := store.localeMap[langName]; ok {
return ErrLocaleAlreadyExist
}
l := &locale{store: store, langName: langName}
if store.reloadMu != nil {
l.reloadMu = &sync.RWMutex{}
l.reloadMu.Lock() // Arguably this is not necessary as AddLocaleByIni isn't concurrent safe - but for consistency we do this
defer l.reloadMu.Unlock()
}
if fileName, ok := source.(string); ok {
l.sourceFileName = fileName
l.sourceFileInfo, _ = os.Stat(fileName) // live-reload only works for regular files. the error can be ignored
}
var err error
l.idxToMsgMap, err = store.readIniToIdxToMsgMap(source)
if err != nil {
return err
}
store.langNames = append(store.langNames, langName)
store.langDescs = append(store.langDescs, langDesc)
store.localeMap[l.langName] = l
return nil
}
// readIniToIdxToMsgMap will read a provided ini and creates an idxToMsgMap
func (store *LocaleStore) readIniToIdxToMsgMap(source interface{}) (map[int]string, error) {
iniFile, err := ini.LoadSources(ini.LoadOptions{
IgnoreInlineComment: true,
UnescapeValueCommentSymbols: true,
}, source)
if err != nil {
return nil, fmt.Errorf("unable to load ini: %w", err)
}
iniFile.BlockMode = false
idxToMsgMap := make(map[int]string)
if store.reloadMu != nil {
store.reloadMu.Lock()
defer store.reloadMu.Unlock()
}
for _, section := range iniFile.Sections() {
for _, key := range section.Keys() {
var trKey string
if section.Name() == "" || section.Name() == "DEFAULT" {
trKey = key.Name()
} else {
trKey = section.Name() + "." + key.Name()
}
// Instead of storing the key strings in multiple different maps we compute a idx which will act as numeric code for key
// This reduces the size of the locale idxToMsgMaps
idx, ok := store.trKeyToIdxMap[trKey]
if !ok {
idx = len(store.trKeyToIdxMap)
store.trKeyToIdxMap[trKey] = idx
}
idxToMsgMap[idx] = key.Value()
}
}
iniFile = nil
return idxToMsgMap, nil
}
func (store *LocaleStore) idxForTrKey(trKey string) (int, bool) {
if store.reloadMu != nil {
store.reloadMu.RLock()
defer store.reloadMu.RUnlock()
}
idx, ok := store.trKeyToIdxMap[trKey]
return idx, ok
}
// HasLang reports if a language is available in the store
func (store *LocaleStore) HasLang(langName string) bool {
_, ok := store.localeMap[langName]
return ok
}
// ListLangNameDesc reports if a language available in the store
func (store *LocaleStore) ListLangNameDesc() (names, desc []string) {
return store.langNames, store.langDescs
}
// SetDefaultLang sets default language as a fallback
func (store *LocaleStore) SetDefaultLang(lang string) {
store.defaultLang = lang
}
// Tr translates content to target language. fall back to default language.
func (store *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string {
l, ok := store.localeMap[lang]
if !ok {
l, ok = store.localeMap[store.defaultLang]
}
if ok {
return l.Tr(trKey, trArgs...)
}
return trKey
}
// reloadIfNeeded will check if the locale needs to be reloaded
// this function will assume that the l.reloadMu has been RLocked if it already exists
func (l *locale) reloadIfNeeded() {
if l.reloadMu == nil {
return
}
now := time.Now()
if now.Sub(l.lastReloadCheckTime) < time.Second || l.sourceFileInfo == nil || l.sourceFileName == "" {
return
}
l.reloadMu.RUnlock()
l.reloadMu.Lock() // (NOTE: a pre-emption can occur between these two locks so we need to recheck)
defer l.reloadMu.RLock()
defer l.reloadMu.Unlock()
if now.Sub(l.lastReloadCheckTime) < time.Second || l.sourceFileInfo == nil || l.sourceFileName == "" {
return
}
l.lastReloadCheckTime = now
sourceFileInfo, err := os.Stat(l.sourceFileName)
if err != nil || sourceFileInfo.ModTime().Equal(l.sourceFileInfo.ModTime()) {
return
}
idxToMsgMap, err := l.store.readIniToIdxToMsgMap(l.sourceFileName)
if err == nil {
l.idxToMsgMap = idxToMsgMap
} else {
log.Error("Unable to live-reload the locale file %q, err: %v", l.sourceFileName, err)
}
// We will set the sourceFileInfo to this file to prevent repeated attempts to re-load this broken file
l.sourceFileInfo = sourceFileInfo
}
// Tr translates content to locale language. fall back to default language.
func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
if l.reloadMu != nil {
l.reloadMu.RLock()
defer l.reloadMu.RUnlock()
l.reloadIfNeeded()
}
msg, _ := l.tryTr(trKey, trArgs...)
return msg
}
func (l *locale) tryTr(trKey string, trArgs ...interface{}) (msg string, found bool) {
trMsg := trKey
// convert the provided trKey to a common idx from the store
idx, ok := l.store.idxForTrKey(trKey)
if ok {
if msg, found = l.idxToMsgMap[idx]; found {
trMsg = msg // use the translation that we have found
} else if l.langName != l.store.defaultLang {
// No translation available in our current language... fallback to the default language
// Attempt to get the default language from the locale store
if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
if def.reloadMu != nil {
def.reloadMu.RLock()
def.reloadIfNeeded()
}
if msg, found = def.idxToMsgMap[idx]; found {
trMsg = msg // use the translation that we have found
}
if def.reloadMu != nil {
def.reloadMu.RUnlock()
}
}
}
}
if !found && !setting.IsProd {
log.Error("missing i18n translation key: %q", trKey)
}
if len(trArgs) == 0 {
return trMsg, found
}
fmtArgs := make([]interface{}, 0, len(trArgs))
for _, arg := range trArgs {
val := reflect.ValueOf(arg)
if val.Kind() == reflect.Slice {
// Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f)
// but this is an unstable behavior.
//
// So we restrict the accepted arguments to either:
//
// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
if len(trArgs) == 1 {
for i := 0; i < val.Len(); i++ {
fmtArgs = append(fmtArgs, val.Index(i).Interface())
}
} else {
log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs)
break
}
} else {
fmtArgs = append(fmtArgs, arg)
}
}
return fmt.Sprintf(trMsg, fmtArgs...), found
// Tr translates a given key and arguments for a language
Tr(lang, trKey string, trArgs ...interface{}) string
// Has reports if a locale has a translation for a given key
Has(lang, trKey string) bool
// SetDefaultLang sets the default language to fall back to
SetDefaultLang(lang string)
// ListLangNameDesc provides paired slices of language names to descriptors
ListLangNameDesc() (names, desc []string)
// Locale return the locale for the provided language or the default language if not found
Locale(langName string) (Locale, bool)
// HasLang returns whether a given language is present in the store
HasLang(langName string) bool
// AddLocaleByIni adds a new language to the store
AddLocaleByIni(langName, langDesc string, source interface{}) error
}
// ResetDefaultLocales resets the current default locales
// NOTE: this is not synchronized
func ResetDefaultLocales(isProd bool) {
DefaultLocales = NewLocaleStore(isProd)
func ResetDefaultLocales() {
if DefaultLocales != nil {
_ = DefaultLocales.Close()
}
DefaultLocales = NewLocaleStore()
}
// Tr use default locales to translate content to target language.
func Tr(lang, trKey string, trArgs ...interface{}) string {
return DefaultLocales.Tr(lang, trKey, trArgs...)
// GetLocales returns the locale from the default locales
func GetLocale(lang string) (Locale, bool) {
return DefaultLocales.Locale(lang)
}