Share HTML template renderers and create a watcher framework (#20218)

The recovery, API, Web and package frameworks all create their own HTML
Renderers. This increases the memory requirements of Gitea
unnecessarily with duplicate templates being kept in memory.

Further the reloading framework in dev mode for these involves locking
and recompiling all of the templates on each load. This will potentially
hide concurrency issues and it is inefficient.

This PR stores the templates renderer in the context and stores this
context in the NormalRoutes, it then creates a fsnotify.Watcher
framework to watch files.

The watching framework is then extended to the mailer templates which
were previously not being reloaded in dev.

Then the locales are simplified to a similar structure.

Fix #20210 
Fix #20211
Fix #20217

Signed-off-by: Andrew Thornton <art27@cantab.net>

20 files changed, 795 insertions, 525 deletions

diff --git a/modules/charset/escape_test.go b/modules/charset/escape_test.go
index 8063e11542..a7232a4658 100644
--- a/modules/charset/escape_test.go
+++ b/modules/charset/escape_test.go
@@ -133,11 +133,18 @@ then resh (ר), and finally heh (ה) (which should appear leftmost).`,
 	},
 }
 
+type nullLocale struct{}
+
+func (nullLocale) Language() string                                                   { return "" }
+func (nullLocale) Tr(key string, _ ...interface{}) string                             { return key }
+func (nullLocale) TrN(cnt interface{}, key1, keyN string, args ...interface{}) string { return "" }
+
+var _ (translation.Locale) = nullLocale{}
+
 func TestEscapeControlString(t *testing.T) {
 	for _, tt := range escapeControlTests {
 		t.Run(tt.name, func(t *testing.T) {
-			locale := translation.NewLocale("en_US")
-			status, result := EscapeControlString(tt.text, locale)
+			status, result := EscapeControlString(tt.text, nullLocale{})
 			if !reflect.DeepEqual(*status, tt.status) {
 				t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status)
 			}
@@ -173,7 +180,7 @@ func TestEscapeControlReader(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			input := strings.NewReader(tt.text)
 			output := &strings.Builder{}
-			status, err := EscapeControlReader(input, output, translation.NewLocale("en_US"))
+			status, err := EscapeControlReader(input, output, nullLocale{})
 			result := output.String()
 			if err != nil {
 				t.Errorf("EscapeControlReader(): err = %v", err)
@@ -195,5 +202,5 @@ func TestEscapeControlReader_panic(t *testing.T) {
 	for i := 0; i < 6826; i++ {
 		bs = append(bs, []byte("—")...)
 	}
-	_, _ = EscapeControlString(string(bs), translation.NewLocale("en_US"))
+	_, _ = EscapeControlString(string(bs), nullLocale{})
 }
diff --git a/modules/context/context.go b/modules/context/context.go
index 45f1978e97..4b6a21b217 100644
--- a/modules/context/context.go
+++ b/modules/context/context.go
@@ -658,8 +658,8 @@ func Auth(authMethod auth.Method) func(*Context) {
 }
 
 // Contexter initializes a classic context for a request.
-func Contexter() func(next http.Handler) http.Handler {
-	rnd := templates.HTMLRenderer()
+func Contexter(ctx context.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	csrfOpts := getCsrfOpts()
 	if !setting.IsProd {
 		CsrfTokenRegenerationInterval = 5 * time.Second // in dev, re-generate the tokens more aggressively for debug purpose
diff --git a/modules/context/package.go b/modules/context/package.go
index 92a97831dd..ad06f4d636 100644
--- a/modules/context/package.go
+++ b/modules/context/package.go
@@ -5,6 +5,7 @@
 package context
 
 import (
+	gocontext "context"
 	"fmt"
 	"net/http"
 
@@ -14,6 +15,7 @@ import (
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/templates"
 )
 
 // Package contains owner, access mode and optional the package descriptor
@@ -118,12 +120,14 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) {
 }
 
 // PackageContexter initializes a package context for a request.
-func PackageContexter() func(next http.Handler) http.Handler {
+func PackageContexter(ctx gocontext.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 			ctx := Context{
-				Resp: NewResponse(resp),
-				Data: map[string]interface{}{},
+				Resp:   NewResponse(resp),
+				Data:   map[string]interface{}{},
+				Render: rnd,
 			}
 			defer ctx.Close()
 
diff --git a/modules/options/base.go b/modules/options/base.go
new file mode 100644
index 0000000000..e1d6efa7f0
--- /dev/null
+++ b/modules/options/base.go
@@ -0,0 +1,40 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package options
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		// name is the path relative to the root
+		name := path[len(root):]
+		if len(name) > 0 && name[0] == '/' {
+			name = name[1:]
+		}
+		if err != nil {
+			if os.IsNotExist(err) {
+				return callback(path, name, d, err)
+			}
+			return err
+		}
+		if util.CommonSkip(d.Name()) {
+			if d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		return callback(path, name, d, err)
+	}); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("unable to get files for assets in %s: %w", root, err)
+	}
+	return nil
+}
diff --git a/modules/options/dynamic.go b/modules/options/dynamic.go
index 5fea337e42..eeef11e8da 100644
--- a/modules/options/dynamic.go
+++ b/modules/options/dynamic.go
@@ -8,8 +8,10 @@ package options
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path"
+	"path/filepath"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -45,7 +47,7 @@ func Dir(name string) ([]string, error) {
 
 	isDir, err = util.IsDir(staticDir)
 	if err != nil {
-		return []string{}, fmt.Errorf("Unabe to check if static directory %s is a directory. %v", staticDir, err)
+		return []string{}, fmt.Errorf("unable to check if static directory %s is a directory. %v", staticDir, err)
 	}
 	if isDir {
 		files, err := util.StatDir(staticDir, true)
@@ -64,6 +66,18 @@ func Locale(name string) ([]byte, error) {
 	return fileFromDir(path.Join("locale", name))
 }
 
+// WalkLocales reads the content of a specific locale from static or custom path.
+func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+	return nil
+}
+
 // Readme reads the content of a specific readme from static or custom path.
 func Readme(name string) ([]byte, error) {
 	return fileFromDir(path.Join("readme", name))
diff --git a/modules/options/static.go b/modules/options/static.go
index 6cad88cb61..d9a6c83664 100644
--- a/modules/options/static.go
+++ b/modules/options/static.go
@@ -9,8 +9,10 @@ package options
 import (
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
+	"path/filepath"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -74,6 +76,14 @@ func Locale(name string) ([]byte, error) {
 	return fileFromDir(path.Join("locale", name))
 }
 
+// WalkLocales reads the content of a specific locale from static or custom path.
+func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+	return nil
+}
+
 // Readme reads the content of a specific readme from bindata or custom path.
 func Readme(name string) ([]byte, error) {
 	return fileFromDir(path.Join("readme", name))
diff --git a/modules/templates/base.go b/modules/templates/base.go
index 9563650e12..d234d531f3 100644
--- a/modules/templates/base.go
+++ b/modules/templates/base.go
@@ -5,15 +5,16 @@
 package templates
 
 import (
+	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
-
-	"github.com/unrolled/render"
 )
 
 // Vars represents variables to be render in golang templates
@@ -47,8 +48,16 @@ func BaseVars() Vars {
 	}
 }
 
-func getDirAssetNames(dir string) []string {
+func getDirTemplateAssetNames(dir string) []string {
+	return getDirAssetNames(dir, false)
+}
+
+func getDirAssetNames(dir string, mailer bool) []string {
 	var tmpls []string
+
+	if mailer {
+		dir += filepath.Join(dir, "mail")
+	}
 	f, err := os.Stat(dir)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -67,8 +76,13 @@ func getDirAssetNames(dir string) []string {
 		log.Warn("Failed to read %s templates dir. %v", dir, err)
 		return tmpls
 	}
+
+	prefix := "templates/"
+	if mailer {
+		prefix += "mail/"
+	}
 	for _, filePath := range files {
-		if strings.HasPrefix(filePath, "mail/") {
+		if !mailer && strings.HasPrefix(filePath, "mail/") {
 			continue
 		}
 
@@ -76,20 +90,39 @@ func getDirAssetNames(dir string) []string {
 			continue
 		}
 
-		tmpls = append(tmpls, "templates/"+filePath)
+		tmpls = append(tmpls, prefix+filePath)
 	}
 	return tmpls
 }
 
-// HTMLRenderer returns a render.
-func HTMLRenderer() *render.Render {
-	return render.New(render.Options{
-		Extensions:                []string{".tmpl"},
-		Directory:                 "templates",
-		Funcs:                     NewFuncMap(),
-		Asset:                     GetAsset,
-		AssetNames:                GetAssetNames,
-		IsDevelopment:             !setting.IsProd,
-		DisableHTTPErrorRendering: true,
-	})
+func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error {
+	mailRoot := filepath.Join(root, "mail")
+	if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		name := path[len(root):]
+		if len(name) > 0 && name[0] == '/' {
+			name = name[1:]
+		}
+		if err != nil {
+			if os.IsNotExist(err) {
+				return callback(path, name, d, err)
+			}
+			return err
+		}
+		if skipMail && path == mailRoot && d.IsDir() {
+			return fs.SkipDir
+		}
+		if util.CommonSkip(d.Name()) {
+			if d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() {
+			return callback(path, name, d, err)
+		}
+		return nil
+	}); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("unable to get files for template assets in %s: %w", root, err)
+	}
+	return nil
 }
diff --git a/modules/templates/dynamic.go b/modules/templates/dynamic.go
index de6968c314..4896580f62 100644
--- a/modules/templates/dynamic.go
+++ b/modules/templates/dynamic.go
@@ -8,15 +8,12 @@ package templates
 
 import (
 	"html/template"
+	"io/fs"
 	"os"
-	"path"
 	"path/filepath"
-	"strings"
 	texttmpl "text/template"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/util"
 )
 
 var (
@@ -36,77 +33,42 @@ func GetAsset(name string) ([]byte, error) {
 	return os.ReadFile(filepath.Join(setting.StaticRootPath, name))
 }
 
-// GetAssetNames returns assets list
-func GetAssetNames() []string {
-	tmpls := getDirAssetNames(filepath.Join(setting.CustomPath, "templates"))
-	tmpls2 := getDirAssetNames(filepath.Join(setting.StaticRootPath, "templates"))
-	return append(tmpls, tmpls2...)
-}
-
-// Mailer provides the templates required for sending notification mails.
-func Mailer() (*texttmpl.Template, *template.Template) {
-	for _, funcs := range NewTextFuncMap() {
-		subjectTemplates.Funcs(funcs)
+// walkTemplateFiles calls a callback for each template asset
+func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
+		return err
 	}
-	for _, funcs := range NewFuncMap() {
-		bodyTemplates.Funcs(funcs)
+	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
+		return err
 	}
+	return nil
+}
 
-	staticDir := path.Join(setting.StaticRootPath, "templates", "mail")
-
-	isDir, err := util.IsDir(staticDir)
-	if err != nil {
-		log.Warn("Unable to check if templates dir %s is a directory. Error: %v", staticDir, err)
-	}
-	if isDir {
-		files, err := util.StatDir(staticDir)
-
-		if err != nil {
-			log.Warn("Failed to read %s templates dir. %v", staticDir, err)
-		} else {
-			for _, filePath := range files {
-				if !strings.HasSuffix(filePath, ".tmpl") {
-					continue
-				}
-
-				content, err := os.ReadFile(path.Join(staticDir, filePath))
-				if err != nil {
-					log.Warn("Failed to read static %s template. %v", filePath, err)
-					continue
-				}
+// GetTemplateAssetNames returns list of template names
+func GetTemplateAssetNames() []string {
+	tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates"))
+	tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates"))
+	return append(tmpls, tmpls2...)
+}
 
-				buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, strings.TrimSuffix(filePath, ".tmpl"), content)
-			}
-		}
+func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
+		return err
 	}
-
-	customDir := path.Join(setting.CustomPath, "templates", "mail")
-
-	isDir, err = util.IsDir(customDir)
-	if err != nil {
-		log.Warn("Unable to check if templates dir %s is a directory. Error: %v", customDir, err)
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
+		return err
 	}
-	if isDir {
-		files, err := util.StatDir(customDir)
-
-		if err != nil {
-			log.Warn("Failed to read %s templates dir. %v", customDir, err)
-		} else {
-			for _, filePath := range files {
-				if !strings.HasSuffix(filePath, ".tmpl") {
-					continue
-				}
-
-				content, err := os.ReadFile(path.Join(customDir, filePath))
-				if err != nil {
-					log.Warn("Failed to read custom %s template. %v", filePath, err)
-					continue
-				}
+	return nil
+}
 
-				buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, strings.TrimSuffix(filePath, ".tmpl"), content)
-			}
-		}
-	}
+// BuiltinAsset will read the provided asset from the embedded assets
+// (This always returns os.ErrNotExist)
+func BuiltinAsset(name string) ([]byte, error) {
+	return nil, os.ErrNotExist
+}
 
-	return subjectTemplates, bodyTemplates
+// BuiltinAssetNames returns the names of the embedded assets
+// (This always returns nil)
+func BuiltinAssetNames() []string {
+	return nil
 }
diff --git a/modules/templates/htmlrenderer.go b/modules/templates/htmlrenderer.go
new file mode 100644
index 0000000000..210bb5e73c
--- /dev/null
+++ b/modules/templates/htmlrenderer.go
@@ -0,0 +1,52 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package templates
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/watcher"
+
+	"github.com/unrolled/render"
+)
+
+var rendererKey interface{} = "templatesHtmlRendereer"
+
+// HTMLRenderer returns the current html renderer for the context or creates and stores one within the context for future use
+func HTMLRenderer(ctx context.Context) (context.Context, *render.Render) {
+	rendererInterface := ctx.Value(rendererKey)
+	if rendererInterface != nil {
+		renderer, ok := rendererInterface.(*render.Render)
+		if ok {
+			return ctx, renderer
+		}
+	}
+
+	rendererType := "static"
+	if !setting.IsProd {
+		rendererType = "auto-reloading"
+	}
+	log.Log(1, log.DEBUG, "Creating "+rendererType+" HTML Renderer")
+
+	renderer := render.New(render.Options{
+		Extensions:                []string{".tmpl"},
+		Directory:                 "templates",
+		Funcs:                     NewFuncMap(),
+		Asset:                     GetAsset,
+		AssetNames:                GetTemplateAssetNames,
+		UseMutexLock:              !setting.IsProd,
+		IsDevelopment:             false,
+		DisableHTTPErrorRendering: true,
+	})
+	if !setting.IsProd {
+		watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{
+			PathsCallback:   walkTemplateFiles,
+			BetweenCallback: renderer.CompileTemplates,
+		})
+	}
+	return context.WithValue(ctx, rendererKey, renderer), renderer
+}
diff --git a/modules/templates/mailer.go b/modules/templates/mailer.go
new file mode 100644
index 0000000000..0cac1280f3
--- /dev/null
+++ b/modules/templates/mailer.go
@@ -0,0 +1,92 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package templates
+
+import (
+	"context"
+	"html/template"
+	"io/fs"
+	"os"
+	"strings"
+	texttmpl "text/template"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/watcher"
+)
+
+// Mailer provides the templates required for sending notification mails.
+func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
+	for _, funcs := range NewTextFuncMap() {
+		subjectTemplates.Funcs(funcs)
+	}
+	for _, funcs := range NewFuncMap() {
+		bodyTemplates.Funcs(funcs)
+	}
+
+	refreshTemplates := func() {
+		for _, assetPath := range BuiltinAssetNames() {
+			if !strings.HasPrefix(assetPath, "mail/") {
+				continue
+			}
+
+			if !strings.HasSuffix(assetPath, ".tmpl") {
+				continue
+			}
+
+			content, err := BuiltinAsset(assetPath)
+			if err != nil {
+				log.Warn("Failed to read embedded %s template. %v", assetPath, err)
+				continue
+			}
+
+			assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
+
+			log.Trace("Adding built-in mailer template for %s", assetName)
+			buildSubjectBodyTemplate(subjectTemplates,
+				bodyTemplates,
+				assetName,
+				content)
+		}
+
+		if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				return nil
+			}
+
+			content, err := os.ReadFile(path)
+			if err != nil {
+				log.Warn("Failed to read custom %s template. %v", path, err)
+				return nil
+			}
+
+			assetName := strings.TrimSuffix(name, ".tmpl")
+			log.Trace("Adding mailer template for %s from %q", assetName, path)
+			buildSubjectBodyTemplate(subjectTemplates,
+				bodyTemplates,
+				assetName,
+				content)
+			return nil
+		}); err != nil && !os.IsNotExist(err) {
+			log.Warn("Error whilst walking mailer templates directories. %v", err)
+		}
+	}
+
+	refreshTemplates()
+
+	if !setting.IsProd {
+		// Now subjectTemplates and bodyTemplates are both synchronized
+		// thus it is safe to call refresh from a different goroutine
+		watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{
+			PathsCallback:   walkMailerTemplates,
+			BetweenCallback: refreshTemplates,
+		})
+	}
+
+	return subjectTemplates, bodyTemplates
+}
diff --git a/modules/templates/static.go b/modules/templates/static.go
index 351e48b4da..3265bd9cfc 100644
--- a/modules/templates/static.go
+++ b/modules/templates/static.go
@@ -9,6 +9,7 @@ package templates
 import (
 	"html/template"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -16,10 +17,8 @@ import (
 	texttmpl "text/template"
 	"time"
 
-	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
-	"code.gitea.io/gitea/modules/util"
 )
 
 var (
@@ -40,95 +39,42 @@ func GetAsset(name string) ([]byte, error) {
 	} else if err == nil {
 		return bs, nil
 	}
-	return Asset(strings.TrimPrefix(name, "templates/"))
+	return BuiltinAsset(strings.TrimPrefix(name, "templates/"))
 }
 
-// GetAssetNames only for chi
-func GetAssetNames() []string {
+// GetFiles calls a callback for each template asset
+func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// GetTemplateAssetNames only for chi
+func GetTemplateAssetNames() []string {
 	realFS := Assets.(vfsgen۰FS)
 	tmpls := make([]string, 0, len(realFS))
 	for k := range realFS {
+		if strings.HasPrefix(k, "/mail/") {
+			continue
+		}
 		tmpls = append(tmpls, "templates/"+k[1:])
 	}
 
 	customDir := path.Join(setting.CustomPath, "templates")
-	customTmpls := getDirAssetNames(customDir)
+	customTmpls := getDirTemplateAssetNames(customDir)
 	return append(tmpls, customTmpls...)
 }
 
-// Mailer provides the templates required for sending notification mails.
-func Mailer() (*texttmpl.Template, *template.Template) {
-	for _, funcs := range NewTextFuncMap() {
-		subjectTemplates.Funcs(funcs)
-	}
-	for _, funcs := range NewFuncMap() {
-		bodyTemplates.Funcs(funcs)
+func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
+		return err
 	}
-
-	for _, assetPath := range AssetNames() {
-		if !strings.HasPrefix(assetPath, "mail/") {
-			continue
-		}
-
-		if !strings.HasSuffix(assetPath, ".tmpl") {
-			continue
-		}
-
-		content, err := Asset(assetPath)
-		if err != nil {
-			log.Warn("Failed to read embedded %s template. %v", assetPath, err)
-			continue
-		}
-
-		buildSubjectBodyTemplate(subjectTemplates,
-			bodyTemplates,
-			strings.TrimPrefix(
-				strings.TrimSuffix(
-					assetPath,
-					".tmpl",
-				),
-				"mail/",
-			),
-			content)
-	}
-
-	customDir := path.Join(setting.CustomPath, "templates", "mail")
-	isDir, err := util.IsDir(customDir)
-	if err != nil {
-		log.Warn("Failed to check if custom directory %s is a directory. %v", err)
-	}
-	if isDir {
-		files, err := util.StatDir(customDir)
-
-		if err != nil {
-			log.Warn("Failed to read %s templates dir. %v", customDir, err)
-		} else {
-			for _, filePath := range files {
-				if !strings.HasSuffix(filePath, ".tmpl") {
-					continue
-				}
-
-				content, err := os.ReadFile(path.Join(customDir, filePath))
-				if err != nil {
-					log.Warn("Failed to read custom %s template. %v", filePath, err)
-					continue
-				}
-
-				buildSubjectBodyTemplate(subjectTemplates,
-					bodyTemplates,
-					strings.TrimSuffix(
-						filePath,
-						".tmpl",
-					),
-					content)
-			}
-		}
-	}
-
-	return subjectTemplates, bodyTemplates
+	return nil
 }
 
-func Asset(name string) ([]byte, error) {
+// BuiltinAsset reads the provided asset from the builtin embedded assets
+func BuiltinAsset(name string) ([]byte, error) {
 	f, err := Assets.Open("/" + name)
 	if err != nil {
 		return nil, err
@@ -137,7 +83,8 @@ func Asset(name string) ([]byte, error) {
 	return io.ReadAll(f)
 }
 
-func AssetNames() []string {
+// BuiltinAssetNames returns the names of the built-in embedded assets
+func BuiltinAssetNames() []string {
 	realFS := Assets.(vfsgen۰FS)
 	results := make([]string, 0, len(realFS))
 	for k := range realFS {
@@ -146,7 +93,8 @@ func AssetNames() []string {
 	return results
 }
 
-func AssetIsDir(name string) (bool, error) {
+// BuiltinAssetIsDir returns if a provided asset is a directory
+func BuiltinAssetIsDir(name string) (bool, error) {
 	if f, err := Assets.Open("/" + name); err != nil {
 		return false, err
 	} else {
diff --git a/modules/timeutil/since_test.go b/modules/timeutil/since_test.go
index dfcf9cb01d..9350b5e96b 100644
--- a/modules/timeutil/since_test.go
+++ b/modules/timeutil/since_test.go
@@ -5,6 +5,7 @@
 package timeutil
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"testing"
@@ -31,7 +32,7 @@ func TestMain(m *testing.M) {
 	setting.Names = []string{"english"}
 	setting.Langs = []string{"en-US"}
 	// setup
-	translation.InitLocales()
+	translation.InitLocales(context.Background())
 	BaseDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)
 
 	// run the tests
diff --git a/modules/translation/i18n/errors.go b/modules/translation/i18n/errors.go
new file mode 100644
index 0000000000..b485badd1d
--- /dev/null
+++ b/modules/translation/i18n/errors.go
@@ -0,0 +1,12 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package i18n
+
+import "errors"
+
+var (
+	ErrLocaleAlreadyExist = errors.New("lang already exists")
+	ErrUncertainArguments = errors.New("arguments to i18n should not contain uncertain slices")
+)
diff --git a/modules/translation/i18n/format.go b/modules/translation/i18n/format.go
new file mode 100644
index 0000000000..3fb9e6d6d0
--- /dev/null
+++ b/modules/translation/i18n/format.go
@@ -0,0 +1,42 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package i18n
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// Format formats provided arguments for a given translated message
+func Format(format string, args ...interface{}) (msg string, err error) {
+	if len(args) == 0 {
+		return format, nil
+	}
+
+	fmtArgs := make([]interface{}, 0, len(args))
+	for _, arg := range args {
+		val := reflect.ValueOf(arg)
+		if val.Kind() == reflect.Slice {
+			// Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f)
+			// but this is an unstable behavior.
+			//
+			// So we restrict the accepted arguments to either:
+			//
+			// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
+			// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
+			if len(args) == 1 {
+				for i := 0; i < val.Len(); i++ {
+					fmtArgs = append(fmtArgs, val.Index(i).Interface())
+				}
+			} else {
+				err = ErrUncertainArguments
+				break
+			}
+		} else {
+			fmtArgs = append(fmtArgs, arg)
+		}
+	}
+	return fmt.Sprintf(format, fmtArgs...), err
+}
diff --git a/modules/translation/i18n/i18n.go b/modules/translation/i18n/i18n.go
index bb906f3c08..23b4e23c76 100644
--- a/modules/translation/i18n/i18n.go
+++ b/modules/translation/i18n/i18n.go
@@ -5,297 +5,48 @@
 package i18n
 
 import (
-	"errors"
-	"fmt"
-	"os"
-	"reflect"
-	"sync"
-	"time"
-
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
-
-	"gopkg.in/ini.v1"
-)
-
-var (
-	ErrLocaleAlreadyExist = errors.New("lang already exists")
-
-	DefaultLocales = NewLocaleStore(true)
+	"io"
 )
 
-type locale struct {
-	// This mutex will be set if we have live-reload enabled (e.g. dev mode)
-	reloadMu *sync.RWMutex
-
-	store    *LocaleStore
-	langName string
-
-	idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
-
-	sourceFileName      string
-	sourceFileInfo      os.FileInfo
-	lastReloadCheckTime time.Time
-}
-
-type LocaleStore struct {
-	// This mutex will be set if we have live-reload enabled (e.g. dev mode)
-	reloadMu *sync.RWMutex
-
-	langNames []string
-	langDescs []string
-	localeMap map[string]*locale
-
-	// this needs to be locked when live-reloading
-	trKeyToIdxMap map[string]int
-
-	defaultLang string
-}
-
-func NewLocaleStore(isProd bool) *LocaleStore {
-	store := &LocaleStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
-	if !isProd {
-		store.reloadMu = &sync.RWMutex{}
-	}
-	return store
-}
-
-// AddLocaleByIni adds locale by ini into the store
-// if source is a string, then the file is loaded. In dev mode, this file will be checked for live-reloading
-// if source is a []byte, then the content is used
-// Note: this is not concurrent safe
-func (store *LocaleStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
-	if _, ok := store.localeMap[langName]; ok {
-		return ErrLocaleAlreadyExist
-	}
-
-	l := &locale{store: store, langName: langName}
-	if store.reloadMu != nil {
-		l.reloadMu = &sync.RWMutex{}
-		l.reloadMu.Lock() // Arguably this is not necessary as AddLocaleByIni isn't concurrent safe - but for consistency we do this
-		defer l.reloadMu.Unlock()
-	}
-
-	if fileName, ok := source.(string); ok {
-		l.sourceFileName = fileName
-		l.sourceFileInfo, _ = os.Stat(fileName) // live-reload only works for regular files. the error can be ignored
-	}
-
-	var err error
-	l.idxToMsgMap, err = store.readIniToIdxToMsgMap(source)
-	if err != nil {
-		return err
-	}
-
-	store.langNames = append(store.langNames, langName)
-	store.langDescs = append(store.langDescs, langDesc)
-
-	store.localeMap[l.langName] = l
-
-	return nil
-}
-
-// readIniToIdxToMsgMap will read a provided ini and creates an idxToMsgMap
-func (store *LocaleStore) readIniToIdxToMsgMap(source interface{}) (map[int]string, error) {
-	iniFile, err := ini.LoadSources(ini.LoadOptions{
-		IgnoreInlineComment:         true,
-		UnescapeValueCommentSymbols: true,
-	}, source)
-	if err != nil {
-		return nil, fmt.Errorf("unable to load ini: %w", err)
-	}
-	iniFile.BlockMode = false
-
-	idxToMsgMap := make(map[int]string)
-
-	if store.reloadMu != nil {
-		store.reloadMu.Lock()
-		defer store.reloadMu.Unlock()
-	}
-
-	for _, section := range iniFile.Sections() {
-		for _, key := range section.Keys() {
-
-			var trKey string
-			if section.Name() == "" || section.Name() == "DEFAULT" {
-				trKey = key.Name()
-			} else {
-				trKey = section.Name() + "." + key.Name()
-			}
-
-			// Instead of storing the key strings in multiple different maps we compute a idx which will act as numeric code for key
-			// This reduces the size of the locale idxToMsgMaps
-			idx, ok := store.trKeyToIdxMap[trKey]
-			if !ok {
-				idx = len(store.trKeyToIdxMap)
-				store.trKeyToIdxMap[trKey] = idx
-			}
-			idxToMsgMap[idx] = key.Value()
-		}
-	}
-	iniFile = nil
-	return idxToMsgMap, nil
-}
-
-func (store *LocaleStore) idxForTrKey(trKey string) (int, bool) {
-	if store.reloadMu != nil {
-		store.reloadMu.RLock()
-		defer store.reloadMu.RUnlock()
-	}
-	idx, ok := store.trKeyToIdxMap[trKey]
-	return idx, ok
-}
-
-// HasLang reports if a language is available in the store
-func (store *LocaleStore) HasLang(langName string) bool {
-	_, ok := store.localeMap[langName]
-	return ok
-}
-
-// ListLangNameDesc reports if a language available in the store
-func (store *LocaleStore) ListLangNameDesc() (names, desc []string) {
-	return store.langNames, store.langDescs
-}
-
-// SetDefaultLang sets default language as a fallback
-func (store *LocaleStore) SetDefaultLang(lang string) {
-	store.defaultLang = lang
-}
-
-// Tr translates content to target language. fall back to default language.
-func (store *LocaleStore) Tr(lang, trKey string, trArgs ...interface{}) string {
-	l, ok := store.localeMap[lang]
-	if !ok {
-		l, ok = store.localeMap[store.defaultLang]
-	}
-
-	if ok {
-		return l.Tr(trKey, trArgs...)
-	}
-	return trKey
-}
-
-// reloadIfNeeded will check if the locale needs to be reloaded
-// this function will assume that the l.reloadMu has been RLocked if it already exists
-func (l *locale) reloadIfNeeded() {
-	if l.reloadMu == nil {
-		return
-	}
-
-	now := time.Now()
-	if now.Sub(l.lastReloadCheckTime) < time.Second || l.sourceFileInfo == nil || l.sourceFileName == "" {
-		return
-	}
-
-	l.reloadMu.RUnlock()
-	l.reloadMu.Lock() // (NOTE: a pre-emption can occur between these two locks so we need to recheck)
-	defer l.reloadMu.RLock()
-	defer l.reloadMu.Unlock()
-
-	if now.Sub(l.lastReloadCheckTime) < time.Second || l.sourceFileInfo == nil || l.sourceFileName == "" {
-		return
-	}
+var DefaultLocales = NewLocaleStore()
 
-	l.lastReloadCheckTime = now
-	sourceFileInfo, err := os.Stat(l.sourceFileName)
-	if err != nil || sourceFileInfo.ModTime().Equal(l.sourceFileInfo.ModTime()) {
-		return
-	}
-
-	idxToMsgMap, err := l.store.readIniToIdxToMsgMap(l.sourceFileName)
-	if err == nil {
-		l.idxToMsgMap = idxToMsgMap
-	} else {
-		log.Error("Unable to live-reload the locale file %q, err: %v", l.sourceFileName, err)
-	}
-
-	// We will set the sourceFileInfo to this file to prevent repeated attempts to re-load this broken file
-	l.sourceFileInfo = sourceFileInfo
-}
-
-// Tr translates content to locale language. fall back to default language.
-func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
-	if l.reloadMu != nil {
-		l.reloadMu.RLock()
-		defer l.reloadMu.RUnlock()
-		l.reloadIfNeeded()
-	}
-
-	msg, _ := l.tryTr(trKey, trArgs...)
-	return msg
+type Locale interface {
+	// Tr translates a given key and arguments for a language
+	Tr(trKey string, trArgs ...interface{}) string
+	// Has reports if a locale has a translation for a given key
+	Has(trKey string) bool
 }
 
-func (l *locale) tryTr(trKey string, trArgs ...interface{}) (msg string, found bool) {
-	trMsg := trKey
-
-	// convert the provided trKey to a common idx from the store
-	idx, ok := l.store.idxForTrKey(trKey)
-
-	if ok {
-		if msg, found = l.idxToMsgMap[idx]; found {
-			trMsg = msg // use the translation that we have found
-		} else if l.langName != l.store.defaultLang {
-			// No translation available in our current language... fallback to the default language
+// LocaleStore provides the functions common to all locale stores
+type LocaleStore interface {
+	io.Closer
 
-			// Attempt to get the default language from the locale store
-			if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
-
-				if def.reloadMu != nil {
-					def.reloadMu.RLock()
-					def.reloadIfNeeded()
-				}
-				if msg, found = def.idxToMsgMap[idx]; found {
-					trMsg = msg // use the translation that we have found
-				}
-				if def.reloadMu != nil {
-					def.reloadMu.RUnlock()
-				}
-			}
-		}
-	}
-
-	if !found && !setting.IsProd {
-		log.Error("missing i18n translation key: %q", trKey)
-	}
-
-	if len(trArgs) == 0 {
-		return trMsg, found
-	}
-
-	fmtArgs := make([]interface{}, 0, len(trArgs))
-	for _, arg := range trArgs {
-		val := reflect.ValueOf(arg)
-		if val.Kind() == reflect.Slice {
-			// Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f)
-			// but this is an unstable behavior.
-			//
-			// So we restrict the accepted arguments to either:
-			//
-			// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
-			// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
-			if len(trArgs) == 1 {
-				for i := 0; i < val.Len(); i++ {
-					fmtArgs = append(fmtArgs, val.Index(i).Interface())
-				}
-			} else {
-				log.Error("the args for i18n shouldn't contain uncertain slices, key=%q, args=%v", trKey, trArgs)
-				break
-			}
-		} else {
-			fmtArgs = append(fmtArgs, arg)
-		}
-	}
-
-	return fmt.Sprintf(trMsg, fmtArgs...), found
+	// Tr translates a given key and arguments for a language
+	Tr(lang, trKey string, trArgs ...interface{}) string
+	// Has reports if a locale has a translation for a given key
+	Has(lang, trKey string) bool
+	// SetDefaultLang sets the default language to fall back to
+	SetDefaultLang(lang string)
+	// ListLangNameDesc provides paired slices of language names to descriptors
+	ListLangNameDesc() (names, desc []string)
+	// Locale return the locale for the provided language or the default language if not found
+	Locale(langName string) (Locale, bool)
+	// HasLang returns whether a given language is present in the store
+	HasLang(langName string) bool
+	// AddLocaleByIni adds a new language to the store
+	AddLocaleByIni(langName, langDesc string, source interface{}) error
 }
 
 // ResetDefaultLocales resets the current default locales
 // NOTE: this is not synchronized
-func ResetDefaultLocales(isProd bool) {
-	DefaultLocales = NewLocaleStore(isProd)
+func ResetDefaultLocales() {
+	if DefaultLocales != nil {
+		_ = DefaultLocales.Close()
+	}
+	DefaultLocales = NewLocaleStore()
 }
 
-// Tr use default locales to translate content to target language.
-func Tr(lang, trKey string, trArgs ...interface{}) string {
-	return DefaultLocales.Tr(lang, trKey, trArgs...)
+// GetLocales returns the locale from the default locales
+func GetLocale(lang string) (Locale, bool) {
+	return DefaultLocales.Locale(lang)
 }
diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go
index 32f7585b32..7940e59c94 100644
--- a/modules/translation/i18n/i18n_test.go
+++ b/modules/translation/i18n/i18n_test.go
@@ -27,36 +27,34 @@ fmt = %[2]s %[1]s
 sub = Changed Sub String
 `)
 
-	for _, isProd := range []bool{true, false} {
-		ls := NewLocaleStore(isProd)
-		assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1))
-		assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2))
-		ls.SetDefaultLang("lang1")
+	ls := NewLocaleStore()
+	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1))
+	assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2))
+	ls.SetDefaultLang("lang1")
 
-		result := ls.Tr("lang1", "fmt", "a", "b")
-		assert.Equal(t, "a b", result)
+	result := ls.Tr("lang1", "fmt", "a", "b")
+	assert.Equal(t, "a b", result)
 
-		result = ls.Tr("lang2", "fmt", "a", "b")
-		assert.Equal(t, "b a", result)
+	result = ls.Tr("lang2", "fmt", "a", "b")
+	assert.Equal(t, "b a", result)
 
-		result = ls.Tr("lang1", "section.sub")
-		assert.Equal(t, "Sub String", result)
+	result = ls.Tr("lang1", "section.sub")
+	assert.Equal(t, "Sub String", result)
 
-		result = ls.Tr("lang2", "section.sub")
-		assert.Equal(t, "Changed Sub String", result)
+	result = ls.Tr("lang2", "section.sub")
+	assert.Equal(t, "Changed Sub String", result)
 
-		result = ls.Tr("", ".dot.name")
-		assert.Equal(t, "Dot Name", result)
+	result = ls.Tr("", ".dot.name")
+	assert.Equal(t, "Dot Name", result)
 
-		result = ls.Tr("lang2", "section.mixed")
-		assert.Equal(t, `test value; <span style="color: red; background: none;">more text</span>`, result)
+	result = ls.Tr("lang2", "section.mixed")
+	assert.Equal(t, `test value; <span style="color: red; background: none;">more text</span>`, result)
 
-		langs, descs := ls.ListLangNameDesc()
-		assert.Equal(t, []string{"lang1", "lang2"}, langs)
-		assert.Equal(t, []string{"Lang1", "Lang2"}, descs)
+	langs, descs := ls.ListLangNameDesc()
+	assert.Equal(t, []string{"lang1", "lang2"}, langs)
+	assert.Equal(t, []string{"Lang1", "Lang2"}, descs)
 
-		result, found := ls.localeMap["lang1"].tryTr("no-such")
-		assert.Equal(t, "no-such", result)
-		assert.False(t, found)
-	}
+	found := ls.Has("lang1", "no-such")
+	assert.False(t, found)
+	assert.NoError(t, ls.Close())
 }
diff --git a/modules/translation/i18n/localestore.go b/modules/translation/i18n/localestore.go
new file mode 100644
index 0000000000..e3b88ad96e
--- /dev/null
+++ b/modules/translation/i18n/localestore.go
@@ -0,0 +1,161 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package i18n
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"gopkg.in/ini.v1"
+)
+
+// This file implements the static LocaleStore that will not watch for changes
+
+type locale struct {
+	store       *localeStore
+	langName    string
+	idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
+}
+
+type localeStore struct {
+	// After initializing has finished, these fields are read-only.
+	langNames []string
+	langDescs []string
+
+	localeMap     map[string]*locale
+	trKeyToIdxMap map[string]int
+
+	defaultLang string
+}
+
+// NewLocaleStore creates a static locale store
+func NewLocaleStore() LocaleStore {
+	return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
+}
+
+// AddLocaleByIni adds locale by ini into the store
+// if source is a string, then the file is loaded
+// if source is a []byte, then the content is used
+func (store *localeStore) AddLocaleByIni(langName, langDesc string, source interface{}) error {
+	if _, ok := store.localeMap[langName]; ok {
+		return ErrLocaleAlreadyExist
+	}
+
+	store.langNames = append(store.langNames, langName)
+	store.langDescs = append(store.langDescs, langDesc)
+
+	l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)}
+	store.localeMap[l.langName] = l
+
+	iniFile, err := ini.LoadSources(ini.LoadOptions{
+		IgnoreInlineComment:         true,
+		UnescapeValueCommentSymbols: true,
+	}, source)
+	if err != nil {
+		return fmt.Errorf("unable to load ini: %w", err)
+	}
+	iniFile.BlockMode = false
+
+	for _, section := range iniFile.Sections() {
+		for _, key := range section.Keys() {
+			var trKey string
+			if section.Name() == "" || section.Name() == "DEFAULT" {
+				trKey = key.Name()
+			} else {
+				trKey = section.Name() + "." + key.Name()
+			}
+			idx, ok := store.trKeyToIdxMap[trKey]
+			if !ok {
+				idx = len(store.trKeyToIdxMap)
+				store.trKeyToIdxMap[trKey] = idx
+			}
+			l.idxToMsgMap[idx] = key.Value()
+		}
+	}
+	iniFile = nil
+
+	return nil
+}
+
+func (store *localeStore) HasLang(langName string) bool {
+	_, ok := store.localeMap[langName]
+	return ok
+}
+
+func (store *localeStore) ListLangNameDesc() (names, desc []string) {
+	return store.langNames, store.langDescs
+}
+
+// SetDefaultLang sets default language as a fallback
+func (store *localeStore) SetDefaultLang(lang string) {
+	store.defaultLang = lang
+}
+
+// Tr translates content to target language. fall back to default language.
+func (store *localeStore) Tr(lang, trKey string, trArgs ...interface{}) string {
+	l, _ := store.Locale(lang)
+
+	return l.Tr(trKey, trArgs...)
+}
+
+// Has returns whether the given language has a translation for the provided key
+func (store *localeStore) Has(lang, trKey string) bool {
+	l, _ := store.Locale(lang)
+
+	return l.Has(trKey)
+}
+
+// Locale returns the locale for the lang or the default language
+func (store *localeStore) Locale(lang string) (Locale, bool) {
+	l, found := store.localeMap[lang]
+	if !found {
+		var ok bool
+		l, ok = store.localeMap[store.defaultLang]
+		if !ok {
+			// no default - return an empty locale
+			l = &locale{store: store, idxToMsgMap: make(map[int]string)}
+		}
+	}
+	return l, found
+}
+
+// Close implements io.Closer
+func (store *localeStore) Close() error {
+	return nil
+}
+
+// Tr translates content to locale language. fall back to default language.
+func (l *locale) Tr(trKey string, trArgs ...interface{}) string {
+	format := trKey
+
+	idx, ok := l.store.trKeyToIdxMap[trKey]
+	if ok {
+		if msg, ok := l.idxToMsgMap[idx]; ok {
+			format = msg // use the found translation
+		} else if def, ok := l.store.localeMap[l.store.defaultLang]; ok {
+			// try to use default locale's translation
+			if msg, ok := def.idxToMsgMap[idx]; ok {
+				format = msg
+			}
+		}
+	}
+
+	msg, err := Format(format, trArgs...)
+	if err != nil {
+		log.Error("Error whilst formatting %q in %s: %v", trKey, l.langName, err)
+	}
+	return msg
+}
+
+// Has returns whether a key is present in this locale or not
+func (l *locale) Has(trKey string) bool {
+	idx, ok := l.store.trKeyToIdxMap[trKey]
+	if !ok {
+		return false
+	}
+	_, ok = l.idxToMsgMap[idx]
+	return ok
+}
diff --git a/modules/translation/translation.go b/modules/translation/translation.go
index fcc101d963..e40a9357fa 100644
--- a/modules/translation/translation.go
+++ b/modules/translation/translation.go
@@ -5,15 +5,16 @@
 package translation
 
 import (
-	"path"
+	"context"
 	"sort"
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/translation/i18n"
-	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/modules/watcher"
 
 	"golang.org/x/text/language"
 )
@@ -31,6 +32,7 @@ type LangType struct {
 }
 
 var (
+	lock          *sync.RWMutex
 	matcher       language.Matcher
 	allLangs      []*LangType
 	allLangMap    map[string]*LangType
@@ -43,58 +45,53 @@ func AllLangs() []*LangType {
 }
 
 // InitLocales loads the locales
-func InitLocales() {
-	i18n.ResetDefaultLocales(setting.IsProd)
-	localeNames, err := options.Dir("locale")
-	if err != nil {
-		log.Fatal("Failed to list locale files: %v", err)
+func InitLocales(ctx context.Context) {
+	if lock != nil {
+		lock.Lock()
+		defer lock.Unlock()
+	} else if !setting.IsProd && lock == nil {
+		lock = &sync.RWMutex{}
 	}
 
-	localFiles := make(map[string]interface{}, len(localeNames))
-	for _, name := range localeNames {
-		if options.IsDynamic() {
-			// Try to check if CustomPath has the file, otherwise fallback to StaticRootPath
-			value := path.Join(setting.CustomPath, "options/locale", name)
-
-			isFile, err := util.IsFile(value)
-			if err != nil {
-				log.Fatal("Failed to load %s locale file. %v", name, err)
-			}
+	refreshLocales := func() {
+		i18n.ResetDefaultLocales()
+		localeNames, err := options.Dir("locale")
+		if err != nil {
+			log.Fatal("Failed to list locale files: %v", err)
+		}
 
-			if isFile {
-				localFiles[name] = value
-			} else {
-				localFiles[name] = path.Join(setting.StaticRootPath, "options/locale", name)
-			}
-		} else {
+		localFiles := make(map[string]interface{}, len(localeNames))
+		for _, name := range localeNames {
 			localFiles[name], err = options.Locale(name)
 			if err != nil {
 				log.Fatal("Failed to load %s locale file. %v", name, err)
 			}
 		}
-	}
 
-	supportedTags = make([]language.Tag, len(setting.Langs))
-	for i, lang := range setting.Langs {
-		supportedTags[i] = language.Raw.Make(lang)
-	}
+		supportedTags = make([]language.Tag, len(setting.Langs))
+		for i, lang := range setting.Langs {
+			supportedTags[i] = language.Raw.Make(lang)
+		}
 
-	matcher = language.NewMatcher(supportedTags)
-	for i := range setting.Names {
-		key := "locale_" + setting.Langs[i] + ".ini"
+		matcher = language.NewMatcher(supportedTags)
+		for i := range setting.Names {
+			key := "locale_" + setting.Langs[i] + ".ini"
 
-		if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil {
-			log.Error("Failed to set messages to %s: %v", setting.Langs[i], err)
+			if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localFiles[key]); err != nil {
+				log.Error("Failed to set messages to %s: %v", setting.Langs[i], err)
+			}
 		}
-	}
-	if len(setting.Langs) != 0 {
-		defaultLangName := setting.Langs[0]
-		if defaultLangName != "en-US" {
-			log.Info("Use the first locale (%s) in LANGS setting option as default", defaultLangName)
+		if len(setting.Langs) != 0 {
+			defaultLangName := setting.Langs[0]
+			if defaultLangName != "en-US" {
+				log.Info("Use the first locale (%s) in LANGS setting option as default", defaultLangName)
+			}
+			i18n.DefaultLocales.SetDefaultLang(defaultLangName)
 		}
-		i18n.DefaultLocales.SetDefaultLang(defaultLangName)
 	}
 
+	refreshLocales()
+
 	langs, descs := i18n.DefaultLocales.ListLangNameDesc()
 	allLangs = make([]*LangType, 0, len(langs))
 	allLangMap = map[string]*LangType{}
@@ -108,6 +105,17 @@ func InitLocales() {
 	sort.Slice(allLangs, func(i, j int) bool {
 		return strings.ToLower(allLangs[i].Name) < strings.ToLower(allLangs[j].Name)
 	})
+
+	if !setting.IsProd {
+		watcher.CreateWatcher(ctx, "Locales", &watcher.CreateWatcherOpts{
+			PathsCallback: options.WalkLocales,
+			BetweenCallback: func() {
+				lock.Lock()
+				defer lock.Unlock()
+				refreshLocales()
+			},
+		})
+	}
 }
 
 // Match matches accept languages
@@ -118,16 +126,24 @@ func Match(tags ...language.Tag) language.Tag {
 
 // locale represents the information of localization.
 type locale struct {
+	i18n.Locale
 	Lang, LangName string // these fields are used directly in templates: .i18n.Lang
 }
 
 // NewLocale return a locale
 func NewLocale(lang string) Locale {
+	if lock != nil {
+		lock.RLock()
+		defer lock.RUnlock()
+	}
+
 	langName := "unknown"
 	if l, ok := allLangMap[lang]; ok {
 		langName = l.Name
 	}
+	i18nLocale, _ := i18n.GetLocale(lang)
 	return &locale{
+		Locale:   i18nLocale,
 		Lang:     lang,
 		LangName: langName,
 	}
@@ -137,11 +153,6 @@ func (l *locale) Language() string {
 	return l.Lang
 }
 
-// Tr translates content to target language.
-func (l *locale) Tr(format string, args ...interface{}) string {
-	return i18n.Tr(l.Lang, format, args...)
-}
-
 // Language specific rules for translating plural texts
 var trNLangRules = map[string]func(int64) int{
 	// the default rule is "en-US" if a language isn't listed here
diff --git a/modules/util/path.go b/modules/util/path.go
index 0ccc7a1dc2..3d4ddec21c 100644
--- a/modules/util/path.go
+++ b/modules/util/path.go
@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
-	"strings"
 )
 
 // EnsureAbsolutePath ensure that a path is absolute, making it
@@ -91,7 +90,7 @@ func statDir(dirPath, recPath string, includeDir, isDirOnly, followSymlinks bool
 
 	statList := make([]string, 0)
 	for _, fi := range fis {
-		if strings.Contains(fi.Name(), ".DS_Store") {
+		if CommonSkip(fi.Name()) {
 			continue
 		}
 
@@ -199,3 +198,21 @@ func HomeDir() (home string, err error) {
 
 	return home, nil
 }
+
+// CommonSkip will check a provided name to see if it represents file or directory that should not be watched
+func CommonSkip(name string) bool {
+	if name == "" {
+		return true
+	}
+
+	switch name[0] {
+	case '.':
+		return true
+	case 't', 'T':
+		return name[1:] == "humbs.db"
+	case 'd', 'D':
+		return name[1:] == "esktop.ini"
+	}
+
+	return false
+}
diff --git a/modules/watcher/watcher.go b/modules/watcher/watcher.go
new file mode 100644
index 0000000000..d737f6ccbb
--- /dev/null
+++ b/modules/watcher/watcher.go
@@ -0,0 +1,115 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package watcher
+
+import (
+	"context"
+	"io/fs"
+	"os"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
+
+	"github.com/fsnotify/fsnotify"
+)
+
+// CreateWatcherOpts are options to configure the watcher
+type CreateWatcherOpts struct {
+	// PathsCallback is used to set the required paths to watch
+	PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error
+
+	// BeforeCallback is called before any files are watched
+	BeforeCallback func()
+
+	// Between Callback is called between after a watched event has occurred
+	BetweenCallback func()
+
+	// AfterCallback is called as this watcher ends
+	AfterCallback func()
+}
+
+// CreateWatcher creates a watcher labelled with the provided description and running with the provided options.
+// The created watcher will create a subcontext from the provided ctx and register it with the process manager.
+func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) {
+	go run(ctx, desc, opts)
+}
+
+func run(ctx context.Context, desc string, opts *CreateWatcherOpts) {
+	if opts.BeforeCallback != nil {
+		opts.BeforeCallback()
+	}
+	if opts.AfterCallback != nil {
+		defer opts.AfterCallback()
+	}
+	ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Watcher: "+desc, process.SystemProcessType, true)
+	defer finished()
+
+	log.Trace("Watcher loop starting for %s", desc)
+	defer log.Trace("Watcher loop ended for %s", desc)
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Error("Unable to create watcher for %s: %v", desc, err)
+		return
+	}
+	if err := opts.PathsCallback(func(path, _ string, d fs.DirEntry, err error) error {
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		log.Trace("Watcher: %s watching %q", desc, path)
+		_ = watcher.Add(path)
+		return nil
+	}); err != nil {
+		log.Error("Unable to create watcher for %s: %v", desc, err)
+		_ = watcher.Close()
+		return
+	}
+
+	// Note we don't call the BetweenCallback here
+
+	for {
+		select {
+		case event, ok := <-watcher.Events:
+			if !ok {
+				_ = watcher.Close()
+				return
+			}
+			log.Debug("Watched file for %s had event: %v", desc, event)
+		case err, ok := <-watcher.Errors:
+			if !ok {
+				_ = watcher.Close()
+				return
+			}
+			log.Error("Error whilst watching files for %s: %v", desc, err)
+		case <-ctx.Done():
+			_ = watcher.Close()
+			return
+		}
+
+		// Recreate the watcher - only call the BetweenCallback after the new watcher is set-up
+		_ = watcher.Close()
+		watcher, err = fsnotify.NewWatcher()
+		if err != nil {
+			log.Error("Unable to create watcher for %s: %v", desc, err)
+			return
+		}
+		if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			_ = watcher.Add(path)
+			return nil
+		}); err != nil {
+			log.Error("Unable to create watcher for %s: %v", desc, err)
+			_ = watcher.Close()
+			return
+		}
+
+		// Inform our BetweenCallback that there has been an event
+		if opts.BetweenCallback != nil {
+			opts.BetweenCallback()
+		}
+	}
+}