From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- modules/translation/i18n/errors.go | 13 ++ modules/translation/i18n/format.go | 41 +++++++ modules/translation/i18n/i18n.go | 50 ++++++++ modules/translation/i18n/i18n_test.go | 204 ++++++++++++++++++++++++++++++++ modules/translation/i18n/localestore.go | 166 ++++++++++++++++++++++++++ 5 files changed, 474 insertions(+) create mode 100644 modules/translation/i18n/errors.go create mode 100644 modules/translation/i18n/format.go create mode 100644 modules/translation/i18n/i18n.go create mode 100644 modules/translation/i18n/i18n_test.go create mode 100644 modules/translation/i18n/localestore.go (limited to 'modules/translation/i18n') diff --git a/modules/translation/i18n/errors.go b/modules/translation/i18n/errors.go new file mode 100644 index 0000000..7f64ccf --- /dev/null +++ b/modules/translation/i18n/errors.go @@ -0,0 +1,13 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package i18n + +import ( + "code.gitea.io/gitea/modules/util" +) + +var ( + ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist} + ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument} +) diff --git a/modules/translation/i18n/format.go b/modules/translation/i18n/format.go new file mode 100644 index 0000000..e5e2218 --- /dev/null +++ b/modules/translation/i18n/format.go @@ -0,0 +1,41 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package i18n + +import ( + "fmt" + "reflect" +) + +// Format formats provided arguments for a given translated message +func Format(format string, args ...any) (msg string, err error) { + if len(args) == 0 { + return format, nil + } + + fmtArgs := make([]any, 0, len(args)) + for _, arg := range args { + val := reflect.ValueOf(arg) + if val.Kind() == reflect.Slice { + // Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f) + // but this is an unstable behavior. + // + // So we restrict the accepted arguments to either: + // + // 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...) + // 2. Tr(lang, key, args...) as Sprintf(msg, args...) + if len(args) == 1 { + for i := 0; i < val.Len(); i++ { + fmtArgs = append(fmtArgs, val.Index(i).Interface()) + } + } else { + err = ErrUncertainArguments + break + } + } else { + fmtArgs = append(fmtArgs, arg) + } + } + return fmt.Sprintf(format, fmtArgs...), err +} diff --git a/modules/translation/i18n/i18n.go b/modules/translation/i18n/i18n.go new file mode 100644 index 0000000..1555cd9 --- /dev/null +++ b/modules/translation/i18n/i18n.go @@ -0,0 +1,50 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package i18n + +import ( + "html/template" + "io" +) + +var DefaultLocales = NewLocaleStore() + +type Locale interface { + // TrString translates a given key and arguments for a language + TrString(trKey string, trArgs ...any) string + // TrHTML translates a given key and arguments for a language, string arguments are escaped to HTML + TrHTML(trKey string, trArgs ...any) template.HTML + // HasKey reports if a locale has a translation for a given key + HasKey(trKey string) bool +} + +// LocaleStore provides the functions common to all locale stores +type LocaleStore interface { + io.Closer + + // SetDefaultLang sets the default language to fall back to + SetDefaultLang(lang string) + // ListLangNameDesc provides paired slices of language names to descriptors + ListLangNameDesc() (names, desc []string) + // Locale return the locale for the provided language or the default language if not found + Locale(langName string) (Locale, bool) + // HasLang returns whether a given language is present in the store + HasLang(langName string) bool + // AddLocaleByIni adds a new language to the store + AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error +} + +// ResetDefaultLocales resets the current default locales +// NOTE: this is not synchronized +func ResetDefaultLocales() { + if DefaultLocales != nil { + _ = DefaultLocales.Close() + } + DefaultLocales = NewLocaleStore() +} + +// GetLocale returns the locale from the default locales +func GetLocale(lang string) (Locale, bool) { + return DefaultLocales.Locale(lang) +} diff --git a/modules/translation/i18n/i18n_test.go b/modules/translation/i18n/i18n_test.go new file mode 100644 index 0000000..244f6ff --- /dev/null +++ b/modules/translation/i18n/i18n_test.go @@ -0,0 +1,204 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package i18n + +import ( + "html/template" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLocaleStore(t *testing.T) { + testData1 := []byte(` +.dot.name = Dot Name +fmt = %[1]s %[2]s + +[section] +sub = Sub String +mixed = test value; %s +`) + + testData2 := []byte(` +fmt = %[2]s %[1]s + +[section] +sub = Changed Sub String +`) + + ls := NewLocaleStore() + require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, nil)) + require.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2, nil)) + ls.SetDefaultLang("lang1") + + lang1, _ := ls.Locale("lang1") + lang2, _ := ls.Locale("lang2") + + result := lang1.TrString("fmt", "a", "b") + assert.Equal(t, "a b", result) + + result = lang2.TrString("fmt", "a", "b") + assert.Equal(t, "b a", result) + + result = lang1.TrString("section.sub") + assert.Equal(t, "Sub String", result) + + result = lang2.TrString("section.sub") + assert.Equal(t, "Changed Sub String", result) + + langNone, _ := ls.Locale("none") + result = langNone.TrString(".dot.name") + assert.Equal(t, "Dot Name", result) + + result2 := lang2.TrHTML("section.mixed", "a&b") + assert.EqualValues(t, `test value; a&b`, result2) + + langs, descs := ls.ListLangNameDesc() + assert.ElementsMatch(t, []string{"lang1", "lang2"}, langs) + assert.ElementsMatch(t, []string{"Lang1", "Lang2"}, descs) + + found := lang1.HasKey("no-such") + assert.False(t, found) + require.NoError(t, ls.Close()) +} + +func TestLocaleStoreMoreSource(t *testing.T) { + testData1 := []byte(` +a=11 +b=12 +`) + + testData2 := []byte(` +b=21 +c=22 +`) + + ls := NewLocaleStore() + require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, testData2)) + lang1, _ := ls.Locale("lang1") + assert.Equal(t, "11", lang1.TrString("a")) + assert.Equal(t, "21", lang1.TrString("b")) + assert.Equal(t, "22", lang1.TrString("c")) +} + +type stringerPointerReceiver struct { + s string +} + +func (s *stringerPointerReceiver) String() string { + return s.s +} + +type stringerStructReceiver struct { + s string +} + +func (s stringerStructReceiver) String() string { + return s.s +} + +type errorStructReceiver struct { + s string +} + +func (e errorStructReceiver) Error() string { + return e.s +} + +type errorPointerReceiver struct { + s string +} + +func (e *errorPointerReceiver) Error() string { + return e.s +} + +func TestLocaleWithTemplate(t *testing.T) { + ls := NewLocaleStore() + require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", []byte(`key=%s`), nil)) + lang1, _ := ls.Locale("lang1") + + tmpl := template.New("test").Funcs(template.FuncMap{"tr": lang1.TrHTML}) + tmpl = template.Must(tmpl.Parse(`{{tr "key" .var}}`)) + + cases := []struct { + in any + want string + }{ + {"", "<str>"}, + {[]byte(""), "[60 98 121 116 101 115 62]"}, + {template.HTML(""), ""}, + {stringerPointerReceiver{""}, "{<stringerPointerReceiver>}"}, + {&stringerPointerReceiver{""}, "<stringerPointerReceiver ptr>"}, + {stringerStructReceiver{""}, "<stringerStructReceiver>"}, + {&stringerStructReceiver{""}, "<stringerStructReceiver ptr>"}, + {errorStructReceiver{""}, "<errorStructReceiver>"}, + {&errorStructReceiver{""}, "<errorStructReceiver ptr>"}, + {errorPointerReceiver{""}, "{<errorPointerReceiver>}"}, + {&errorPointerReceiver{""}, "<errorPointerReceiver ptr>"}, + } + + buf := &strings.Builder{} + for _, c := range cases { + buf.Reset() + require.NoError(t, tmpl.Execute(buf, map[string]any{"var": c.in})) + assert.Equal(t, c.want, buf.String()) + } +} + +func TestLocaleStoreQuirks(t *testing.T) { + const nl = "\n" + q := func(q1, s string, q2 ...string) string { + return q1 + s + strings.Join(q2, "") + } + testDataList := []struct { + in string + out string + hint string + }{ + {` xx`, `xx`, "simple, no quote"}, + {`" xx"`, ` xx`, "simple, double-quote"}, + {`' xx'`, ` xx`, "simple, single-quote"}, + {"` xx`", ` xx`, "simple, back-quote"}, + + {`x\"y`, `x\"y`, "no unescape, simple"}, + {q(`"`, `x\"y`, `"`), `"x\"y"`, "unescape, double-quote"}, + {q(`'`, `x\"y`, `'`), `x\"y`, "no unescape, single-quote"}, + {q("`", `x\"y`, "`"), `x\"y`, "no unescape, back-quote"}, + + {q(`"`, `x\"y`) + nl + "b=", `"x\"y`, "half open, double-quote"}, + {q(`'`, `x\"y`) + nl + "b=", `'x\"y`, "half open, single-quote"}, + {q("`", `x\"y`) + nl + "b=`", `x\"y` + nl + "b=", "half open, back-quote, multi-line"}, + + {`x ; y`, `x ; y`, "inline comment (;)"}, + {`x # y`, `x # y`, "inline comment (#)"}, + {`x \; y`, `x ; y`, `inline comment (\;)`}, + {`x \# y`, `x # y`, `inline comment (\#)`}, + } + + for _, testData := range testDataList { + ls := NewLocaleStore() + err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil) + lang1, _ := ls.Locale("lang1") + require.NoError(t, err, testData.hint) + assert.Equal(t, testData.out, lang1.TrString("a"), testData.hint) + require.NoError(t, ls.Close()) + } + + // TODO: Crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes + // and Crowdin always outputs quoted strings if there are quotes in the strings. + // So, Gitea's `key="quoted" unquoted` content shouldn't be used on Crowdin directly, + // it should be converted to `key="\"quoted\" unquoted"` first. + // TODO: We can not use UnescapeValueDoubleQuotes=true, because there are a lot of back-quotes in en-US.ini, + // then Crowdin will output: + // > key = "`x \" y`" + // Then Gitea will read a string with back-quotes, which is incorrect. + // TODO: Crowdin might generate multi-line strings, quoted by double-quote, it's not supported by LocaleStore + // LocaleStore uses back-quote for multi-line strings, it's not supported by Crowdin. + // TODO: Crowdin doesn't support back-quote as string quoter, it mainly uses double-quote + // so, the following line will be parsed as: value="`first", comment="second`" on Crowdin + // > a = `first; second` +} diff --git a/modules/translation/i18n/localestore.go b/modules/translation/i18n/localestore.go new file mode 100644 index 0000000..0e6ddab --- /dev/null +++ b/modules/translation/i18n/localestore.go @@ -0,0 +1,166 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package i18n + +import ( + "fmt" + "html/template" + "slices" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +// This file implements the static LocaleStore that will not watch for changes + +type locale struct { + store *localeStore + langName string + idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap +} + +var _ Locale = (*locale)(nil) + +type localeStore struct { + // After initializing has finished, these fields are read-only. + langNames []string + langDescs []string + + localeMap map[string]*locale + trKeyToIdxMap map[string]int + + defaultLang string +} + +// NewLocaleStore creates a static locale store +func NewLocaleStore() LocaleStore { + return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)} +} + +// AddLocaleByIni adds locale by ini into the store +func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error { + if _, ok := store.localeMap[langName]; ok { + return ErrLocaleAlreadyExist + } + + store.langNames = append(store.langNames, langName) + store.langDescs = append(store.langDescs, langDesc) + + l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)} + store.localeMap[l.langName] = l + + iniFile, err := setting.NewConfigProviderForLocale(source, moreSource) + if err != nil { + return fmt.Errorf("unable to load ini: %w", err) + } + + for _, section := range iniFile.Sections() { + for _, key := range section.Keys() { + var trKey string + // see https://codeberg.org/forgejo/discussions/issues/104 + // https://github.com/WeblateOrg/weblate/issues/10831 + // for an explanation of why "common" is an alternative + if section.Name() == "" || section.Name() == "DEFAULT" || section.Name() == "common" { + trKey = key.Name() + } else { + trKey = section.Name() + "." + key.Name() + } + idx, ok := store.trKeyToIdxMap[trKey] + if !ok { + idx = len(store.trKeyToIdxMap) + store.trKeyToIdxMap[trKey] = idx + } + l.idxToMsgMap[idx] = key.Value() + } + } + + return nil +} + +func (store *localeStore) HasLang(langName string) bool { + _, ok := store.localeMap[langName] + return ok +} + +func (store *localeStore) ListLangNameDesc() (names, desc []string) { + return store.langNames, store.langDescs +} + +// SetDefaultLang sets default language as a fallback +func (store *localeStore) SetDefaultLang(lang string) { + store.defaultLang = lang +} + +// Locale returns the locale for the lang or the default language +func (store *localeStore) Locale(lang string) (Locale, bool) { + l, found := store.localeMap[lang] + if !found { + var ok bool + l, ok = store.localeMap[store.defaultLang] + if !ok { + // no default - return an empty locale + l = &locale{store: store, idxToMsgMap: make(map[int]string)} + } + } + return l, found +} + +func (store *localeStore) Close() error { + return nil +} + +func (l *locale) TrString(trKey string, trArgs ...any) string { + format := trKey + + idx, ok := l.store.trKeyToIdxMap[trKey] + found := false + if ok { + if msg, ok := l.idxToMsgMap[idx]; ok { + format = msg // use the found translation + found = true + } else if def, ok := l.store.localeMap[l.store.defaultLang]; ok { + // try to use default locale's translation + if msg, ok := def.idxToMsgMap[idx]; ok { + format = msg + found = true + } + } + } + if !found { + log.Error("Missing translation %q", trKey) + } + + msg, err := Format(format, trArgs...) + if err != nil { + log.Error("Error whilst formatting %q in %s: %v", trKey, l.langName, err) + } + return msg +} + +func (l *locale) TrHTML(trKey string, trArgs ...any) template.HTML { + args := slices.Clone(trArgs) + for i, v := range args { + switch v := v.(type) { + case nil, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, template.HTML: + // for most basic types (including template.HTML which is safe), just do nothing and use it + case string: + args[i] = template.HTMLEscapeString(v) + case fmt.Stringer: + args[i] = template.HTMLEscapeString(v.String()) + default: + args[i] = template.HTMLEscapeString(fmt.Sprint(v)) + } + } + return template.HTML(l.TrString(trKey, args...)) +} + +// HasKey returns whether a key is present in this locale or not +func (l *locale) HasKey(trKey string) bool { + idx, ok := l.store.trKeyToIdxMap[trKey] + if !ok { + return false + } + _, ok = l.idxToMsgMap[idx] + return ok +} -- cgit v1.2.3