summaryrefslogtreecommitdiffstats
path: root/modules/indexer/internal
diff options
context:
space:
mode:
authorDaniel Baumann <daniel@debian.org>2024-10-18 20:33:49 +0200
committerDaniel Baumann <daniel@debian.org>2024-12-12 23:57:56 +0100
commite68b9d00a6e05b3a941f63ffb696f91e554ac5ec (patch)
tree97775d6c13b0f416af55314eb6a89ef792474615 /modules/indexer/internal
parentInitial commit. (diff)
downloadforgejo-e68b9d00a6e05b3a941f63ffb696f91e554ac5ec.tar.xz
forgejo-e68b9d00a6e05b3a941f63ffb696f91e554ac5ec.zip
Adding upstream version 9.0.3.
Signed-off-by: Daniel Baumann <daniel@debian.org>
Diffstat (limited to '')
-rw-r--r--modules/indexer/internal/base32.go21
-rw-r--r--modules/indexer/internal/bleve/batch.go58
-rw-r--r--modules/indexer/internal/bleve/indexer.go102
-rw-r--r--modules/indexer/internal/bleve/metadata.go55
-rw-r--r--modules/indexer/internal/bleve/metadata_test.go28
-rw-r--r--modules/indexer/internal/bleve/query.go56
-rw-r--r--modules/indexer/internal/bleve/util.go48
-rw-r--r--modules/indexer/internal/db/indexer.go34
-rw-r--r--modules/indexer/internal/elasticsearch/indexer.go93
-rw-r--r--modules/indexer/internal/elasticsearch/util.go68
-rw-r--r--modules/indexer/internal/indexer.go37
-rw-r--r--modules/indexer/internal/meilisearch/filter.go119
-rw-r--r--modules/indexer/internal/meilisearch/indexer.go88
-rw-r--r--modules/indexer/internal/meilisearch/util.go38
-rw-r--r--modules/indexer/internal/paginator.go34
15 files changed, 879 insertions, 0 deletions
diff --git a/modules/indexer/internal/base32.go b/modules/indexer/internal/base32.go
new file mode 100644
index 0000000..aca756c
--- /dev/null
+++ b/modules/indexer/internal/base32.go
@@ -0,0 +1,21 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+ "fmt"
+ "strconv"
+)
+
+func Base36(i int64) string {
+ return strconv.FormatInt(i, 36)
+}
+
+func ParseBase36(s string) (int64, error) {
+ i, err := strconv.ParseInt(s, 36, 64)
+ if err != nil {
+ return 0, fmt.Errorf("invalid base36 integer %q: %w", s, err)
+ }
+ return i, nil
+}
diff --git a/modules/indexer/internal/bleve/batch.go b/modules/indexer/internal/bleve/batch.go
new file mode 100644
index 0000000..ed5ef07
--- /dev/null
+++ b/modules/indexer/internal/bleve/batch.go
@@ -0,0 +1,58 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+ "github.com/blevesearch/bleve/v2"
+)
+
+// FlushingBatch is a batch of operations that automatically flushes to the
+// underlying index once it reaches a certain size.
+type FlushingBatch struct {
+ maxBatchSize int
+ batch *bleve.Batch
+ index bleve.Index
+}
+
+// NewFlushingBatch creates a new flushing batch for the specified index. Once
+// the number of operations in the batch reaches the specified limit, the batch
+// automatically flushes its operations to the index.
+func NewFlushingBatch(index bleve.Index, maxBatchSize int) *FlushingBatch {
+ return &FlushingBatch{
+ maxBatchSize: maxBatchSize,
+ batch: index.NewBatch(),
+ index: index,
+ }
+}
+
+// Index add a new index to batch
+func (b *FlushingBatch) Index(id string, data any) error {
+ if err := b.batch.Index(id, data); err != nil {
+ return err
+ }
+ return b.flushIfFull()
+}
+
+// Delete add a delete index to batch
+func (b *FlushingBatch) Delete(id string) error {
+ b.batch.Delete(id)
+ return b.flushIfFull()
+}
+
+func (b *FlushingBatch) flushIfFull() error {
+ if b.batch.Size() < b.maxBatchSize {
+ return nil
+ }
+ return b.Flush()
+}
+
+// Flush submit the batch and create a new one
+func (b *FlushingBatch) Flush() error {
+ err := b.index.Batch(b.batch)
+ if err != nil {
+ return err
+ }
+ b.batch = b.index.NewBatch()
+ return nil
+}
diff --git a/modules/indexer/internal/bleve/indexer.go b/modules/indexer/internal/bleve/indexer.go
new file mode 100644
index 0000000..1435d2f
--- /dev/null
+++ b/modules/indexer/internal/bleve/indexer.go
@@ -0,0 +1,102 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+ "context"
+ "fmt"
+
+ "code.gitea.io/gitea/modules/indexer/internal"
+ "code.gitea.io/gitea/modules/log"
+
+ "github.com/blevesearch/bleve/v2"
+ "github.com/blevesearch/bleve/v2/mapping"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic bleve indexer implementation
+type Indexer struct {
+ Indexer bleve.Index
+
+ indexDir string
+ version int
+ mappingGetter MappingGetter
+}
+
+type MappingGetter func() (mapping.IndexMapping, error)
+
+func NewIndexer(indexDir string, version int, mappingGetter func() (mapping.IndexMapping, error)) *Indexer {
+ return &Indexer{
+ indexDir: indexDir,
+ version: version,
+ mappingGetter: mappingGetter,
+ }
+}
+
+// Init initializes the indexer
+func (i *Indexer) Init(_ context.Context) (bool, error) {
+ if i == nil {
+ return false, fmt.Errorf("cannot init nil indexer")
+ }
+
+ if i.Indexer != nil {
+ return false, fmt.Errorf("indexer is already initialized")
+ }
+
+ indexer, version, err := openIndexer(i.indexDir, i.version)
+ if err != nil {
+ return false, err
+ }
+ if indexer != nil {
+ i.Indexer = indexer
+ return true, nil
+ }
+
+ if version != 0 {
+ log.Warn("Found older bleve index with version %d, Forgejo will remove it and rebuild", version)
+ }
+
+ indexMapping, err := i.mappingGetter()
+ if err != nil {
+ return false, err
+ }
+
+ indexer, err = bleve.New(i.indexDir, indexMapping)
+ if err != nil {
+ return false, err
+ }
+
+ if err = writeIndexMetadata(i.indexDir, &IndexMetadata{
+ Version: i.version,
+ }); err != nil {
+ return false, err
+ }
+
+ i.Indexer = indexer
+
+ return false, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(_ context.Context) error {
+ if i == nil {
+ return fmt.Errorf("cannot ping nil indexer")
+ }
+ if i.Indexer == nil {
+ return fmt.Errorf("indexer is not initialized")
+ }
+ return nil
+}
+
+func (i *Indexer) Close() {
+ if i == nil || i.Indexer == nil {
+ return
+ }
+
+ if err := i.Indexer.Close(); err != nil {
+ log.Error("Failed to close bleve indexer in %q: %v", i.indexDir, err)
+ }
+ i.Indexer = nil
+}
diff --git a/modules/indexer/internal/bleve/metadata.go b/modules/indexer/internal/bleve/metadata.go
new file mode 100644
index 0000000..3c570ab
--- /dev/null
+++ b/modules/indexer/internal/bleve/metadata.go
@@ -0,0 +1,55 @@
+// Copyright 2024 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// Copied and modified from https://github.com/ethantkoenig/rupture (MIT License)
+
+package bleve
+
+import (
+ "os"
+ "path/filepath"
+
+ "code.gitea.io/gitea/modules/json"
+)
+
+const metaFilename = "rupture_meta.json"
+
+func indexMetadataPath(dir string) string {
+ return filepath.Join(dir, metaFilename)
+}
+
+// IndexMetadata contains metadata about a bleve index.
+type IndexMetadata struct {
+ // The version of the data in the index. This can be useful for tracking
+ // schema changes or data migrations.
+ Version int `json:"version"`
+}
+
+// readIndexMetadata returns the metadata for the index at the specified path.
+// If no such index metadata exists, an empty metadata and a nil error are
+// returned.
+func readIndexMetadata(path string) (*IndexMetadata, error) {
+ meta := &IndexMetadata{}
+ metaPath := indexMetadataPath(path)
+ if _, err := os.Stat(metaPath); os.IsNotExist(err) {
+ return meta, nil
+ } else if err != nil {
+ return nil, err
+ }
+
+ metaBytes, err := os.ReadFile(metaPath)
+ if err != nil {
+ return nil, err
+ }
+ return meta, json.Unmarshal(metaBytes, &meta)
+}
+
+// writeIndexMetadata writes metadata for the index at the specified path.
+func writeIndexMetadata(path string, meta *IndexMetadata) error {
+ metaBytes, err := json.Marshal(meta)
+ if err != nil {
+ return err
+ }
+
+ return os.WriteFile(indexMetadataPath(path), metaBytes, 0o644)
+}
diff --git a/modules/indexer/internal/bleve/metadata_test.go b/modules/indexer/internal/bleve/metadata_test.go
new file mode 100644
index 0000000..31603a9
--- /dev/null
+++ b/modules/indexer/internal/bleve/metadata_test.go
@@ -0,0 +1,28 @@
+// Copyright 2024 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// Copied and modified from https://github.com/ethantkoenig/rupture (MIT License)
+
+package bleve
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestMetadata(t *testing.T) {
+ dir := t.TempDir()
+
+ meta, err := readIndexMetadata(dir)
+ require.NoError(t, err)
+ assert.Equal(t, &IndexMetadata{}, meta)
+
+ meta.Version = 24
+ require.NoError(t, writeIndexMetadata(dir, meta))
+
+ meta, err = readIndexMetadata(dir)
+ require.NoError(t, err)
+ assert.EqualValues(t, 24, meta.Version)
+}
diff --git a/modules/indexer/internal/bleve/query.go b/modules/indexer/internal/bleve/query.go
new file mode 100644
index 0000000..90626da
--- /dev/null
+++ b/modules/indexer/internal/bleve/query.go
@@ -0,0 +1,56 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+ "code.gitea.io/gitea/modules/optional"
+
+ "github.com/blevesearch/bleve/v2"
+ "github.com/blevesearch/bleve/v2/search/query"
+)
+
+// NumericEqualityQuery generates a numeric equality query for the given value and field
+func NumericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
+ f := float64(value)
+ tru := true // codespell-ignore
+ q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru) // codespell-ignore
+ q.SetField(field)
+ return q
+}
+
+// MatchPhraseQuery generates a match phrase query for the given phrase, field and analyzer
+func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query.MatchPhraseQuery {
+ q := bleve.NewMatchPhraseQuery(matchPhrase)
+ q.FieldVal = field
+ q.Analyzer = analyzer
+ q.Fuzziness = fuzziness
+ return q
+}
+
+// BoolFieldQuery generates a bool field query for the given value and field
+func BoolFieldQuery(value bool, field string) *query.BoolFieldQuery {
+ q := bleve.NewBoolFieldQuery(value)
+ q.SetField(field)
+ return q
+}
+
+func NumericRangeInclusiveQuery(min, max optional.Option[int64], field string) *query.NumericRangeQuery {
+ var minF, maxF *float64
+ var minI, maxI *bool
+ if min.Has() {
+ minF = new(float64)
+ *minF = float64(min.Value())
+ minI = new(bool)
+ *minI = true
+ }
+ if max.Has() {
+ maxF = new(float64)
+ *maxF = float64(max.Value())
+ maxI = new(bool)
+ *maxI = true
+ }
+ q := bleve.NewNumericRangeInclusiveQuery(minF, maxF, minI, maxI)
+ q.SetField(field)
+ return q
+}
diff --git a/modules/indexer/internal/bleve/util.go b/modules/indexer/internal/bleve/util.go
new file mode 100644
index 0000000..d05b679
--- /dev/null
+++ b/modules/indexer/internal/bleve/util.go
@@ -0,0 +1,48 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+ "errors"
+ "os"
+
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/util"
+
+ "github.com/blevesearch/bleve/v2"
+ "github.com/blevesearch/bleve/v2/index/upsidedown"
+)
+
+// openIndexer open the index at the specified path, checking for metadata
+// updates and bleve version updates. If index needs to be created (or
+// re-created), returns (nil, nil)
+func openIndexer(path string, latestVersion int) (bleve.Index, int, error) {
+ _, err := os.Stat(path)
+ if err != nil && os.IsNotExist(err) {
+ return nil, 0, nil
+ } else if err != nil {
+ return nil, 0, err
+ }
+
+ metadata, err := readIndexMetadata(path)
+ if err != nil {
+ return nil, 0, err
+ }
+ if metadata.Version < latestVersion {
+ // the indexer is using a previous version, so we should delete it and
+ // re-populate
+ return nil, metadata.Version, util.RemoveAll(path)
+ }
+
+ index, err := bleve.Open(path)
+ if err != nil {
+ if errors.Is(err, upsidedown.IncompatibleVersion) {
+ log.Warn("Indexer was built with a previous version of bleve, deleting and rebuilding")
+ return nil, 0, util.RemoveAll(path)
+ }
+ return nil, 0, err
+ }
+
+ return index, 0, nil
+}
diff --git a/modules/indexer/internal/db/indexer.go b/modules/indexer/internal/db/indexer.go
new file mode 100644
index 0000000..3deec83
--- /dev/null
+++ b/modules/indexer/internal/db/indexer.go
@@ -0,0 +1,34 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package db
+
+import (
+ "context"
+
+ "code.gitea.io/gitea/modules/indexer/internal"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic db indexer implementation
+type Indexer struct{}
+
+// Init initializes the indexer
+func (i *Indexer) Init(_ context.Context) (bool, error) {
+ // Return true to indicate that the index was opened/existed.
+ // So that the indexer will not try to populate the index, the data is already there.
+ return true, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(_ context.Context) error {
+ // No need to ping database to check if it is available.
+ // If the database goes down, Gitea will go down, so nobody will care if the indexer is available.
+ return nil
+}
+
+// Close closes the indexer
+func (i *Indexer) Close() {
+ // nothing to do
+}
diff --git a/modules/indexer/internal/elasticsearch/indexer.go b/modules/indexer/internal/elasticsearch/indexer.go
new file mode 100644
index 0000000..395eea3
--- /dev/null
+++ b/modules/indexer/internal/elasticsearch/indexer.go
@@ -0,0 +1,93 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package elasticsearch
+
+import (
+ "context"
+ "fmt"
+
+ "code.gitea.io/gitea/modules/indexer/internal"
+
+ "github.com/olivere/elastic/v7"
+)
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a basic elasticsearch indexer implementation
+type Indexer struct {
+ Client *elastic.Client
+
+ url string
+ indexName string
+ version int
+ mapping string
+}
+
+func NewIndexer(url, indexName string, version int, mapping string) *Indexer {
+ return &Indexer{
+ url: url,
+ indexName: indexName,
+ version: version,
+ mapping: mapping,
+ }
+}
+
+// Init initializes the indexer
+func (i *Indexer) Init(ctx context.Context) (bool, error) {
+ if i == nil {
+ return false, fmt.Errorf("cannot init nil indexer")
+ }
+ if i.Client != nil {
+ return false, fmt.Errorf("indexer is already initialized")
+ }
+
+ client, err := i.initClient()
+ if err != nil {
+ return false, err
+ }
+ i.Client = client
+
+ exists, err := i.Client.IndexExists(i.VersionedIndexName()).Do(ctx)
+ if err != nil {
+ return false, err
+ }
+ if exists {
+ return true, nil
+ }
+
+ if err := i.createIndex(ctx); err != nil {
+ return false, err
+ }
+
+ return exists, nil
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(ctx context.Context) error {
+ if i == nil {
+ return fmt.Errorf("cannot ping nil indexer")
+ }
+ if i.Client == nil {
+ return fmt.Errorf("indexer is not initialized")
+ }
+
+ resp, err := i.Client.ClusterHealth().Do(ctx)
+ if err != nil {
+ return err
+ }
+ if resp.Status != "green" && resp.Status != "yellow" {
+ // It's healthy if the status is green, and it's available if the status is yellow,
+ // see https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html
+ return fmt.Errorf("status of elasticsearch cluster is %s", resp.Status)
+ }
+ return nil
+}
+
+// Close closes the indexer
+func (i *Indexer) Close() {
+ if i == nil {
+ return
+ }
+ i.Client = nil
+}
diff --git a/modules/indexer/internal/elasticsearch/util.go b/modules/indexer/internal/elasticsearch/util.go
new file mode 100644
index 0000000..18cb152
--- /dev/null
+++ b/modules/indexer/internal/elasticsearch/util.go
@@ -0,0 +1,68 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package elasticsearch
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "code.gitea.io/gitea/modules/log"
+
+ "github.com/olivere/elastic/v7"
+)
+
+// VersionedIndexName returns the full index name with version
+func (i *Indexer) VersionedIndexName() string {
+ return versionedIndexName(i.indexName, i.version)
+}
+
+func versionedIndexName(indexName string, version int) string {
+ if version == 0 {
+ // Old index name without version
+ return indexName
+ }
+ return fmt.Sprintf("%s.v%d", indexName, version)
+}
+
+func (i *Indexer) createIndex(ctx context.Context) error {
+ createIndex, err := i.Client.CreateIndex(i.VersionedIndexName()).BodyString(i.mapping).Do(ctx)
+ if err != nil {
+ return err
+ }
+ if !createIndex.Acknowledged {
+ return fmt.Errorf("create index %s with %s failed", i.VersionedIndexName(), i.mapping)
+ }
+
+ i.checkOldIndexes(ctx)
+
+ return nil
+}
+
+func (i *Indexer) initClient() (*elastic.Client, error) {
+ opts := []elastic.ClientOptionFunc{
+ elastic.SetURL(i.url),
+ elastic.SetSniff(false),
+ elastic.SetHealthcheckInterval(10 * time.Second),
+ elastic.SetGzip(false),
+ }
+
+ logger := log.GetLogger(log.DEFAULT)
+
+ opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace}))
+ opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info}))
+ opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error}))
+
+ return elastic.NewClient(opts...)
+}
+
+func (i *Indexer) checkOldIndexes(ctx context.Context) {
+ for v := 0; v < i.version; v++ {
+ indexName := versionedIndexName(i.indexName, v)
+ exists, err := i.Client.IndexExists(indexName).Do(ctx)
+ if err == nil && exists {
+ log.Warn("Found older elasticsearch index named %q, Forgejo will keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", indexName)
+ }
+ }
+}
diff --git a/modules/indexer/internal/indexer.go b/modules/indexer/internal/indexer.go
new file mode 100644
index 0000000..c7f356d
--- /dev/null
+++ b/modules/indexer/internal/indexer.go
@@ -0,0 +1,37 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+ "context"
+ "fmt"
+)
+
+// Indexer defines an basic indexer interface
+type Indexer interface {
+ // Init initializes the indexer
+ // returns true if the index was opened/existed (with data populated), false if it was created/not-existed (with no data)
+ Init(ctx context.Context) (bool, error)
+ // Ping checks if the indexer is available
+ Ping(ctx context.Context) error
+ // Close closes the indexer
+ Close()
+}
+
+// NewDummyIndexer returns a dummy indexer
+func NewDummyIndexer() Indexer {
+ return &dummyIndexer{}
+}
+
+type dummyIndexer struct{}
+
+func (d *dummyIndexer) Init(ctx context.Context) (bool, error) {
+ return false, fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Ping(ctx context.Context) error {
+ return fmt.Errorf("indexer is not ready")
+}
+
+func (d *dummyIndexer) Close() {}
diff --git a/modules/indexer/internal/meilisearch/filter.go b/modules/indexer/internal/meilisearch/filter.go
new file mode 100644
index 0000000..593177f
--- /dev/null
+++ b/modules/indexer/internal/meilisearch/filter.go
@@ -0,0 +1,119 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package meilisearch
+
+import (
+ "fmt"
+ "strings"
+)
+
+// Filter represents a filter for meilisearch queries.
+// It's just a simple wrapper around a string.
+// DO NOT assume that it is a complete implementation.
+type Filter interface {
+ Statement() string
+}
+
+type FilterAnd struct {
+ filters []Filter
+}
+
+func (f *FilterAnd) Statement() string {
+ var statements []string
+ for _, filter := range f.filters {
+ if s := filter.Statement(); s != "" {
+ statements = append(statements, fmt.Sprintf("(%s)", s))
+ }
+ }
+ return strings.Join(statements, " AND ")
+}
+
+func (f *FilterAnd) And(filter Filter) *FilterAnd {
+ f.filters = append(f.filters, filter)
+ return f
+}
+
+type FilterOr struct {
+ filters []Filter
+}
+
+func (f *FilterOr) Statement() string {
+ var statements []string
+ for _, filter := range f.filters {
+ if s := filter.Statement(); s != "" {
+ statements = append(statements, fmt.Sprintf("(%s)", s))
+ }
+ }
+ return strings.Join(statements, " OR ")
+}
+
+func (f *FilterOr) Or(filter Filter) *FilterOr {
+ f.filters = append(f.filters, filter)
+ return f
+}
+
+type FilterIn string
+
+// NewFilterIn creates a new FilterIn.
+// It supports int64 only, to avoid extra works to handle strings with special characters.
+func NewFilterIn[T int64](field string, values ...T) FilterIn {
+ if len(values) == 0 {
+ return ""
+ }
+ vs := make([]string, len(values))
+ for i, v := range values {
+ vs[i] = fmt.Sprintf("%v", v)
+ }
+ return FilterIn(fmt.Sprintf("%s IN [%v]", field, strings.Join(vs, ", ")))
+}
+
+func (f FilterIn) Statement() string {
+ return string(f)
+}
+
+type FilterEq string
+
+// NewFilterEq creates a new FilterEq.
+// It supports int64 and bool only, to avoid extra works to handle strings with special characters.
+func NewFilterEq[T bool | int64](field string, value T) FilterEq {
+ return FilterEq(fmt.Sprintf("%s = %v", field, value))
+}
+
+func (f FilterEq) Statement() string {
+ return string(f)
+}
+
+type FilterNot string
+
+func NewFilterNot(filter Filter) FilterNot {
+ return FilterNot(fmt.Sprintf("NOT (%s)", filter.Statement()))
+}
+
+func (f FilterNot) Statement() string {
+ return string(f)
+}
+
+type FilterGte string
+
+// NewFilterGte creates a new FilterGte.
+// It supports int64 only, to avoid extra works to handle strings with special characters.
+func NewFilterGte[T int64](field string, value T) FilterGte {
+ return FilterGte(fmt.Sprintf("%s >= %v", field, value))
+}
+
+func (f FilterGte) Statement() string {
+ return string(f)
+}
+
+type FilterLte string
+
+// NewFilterLte creates a new FilterLte.
+// It supports int64 only, to avoid extra works to handle strings with special characters.
+func NewFilterLte[T int64](field string, value T) FilterLte {
+ return FilterLte(fmt.Sprintf("%s <= %v", field, value))
+}
+
+func (f FilterLte) Statement() string {
+ return string(f)
+}
diff --git a/modules/indexer/internal/meilisearch/indexer.go b/modules/indexer/internal/meilisearch/indexer.go
new file mode 100644
index 0000000..feac1d0
--- /dev/null
+++ b/modules/indexer/internal/meilisearch/indexer.go
@@ -0,0 +1,88 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package meilisearch
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/meilisearch/meilisearch-go"
+)
+
+// Indexer represents a basic meilisearch indexer implementation
+type Indexer struct {
+ Client meilisearch.ServiceManager
+
+ url, apiKey string
+ indexName string
+ version int
+ settings *meilisearch.Settings
+}
+
+func NewIndexer(url, apiKey, indexName string, version int, settings *meilisearch.Settings) *Indexer {
+ return &Indexer{
+ url: url,
+ apiKey: apiKey,
+ indexName: indexName,
+ version: version,
+ settings: settings,
+ }
+}
+
+// Init initializes the indexer
+func (i *Indexer) Init(_ context.Context) (bool, error) {
+ if i == nil {
+ return false, fmt.Errorf("cannot init nil indexer")
+ }
+
+ if i.Client != nil {
+ return false, fmt.Errorf("indexer is already initialized")
+ }
+
+ i.Client = meilisearch.New(i.url, meilisearch.WithAPIKey(i.apiKey))
+
+ _, err := i.Client.GetIndex(i.VersionedIndexName())
+ if err == nil {
+ return true, nil
+ }
+ _, err = i.Client.CreateIndex(&meilisearch.IndexConfig{
+ Uid: i.VersionedIndexName(),
+ PrimaryKey: "id",
+ })
+ if err != nil {
+ return false, err
+ }
+
+ i.checkOldIndexes()
+
+ _, err = i.Client.Index(i.VersionedIndexName()).UpdateSettings(i.settings)
+ return false, err
+}
+
+// Ping checks if the indexer is available
+func (i *Indexer) Ping(ctx context.Context) error {
+ if i == nil {
+ return fmt.Errorf("cannot ping nil indexer")
+ }
+ if i.Client == nil {
+ return fmt.Errorf("indexer is not initialized")
+ }
+ resp, err := i.Client.Health()
+ if err != nil {
+ return err
+ }
+ if resp.Status != "available" {
+ // See https://docs.meilisearch.com/reference/api/health.html#status
+ return fmt.Errorf("status of meilisearch is not available: %s", resp.Status)
+ }
+ return nil
+}
+
+// Close closes the indexer
+func (i *Indexer) Close() {
+ if i == nil {
+ return
+ }
+ i.Client = nil
+}
diff --git a/modules/indexer/internal/meilisearch/util.go b/modules/indexer/internal/meilisearch/util.go
new file mode 100644
index 0000000..845bdb6
--- /dev/null
+++ b/modules/indexer/internal/meilisearch/util.go
@@ -0,0 +1,38 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package meilisearch
+
+import (
+ "fmt"
+
+ "code.gitea.io/gitea/modules/log"
+)
+
+// VersionedIndexName returns the full index name with version
+func (i *Indexer) VersionedIndexName() string {
+ return versionedIndexName(i.indexName, i.version)
+}
+
+func versionedIndexName(indexName string, version int) string {
+ if version == 0 {
+ // Old index name without version
+ return indexName
+ }
+
+ // The format of the index name is <index_name>_v<version>, not <index_name>.v<version> like elasticsearch.
+ // Because meilisearch does not support "." in index name, it should contain only alphanumeric characters, hyphens (-) and underscores (_).
+ // See https://www.meilisearch.com/docs/learn/core_concepts/indexes#index-uid
+
+ return fmt.Sprintf("%s_v%d", indexName, version)
+}
+
+func (i *Indexer) checkOldIndexes() {
+ for v := 0; v < i.version; v++ {
+ indexName := versionedIndexName(i.indexName, v)
+ _, err := i.Client.GetIndex(indexName)
+ if err == nil {
+ log.Warn("Found older meilisearch index named %q, Forgejo will keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", indexName)
+ }
+ }
+}
diff --git a/modules/indexer/internal/paginator.go b/modules/indexer/internal/paginator.go
new file mode 100644
index 0000000..ee204bf
--- /dev/null
+++ b/modules/indexer/internal/paginator.go
@@ -0,0 +1,34 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package internal
+
+import (
+ "math"
+
+ "code.gitea.io/gitea/models/db"
+)
+
+// ParsePaginator parses a db.Paginator into a skip and limit
+func ParsePaginator(paginator *db.ListOptions, max ...int) (int, int) {
+ // Use a very large number to indicate no limit
+ unlimited := math.MaxInt32
+ if len(max) > 0 {
+ // Some indexer engines have a limit on the page size, respect that
+ unlimited = max[0]
+ }
+
+ if paginator == nil || paginator.IsListAll() {
+ // It shouldn't happen. In actual usage scenarios, there should not be requests to search all.
+ // But if it does happen, respect it and return "unlimited".
+ // And it's also useful for testing.
+ return 0, unlimited
+ }
+
+ if paginator.PageSize == 0 {
+ // Do not return any results when searching, it's used to get the total count only.
+ return 0, 0
+ }
+
+ return paginator.GetSkipTake()
+}