diff options
author | dark-angel <70754989+inferno-umar@users.noreply.github.com> | 2024-02-07 09:57:16 +0100 |
---|---|---|
committer | Earl Warren <contact@earl-warren.org> | 2024-02-10 10:53:43 +0100 |
commit | 00370f17a4f7cba31e8a2617ebdbff139739d6d4 (patch) | |
tree | 3e4e0609057e0fcd0254d50f22bf22c58f9d1956 /modules/indexer/code | |
parent | [gitea] Remove lightningcss (#29070) (diff) | |
download | forgejo-00370f17a4f7cba31e8a2617ebdbff139739d6d4.tar.xz forgejo-00370f17a4f7cba31e8a2617ebdbff139739d6d4.zip |
[gitea] fix: Elasticsearch: Request Entity Too Large #28117 (#29062)
Fix for gitea putting everything into one request without batching and
sending it to Elasticsearch for indexing as issued in #28117
This issue occured in large repositories while Gitea tries to
index the code using ElasticSearch.
I've applied necessary changes that takes batch length from below config
(app.ini)
```
[queue.code_indexer]
BATCH_LENGTH=<length_int>
```
and batches all requests to Elasticsearch in chunks as configured in the
above config
(cherry picked from commit 5c0fc9087211f01375f208d679a1e6de0685320c)
Diffstat (limited to 'modules/indexer/code')
-rw-r--r-- | modules/indexer/code/elasticsearch/elasticsearch.go | 16 |
1 files changed, 11 insertions, 5 deletions
diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index 2fadbfeb06..0f70f13485 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -180,11 +180,17 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st } if len(reqs) > 0 { - _, err := b.inner.Client.Bulk(). - Index(b.inner.VersionedIndexName()). - Add(reqs...). - Do(ctx) - return err + esBatchSize := 50 + + for i := 0; i < len(reqs); i += esBatchSize { + _, err := b.inner.Client.Bulk(). + Index(b.inner.VersionedIndexName()). + Add(reqs[i:min(i+esBatchSize, len(reqs))]...). + Do(ctx) + if err != nil { + return err + } + } } return nil } |