summaryrefslogtreecommitdiffstats
path: root/modules/indexer/code
diff options
context:
space:
mode:
authordark-angel <70754989+inferno-umar@users.noreply.github.com>2024-02-07 09:57:16 +0100
committerEarl Warren <contact@earl-warren.org>2024-02-10 10:53:43 +0100
commit00370f17a4f7cba31e8a2617ebdbff139739d6d4 (patch)
tree3e4e0609057e0fcd0254d50f22bf22c58f9d1956 /modules/indexer/code
parent[gitea] Remove lightningcss (#29070) (diff)
downloadforgejo-00370f17a4f7cba31e8a2617ebdbff139739d6d4.tar.xz
forgejo-00370f17a4f7cba31e8a2617ebdbff139739d6d4.zip
[gitea] fix: Elasticsearch: Request Entity Too Large #28117 (#29062)
Fix for gitea putting everything into one request without batching and sending it to Elasticsearch for indexing as issued in #28117 This issue occured in large repositories while Gitea tries to index the code using ElasticSearch. I've applied necessary changes that takes batch length from below config (app.ini) ``` [queue.code_indexer] BATCH_LENGTH=<length_int> ``` and batches all requests to Elasticsearch in chunks as configured in the above config (cherry picked from commit 5c0fc9087211f01375f208d679a1e6de0685320c)
Diffstat (limited to 'modules/indexer/code')
-rw-r--r--modules/indexer/code/elasticsearch/elasticsearch.go16
1 files changed, 11 insertions, 5 deletions
diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go
index 2fadbfeb06..0f70f13485 100644
--- a/modules/indexer/code/elasticsearch/elasticsearch.go
+++ b/modules/indexer/code/elasticsearch/elasticsearch.go
@@ -180,11 +180,17 @@ func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha st
}
if len(reqs) > 0 {
- _, err := b.inner.Client.Bulk().
- Index(b.inner.VersionedIndexName()).
- Add(reqs...).
- Do(ctx)
- return err
+ esBatchSize := 50
+
+ for i := 0; i < len(reqs); i += esBatchSize {
+ _, err := b.inner.Client.Bulk().
+ Index(b.inner.VersionedIndexName()).
+ Add(reqs[i:min(i+esBatchSize, len(reqs))]...).
+ Do(ctx)
+ if err != nil {
+ return err
+ }
+ }
}
return nil
}