From fe5adbbbdc0862736de37026808d4cd72adf4dab Mon Sep 17 00:00:00 2001 From: Royce Remer Date: Tue, 29 Oct 2024 22:41:55 -0700 Subject: Add new [lfs_client].BATCH_SIZE and [server].LFS_MAX_BATCH_SIZE config settings. (#32307) This contains two backwards-compatible changes: * in the lfs http_client, the number of lfs oids requested per batch is loaded from lfs_client#BATCH_SIZE and defaulted to the previous value of 20 * in the lfs server/service, the max number of lfs oids allowed in a batch api request is loaded from server#LFS_MAX_BATCH_SIZE and defaults to 'nil' which equates to the previous behavior of 'infinite' This fixes #32306 --------- Signed-off-by: Royce Remer Co-authored-by: wxiaoguang (cherry picked from commit c60e4dc1095ef90a790582cacfad27c972637bb2) Conflicts: - services/lfs/server.go Conflict due to our Quota implementation. Resolved by manually adding the change after the quota check. --- services/lfs/server.go | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'services') diff --git a/services/lfs/server.go b/services/lfs/server.go index a300de19c4..225dfdb024 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -192,6 +192,11 @@ func BatchHandler(ctx *context.Context) { } } + if setting.LFS.MaxBatchSize != 0 && len(br.Objects) > setting.LFS.MaxBatchSize { + writeStatus(ctx, http.StatusRequestEntityTooLarge) + return + } + contentStore := lfs_module.NewContentStore() var responseObjects []*lfs_module.ObjectResponse -- cgit v1.2.3 From 4aa61601c3db92fcd6f0eac55c774d6ab72156b8 Mon Sep 17 00:00:00 2001 From: Oleksandr Redko Date: Wed, 30 Oct 2024 21:36:24 +0200 Subject: refactor: remove redundant err declarations (#32381) (cherry picked from commit f4d3aaeeb9e1b11c5495e4608a3f52f316c35758) Conflicts: - modules/charset/charset_test.go Resolved by manually changing a `=` to `:=`, as per the original patch. Conflict was due to `require.NoError`. --- build/generate-emoji.go | 4 ---- models/git/lfs.go | 2 -- models/issues/label_test.go | 3 +-- modules/charset/charset_test.go | 4 +--- modules/markup/markdown/goldmark.go | 3 +-- routers/api/v1/repo/repo.go | 1 - routers/api/v1/user/repo.go | 1 - routers/web/repo/actions/view.go | 1 - routers/web/repo/activity.go | 1 - routers/web/repo/view.go | 1 - services/issue/template.go | 2 -- 11 files changed, 3 insertions(+), 20 deletions(-) (limited to 'services') diff --git a/build/generate-emoji.go b/build/generate-emoji.go index 5a88e456ee..98c2f15d75 100644 --- a/build/generate-emoji.go +++ b/build/generate-emoji.go @@ -53,8 +53,6 @@ func (e Emoji) MarshalJSON() ([]byte, error) { } func main() { - var err error - flag.Parse() // generate data @@ -83,8 +81,6 @@ var replacer = strings.NewReplacer( var emojiRE = regexp.MustCompile(`\{Emoji:"([^"]*)"`) func generate() ([]byte, error) { - var err error - // load gemoji data res, err := http.Get(gemojiURL) if err != nil { diff --git a/models/git/lfs.go b/models/git/lfs.go index 44b741c4c8..635d70d9bc 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -136,8 +136,6 @@ var ErrLFSObjectNotExist = db.ErrNotExist{Resource: "LFS Meta object"} // NewLFSMetaObject stores a given populated LFSMetaObject structure in the database // if it is not already present. func NewLFSMetaObject(ctx context.Context, repoID int64, p lfs.Pointer) (*LFSMetaObject, error) { - var err error - ctx, committer, err := db.TxContext(ctx) if err != nil { return nil, err diff --git a/models/issues/label_test.go b/models/issues/label_test.go index b03fc1cd20..3e3d097458 100644 --- a/models/issues/label_test.go +++ b/models/issues/label_test.go @@ -231,8 +231,7 @@ func TestGetLabelsByOrgID(t *testing.T) { testSuccess(3, "reversealphabetically", []int64{4, 3}) testSuccess(3, "default", []int64{3, 4}) - var err error - _, err = issues_model.GetLabelsByOrgID(db.DefaultContext, 0, "leastissues", db.ListOptions{}) + _, err := issues_model.GetLabelsByOrgID(db.DefaultContext, 0, "leastissues", db.ListOptions{}) assert.True(t, issues_model.IsErrOrgLabelNotExist(err)) _, err = issues_model.GetLabelsByOrgID(db.DefaultContext, -1, "leastissues", db.ListOptions{}) diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index 42c8415376..f7ea2beccf 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -41,14 +41,12 @@ func TestMaybeRemoveBOM(t *testing.T) { func TestToUTF8(t *testing.T) { resetDefaultCharsetsOrder() - var res string - var err error // Note: golang compiler seems so behave differently depending on the current // locale, so some conversions might behave differently. For that reason, we don't // depend on particular conversions but in expected behaviors. - res, err = ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) + res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) require.NoError(t, err) assert.Equal(t, "ABC", res) diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 0290e1312d..1d3e04224f 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -203,8 +203,7 @@ func (r *HTMLRenderer) renderIcon(w util.BufWriter, source []byte, node ast.Node return ast.WalkContinue, nil } - var err error - _, err = w.WriteString(fmt.Sprintf(``, name)) + _, err := w.WriteString(fmt.Sprintf(``, name)) if err != nil { return ast.WalkStop, err } diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index e2e784a671..2c4a65934d 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -202,7 +202,6 @@ func Search(ctx *context.APIContext) { } } - var err error repos, count, err := repo_model.SearchRepository(ctx, opts) if err != nil { ctx.JSON(http.StatusInternalServerError, api.SearchError{ diff --git a/routers/api/v1/user/repo.go b/routers/api/v1/user/repo.go index 86716ff44f..3b304c30b0 100644 --- a/routers/api/v1/user/repo.go +++ b/routers/api/v1/user/repo.go @@ -130,7 +130,6 @@ func ListMyRepos(ctx *context.APIContext) { return } - var err error repos, count, err := repo_model.SearchRepository(ctx, opts) if err != nil { ctx.Error(http.StatusInternalServerError, "SearchRepository", err) diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index bc1ecbfc1e..a343f60a98 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -307,7 +307,6 @@ func ViewPost(ctx *context_module.Context) { if validCursor { length := step.LogLength - cursor.Cursor offset := task.LogIndexes[index] - var err error logRows, err := actions.ReadLogs(ctx, task.LogInStorage, task.LogFilename, offset, length) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) diff --git a/routers/web/repo/activity.go b/routers/web/repo/activity.go index ba776c84d3..351578e1d6 100644 --- a/routers/web/repo/activity.go +++ b/routers/web/repo/activity.go @@ -94,7 +94,6 @@ func ActivityAuthors(ctx *context.Context) { timeFrom = timeUntil.Add(-time.Hour * 168) } - var err error authors, err := activities_model.GetActivityStatsTopAuthors(ctx, ctx.Repo.Repository, timeFrom, 10) if err != nil { ctx.ServerError("GetActivityStatsTopAuthors", err) diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index f1445c580a..41ff5f97f7 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -147,7 +147,6 @@ func FindReadmeFileInEntries(ctx *context.Context, entries []*git.TreeEntry, try // this should be impossible; if subTreeEntry exists so should this. continue } - var err error childEntries, err := subTree.ListEntries() if err != nil { return "", nil, err diff --git a/services/issue/template.go b/services/issue/template.go index 47633e5d85..9a2b048401 100644 --- a/services/issue/template.go +++ b/services/issue/template.go @@ -56,8 +56,6 @@ func GetTemplateConfig(gitRepo *git.Repository, path string, commit *git.Commit) return GetDefaultTemplateConfig(), nil } - var err error - treeEntry, err := commit.GetTreeEntryByPath(path) if err != nil { return GetDefaultTemplateConfig(), err -- cgit v1.2.3 From 6b74043b85ee03215705e789cebf79bc6f604bda Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Thu, 31 Oct 2024 23:28:25 +0800 Subject: Fix `missing signature key` error when pulling Docker images with `SERVE_DIRECT` enabled (#32365) Fix #28121 I did some tests and found that the `missing signature key` error is caused by an incorrect `Content-Type` header. Gitea correctly sets the `Content-Type` header when serving files. https://github.com/go-gitea/gitea/blob/348d1d0f322ca57c459acd902f54821d687ca804/routers/api/packages/container/container.go#L712-L717 However, when `SERVE_DIRECT` is enabled, the `Content-Type` header may be set to an incorrect value by the storage service. To fix this issue, we can use query parameters to override response header values. https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html In this PR, I introduced a new parameter to the `URL` method to support additional parameters. ``` URL(path, name string, reqParams url.Values) (*url.URL, error) ``` --- Most S3-like services support specifying the content type when storing objects. However, Gitea always use `application/octet-stream`. Therefore, I believe we also need to improve the `Save` method to support storing objects with the correct content type. https://github.com/go-gitea/gitea/blob/b7fb20e73e63b8edc9b90c52073e248bef428fcc/modules/storage/minio.go#L214-L221 (cherry picked from commit 0690cb076bf63f71988a709f62a9c04660b51a4f) Conflicts: - modules/storage/azureblob.go Dropped the change, as we do not support Azure blob storage. - modules/storage/helper.go Resolved by adjusting their `discardStorage` to our `DiscardStorage` - routers/api/actions/artifacts.go routers/api/actions/artifactsv4.go routers/web/repo/actions/view.go routers/web/repo/download.go Resolved the conflicts by manually adding the new `nil` parameter to the `storage.Attachments.URL()` calls. Originally conflicted due to differences in the if expression above these calls. --- modules/packages/content_store.go | 4 ++-- modules/storage/helper.go | 2 +- modules/storage/helper_test.go | 2 +- modules/storage/local.go | 2 +- modules/storage/minio.go | 8 ++++++-- modules/storage/storage.go | 2 +- routers/api/actions/artifacts.go | 2 +- routers/api/actions/artifactsv4.go | 2 +- routers/api/packages/container/container.go | 4 +++- routers/api/packages/maven/maven.go | 2 +- routers/api/v1/repo/file.go | 4 ++-- routers/web/base.go | 2 +- routers/web/repo/actions/view.go | 3 ++- routers/web/repo/attachment.go | 2 +- routers/web/repo/download.go | 4 ++-- routers/web/repo/repo.go | 2 +- services/lfs/server.go | 2 +- services/packages/packages.go | 6 +++--- 18 files changed, 31 insertions(+), 24 deletions(-) (limited to 'services') diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index da93e6cf6b..6438fb174f 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -37,8 +37,8 @@ func (s *ContentStore) ShouldServeDirect() bool { return setting.Packages.Storage.MinioConfig.ServeDirect } -func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename string) (*url.URL, error) { - return s.store.URL(KeyToRelativePath(key), filename) +func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename string, reqParams url.Values) (*url.URL, error) { + return s.store.URL(KeyToRelativePath(key), filename, reqParams) } // FIXME: Workaround to be removed in v1.20 diff --git a/modules/storage/helper.go b/modules/storage/helper.go index 95f1c7b9a8..8bec3a0042 100644 --- a/modules/storage/helper.go +++ b/modules/storage/helper.go @@ -30,7 +30,7 @@ func (s DiscardStorage) Delete(_ string) error { return fmt.Errorf("%s", s) } -func (s DiscardStorage) URL(_, _ string) (*url.URL, error) { +func (s DiscardStorage) URL(_, _ string, _ url.Values) (*url.URL, error) { return nil, fmt.Errorf("%s", s) } diff --git a/modules/storage/helper_test.go b/modules/storage/helper_test.go index 60a7c61289..dd30c9b8ac 100644 --- a/modules/storage/helper_test.go +++ b/modules/storage/helper_test.go @@ -38,7 +38,7 @@ func Test_discardStorage(t *testing.T) { require.Error(t, err, string(tt)) } { - got, err := tt.URL("path", "name") + got, err := tt.URL("path", "name", nil) assert.Nil(t, got) require.Errorf(t, err, string(tt)) } diff --git a/modules/storage/local.go b/modules/storage/local.go index 9bb532f1df..00c7f668aa 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -114,7 +114,7 @@ func (l *LocalStorage) Delete(path string) error { } // URL gets the redirect URL to a file -func (l *LocalStorage) URL(path, name string) (*url.URL, error) { +func (l *LocalStorage) URL(path, name string, reqParams url.Values) (*url.URL, error) { return nil, ErrURLNotSupported } diff --git a/modules/storage/minio.go b/modules/storage/minio.go index d0c2dec65b..b02eec7aa0 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -276,8 +276,12 @@ func (m *MinioStorage) Delete(path string) error { } // URL gets the redirect URL to a file. The presigned link is valid for 5 minutes. -func (m *MinioStorage) URL(path, name string) (*url.URL, error) { - reqParams := make(url.Values) +func (m *MinioStorage) URL(path, name string, serveDirectReqParams url.Values) (*url.URL, error) { + // copy serveDirectReqParams + reqParams, err := url.ParseQuery(serveDirectReqParams.Encode()) + if err != nil { + return nil, err + } // TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we? reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"") u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams) diff --git a/modules/storage/storage.go b/modules/storage/storage.go index b83b1c7929..9cc6949256 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -63,7 +63,7 @@ type ObjectStorage interface { Save(path string, r io.Reader, size int64) (int64, error) Stat(path string) (os.FileInfo, error) Delete(path string) error - URL(path, name string) (*url.URL, error) + URL(path, name string, reqParams url.Values) (*url.URL, error) IterateObjects(path string, iterator func(path string, obj Object) error) error } diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go index bc29e4481d..405686a058 100644 --- a/routers/api/actions/artifacts.go +++ b/routers/api/actions/artifacts.go @@ -437,7 +437,7 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) { for _, artifact := range artifacts { var downloadURL string if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect { - u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName) + u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName, nil) if err != nil && !errors.Is(err, storage.ErrURLNotSupported) { log.Error("Error getting serve direct url: %v", err) } diff --git a/routers/api/actions/artifactsv4.go b/routers/api/actions/artifactsv4.go index 677e89da2f..0417f98242 100644 --- a/routers/api/actions/artifactsv4.go +++ b/routers/api/actions/artifactsv4.go @@ -530,7 +530,7 @@ func (r *artifactV4Routes) getSignedArtifactURL(ctx *ArtifactContext) { respData := GetSignedArtifactURLResponse{} if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect { - u, err := storage.ActionsArtifacts.URL(artifact.StoragePath, artifact.ArtifactPath) + u, err := storage.ActionsArtifacts.URL(artifact.StoragePath, artifact.ArtifactPath, nil) if u != nil && err == nil { respData.SignedUrl = u.String() } diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index f376e7bc59..9c9da38424 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -689,7 +689,9 @@ func DeleteManifest(ctx *context.Context) { } func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor) { - s, u, _, err := packages_service.GetPackageBlobStream(ctx, pfd.File, pfd.Blob) + serveDirectReqParams := make(url.Values) + serveDirectReqParams.Set("response-content-type", pfd.Properties.GetByName(container_module.PropertyMediaType)) + s, u, _, err := packages_service.GetPackageBlobStream(ctx, pfd.File, pfd.Blob, serveDirectReqParams) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index 4181577454..521ef2209a 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -217,7 +217,7 @@ func servePackageFile(ctx *context.Context, params parameters, serveContent bool return } - s, u, _, err := packages_service.GetPackageBlobStream(ctx, pf, pb) + s, u, _, err := packages_service.GetPackageBlobStream(ctx, pf, pb, nil) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 1fa44d50c4..50d2786ec8 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -214,7 +214,7 @@ func GetRawFileOrLFS(ctx *context.APIContext) { if setting.LFS.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name()) + u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name(), nil) if u != nil && err == nil { ctx.Redirect(u.String()) return @@ -341,7 +341,7 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model. rPath := archiver.RelativePath() if setting.RepoArchive.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.RepoArchives.URL(rPath, downloadName) + u, err := storage.RepoArchives.URL(rPath, downloadName, nil) if u != nil && err == nil { ctx.Redirect(u.String()) return diff --git a/routers/web/base.go b/routers/web/base.go index 78dde57fa6..285d1ecddc 100644 --- a/routers/web/base.go +++ b/routers/web/base.go @@ -39,7 +39,7 @@ func storageHandler(storageSetting *setting.Storage, prefix string, objStore sto rPath := strings.TrimPrefix(req.URL.Path, "/"+prefix+"/") rPath = util.PathJoinRelX(rPath) - u, err := objStore.URL(rPath, path.Base(rPath)) + u, err := objStore.URL(rPath, path.Base(rPath), nil) if err != nil { if os.IsNotExist(err) || errors.Is(err, os.ErrNotExist) { log.Warn("Unable to find %s %s", prefix, rPath) diff --git a/routers/web/repo/actions/view.go b/routers/web/repo/actions/view.go index a343f60a98..e7dbb6d975 100644 --- a/routers/web/repo/actions/view.go +++ b/routers/web/repo/actions/view.go @@ -688,7 +688,8 @@ func ArtifactsDownloadView(ctx *context_module.Context) { if len(artifacts) == 1 && artifacts[0].ArtifactName+".zip" == artifacts[0].ArtifactPath && artifacts[0].ContentEncoding == "application/zip" { art := artifacts[0] if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect { - u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath) + u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath, nil) + if u != nil && err == nil { ctx.Redirect(u.String()) return diff --git a/routers/web/repo/attachment.go b/routers/web/repo/attachment.go index b42effd8c3..b5078e1f63 100644 --- a/routers/web/repo/attachment.go +++ b/routers/web/repo/attachment.go @@ -134,7 +134,7 @@ func ServeAttachment(ctx *context.Context, uuid string) { if setting.Attachment.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name) + u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name, nil) if u != nil && err == nil { ctx.Redirect(u.String()) diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index c4a8baecca..1e87bbf015 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -54,8 +54,8 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim } if setting.LFS.Storage.MinioConfig.ServeDirect { - // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name()) + // If we have a signed url (S3, object storage, blob storage), redirect to this directly. + u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name(), nil) if u != nil && err == nil { ctx.Redirect(u.String()) return nil diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 9562491440..8036bcae67 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -505,7 +505,7 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep rPath := archiver.RelativePath() if setting.RepoArchive.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.RepoArchives.URL(rPath, downloadName) + u, err := storage.RepoArchives.URL(rPath, downloadName, nil) if u != nil && err == nil { if archiver.ReleaseID != 0 { err = repo_model.CountArchiveDownload(ctx, ctx.Repo.Repository.ID, archiver.ReleaseID, archiver.Type) diff --git a/services/lfs/server.go b/services/lfs/server.go index 225dfdb024..51d6f42776 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -485,7 +485,7 @@ func buildObjectResponse(rc *requestContext, pointer lfs_module.Pointer, downloa var link *lfs_module.Link if setting.LFS.Storage.MinioConfig.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. - u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid) + u, err := storage.LFS.URL(pointer.RelativePath(), pointer.Oid, nil) if u != nil && err == nil { // Presigned url does not need the Authorization header // https://github.com/go-gitea/gitea/issues/21525 diff --git a/services/packages/packages.go b/services/packages/packages.go index a5b84506de..72ab19ee27 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -602,12 +602,12 @@ func GetPackageFileStream(ctx context.Context, pf *packages_model.PackageFile) ( return nil, nil, nil, err } - return GetPackageBlobStream(ctx, pf, pb) + return GetPackageBlobStream(ctx, pf, pb, nil) } // GetPackageBlobStream returns the content of the specific package blob // If the storage supports direct serving and it's enabled, only the direct serving url is returned. -func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { +func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, serveDirectReqParams url.Values) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { key := packages_module.BlobHash256Key(pb.HashSHA256) cs := packages_module.NewContentStore() @@ -617,7 +617,7 @@ func GetPackageBlobStream(ctx context.Context, pf *packages_model.PackageFile, p var err error if cs.ShouldServeDirect() { - u, err = cs.GetServeDirectURL(key, pf.Name) + u, err = cs.GetServeDirectURL(key, pf.Name, serveDirectReqParams) if err != nil && !errors.Is(err, storage.ErrURLNotSupported) { log.Error("Error getting serve direct url: %v", err) } -- cgit v1.2.3 From befafe9a05f118b34990db16a57a7ebc188aaa9a Mon Sep 17 00:00:00 2001 From: Rowan Bohde Date: Fri, 1 Nov 2024 22:29:37 -0500 Subject: improve performance of diffs (#32393) This has two major changes that significantly reduce the amount of work done for large diffs: * Kill a running git process when reaching the maximum number of files in a diff, preventing it from processing the entire diff. * When loading a diff with the URL param `file-only=true`, skip loading stats. This speeds up loading both hidden files of a diff and sections of a diff when clicking the "Show More" button. A couple of minor things from profiling are also included: * Reuse existing repo in `PrepareViewPullInfo` if head and base are the same. The performance impact is going to depend heavily on the individual diff and the hardware it runs on, but when testing locally on a diff changing 100k+ lines over hundreds of files, I'm seeing a roughly 75% reduction in time to load the result of "Show More" --------- Co-authored-by: wxiaoguang (cherry picked from commit 7dcccc3bb19655a6f83dd495ffc332708d0c8678) --- routers/web/repo/commit.go | 1 + routers/web/repo/compare.go | 3 +++ routers/web/repo/pull.go | 7 +++--- services/gitdiff/gitdiff.go | 60 ++++++++++++++++++++------------------------- 4 files changed, 34 insertions(+), 37 deletions(-) (limited to 'services') diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go index 0e5d1f0a1f..1428238074 100644 --- a/routers/web/repo/commit.go +++ b/routers/web/repo/commit.go @@ -338,6 +338,7 @@ func Diff(ctx *context.Context) { MaxLineCharacters: setting.Git.MaxGitDiffLineCharacters, MaxFiles: maxFiles, WhitespaceBehavior: gitdiff.GetWhitespaceFlag(ctx.Data["WhitespaceBehavior"].(string)), + FileOnly: fileOnly, }, files...) if err != nil { ctx.NotFound("GetDiff", err) diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go index 38d6004ec6..e5eab2bffa 100644 --- a/routers/web/repo/compare.go +++ b/routers/web/repo/compare.go @@ -611,6 +611,8 @@ func PrepareCompareDiff( maxLines, maxFiles = -1, -1 } + fileOnly := ctx.FormBool("file-only") + diff, err := gitdiff.GetDiff(ctx, ci.HeadGitRepo, &gitdiff.DiffOptions{ BeforeCommitID: beforeCommitID, @@ -621,6 +623,7 @@ func PrepareCompareDiff( MaxFiles: maxFiles, WhitespaceBehavior: whitespaceBehavior, DirectComparison: ci.DirectComparison, + FileOnly: fileOnly, }, ctx.FormStrings("files")...) if err != nil { ctx.ServerError("GetDiffRangeWithWhitespaceBehavior", err) diff --git a/routers/web/repo/pull.go b/routers/web/repo/pull.go index 9d0dcad61e..98dacc1a0d 100644 --- a/routers/web/repo/pull.go +++ b/routers/web/repo/pull.go @@ -614,12 +614,12 @@ func PrepareViewPullInfo(ctx *context.Context, issue *issues_model.Issue) *git.C var headBranchSha string // HeadRepo may be missing if pull.HeadRepo != nil { - headGitRepo, err := gitrepo.OpenRepository(ctx, pull.HeadRepo) + headGitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, pull.HeadRepo) if err != nil { - ctx.ServerError("OpenRepository", err) + ctx.ServerError("RepositoryFromContextOrOpen", err) return nil } - defer headGitRepo.Close() + defer closer.Close() if pull.Flow == issues_model.PullRequestFlowGithub { headBranchExist = headGitRepo.IsBranchExist(pull.HeadBranch) @@ -966,6 +966,7 @@ func viewPullFiles(ctx *context.Context, specifiedStartCommit, specifiedEndCommi MaxLineCharacters: setting.Git.MaxGitDiffLineCharacters, MaxFiles: maxFiles, WhitespaceBehavior: gitdiff.GetWhitespaceFlag(ctx.Data["WhitespaceBehavior"].(string)), + FileOnly: fileOnly, } if !willShowSpecifiedCommit { diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index 91b1f135c4..7d137fb214 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -379,18 +379,11 @@ func (diffFile *DiffFile) GetType() int { } // GetTailSection creates a fake DiffLineSection if the last section is not the end of the file -func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommitID, rightCommitID string) *DiffSection { +func (diffFile *DiffFile) GetTailSection(gitRepo *git.Repository, leftCommit, rightCommit *git.Commit) *DiffSection { if len(diffFile.Sections) == 0 || diffFile.Type != DiffFileChange || diffFile.IsBin || diffFile.IsLFSFile { return nil } - leftCommit, err := gitRepo.GetCommit(leftCommitID) - if err != nil { - return nil - } - rightCommit, err := gitRepo.GetCommit(rightCommitID) - if err != nil { - return nil - } + lastSection := diffFile.Sections[len(diffFile.Sections)-1] lastLine := lastSection.Lines[len(lastSection.Lines)-1] leftLineCount := getCommitFileLineCount(leftCommit, diffFile.Name) @@ -532,11 +525,6 @@ parsingLoop: lastFile := createDiffFile(diff, line) diff.End = lastFile.Name diff.IsIncomplete = true - _, err := io.Copy(io.Discard, reader) - if err != nil { - // By the definition of io.Copy this never returns io.EOF - return diff, fmt.Errorf("error during io.Copy: %w", err) - } break parsingLoop } @@ -1097,6 +1085,7 @@ type DiffOptions struct { MaxFiles int WhitespaceBehavior git.TrustedCmdArgs DirectComparison bool + FileOnly bool } // GetDiff builds a Diff between two commits of a repository. @@ -1105,12 +1094,16 @@ type DiffOptions struct { func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, files ...string) (*Diff, error) { repoPath := gitRepo.Path + var beforeCommit *git.Commit commit, err := gitRepo.GetCommit(opts.AfterCommitID) if err != nil { return nil, err } - cmdDiff := git.NewCommand(gitRepo.Ctx) + cmdCtx, cmdCancel := context.WithCancel(ctx) + defer cmdCancel() + + cmdDiff := git.NewCommand(cmdCtx) objectFormat, err := gitRepo.GetObjectFormat() if err != nil { return nil, err @@ -1132,6 +1125,12 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi AddArguments(opts.WhitespaceBehavior...). AddDynamicArguments(actualBeforeCommitID, opts.AfterCommitID) opts.BeforeCommitID = actualBeforeCommitID + + var err error + beforeCommit, err = gitRepo.GetCommit(opts.BeforeCommitID) + if err != nil { + return nil, err + } } // In git 2.31, git diff learned --skip-to which we can use to shortcut skip to file @@ -1166,7 +1165,9 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi _ = writer.Close() }() - diff, err := ParsePatch(ctx, opts.MaxLines, opts.MaxLineCharacters, opts.MaxFiles, reader, parsePatchSkipToFile) + diff, err := ParsePatch(cmdCtx, opts.MaxLines, opts.MaxLineCharacters, opts.MaxFiles, reader, parsePatchSkipToFile) + // Ensure the git process is killed if it didn't exit already + cmdCancel() if err != nil { return nil, fmt.Errorf("unable to ParsePatch: %w", err) } @@ -1207,37 +1208,28 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi diffFile.IsGenerated = analyze.IsGenerated(diffFile.Name) } - tailSection := diffFile.GetTailSection(gitRepo, opts.BeforeCommitID, opts.AfterCommitID) + tailSection := diffFile.GetTailSection(gitRepo, beforeCommit, commit) if tailSection != nil { diffFile.Sections = append(diffFile.Sections, tailSection) } } - separator := "..." - if opts.DirectComparison { - separator = ".." + if opts.FileOnly { + return diff, nil } - diffPaths := []string{opts.BeforeCommitID + separator + opts.AfterCommitID} - if len(opts.BeforeCommitID) == 0 || opts.BeforeCommitID == objectFormat.EmptyObjectID().String() { - diffPaths = []string{objectFormat.EmptyTree().String(), opts.AfterCommitID} - } - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) - if err != nil && strings.Contains(err.Error(), "no merge base") { - // git >= 2.28 now returns an error if base and head have become unrelated. - // previously it would return the results of git diff --shortstat base head so let's try that... - diffPaths = []string{opts.BeforeCommitID, opts.AfterCommitID} - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) - } + stats, err := GetPullDiffStats(gitRepo, opts) if err != nil { return nil, err } + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion = stats.NumFiles, stats.TotalAddition, stats.TotalDeletion + return diff, nil } type PullDiffStats struct { - TotalAddition, TotalDeletion int + NumFiles, TotalAddition, TotalDeletion int } // GetPullDiffStats @@ -1261,12 +1253,12 @@ func GetPullDiffStats(gitRepo *git.Repository, opts *DiffOptions) (*PullDiffStat diffPaths = []string{objectFormat.EmptyTree().String(), opts.AfterCommitID} } - _, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) if err != nil && strings.Contains(err.Error(), "no merge base") { // git >= 2.28 now returns an error if base and head have become unrelated. // previously it would return the results of git diff --shortstat base head so let's try that... diffPaths = []string{opts.BeforeCommitID, opts.AfterCommitID} - _, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(gitRepo.Ctx, repoPath, nil, diffPaths...) } if err != nil { return nil, err -- cgit v1.2.3