From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- services/repository/adopt.go | 370 +++++++++++++++ services/repository/adopt_test.go | 115 +++++ services/repository/archiver/archiver.go | 377 +++++++++++++++ services/repository/archiver/archiver_test.go | 134 ++++++ services/repository/avatar.go | 116 +++++ services/repository/avatar_test.go | 64 +++ services/repository/branch.go | 565 +++++++++++++++++++++++ services/repository/cache.go | 30 ++ services/repository/check.go | 202 ++++++++ services/repository/collaboration.go | 52 +++ services/repository/collaboration_test.go | 28 ++ services/repository/commit.go | 55 +++ services/repository/commitstatus/commitstatus.go | 202 ++++++++ services/repository/contributors_graph.go | 321 +++++++++++++ services/repository/contributors_graph_test.go | 101 ++++ services/repository/create.go | 318 +++++++++++++ services/repository/create_test.go | 149 ++++++ services/repository/delete.go | 471 +++++++++++++++++++ services/repository/files/cherry_pick.go | 128 +++++ services/repository/files/commit.go | 44 ++ services/repository/files/content.go | 278 +++++++++++ services/repository/files/content_test.go | 201 ++++++++ services/repository/files/diff.go | 42 ++ services/repository/files/diff_test.go | 166 +++++++ services/repository/files/file.go | 174 +++++++ services/repository/files/file_test.go | 115 +++++ services/repository/files/patch.go | 199 ++++++++ services/repository/files/temp_repo.go | 406 ++++++++++++++++ services/repository/files/temp_repo_test.go | 28 ++ services/repository/files/tree.go | 101 ++++ services/repository/files/tree_test.go | 52 +++ services/repository/files/update.go | 501 ++++++++++++++++++++ services/repository/files/upload.go | 248 ++++++++++ services/repository/fork.go | 248 ++++++++++ services/repository/fork_test.go | 49 ++ services/repository/generate.go | 391 ++++++++++++++++ services/repository/generate_test.go | 67 +++ services/repository/hooks.go | 110 +++++ services/repository/init.go | 83 ++++ services/repository/lfs.go | 123 +++++ services/repository/lfs_test.go | 75 +++ services/repository/main_test.go | 14 + services/repository/migrate.go | 289 ++++++++++++ services/repository/push.go | 420 +++++++++++++++++ services/repository/repository.go | 153 ++++++ services/repository/repository_test.go | 43 ++ services/repository/review.go | 24 + services/repository/review_test.go | 29 ++ services/repository/setting.go | 57 +++ services/repository/star.go | 27 ++ services/repository/template.go | 135 ++++++ services/repository/transfer.go | 434 +++++++++++++++++ services/repository/transfer_test.go | 124 +++++ 53 files changed, 9248 insertions(+) create mode 100644 services/repository/adopt.go create mode 100644 services/repository/adopt_test.go create mode 100644 services/repository/archiver/archiver.go create mode 100644 services/repository/archiver/archiver_test.go create mode 100644 services/repository/avatar.go create mode 100644 services/repository/avatar_test.go create mode 100644 services/repository/branch.go create mode 100644 services/repository/cache.go create mode 100644 services/repository/check.go create mode 100644 services/repository/collaboration.go create mode 100644 services/repository/collaboration_test.go create mode 100644 services/repository/commit.go create mode 100644 services/repository/commitstatus/commitstatus.go create mode 100644 services/repository/contributors_graph.go create mode 100644 services/repository/contributors_graph_test.go create mode 100644 services/repository/create.go create mode 100644 services/repository/create_test.go create mode 100644 services/repository/delete.go create mode 100644 services/repository/files/cherry_pick.go create mode 100644 services/repository/files/commit.go create mode 100644 services/repository/files/content.go create mode 100644 services/repository/files/content_test.go create mode 100644 services/repository/files/diff.go create mode 100644 services/repository/files/diff_test.go create mode 100644 services/repository/files/file.go create mode 100644 services/repository/files/file_test.go create mode 100644 services/repository/files/patch.go create mode 100644 services/repository/files/temp_repo.go create mode 100644 services/repository/files/temp_repo_test.go create mode 100644 services/repository/files/tree.go create mode 100644 services/repository/files/tree_test.go create mode 100644 services/repository/files/update.go create mode 100644 services/repository/files/upload.go create mode 100644 services/repository/fork.go create mode 100644 services/repository/fork_test.go create mode 100644 services/repository/generate.go create mode 100644 services/repository/generate_test.go create mode 100644 services/repository/hooks.go create mode 100644 services/repository/init.go create mode 100644 services/repository/lfs.go create mode 100644 services/repository/lfs_test.go create mode 100644 services/repository/main_test.go create mode 100644 services/repository/migrate.go create mode 100644 services/repository/push.go create mode 100644 services/repository/repository.go create mode 100644 services/repository/repository_test.go create mode 100644 services/repository/review.go create mode 100644 services/repository/review_test.go create mode 100644 services/repository/setting.go create mode 100644 services/repository/star.go create mode 100644 services/repository/template.go create mode 100644 services/repository/transfer.go create mode 100644 services/repository/transfer_test.go (limited to 'services/repository') diff --git a/services/repository/adopt.go b/services/repository/adopt.go new file mode 100644 index 0000000..3d6fe71 --- /dev/null +++ b/services/repository/adopt.go @@ -0,0 +1,370 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + notify_service "code.gitea.io/gitea/services/notify" + + "github.com/gobwas/glob" +) + +// AdoptRepository adopts pre-existing repository files for the user/organization. +func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { + if !doer.IsAdmin && !u.CanCreateRepo() { + return nil, repo_model.ErrReachLimitOfRepo{ + Limit: u.MaxRepoCreation, + } + } + + repo := &repo_model.Repository{ + OwnerID: u.ID, + Owner: u, + OwnerName: u.Name, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + OriginalURL: opts.OriginalURL, + OriginalServiceType: opts.GitServiceType, + IsPrivate: opts.IsPrivate, + IsFsckEnabled: !opts.IsMirror, + CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, + Status: opts.Status, + IsEmpty: !opts.AutoInit, + } + + if err := db.WithTx(ctx, func(ctx context.Context) error { + repoPath := repo_model.RepoPath(u.Name, repo.Name) + isExist, err := util.IsExist(repoPath) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + return err + } + if !isExist { + return repo_model.ErrRepoNotExist{ + OwnerName: u.Name, + Name: repo.Name, + } + } + + if err := repo_module.CreateRepositoryByExample(ctx, doer, u, repo, true, false); err != nil { + return err + } + + // Re-fetch the repository from database before updating it (else it would + // override changes that were done earlier with sql) + if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + return fmt.Errorf("getRepositoryByID: %w", err) + } + + if err := adoptRepository(ctx, repoPath, repo, opts.DefaultBranch); err != nil { + return fmt.Errorf("adoptRepository: %w", err) + } + + if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { + return fmt.Errorf("checkDaemonExportOK: %w", err) + } + + // Initialize Issue Labels if selected + if len(opts.IssueLabels) > 0 { + if err := repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { + return fmt.Errorf("InitializeLabels: %w", err) + } + } + + if stdout, _, err := git.NewCommand(ctx, "update-server-info"). + SetDescription(fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath)). + RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return fmt.Errorf("CreateRepository(git update-server-info): %w", err) + } + return nil + }); err != nil { + return nil, err + } + + notify_service.AdoptRepository(ctx, doer, u, repo) + + return repo, nil +} + +func adoptRepository(ctx context.Context, repoPath string, repo *repo_model.Repository, defaultBranch string) (err error) { + isExist, err := util.IsExist(repoPath) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + return err + } + if !isExist { + return fmt.Errorf("adoptRepository: path does not already exist: %s", repoPath) + } + + if err := repo_module.CreateDelegateHooks(repoPath); err != nil { + return fmt.Errorf("createDelegateHooks: %w", err) + } + + repo.IsEmpty = false + + if len(defaultBranch) > 0 { + repo.DefaultBranch = defaultBranch + + if err = gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + return fmt.Errorf("setDefaultBranch: %w", err) + } + } else { + repo.DefaultBranch, err = gitrepo.GetDefaultBranch(ctx, repo) + if err != nil { + repo.DefaultBranch = setting.Repository.DefaultBranch + if err = gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + return fmt.Errorf("setDefaultBranch: %w", err) + } + } + } + + // Don't bother looking this repo in the context it won't be there + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + return fmt.Errorf("openRepository: %w", err) + } + defer gitRepo.Close() + + if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, 0); err != nil { + return fmt.Errorf("SyncRepoBranchesWithRepo: %w", err) + } + + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { + return fmt.Errorf("SyncReleasesWithTags: %w", err) + } + + branches, _ := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{ + RepoID: repo.ID, + ListOptions: db.ListOptionsAll, + IsDeletedBranch: optional.Some(false), + }) + + found := false + hasDefault := false + hasMaster := false + hasMain := false + for _, branch := range branches { + if branch == repo.DefaultBranch { + found = true + break + } else if branch == setting.Repository.DefaultBranch { + hasDefault = true + } else if branch == "master" { + hasMaster = true + } else if branch == "main" { + hasMain = true + } + } + if !found { + if hasDefault { + repo.DefaultBranch = setting.Repository.DefaultBranch + } else if hasMaster { + repo.DefaultBranch = "master" + } else if hasMain { + repo.DefaultBranch = "main" + } else if len(branches) > 0 { + repo.DefaultBranch = branches[0] + } else { + repo.IsEmpty = true + repo.DefaultBranch = setting.Repository.DefaultBranch + } + + if err = gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + return fmt.Errorf("setDefaultBranch: %w", err) + } + } + if err = repo_module.UpdateRepository(ctx, repo, false); err != nil { + return fmt.Errorf("updateRepository: %w", err) + } + + return nil +} + +// DeleteUnadoptedRepository deletes unadopted repository files from the filesystem +func DeleteUnadoptedRepository(ctx context.Context, doer, u *user_model.User, repoName string) error { + if err := repo_model.IsUsableRepoName(repoName); err != nil { + return err + } + + repoPath := repo_model.RepoPath(u.Name, repoName) + isExist, err := util.IsExist(repoPath) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + return err + } + if !isExist { + return repo_model.ErrRepoNotExist{ + OwnerName: u.Name, + Name: repoName, + } + } + + if exist, err := repo_model.IsRepositoryModelExist(ctx, u, repoName); err != nil { + return err + } else if exist { + return repo_model.ErrRepoAlreadyExist{ + Uname: u.Name, + Name: repoName, + } + } + + return util.RemoveAll(repoPath) +} + +type unadoptedRepositories struct { + repositories []string + index int + start int + end int +} + +func (unadopted *unadoptedRepositories) add(repository string) { + if unadopted.index >= unadopted.start && unadopted.index < unadopted.end { + unadopted.repositories = append(unadopted.repositories, repository) + } + unadopted.index++ +} + +func checkUnadoptedRepositories(ctx context.Context, userName string, repoNamesToCheck []string, unadopted *unadoptedRepositories) error { + if len(repoNamesToCheck) == 0 { + return nil + } + ctxUser, err := user_model.GetUserByName(ctx, userName) + if err != nil { + if user_model.IsErrUserNotExist(err) { + log.Debug("Missing user: %s", userName) + return nil + } + return err + } + repos, _, err := repo_model.GetUserRepositories(ctx, &repo_model.SearchRepoOptions{ + Actor: ctxUser, + Private: true, + ListOptions: db.ListOptions{ + Page: 1, + PageSize: len(repoNamesToCheck), + }, LowerNames: repoNamesToCheck, + }) + if err != nil { + return err + } + if len(repos) == len(repoNamesToCheck) { + return nil + } + repoNames := make(container.Set[string], len(repos)) + for _, repo := range repos { + repoNames.Add(repo.LowerName) + } + for _, repoName := range repoNamesToCheck { + if !repoNames.Contains(repoName) { + unadopted.add(path.Join(userName, repoName)) // These are not used as filepaths - but as reponames - therefore use path.Join not filepath.Join + } + } + return nil +} + +// ListUnadoptedRepositories lists all the unadopted repositories that match the provided query +func ListUnadoptedRepositories(ctx context.Context, query string, opts *db.ListOptions) ([]string, int, error) { + globUser, _ := glob.Compile("*") + globRepo, _ := glob.Compile("*") + + qsplit := strings.SplitN(query, "/", 2) + if len(qsplit) > 0 && len(query) > 0 { + var err error + globUser, err = glob.Compile(qsplit[0]) + if err != nil { + log.Info("Invalid glob expression '%s' (skipped): %v", qsplit[0], err) + } + if len(qsplit) > 1 { + globRepo, err = glob.Compile(qsplit[1]) + if err != nil { + log.Info("Invalid glob expression '%s' (skipped): %v", qsplit[1], err) + } + } + } + var repoNamesToCheck []string + + start := (opts.Page - 1) * opts.PageSize + unadopted := &unadoptedRepositories{ + repositories: make([]string, 0, opts.PageSize), + start: start, + end: start + opts.PageSize, + index: 0, + } + + var userName string + + // We're going to iterate by pagesize. + root := filepath.Clean(setting.RepoRootPath) + if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() || path == root { + return nil + } + + name := d.Name() + + if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) { + // Got a new user + if err = checkUnadoptedRepositories(ctx, userName, repoNamesToCheck, unadopted); err != nil { + return err + } + repoNamesToCheck = repoNamesToCheck[:0] + + if !globUser.Match(name) { + return filepath.SkipDir + } + + userName = name + return nil + } + + if !strings.HasSuffix(name, ".git") { + return filepath.SkipDir + } + name = name[:len(name)-4] + if repo_model.IsUsableRepoName(name) != nil || strings.ToLower(name) != name || !globRepo.Match(name) { + return filepath.SkipDir + } + + repoNamesToCheck = append(repoNamesToCheck, name) + if len(repoNamesToCheck) >= setting.Database.IterateBufferSize { + if err = checkUnadoptedRepositories(ctx, userName, repoNamesToCheck, unadopted); err != nil { + return err + } + repoNamesToCheck = repoNamesToCheck[:0] + } + return filepath.SkipDir + }); err != nil { + return nil, 0, err + } + + if err := checkUnadoptedRepositories(ctx, userName, repoNamesToCheck, unadopted); err != nil { + return nil, 0, err + } + + return unadopted.repositories, unadopted.index, nil +} diff --git a/services/repository/adopt_test.go b/services/repository/adopt_test.go new file mode 100644 index 0000000..71fb1fc --- /dev/null +++ b/services/repository/adopt_test.go @@ -0,0 +1,115 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "os" + "path" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckUnadoptedRepositories_Add(t *testing.T) { + start := 10 + end := 20 + unadopted := &unadoptedRepositories{ + start: start, + end: end, + index: 0, + } + + total := 30 + for i := 0; i < total; i++ { + unadopted.add("something") + } + + assert.Equal(t, total, unadopted.index) + assert.Len(t, unadopted.repositories, end-start) +} + +func TestCheckUnadoptedRepositories(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + // + // Non existent user + // + unadopted := &unadoptedRepositories{start: 0, end: 100} + err := checkUnadoptedRepositories(db.DefaultContext, "notauser", []string{"repo"}, unadopted) + require.NoError(t, err) + assert.Empty(t, unadopted.repositories) + // + // Unadopted repository is returned + // Existing (adopted) repository is not returned + // + userName := "user2" + repoName := "repo2" + unadoptedRepoName := "unadopted" + unadopted = &unadoptedRepositories{start: 0, end: 100} + err = checkUnadoptedRepositories(db.DefaultContext, userName, []string{repoName, unadoptedRepoName}, unadopted) + require.NoError(t, err) + assert.Equal(t, []string{path.Join(userName, unadoptedRepoName)}, unadopted.repositories) + // + // Existing (adopted) repository is not returned + // + unadopted = &unadoptedRepositories{start: 0, end: 100} + err = checkUnadoptedRepositories(db.DefaultContext, userName, []string{repoName}, unadopted) + require.NoError(t, err) + assert.Empty(t, unadopted.repositories) + assert.Equal(t, 0, unadopted.index) +} + +func TestListUnadoptedRepositories_ListOptions(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + username := "user2" + unadoptedList := []string{path.Join(username, "unadopted1"), path.Join(username, "unadopted2")} + for _, unadopted := range unadoptedList { + _ = os.Mkdir(path.Join(setting.RepoRootPath, unadopted+".git"), 0o755) + } + + opts := db.ListOptions{Page: 1, PageSize: 1} + repoNames, count, err := ListUnadoptedRepositories(db.DefaultContext, "", &opts) + require.NoError(t, err) + assert.Equal(t, 2, count) + assert.Equal(t, unadoptedList[0], repoNames[0]) + + opts = db.ListOptions{Page: 2, PageSize: 1} + repoNames, count, err = ListUnadoptedRepositories(db.DefaultContext, "", &opts) + require.NoError(t, err) + assert.Equal(t, 2, count) + assert.Equal(t, unadoptedList[1], repoNames[0]) +} + +func TestAdoptRepository(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + username := "user2" + + unadopted := "unadopted" + require.NoError(t, unittest.CopyDir( + "../../modules/git/tests/repos/repo1_bare", + path.Join(setting.RepoRootPath, username, unadopted+".git"), + )) + + opts := db.ListOptions{Page: 1, PageSize: 1} + repoNames, _, err := ListUnadoptedRepositories(db.DefaultContext, "", &opts) + require.NoError(t, err) + require.Contains(t, repoNames, path.Join(username, unadopted)) + + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo, err := AdoptRepository(db.DefaultContext, doer, owner, CreateRepoOptions{ + Name: unadopted, + Description: "description", + IsPrivate: false, + AutoInit: true, + }) + require.NoError(t, err) + assert.Equal(t, git.Sha1ObjectFormat.Name(), repo.ObjectFormatName) +} diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go new file mode 100644 index 0000000..c74712b --- /dev/null +++ b/services/repository/archiver/archiver.go @@ -0,0 +1,377 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package archiver + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +// ArchiveRequest defines the parameters of an archive request, which notably +// includes the specific repository being archived as well as the commit, the +// name by which it was requested, and the kind of archive being requested. +// This is entirely opaque to external entities, though, and mostly used as a +// handle elsewhere. +type ArchiveRequest struct { + RepoID int64 + refName string + Type git.ArchiveType + CommitID string + ReleaseID int64 +} + +// ErrUnknownArchiveFormat request archive format is not supported +type ErrUnknownArchiveFormat struct { + RequestFormat string +} + +// Error implements error +func (err ErrUnknownArchiveFormat) Error() string { + return fmt.Sprintf("unknown format: %s", err.RequestFormat) +} + +// Is implements error +func (ErrUnknownArchiveFormat) Is(err error) bool { + _, ok := err.(ErrUnknownArchiveFormat) + return ok +} + +// RepoRefNotFoundError is returned when a requested reference (commit, tag) was not found. +type RepoRefNotFoundError struct { + RefName string +} + +// Error implements error. +func (e RepoRefNotFoundError) Error() string { + return fmt.Sprintf("unrecognized repository reference: %s", e.RefName) +} + +func (e RepoRefNotFoundError) Is(err error) bool { + _, ok := err.(RepoRefNotFoundError) + return ok +} + +// NewRequest creates an archival request, based on the URI. The +// resulting ArchiveRequest is suitable for being passed to ArchiveRepository() +// if it's determined that the request still needs to be satisfied. +func NewRequest(ctx context.Context, repoID int64, repo *git.Repository, uri string) (*ArchiveRequest, error) { + r := &ArchiveRequest{ + RepoID: repoID, + } + + var ext string + switch { + case strings.HasSuffix(uri, ".zip"): + ext = ".zip" + r.Type = git.ZIP + case strings.HasSuffix(uri, ".tar.gz"): + ext = ".tar.gz" + r.Type = git.TARGZ + case strings.HasSuffix(uri, ".bundle"): + ext = ".bundle" + r.Type = git.BUNDLE + default: + return nil, ErrUnknownArchiveFormat{RequestFormat: uri} + } + + r.refName = strings.TrimSuffix(uri, ext) + + // Get corresponding commit. + commitID, err := repo.ConvertToGitID(r.refName) + if err != nil { + return nil, RepoRefNotFoundError{RefName: r.refName} + } + + r.CommitID = commitID.String() + + release, err := repo_model.GetRelease(ctx, repoID, r.refName) + if err != nil { + if !repo_model.IsErrReleaseNotExist(err) { + return nil, err + } + } + if release != nil { + r.ReleaseID = release.ID + } + + return r, nil +} + +// GetArchiveName returns the name of the caller, based on the ref used by the +// caller to create this request. +func (aReq *ArchiveRequest) GetArchiveName() string { + return strings.ReplaceAll(aReq.refName, "/", "-") + "." + aReq.Type.String() +} + +// Await awaits the completion of an ArchiveRequest. If the archive has +// already been prepared the method returns immediately. Otherwise an archiver +// process will be started and its completion awaited. On success the returned +// RepoArchiver may be used to download the archive. Note that even if the +// context is cancelled/times out a started archiver will still continue to run +// in the background. +func (aReq *ArchiveRequest) Await(ctx context.Context) (*repo_model.RepoArchiver, error) { + archiver, err := repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) + if err != nil { + return nil, fmt.Errorf("models.GetRepoArchiver: %w", err) + } + + if archiver != nil { + archiver.ReleaseID = aReq.ReleaseID + } + + if archiver != nil && archiver.Status == repo_model.ArchiverReady { + // Archive already generated, we're done. + return archiver, nil + } + + if err := StartArchive(aReq); err != nil { + return nil, fmt.Errorf("archiver.StartArchive: %w", err) + } + + poll := time.NewTicker(time.Second * 1) + defer poll.Stop() + + for { + select { + case <-graceful.GetManager().HammerContext().Done(): + // System stopped. + return nil, graceful.GetManager().HammerContext().Err() + case <-ctx.Done(): + return nil, ctx.Err() + case <-poll.C: + archiver, err = repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) + if err != nil { + return nil, fmt.Errorf("repo_model.GetRepoArchiver: %w", err) + } + if archiver != nil && archiver.Status == repo_model.ArchiverReady { + archiver.ReleaseID = aReq.ReleaseID + return archiver, nil + } + } + } +} + +func doArchive(ctx context.Context, r *ArchiveRequest) (*repo_model.RepoArchiver, error) { + txCtx, committer, err := db.TxContext(ctx) + if err != nil { + return nil, err + } + defer committer.Close() + ctx, _, finished := process.GetManager().AddContext(txCtx, fmt.Sprintf("ArchiveRequest[%d]: %s", r.RepoID, r.GetArchiveName())) + defer finished() + + archiver, err := repo_model.GetRepoArchiver(ctx, r.RepoID, r.Type, r.CommitID) + if err != nil { + return nil, err + } + + if archiver != nil { + // FIXME: If another process are generating it, we think it's not ready and just return + // Or we should wait until the archive generated. + if archiver.Status == repo_model.ArchiverGenerating { + return nil, nil + } + } else { + archiver = &repo_model.RepoArchiver{ + RepoID: r.RepoID, + Type: r.Type, + CommitID: r.CommitID, + Status: repo_model.ArchiverGenerating, + } + if err := db.Insert(ctx, archiver); err != nil { + return nil, err + } + } + + rPath := archiver.RelativePath() + _, err = storage.RepoArchives.Stat(rPath) + if err == nil { + if archiver.Status == repo_model.ArchiverGenerating { + archiver.Status = repo_model.ArchiverReady + if err = repo_model.UpdateRepoArchiverStatus(ctx, archiver); err != nil { + return nil, err + } + } + return archiver, committer.Commit() + } + + if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("unable to stat archive: %w", err) + } + + rd, w := io.Pipe() + defer func() { + w.Close() + rd.Close() + }() + done := make(chan error, 1) // Ensure that there is some capacity which will ensure that the goroutine below can always finish + repo, err := repo_model.GetRepositoryByID(ctx, archiver.RepoID) + if err != nil { + return nil, fmt.Errorf("archiver.LoadRepo failed: %w", err) + } + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + return nil, err + } + defer gitRepo.Close() + + go func(done chan error, w *io.PipeWriter, archiver *repo_model.RepoArchiver, gitRepo *git.Repository) { + defer func() { + if r := recover(); r != nil { + done <- fmt.Errorf("%v", r) + } + }() + + if archiver.Type == git.BUNDLE { + err = gitRepo.CreateBundle( + ctx, + archiver.CommitID, + w, + ) + } else { + err = gitRepo.CreateArchive( + ctx, + archiver.Type, + w, + setting.Repository.PrefixArchiveFiles, + archiver.CommitID, + ) + } + _ = w.CloseWithError(err) + done <- err + }(done, w, archiver, gitRepo) + + // TODO: add lfs data to zip + // TODO: add submodule data to zip + + if _, err := storage.RepoArchives.Save(rPath, rd, -1); err != nil { + return nil, fmt.Errorf("unable to write archive: %w", err) + } + + err = <-done + if err != nil { + return nil, err + } + + if archiver.Status == repo_model.ArchiverGenerating { + archiver.Status = repo_model.ArchiverReady + if err = repo_model.UpdateRepoArchiverStatus(ctx, archiver); err != nil { + return nil, err + } + } + + return archiver, committer.Commit() +} + +// ArchiveRepository satisfies the ArchiveRequest being passed in. Processing +// will occur in a separate goroutine, as this phase may take a while to +// complete. If the archive already exists, ArchiveRepository will not do +// anything. In all cases, the caller should be examining the *ArchiveRequest +// being returned for completion, as it may be different than the one they passed +// in. +func ArchiveRepository(ctx context.Context, request *ArchiveRequest) (*repo_model.RepoArchiver, error) { + return doArchive(ctx, request) +} + +var archiverQueue *queue.WorkerPoolQueue[*ArchiveRequest] + +// Init initializes archiver +func Init(ctx context.Context) error { + handler := func(items ...*ArchiveRequest) []*ArchiveRequest { + for _, archiveReq := range items { + log.Trace("ArchiverData Process: %#v", archiveReq) + if _, err := doArchive(ctx, archiveReq); err != nil { + log.Error("Archive %v failed: %v", archiveReq, err) + } + } + return nil + } + + archiverQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo-archive", handler) + if archiverQueue == nil { + return errors.New("unable to create repo-archive queue") + } + go graceful.GetManager().RunWithCancel(archiverQueue) + + return nil +} + +// StartArchive push the archive request to the queue +func StartArchive(request *ArchiveRequest) error { + has, err := archiverQueue.Has(request) + if err != nil { + return err + } + if has { + return nil + } + return archiverQueue.Push(request) +} + +func deleteOldRepoArchiver(ctx context.Context, archiver *repo_model.RepoArchiver) error { + if _, err := db.DeleteByID[repo_model.RepoArchiver](ctx, archiver.ID); err != nil { + return err + } + p := archiver.RelativePath() + if err := storage.RepoArchives.Delete(p); err != nil { + log.Error("delete repo archive file failed: %v", err) + } + return nil +} + +// DeleteOldRepositoryArchives deletes old repository archives. +func DeleteOldRepositoryArchives(ctx context.Context, olderThan time.Duration) error { + log.Trace("Doing: ArchiveCleanup") + + for { + archivers, err := db.Find[repo_model.RepoArchiver](ctx, repo_model.FindRepoArchiversOption{ + ListOptions: db.ListOptions{ + PageSize: 100, + Page: 1, + }, + OlderThan: olderThan, + }) + if err != nil { + log.Trace("Error: ArchiveClean: %v", err) + return err + } + + for _, archiver := range archivers { + if err := deleteOldRepoArchiver(ctx, archiver); err != nil { + return err + } + } + if len(archivers) < 100 { + break + } + } + + log.Trace("Finished: ArchiveCleanup") + return nil +} + +// DeleteRepositoryArchives deletes all repositories' archives. +func DeleteRepositoryArchives(ctx context.Context) error { + if err := repo_model.DeleteAllRepoArchives(ctx); err != nil { + return err + } + return storage.Clean(storage.RepoArchives) +} diff --git a/services/repository/archiver/archiver_test.go b/services/repository/archiver/archiver_test.go new file mode 100644 index 0000000..9f822a3 --- /dev/null +++ b/services/repository/archiver/archiver_test.go @@ -0,0 +1,134 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package archiver + +import ( + "testing" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/services/contexttest" + + _ "code.gitea.io/gitea/models/actions" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m) +} + +func TestArchive_Basic(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + ctx, _ := contexttest.MockContext(t, "user27/repo49") + firstCommit, secondCommit := "51f84af23134", "aacbdfe9e1c4" + + contexttest.LoadRepo(t, ctx, 49) + contexttest.LoadGitRepo(t, ctx) + defer ctx.Repo.GitRepo.Close() + + bogusReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + require.NoError(t, err) + assert.NotNil(t, bogusReq) + assert.EqualValues(t, firstCommit+".zip", bogusReq.GetArchiveName()) + + // Check a series of bogus requests. + // Step 1, valid commit with a bad extension. + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".dilbert") + require.Error(t, err) + assert.Nil(t, bogusReq) + + // Step 2, missing commit. + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "dbffff.zip") + require.Error(t, err) + assert.Nil(t, bogusReq) + + // Step 3, doesn't look like branch/tag/commit. + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "db.zip") + require.Error(t, err) + assert.Nil(t, bogusReq) + + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master.zip") + require.NoError(t, err) + assert.NotNil(t, bogusReq) + assert.EqualValues(t, "master.zip", bogusReq.GetArchiveName()) + + bogusReq, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive.zip") + require.NoError(t, err) + assert.NotNil(t, bogusReq) + assert.EqualValues(t, "test-archive.zip", bogusReq.GetArchiveName()) + + // Now two valid requests, firstCommit with valid extensions. + zipReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + require.NoError(t, err) + assert.NotNil(t, zipReq) + + tgzReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".tar.gz") + require.NoError(t, err) + assert.NotNil(t, tgzReq) + + secondReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".zip") + require.NoError(t, err) + assert.NotNil(t, secondReq) + + inFlight := make([]*ArchiveRequest, 3) + inFlight[0] = zipReq + inFlight[1] = tgzReq + inFlight[2] = secondReq + + ArchiveRepository(db.DefaultContext, zipReq) + ArchiveRepository(db.DefaultContext, tgzReq) + ArchiveRepository(db.DefaultContext, secondReq) + + // Make sure sending an unprocessed request through doesn't affect the queue + // count. + ArchiveRepository(db.DefaultContext, zipReq) + + // Sleep two seconds to make sure the queue doesn't change. + time.Sleep(2 * time.Second) + + zipReq2, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + require.NoError(t, err) + // This zipReq should match what's sitting in the queue, as we haven't + // let it release yet. From the consumer's point of view, this looks like + // a long-running archive task. + assert.Equal(t, zipReq, zipReq2) + + // We still have the other three stalled at completion, waiting to remove + // from archiveInProgress. Try to submit this new one before its + // predecessor has cleared out of the queue. + ArchiveRepository(db.DefaultContext, zipReq2) + + // Now we'll submit a request and TimedWaitForCompletion twice, before and + // after we release it. We should trigger both the timeout and non-timeout + // cases. + timedReq, err := NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".tar.gz") + require.NoError(t, err) + assert.NotNil(t, timedReq) + ArchiveRepository(db.DefaultContext, timedReq) + + zipReq2, err = NewRequest(ctx, ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") + require.NoError(t, err) + // Now, we're guaranteed to have released the original zipReq from the queue. + // Ensure that we don't get handed back the released entry somehow, but they + // should remain functionally equivalent in all fields. The exception here + // is zipReq.cchan, which will be non-nil because it's a completed request. + // It's fine to go ahead and set it to nil now. + + assert.Equal(t, zipReq, zipReq2) + assert.NotSame(t, zipReq, zipReq2) + + // Same commit, different compression formats should have different names. + // Ideally, the extension would match what we originally requested. + assert.NotEqual(t, zipReq.GetArchiveName(), tgzReq.GetArchiveName()) + assert.NotEqual(t, zipReq.GetArchiveName(), secondReq.GetArchiveName()) +} + +func TestErrUnknownArchiveFormat(t *testing.T) { + err := ErrUnknownArchiveFormat{RequestFormat: "master"} + assert.ErrorIs(t, err, ErrUnknownArchiveFormat{}) +} diff --git a/services/repository/avatar.go b/services/repository/avatar.go new file mode 100644 index 0000000..38c2621 --- /dev/null +++ b/services/repository/avatar.go @@ -0,0 +1,116 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "io" + "strconv" + "strings" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/avatar" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" +) + +// UploadAvatar saves custom avatar for repository. +// FIXME: split uploads to different subdirs in case we have massive number of repos. +func UploadAvatar(ctx context.Context, repo *repo_model.Repository, data []byte) error { + avatarData, err := avatar.ProcessAvatarImage(data) + if err != nil { + return err + } + + newAvatar := avatar.HashAvatar(repo.ID, data) + if repo.Avatar == newAvatar { // upload the same picture + return nil + } + + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + oldAvatarPath := repo.CustomAvatarRelativePath() + + // Users can upload the same image to other repo - prefix it with ID + // Then repo will be removed - only it avatar file will be removed + repo.Avatar = newAvatar + if err := repo_model.UpdateRepositoryCols(ctx, repo, "avatar"); err != nil { + return fmt.Errorf("UploadAvatar: Update repository avatar: %w", err) + } + + if err := storage.SaveFrom(storage.RepoAvatars, repo.CustomAvatarRelativePath(), func(w io.Writer) error { + _, err := w.Write(avatarData) + return err + }); err != nil { + return fmt.Errorf("UploadAvatar %s failed: Failed to remove old repo avatar %s: %w", repo.RepoPath(), newAvatar, err) + } + + if len(oldAvatarPath) > 0 { + if err := storage.RepoAvatars.Delete(oldAvatarPath); err != nil { + return fmt.Errorf("UploadAvatar: Failed to remove old repo avatar %s: %w", oldAvatarPath, err) + } + } + + return committer.Commit() +} + +// DeleteAvatar deletes the repos's custom avatar. +func DeleteAvatar(ctx context.Context, repo *repo_model.Repository) error { + // Avatar not exists + if len(repo.Avatar) == 0 { + return nil + } + + avatarPath := repo.CustomAvatarRelativePath() + log.Trace("DeleteAvatar[%d]: %s", repo.ID, avatarPath) + + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + repo.Avatar = "" + if err := repo_model.UpdateRepositoryCols(ctx, repo, "avatar"); err != nil { + return fmt.Errorf("DeleteAvatar: Update repository avatar: %w", err) + } + + if err := storage.RepoAvatars.Delete(avatarPath); err != nil { + return fmt.Errorf("DeleteAvatar: Failed to remove %s: %w", avatarPath, err) + } + + return committer.Commit() +} + +// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories +func RemoveRandomAvatars(ctx context.Context) error { + return db.Iterate(ctx, nil, func(ctx context.Context, repository *repo_model.Repository) error { + select { + case <-ctx.Done(): + return db.ErrCancelledf("before random avatars removed for %s", repository.FullName()) + default: + } + stringifiedID := strconv.FormatInt(repository.ID, 10) + if repository.Avatar == stringifiedID { + return DeleteAvatar(ctx, repository) + } + return nil + }) +} + +// generateAvatar generates the avatar from a template repository +func generateAvatar(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { + generateRepo.Avatar = strings.Replace(templateRepo.Avatar, strconv.FormatInt(templateRepo.ID, 10), strconv.FormatInt(generateRepo.ID, 10), 1) + if _, err := storage.Copy(storage.RepoAvatars, generateRepo.CustomAvatarRelativePath(), storage.RepoAvatars, templateRepo.CustomAvatarRelativePath()); err != nil { + return err + } + + return repo_model.UpdateRepositoryCols(ctx, generateRepo, "avatar") +} diff --git a/services/repository/avatar_test.go b/services/repository/avatar_test.go new file mode 100644 index 0000000..f0fe991 --- /dev/null +++ b/services/repository/avatar_test.go @@ -0,0 +1,64 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "bytes" + "image" + "image/png" + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/avatar" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUploadAvatar(t *testing.T) { + // Generate image + myImage := image.NewRGBA(image.Rect(0, 0, 1, 1)) + var buff bytes.Buffer + png.Encode(&buff, myImage) + + require.NoError(t, unittest.PrepareTestDatabase()) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}) + + err := UploadAvatar(db.DefaultContext, repo, buff.Bytes()) + require.NoError(t, err) + assert.Equal(t, avatar.HashAvatar(10, buff.Bytes()), repo.Avatar) +} + +func TestUploadBigAvatar(t *testing.T) { + // Generate BIG image + myImage := image.NewRGBA(image.Rect(0, 0, 5000, 1)) + var buff bytes.Buffer + png.Encode(&buff, myImage) + + require.NoError(t, unittest.PrepareTestDatabase()) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}) + + err := UploadAvatar(db.DefaultContext, repo, buff.Bytes()) + require.Error(t, err) +} + +func TestDeleteAvatar(t *testing.T) { + // Generate image + myImage := image.NewRGBA(image.Rect(0, 0, 1, 1)) + var buff bytes.Buffer + png.Encode(&buff, myImage) + + require.NoError(t, unittest.PrepareTestDatabase()) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}) + + err := UploadAvatar(db.DefaultContext, repo, buff.Bytes()) + require.NoError(t, err) + + err = DeleteAvatar(db.DefaultContext, repo) + require.NoError(t, err) + + assert.Equal(t, "", repo.Avatar) +} diff --git a/services/repository/branch.go b/services/repository/branch.go new file mode 100644 index 0000000..f0e7120 --- /dev/null +++ b/services/repository/branch.go @@ -0,0 +1,565 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "errors" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/queue" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/timeutil" + webhook_module "code.gitea.io/gitea/modules/webhook" + notify_service "code.gitea.io/gitea/services/notify" + files_service "code.gitea.io/gitea/services/repository/files" + + "xorm.io/builder" +) + +// CreateNewBranch creates a new repository branch +func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, gitRepo *git.Repository, oldBranchName, branchName string) (err error) { + branch, err := git_model.GetBranch(ctx, repo.ID, oldBranchName) + if err != nil { + return err + } + + return CreateNewBranchFromCommit(ctx, doer, repo, gitRepo, branch.CommitID, branchName) +} + +// Branch contains the branch information +type Branch struct { + DBBranch *git_model.Branch + IsProtected bool + IsIncluded bool + CommitsAhead int + CommitsBehind int + LatestPullRequest *issues_model.PullRequest + MergeMovedOn bool +} + +// LoadBranches loads branches from the repository limited by page & pageSize. +func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, isDeletedBranch optional.Option[bool], keyword string, page, pageSize int) (*Branch, []*Branch, int64, error) { + defaultDBBranch, err := git_model.GetBranch(ctx, repo.ID, repo.DefaultBranch) + if err != nil { + return nil, nil, 0, err + } + + branchOpts := git_model.FindBranchOptions{ + RepoID: repo.ID, + IsDeletedBranch: isDeletedBranch, + ListOptions: db.ListOptions{ + Page: page, + PageSize: pageSize, + }, + Keyword: keyword, + ExcludeBranchNames: []string{repo.DefaultBranch}, + } + + dbBranches, totalNumOfBranches, err := db.FindAndCount[git_model.Branch](ctx, branchOpts) + if err != nil { + return nil, nil, 0, err + } + + if err := git_model.BranchList(dbBranches).LoadDeletedBy(ctx); err != nil { + return nil, nil, 0, err + } + if err := git_model.BranchList(dbBranches).LoadPusher(ctx); err != nil { + return nil, nil, 0, err + } + + rules, err := git_model.FindRepoProtectedBranchRules(ctx, repo.ID) + if err != nil { + return nil, nil, 0, err + } + + repoIDToRepo := map[int64]*repo_model.Repository{} + repoIDToRepo[repo.ID] = repo + + repoIDToGitRepo := map[int64]*git.Repository{} + repoIDToGitRepo[repo.ID] = gitRepo + + branches := make([]*Branch, 0, len(dbBranches)) + for i := range dbBranches { + branch, err := loadOneBranch(ctx, repo, dbBranches[i], &rules, repoIDToRepo, repoIDToGitRepo) + if err != nil { + log.Error("loadOneBranch() on repo #%d, branch '%s' failed: %v", repo.ID, dbBranches[i].Name, err) + + // TODO: Ideally, we would only do this if the branch doesn't exist + // anymore. That is not practical to check here currently, so we do + // this for all kinds of errors. + totalNumOfBranches-- + continue + } + + branches = append(branches, branch) + } + + // Always add the default branch + log.Debug("loadOneBranch: load default: '%s'", defaultDBBranch.Name) + defaultBranch, err := loadOneBranch(ctx, repo, defaultDBBranch, &rules, repoIDToRepo, repoIDToGitRepo) + if err != nil { + return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err) + } + + return defaultBranch, branches, totalNumOfBranches, nil +} + +func loadOneBranch(ctx context.Context, repo *repo_model.Repository, dbBranch *git_model.Branch, protectedBranches *git_model.ProtectedBranchRules, + repoIDToRepo map[int64]*repo_model.Repository, + repoIDToGitRepo map[int64]*git.Repository, +) (*Branch, error) { + log.Trace("loadOneBranch: '%s'", dbBranch.Name) + + branchName := dbBranch.Name + p := protectedBranches.GetFirstMatched(branchName) + isProtected := p != nil + + var divergence *git.DivergeObject + + // it's not default branch + if repo.DefaultBranch != dbBranch.Name && !dbBranch.IsDeleted { + var err error + divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName) + if err != nil { + return nil, fmt.Errorf("CountDivergingCommits: %v", err) + } + } + + if divergence == nil { + // tolerate the error that we cannot get divergence + divergence = &git.DivergeObject{Ahead: -1, Behind: -1} + } + + pr, err := issues_model.GetLatestPullRequestByHeadInfo(ctx, repo.ID, branchName) + if err != nil { + return nil, fmt.Errorf("GetLatestPullRequestByHeadInfo: %v", err) + } + headCommit := dbBranch.CommitID + + mergeMovedOn := false + if pr != nil { + pr.HeadRepo = repo + if err := pr.LoadIssue(ctx); err != nil { + return nil, fmt.Errorf("LoadIssue: %v", err) + } + if repo, ok := repoIDToRepo[pr.BaseRepoID]; ok { + pr.BaseRepo = repo + } else if err := pr.LoadBaseRepo(ctx); err != nil { + return nil, fmt.Errorf("LoadBaseRepo: %v", err) + } else { + repoIDToRepo[pr.BaseRepoID] = pr.BaseRepo + } + pr.Issue.Repo = pr.BaseRepo + + if pr.HasMerged { + baseGitRepo, ok := repoIDToGitRepo[pr.BaseRepoID] + if !ok { + baseGitRepo, err = gitrepo.OpenRepository(ctx, pr.BaseRepo) + if err != nil { + return nil, fmt.Errorf("OpenRepository: %v", err) + } + defer baseGitRepo.Close() + repoIDToGitRepo[pr.BaseRepoID] = baseGitRepo + } + pullCommit, err := baseGitRepo.GetRefCommitID(pr.GetGitRefName()) + if err != nil && !git.IsErrNotExist(err) { + return nil, fmt.Errorf("GetBranchCommitID: %v", err) + } + if err == nil && headCommit != pullCommit { + // the head has moved on from the merge - we shouldn't delete + mergeMovedOn = true + } + } + } + + isIncluded := divergence.Ahead == 0 && repo.DefaultBranch != branchName + return &Branch{ + DBBranch: dbBranch, + IsProtected: isProtected, + IsIncluded: isIncluded, + CommitsAhead: divergence.Ahead, + CommitsBehind: divergence.Behind, + LatestPullRequest: pr, + MergeMovedOn: mergeMovedOn, + }, nil +} + +// checkBranchName validates branch name with existing repository branches +func checkBranchName(ctx context.Context, repo *repo_model.Repository, name string) error { + _, err := gitrepo.WalkReferences(ctx, repo, func(_, refName string) error { + branchRefName := strings.TrimPrefix(refName, git.BranchPrefix) + switch { + case branchRefName == name: + return git_model.ErrBranchAlreadyExists{ + BranchName: name, + } + // If branchRefName like a/b but we want to create a branch named a then we have a conflict + case strings.HasPrefix(branchRefName, name+"/"): + return git_model.ErrBranchNameConflict{ + BranchName: branchRefName, + } + // Conversely if branchRefName like a but we want to create a branch named a/b then we also have a conflict + case strings.HasPrefix(name, branchRefName+"/"): + return git_model.ErrBranchNameConflict{ + BranchName: branchRefName, + } + case refName == git.TagPrefix+name: + return models.ErrTagAlreadyExists{ + TagName: name, + } + } + return nil + }) + + return err +} + +// SyncBranchesToDB sync the branch information in the database. +// It will check whether the branches of the repository have never been synced before. +// If so, it will sync all branches of the repository. +// Otherwise, it will sync the branches that need to be updated. +func SyncBranchesToDB(ctx context.Context, repoID, pusherID int64, branchNames, commitIDs []string, getCommit func(commitID string) (*git.Commit, error)) error { + // Some designs that make the code look strange but are made for performance optimization purposes: + // 1. Sync branches in a batch to reduce the number of DB queries. + // 2. Lazy load commit information since it may be not necessary. + // 3. Exit early if synced all branches of git repo when there's no branch in DB. + // 4. Check the branches in DB if they are already synced. + // + // If the user pushes many branches at once, the Git hook will call the internal API in batches, rather than all at once. + // See https://github.com/go-gitea/gitea/blob/cb52b17f92e2d2293f7c003649743464492bca48/cmd/hook.go#L27 + // For the first batch, it will hit optimization 3. + // For other batches, it will hit optimization 4. + + if len(branchNames) != len(commitIDs) { + return fmt.Errorf("branchNames and commitIDs length not match") + } + + return db.WithTx(ctx, func(ctx context.Context) error { + branches, err := git_model.GetBranches(ctx, repoID, branchNames) + if err != nil { + return fmt.Errorf("git_model.GetBranches: %v", err) + } + + if len(branches) == 0 { + // if user haven't visit UI but directly push to a branch after upgrading from 1.20 -> 1.21, + // we cannot simply insert the branch but need to check we have branches or not + hasBranch, err := db.Exist[git_model.Branch](ctx, git_model.FindBranchOptions{ + RepoID: repoID, + IsDeletedBranch: optional.Some(false), + }.ToConds()) + if err != nil { + return err + } + if !hasBranch { + if _, err = repo_module.SyncRepoBranches(ctx, repoID, pusherID); err != nil { + return fmt.Errorf("repo_module.SyncRepoBranches %d failed: %v", repoID, err) + } + return nil + } + } + + branchMap := make(map[string]*git_model.Branch, len(branches)) + for _, branch := range branches { + branchMap[branch.Name] = branch + } + + newBranches := make([]*git_model.Branch, 0, len(branchNames)) + + for i, branchName := range branchNames { + commitID := commitIDs[i] + branch, exist := branchMap[branchName] + if exist && branch.CommitID == commitID && !branch.IsDeleted { + continue + } + + commit, err := getCommit(commitID) + if err != nil { + return fmt.Errorf("get commit of %s failed: %v", branchName, err) + } + + if exist { + if _, err := git_model.UpdateBranch(ctx, repoID, pusherID, branchName, commit); err != nil { + return fmt.Errorf("git_model.UpdateBranch %d:%s failed: %v", repoID, branchName, err) + } + continue + } + + // if database have branches but not this branch, it means this is a new branch + newBranches = append(newBranches, &git_model.Branch{ + RepoID: repoID, + Name: branchName, + CommitID: commit.ID.String(), + CommitMessage: commit.Summary(), + PusherID: pusherID, + CommitTime: timeutil.TimeStamp(commit.Committer.When.Unix()), + }) + } + + if len(newBranches) > 0 { + return db.Insert(ctx, newBranches) + } + return nil + }) +} + +// CreateNewBranchFromCommit creates a new repository branch +func CreateNewBranchFromCommit(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, gitRepo *git.Repository, commitID, branchName string) (err error) { + err = repo.MustNotBeArchived() + if err != nil { + return err + } + + // Check if branch name can be used + if err := checkBranchName(ctx, repo, branchName); err != nil { + return err + } + + if err := git.Push(ctx, repo.RepoPath(), git.PushOptions{ + Remote: repo.RepoPath(), + Branch: fmt.Sprintf("%s:%s%s", commitID, git.BranchPrefix, branchName), + Env: repo_module.PushingEnvironment(doer, repo), + }); err != nil { + if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) { + return err + } + return fmt.Errorf("push: %w", err) + } + return nil +} + +// RenameBranch rename a branch +func RenameBranch(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, gitRepo *git.Repository, from, to string) (string, error) { + err := repo.MustNotBeArchived() + if err != nil { + return "", err + } + + if from == to { + return "target_exist", nil + } + + if gitRepo.IsBranchExist(to) { + return "target_exist", nil + } + + if !gitRepo.IsBranchExist(from) { + return "from_not_exist", nil + } + + if err := git_model.RenameBranch(ctx, repo, from, to, func(ctx context.Context, isDefault bool) error { + err2 := gitRepo.RenameBranch(from, to) + if err2 != nil { + return err2 + } + + if isDefault { + // if default branch changed, we need to delete all schedules and cron jobs + if err := actions_model.DeleteScheduleTaskByRepo(ctx, repo.ID); err != nil { + log.Error("DeleteCronTaskByRepo: %v", err) + } + // cancel running cron jobs of this repository and delete old schedules + if err := actions_model.CancelPreviousJobs( + ctx, + repo.ID, + from, + "", + webhook_module.HookEventSchedule, + ); err != nil { + log.Error("CancelPreviousJobs: %v", err) + } + + err2 = gitrepo.SetDefaultBranch(ctx, repo, to) + if err2 != nil { + return err2 + } + } + + return nil + }); err != nil { + return "", err + } + refNameTo := git.RefNameFromBranch(to) + refID, err := gitRepo.GetRefCommitID(refNameTo.String()) + if err != nil { + return "", err + } + + notify_service.DeleteRef(ctx, doer, repo, git.RefNameFromBranch(from)) + notify_service.CreateRef(ctx, doer, repo, refNameTo, refID) + + return "", nil +} + +// enmuerates all branch related errors +var ( + ErrBranchIsDefault = errors.New("branch is default") +) + +// DeleteBranch delete branch +func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, gitRepo *git.Repository, branchName string) error { + err := repo.MustNotBeArchived() + if err != nil { + return err + } + + if branchName == repo.DefaultBranch { + return ErrBranchIsDefault + } + + isProtected, err := git_model.IsBranchProtected(ctx, repo.ID, branchName) + if err != nil { + return err + } + if isProtected { + return git_model.ErrBranchIsProtected + } + + rawBranch, err := git_model.GetBranch(ctx, repo.ID, branchName) + if err != nil && !git_model.IsErrBranchNotExist(err) { + return fmt.Errorf("GetBranch: %v", err) + } + + // database branch record not exist or it's a deleted branch + notExist := git_model.IsErrBranchNotExist(err) || rawBranch.IsDeleted + + commit, err := gitRepo.GetBranchCommit(branchName) + if err != nil { + return err + } + + if err := db.WithTx(ctx, func(ctx context.Context) error { + if !notExist { + if err := git_model.AddDeletedBranch(ctx, repo.ID, branchName, doer.ID); err != nil { + return err + } + } + + return gitRepo.DeleteBranch(branchName, git.DeleteBranchOptions{ + Force: true, + }) + }); err != nil { + return err + } + + objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + + // Don't return error below this + if err := PushUpdate( + &repo_module.PushUpdateOptions{ + RefFullName: git.RefNameFromBranch(branchName), + OldCommitID: commit.ID.String(), + NewCommitID: objectFormat.EmptyObjectID().String(), + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.OwnerName, + RepoName: repo.Name, + }); err != nil { + log.Error("Update: %v", err) + } + + return nil +} + +type BranchSyncOptions struct { + RepoID int64 +} + +// branchSyncQueue represents a queue to handle branch sync jobs. +var branchSyncQueue *queue.WorkerPoolQueue[*BranchSyncOptions] + +func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions { + for _, opts := range items { + _, err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, 0) + if err != nil { + log.Error("syncRepoBranches [%d] failed: %v", opts.RepoID, err) + } + } + return nil +} + +func addRepoToBranchSyncQueue(repoID int64) error { + return branchSyncQueue.Push(&BranchSyncOptions{ + RepoID: repoID, + }) +} + +func initBranchSyncQueue(ctx context.Context) error { + branchSyncQueue = queue.CreateUniqueQueue(ctx, "branch_sync", handlerBranchSync) + if branchSyncQueue == nil { + return errors.New("unable to create branch_sync queue") + } + go graceful.GetManager().RunWithCancel(branchSyncQueue) + + return nil +} + +func AddAllRepoBranchesToSyncQueue(ctx context.Context) error { + if err := db.Iterate(ctx, builder.Eq{"is_empty": false}, func(ctx context.Context, repo *repo_model.Repository) error { + return addRepoToBranchSyncQueue(repo.ID) + }); err != nil { + return fmt.Errorf("run sync all branches failed: %v", err) + } + return nil +} + +func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, newBranchName string) error { + if repo.DefaultBranch == newBranchName { + return nil + } + + if !gitRepo.IsBranchExist(newBranchName) { + return git_model.ErrBranchNotExist{ + BranchName: newBranchName, + } + } + + oldDefaultBranchName := repo.DefaultBranch + repo.DefaultBranch = newBranchName + if err := db.WithTx(ctx, func(ctx context.Context) error { + if err := repo_model.UpdateDefaultBranch(ctx, repo); err != nil { + return err + } + + if err := actions_model.DeleteScheduleTaskByRepo(ctx, repo.ID); err != nil { + log.Error("DeleteCronTaskByRepo: %v", err) + } + // cancel running cron jobs of this repository and delete old schedules + if err := actions_model.CancelPreviousJobs( + ctx, + repo.ID, + oldDefaultBranchName, + "", + webhook_module.HookEventSchedule, + ); err != nil { + log.Error("CancelPreviousJobs: %v", err) + } + + if err := gitrepo.SetDefaultBranch(ctx, repo, newBranchName); err != nil { + if !git.IsErrUnsupportedVersion(err) { + return err + } + } + return nil + }); err != nil { + return err + } + + notify_service.ChangeDefaultBranch(ctx, repo) + + return nil +} diff --git a/services/repository/cache.go b/services/repository/cache.go new file mode 100644 index 0000000..b0811a9 --- /dev/null +++ b/services/repository/cache.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" +) + +// CacheRef cachhe last commit information of the branch or the tag +func CacheRef(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, fullRefName git.RefName) error { + commit, err := gitRepo.GetCommit(fullRefName.String()) + if err != nil { + return err + } + + if gitRepo.LastCommitCache == nil { + commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(fullRefName.ShortName(), true), commit.CommitsCount) + if err != nil { + return err + } + gitRepo.LastCommitCache = git.NewLastCommitCache(commitsCount, repo.FullName(), gitRepo, cache.GetCache()) + } + + return commit.CacheCommit(ctx) +} diff --git a/services/repository/check.go b/services/repository/check.go new file mode 100644 index 0000000..5cdcc14 --- /dev/null +++ b/services/repository/check.go @@ -0,0 +1,202 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + system_model "code.gitea.io/gitea/models/system" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/util" + + "xorm.io/builder" +) + +// GitFsckRepos calls 'git fsck' to check repository health. +func GitFsckRepos(ctx context.Context, timeout time.Duration, args git.TrustedCmdArgs) error { + log.Trace("Doing: GitFsck") + + if err := db.Iterate( + ctx, + builder.Expr("id>0 AND is_fsck_enabled=?", true), + func(ctx context.Context, repo *repo_model.Repository) error { + select { + case <-ctx.Done(): + return db.ErrCancelledf("before fsck of %s", repo.FullName()) + default: + } + return GitFsckRepo(ctx, repo, timeout, args) + }, + ); err != nil { + log.Trace("Error: GitFsck: %v", err) + return err + } + + log.Trace("Finished: GitFsck") + return nil +} + +// GitFsckRepo calls 'git fsck' to check an individual repository's health. +func GitFsckRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args git.TrustedCmdArgs) error { + log.Trace("Running health check on repository %-v", repo) + repoPath := repo.RepoPath() + if err := git.Fsck(ctx, repoPath, timeout, args); err != nil { + log.Warn("Failed to health check repository (%-v): %v", repo, err) + if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } + return nil +} + +// GitGcRepos calls 'git gc' to remove unnecessary files and optimize the local repository +func GitGcRepos(ctx context.Context, timeout time.Duration, args git.TrustedCmdArgs) error { + log.Trace("Doing: GitGcRepos") + + if err := db.Iterate( + ctx, + builder.Gt{"id": 0}, + func(ctx context.Context, repo *repo_model.Repository) error { + select { + case <-ctx.Done(): + return db.ErrCancelledf("before GC of %s", repo.FullName()) + default: + } + // we can ignore the error here because it will be logged in GitGCRepo + _ = GitGcRepo(ctx, repo, timeout, args) + return nil + }, + ); err != nil { + return err + } + + log.Trace("Finished: GitGcRepos") + return nil +} + +// GitGcRepo calls 'git gc' to remove unnecessary files and optimize the local repository +func GitGcRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args git.TrustedCmdArgs) error { + log.Trace("Running git gc on %-v", repo) + command := git.NewCommand(ctx, "gc").AddArguments(args...). + SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())) + var stdout string + var err error + stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) + if err != nil { + log.Error("Repository garbage collection failed for %-v. Stdout: %s\nError: %v", repo, stdout, err) + desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) + if err := system_model.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) + } + + // Now update the size of the repository + if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Updating size as part of garbage collection failed for %-v. Stdout: %s\nError: %v", repo, stdout, err) + desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) + if err := system_model.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) + } + + return nil +} + +func gatherMissingRepoRecords(ctx context.Context) (repo_model.RepositoryList, error) { + repos := make([]*repo_model.Repository, 0, 10) + if err := db.Iterate( + ctx, + builder.Gt{"id": 0}, + func(ctx context.Context, repo *repo_model.Repository) error { + select { + case <-ctx.Done(): + return db.ErrCancelledf("during gathering missing repo records before checking %s", repo.FullName()) + default: + } + isDir, err := util.IsDir(repo.RepoPath()) + if err != nil { + return fmt.Errorf("Unable to check dir for %s. %w", repo.FullName(), err) + } + if !isDir { + repos = append(repos, repo) + } + return nil + }, + ); err != nil { + if strings.HasPrefix(err.Error(), "Aborted gathering missing repo") { + return nil, err + } + if err2 := system_model.CreateRepositoryNotice("gatherMissingRepoRecords: %v", err); err2 != nil { + log.Error("CreateRepositoryNotice: %v", err2) + } + return nil, err + } + return repos, nil +} + +// DeleteMissingRepositories deletes all repository records that lost Git files. +func DeleteMissingRepositories(ctx context.Context, doer *user_model.User) error { + repos, err := gatherMissingRepoRecords(ctx) + if err != nil { + return err + } + + if len(repos) == 0 { + return nil + } + + for _, repo := range repos { + select { + case <-ctx.Done(): + return db.ErrCancelledf("during DeleteMissingRepositories before %s", repo.FullName()) + default: + } + log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID) + if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { + log.Error("Failed to DeleteRepository %-v: Error: %v", repo, err) + if err2 := system_model.CreateRepositoryNotice("Failed to DeleteRepository %s [%d]: Error: %v", repo.FullName(), repo.ID, err); err2 != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } + } + return nil +} + +// ReinitMissingRepositories reinitializes all repository records that lost Git files. +func ReinitMissingRepositories(ctx context.Context) error { + repos, err := gatherMissingRepoRecords(ctx) + if err != nil { + return err + } + + if len(repos) == 0 { + return nil + } + + for _, repo := range repos { + select { + case <-ctx.Done(): + return db.ErrCancelledf("during ReinitMissingRepositories before %s", repo.FullName()) + default: + } + log.Trace("Initializing %d/%d...", repo.OwnerID, repo.ID) + if err := git.InitRepository(ctx, repo.RepoPath(), true, repo.ObjectFormatName); err != nil { + log.Error("Unable (re)initialize repository %d at %s. Error: %v", repo.ID, repo.RepoPath(), err) + if err2 := system_model.CreateRepositoryNotice("InitRepository [%d]: %v", repo.ID, err); err2 != nil { + log.Error("CreateRepositoryNotice: %v", err2) + } + } + } + return nil +} diff --git a/services/repository/collaboration.go b/services/repository/collaboration.go new file mode 100644 index 0000000..dccc124 --- /dev/null +++ b/services/repository/collaboration.go @@ -0,0 +1,52 @@ +// Copyright 2016 The Gogs Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" + access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" +) + +// DeleteCollaboration removes collaboration relation between the user and repository. +func DeleteCollaboration(ctx context.Context, repo *repo_model.Repository, uid int64) (err error) { + collaboration := &repo_model.Collaboration{ + RepoID: repo.ID, + UserID: uid, + } + + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + if has, err := db.GetEngine(ctx).Delete(collaboration); err != nil { + return err + } else if has == 0 { + return committer.Commit() + } + if err = access_model.RecalculateAccesses(ctx, repo); err != nil { + return err + } + + if err = repo_model.WatchRepo(ctx, uid, repo.ID, false); err != nil { + return err + } + + if err = models.ReconsiderWatches(ctx, repo, uid); err != nil { + return err + } + + // Unassign a user from any issue (s)he has been assigned to in the repository + if err := models.ReconsiderRepoIssuesAssignee(ctx, repo, uid); err != nil { + return err + } + + return committer.Commit() +} diff --git a/services/repository/collaboration_test.go b/services/repository/collaboration_test.go new file mode 100644 index 0000000..c087018 --- /dev/null +++ b/services/repository/collaboration_test.go @@ -0,0 +1,28 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + + "github.com/stretchr/testify/require" +) + +func TestRepository_DeleteCollaboration(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 4}) + require.NoError(t, repo.LoadOwner(db.DefaultContext)) + require.NoError(t, DeleteCollaboration(db.DefaultContext, repo, 4)) + unittest.AssertNotExistsBean(t, &repo_model.Collaboration{RepoID: repo.ID, UserID: 4}) + + require.NoError(t, DeleteCollaboration(db.DefaultContext, repo, 4)) + unittest.AssertNotExistsBean(t, &repo_model.Collaboration{RepoID: repo.ID, UserID: 4}) + + unittest.CheckConsistencyFor(t, &repo_model.Repository{ID: repo.ID}) +} diff --git a/services/repository/commit.go b/services/repository/commit.go new file mode 100644 index 0000000..e8c0262 --- /dev/null +++ b/services/repository/commit.go @@ -0,0 +1,55 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/modules/util" + gitea_ctx "code.gitea.io/gitea/services/context" +) + +type ContainedLinks struct { // TODO: better name? + Branches []*namedLink `json:"branches"` + Tags []*namedLink `json:"tags"` + DefaultBranch string `json:"default_branch"` +} + +type namedLink struct { // TODO: better name? + Name string `json:"name"` + WebLink string `json:"web_link"` +} + +// LoadBranchesAndTags creates a new repository branch +func LoadBranchesAndTags(ctx context.Context, baseRepo *gitea_ctx.Repository, commitSHA string) (*ContainedLinks, error) { + containedTags, err := baseRepo.GitRepo.ListOccurrences(ctx, "tag", commitSHA) + if err != nil { + return nil, fmt.Errorf("encountered a problem while querying %s: %w", "tags", err) + } + containedBranches, err := baseRepo.GitRepo.ListOccurrences(ctx, "branch", commitSHA) + if err != nil { + return nil, fmt.Errorf("encountered a problem while querying %s: %w", "branches", err) + } + + result := &ContainedLinks{ + DefaultBranch: baseRepo.Repository.DefaultBranch, + Branches: make([]*namedLink, 0, len(containedBranches)), + Tags: make([]*namedLink, 0, len(containedTags)), + } + for _, tag := range containedTags { + // TODO: Use a common method to get the link to a branch/tag instead of hard-coding it here + result.Tags = append(result.Tags, &namedLink{ + Name: tag, + WebLink: fmt.Sprintf("%s/src/tag/%s", baseRepo.RepoLink, util.PathEscapeSegments(tag)), + }) + } + for _, branch := range containedBranches { + result.Branches = append(result.Branches, &namedLink{ + Name: branch, + WebLink: fmt.Sprintf("%s/src/branch/%s", baseRepo.RepoLink, util.PathEscapeSegments(branch)), + }) + } + return result, nil +} diff --git a/services/repository/commitstatus/commitstatus.go b/services/repository/commitstatus/commitstatus.go new file mode 100644 index 0000000..5c63020 --- /dev/null +++ b/services/repository/commitstatus/commitstatus.go @@ -0,0 +1,202 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package commitstatus + +import ( + "context" + "crypto/sha256" + "fmt" + "slices" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/services/automerge" +) + +func getCacheKey(repoID int64, brancheName string) string { + hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%d:%s", repoID, brancheName))) + return fmt.Sprintf("commit_status:%x", hashBytes) +} + +type commitStatusCacheValue struct { + State string `json:"state"` + TargetURL string `json:"target_url"` +} + +func getCommitStatusCache(repoID int64, branchName string) *commitStatusCacheValue { + c := cache.GetCache() + statusStr, ok := c.Get(getCacheKey(repoID, branchName)).(string) + if ok && statusStr != "" { + var cv commitStatusCacheValue + err := json.Unmarshal([]byte(statusStr), &cv) + if err == nil && cv.State != "" { + return &cv + } + if err != nil { + log.Warn("getCommitStatusCache: json.Unmarshal failed: %v", err) + } + } + return nil +} + +func updateCommitStatusCache(repoID int64, branchName string, state api.CommitStatusState, targetURL string) error { + c := cache.GetCache() + bs, err := json.Marshal(commitStatusCacheValue{ + State: state.String(), + TargetURL: targetURL, + }) + if err != nil { + log.Warn("updateCommitStatusCache: json.Marshal failed: %v", err) + return nil + } + return c.Put(getCacheKey(repoID, branchName), string(bs), 3*24*60) +} + +func deleteCommitStatusCache(repoID int64, branchName string) error { + c := cache.GetCache() + return c.Delete(getCacheKey(repoID, branchName)) +} + +// CreateCommitStatus creates a new CommitStatus given a bunch of parameters +// NOTE: All text-values will be trimmed from whitespaces. +// Requires: Repo, Creator, SHA +func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creator *user_model.User, sha string, status *git_model.CommitStatus) error { + repoPath := repo.RepoPath() + + // confirm that commit is exist + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return fmt.Errorf("OpenRepository[%s]: %w", repoPath, err) + } + defer closer.Close() + + objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + + commit, err := gitRepo.GetCommit(sha) + if err != nil { + return fmt.Errorf("GetCommit[%s]: %w", sha, err) + } + if len(sha) != objectFormat.FullLength() { + // use complete commit sha + sha = commit.ID.String() + } + + if err := db.WithTx(ctx, func(ctx context.Context) error { + if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{ + Repo: repo, + Creator: creator, + SHA: commit.ID, + CommitStatus: status, + }); err != nil { + return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err) + } + + return git_model.UpdateCommitStatusSummary(ctx, repo.ID, commit.ID.String()) + }); err != nil { + return err + } + + defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) + if err != nil { + return fmt.Errorf("GetBranchCommit[%s]: %w", repo.DefaultBranch, err) + } + + if commit.ID.String() == defaultBranchCommit.ID.String() { // since one commit status updated, the combined commit status should be invalid + if err := deleteCommitStatusCache(repo.ID, repo.DefaultBranch); err != nil { + log.Error("deleteCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err) + } + } + + if status.State.IsSuccess() { + if err := automerge.StartPRCheckAndAutoMergeBySHA(ctx, sha, repo); err != nil { + return fmt.Errorf("MergeScheduledPullRequest[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err) + } + } + + return nil +} + +// FindReposLastestCommitStatuses loading repository default branch latest combined commit status with cache +func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Repository) ([]*git_model.CommitStatus, error) { + if len(repos) == 0 { + return nil, nil + } + results := make([]*git_model.CommitStatus, len(repos)) + for i, repo := range repos { + if cv := getCommitStatusCache(repo.ID, repo.DefaultBranch); cv != nil { + results[i] = &git_model.CommitStatus{ + State: api.CommitStatusState(cv.State), + TargetURL: cv.TargetURL, + } + } + } + + // collect the latest commit of each repo + // at most there are dozens of repos (limited by MaxResponseItems), so it's not a big problem at the moment + repoBranchNames := make(map[int64]string, len(repos)) + for i, repo := range repos { + if results[i] == nil { + repoBranchNames[repo.ID] = repo.DefaultBranch + } + } + + repoIDsToLatestCommitSHAs, err := git_model.FindBranchesByRepoAndBranchName(ctx, repoBranchNames) + if err != nil { + return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err) + } + + var repoSHAs []git_model.RepoSHA + for id, sha := range repoIDsToLatestCommitSHAs { + repoSHAs = append(repoSHAs, git_model.RepoSHA{RepoID: id, SHA: sha}) + } + + summaryResults, err := git_model.GetLatestCommitStatusForRepoAndSHAs(ctx, repoSHAs) + if err != nil { + return nil, fmt.Errorf("GetLatestCommitStatusForRepoAndSHAs: %v", err) + } + + for _, summary := range summaryResults { + for i, repo := range repos { + if repo.ID == summary.RepoID { + results[i] = summary + _ = slices.DeleteFunc(repoSHAs, func(repoSHA git_model.RepoSHA) bool { + return repoSHA.RepoID == repo.ID + }) + if results[i].State != "" { + if err := updateCommitStatusCache(repo.ID, repo.DefaultBranch, results[i].State, results[i].TargetURL); err != nil { + log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err) + } + } + break + } + } + } + + // call the database O(1) times to get the commit statuses for all repos + repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoSHAs) + if err != nil { + return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err) + } + + for i, repo := range repos { + if results[i] == nil { + results[i] = git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID]) + if results[i] != nil && results[i].State != "" { + if err := updateCommitStatusCache(repo.ID, repo.DefaultBranch, results[i].State, results[i].TargetURL); err != nil { + log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err) + } + } + } + } + + return results, nil +} diff --git a/services/repository/contributors_graph.go b/services/repository/contributors_graph.go new file mode 100644 index 0000000..4887181 --- /dev/null +++ b/services/repository/contributors_graph.go @@ -0,0 +1,321 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/models/avatars" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + + "code.forgejo.org/go-chi/cache" +) + +const contributorStatsCacheKey = "GetContributorStats/%s/%s" + +var ( + ErrAwaitGeneration = errors.New("generation took longer than ") + awaitGenerationTime = time.Second * 5 + generateLock = sync.Map{} +) + +type WeekData struct { + Week int64 `json:"week"` // Starting day of the week as Unix timestamp + Additions int `json:"additions"` // Number of additions in that week + Deletions int `json:"deletions"` // Number of deletions in that week + Commits int `json:"commits"` // Number of commits in that week +} + +// ContributorData represents statistical git commit count data +type ContributorData struct { + Name string `json:"name"` // Display name of the contributor + Login string `json:"login"` // Login name of the contributor in case it exists + AvatarLink string `json:"avatar_link"` + HomeLink string `json:"home_link"` + TotalCommits int64 `json:"total_commits"` + Weeks map[int64]*WeekData `json:"weeks"` +} + +// ExtendedCommitStats contains information for commit stats with author data +type ExtendedCommitStats struct { + Author *api.CommitUser `json:"author"` + Stats *api.CommitStats `json:"stats"` +} + +const layout = time.DateOnly + +func findLastSundayBeforeDate(dateStr string) (string, error) { + date, err := time.Parse(layout, dateStr) + if err != nil { + return "", err + } + + weekday := date.Weekday() + daysToSubtract := int(weekday) - int(time.Sunday) + if daysToSubtract < 0 { + daysToSubtract += 7 + } + + lastSunday := date.AddDate(0, 0, -daysToSubtract) + return lastSunday.Format(layout), nil +} + +// GetContributorStats returns contributors stats for git commits for given revision or default branch +func GetContributorStats(ctx context.Context, cache cache.Cache, repo *repo_model.Repository, revision string) (map[string]*ContributorData, error) { + // as GetContributorStats is resource intensive we cache the result + cacheKey := fmt.Sprintf(contributorStatsCacheKey, repo.FullName(), revision) + if !cache.IsExist(cacheKey) { + genReady := make(chan struct{}) + + // dont start multiple async generations + _, run := generateLock.Load(cacheKey) + if run { + return nil, ErrAwaitGeneration + } + + generateLock.Store(cacheKey, struct{}{}) + // run generation async + go generateContributorStats(genReady, cache, cacheKey, repo, revision) + + select { + case <-time.After(awaitGenerationTime): + return nil, ErrAwaitGeneration + case <-genReady: + // we got generation ready before timeout + break + } + } + // TODO: renew timeout of cache cache.UpdateTimeout(cacheKey, contributorStatsCacheTimeout) + + switch v := cache.Get(cacheKey).(type) { + case error: + return nil, v + case string: + var cachedStats map[string]*ContributorData + return cachedStats, json.Unmarshal([]byte(v), &cachedStats) + default: + return nil, fmt.Errorf("unexpected type in cache detected") + } +} + +// getExtendedCommitStats return the list of *ExtendedCommitStats for the given revision +func getExtendedCommitStats(repo *git.Repository, revision string /*, limit int */) ([]*ExtendedCommitStats, error) { + baseCommit, err := repo.GetCommit(revision) + if err != nil { + return nil, err + } + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + return nil, err + } + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + gitCmd := git.NewCommand(repo.Ctx, "log", "--shortstat", "--no-merges", "--pretty=format:---%n%aN%n%aE%n%as", "--reverse") + // AddOptionFormat("--max-count=%d", limit) + gitCmd.AddDynamicArguments(baseCommit.ID.String()) + + var extendedCommitStats []*ExtendedCommitStats + stderr := new(strings.Builder) + err = gitCmd.Run(&git.RunOpts{ + Dir: repo.Path, + Stdout: stdoutWriter, + Stderr: stderr, + PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { + _ = stdoutWriter.Close() + scanner := bufio.NewScanner(stdoutReader) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "---" { + continue + } + scanner.Scan() + authorName := strings.TrimSpace(scanner.Text()) + scanner.Scan() + authorEmail := strings.TrimSpace(scanner.Text()) + scanner.Scan() + date := strings.TrimSpace(scanner.Text()) + scanner.Scan() + stats := strings.TrimSpace(scanner.Text()) + if authorName == "" || authorEmail == "" || date == "" || stats == "" { + // FIXME: find a better way to parse the output so that we will handle this properly + log.Warn("Something is wrong with git log output, skipping...") + log.Warn("authorName: %s, authorEmail: %s, date: %s, stats: %s", authorName, authorEmail, date, stats) + continue + } + // 1 file changed, 1 insertion(+), 1 deletion(-) + fields := strings.Split(stats, ",") + + commitStats := api.CommitStats{} + for _, field := range fields[1:] { + parts := strings.Split(strings.TrimSpace(field), " ") + value, contributionType := parts[0], parts[1] + amount, _ := strconv.Atoi(value) + + if strings.HasPrefix(contributionType, "insertion") { + commitStats.Additions = amount + } else { + commitStats.Deletions = amount + } + } + commitStats.Total = commitStats.Additions + commitStats.Deletions + scanner.Text() // empty line at the end + + res := &ExtendedCommitStats{ + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: authorName, + Email: authorEmail, + }, + Date: date, + }, + Stats: &commitStats, + } + extendedCommitStats = append(extendedCommitStats, res) + } + _ = stdoutReader.Close() + return nil + }, + }) + if err != nil { + return nil, fmt.Errorf("Failed to get ContributorsCommitStats for repository.\nError: %w\nStderr: %s", err, stderr) + } + + return extendedCommitStats, nil +} + +func generateContributorStats(genDone chan struct{}, cache cache.Cache, cacheKey string, repo *repo_model.Repository, revision string) { + ctx := graceful.GetManager().HammerContext() + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + log.Error("OpenRepository[repo=%q]: %v", repo.FullName(), err) + return + } + defer closer.Close() + + if len(revision) == 0 { + revision = repo.DefaultBranch + } + extendedCommitStats, err := getExtendedCommitStats(gitRepo, revision) + if err != nil { + log.Error("getExtendedCommitStats[repo=%q revision=%q]: %v", repo.FullName(), revision, err) + return + } + if len(extendedCommitStats) == 0 { + log.Error("No commit stats were returned [repo=%q revision=%q]", repo.FullName(), revision) + return + } + + layout := time.DateOnly + + unknownUserAvatarLink := user_model.NewGhostUser().AvatarLinkWithSize(ctx, 0) + contributorsCommitStats := make(map[string]*ContributorData) + contributorsCommitStats["total"] = &ContributorData{ + Name: "Total", + Weeks: make(map[int64]*WeekData), + } + total := contributorsCommitStats["total"] + + for _, v := range extendedCommitStats { + userEmail := v.Author.Email + if len(userEmail) == 0 { + continue + } + u, _ := user_model.GetUserByEmail(ctx, userEmail) + if u != nil { + // update userEmail with user's primary email address so + // that different mail addresses will linked to same account + userEmail = u.GetEmail() + } + // duplicated logic + if _, ok := contributorsCommitStats[userEmail]; !ok { + if u == nil { + avatarLink := avatars.GenerateEmailAvatarFastLink(ctx, userEmail, 0) + if avatarLink == "" { + avatarLink = unknownUserAvatarLink + } + contributorsCommitStats[userEmail] = &ContributorData{ + Name: v.Author.Name, + AvatarLink: avatarLink, + Weeks: make(map[int64]*WeekData), + } + } else { + contributorsCommitStats[userEmail] = &ContributorData{ + Name: u.DisplayName(), + Login: u.LowerName, + AvatarLink: u.AvatarLinkWithSize(ctx, 0), + HomeLink: u.HomeLink(), + Weeks: make(map[int64]*WeekData), + } + } + } + // Update user statistics + user := contributorsCommitStats[userEmail] + startingOfWeek, _ := findLastSundayBeforeDate(v.Author.Date) + + val, _ := time.Parse(layout, startingOfWeek) + week := val.UnixMilli() + + if user.Weeks[week] == nil { + user.Weeks[week] = &WeekData{ + Additions: 0, + Deletions: 0, + Commits: 0, + Week: week, + } + } + if total.Weeks[week] == nil { + total.Weeks[week] = &WeekData{ + Additions: 0, + Deletions: 0, + Commits: 0, + Week: week, + } + } + user.Weeks[week].Additions += v.Stats.Additions + user.Weeks[week].Deletions += v.Stats.Deletions + user.Weeks[week].Commits++ + user.TotalCommits++ + + // Update overall statistics + total.Weeks[week].Additions += v.Stats.Additions + total.Weeks[week].Deletions += v.Stats.Deletions + total.Weeks[week].Commits++ + total.TotalCommits++ + } + + data, err := json.Marshal(contributorsCommitStats) + if err != nil { + log.Error("json.Marshal[repo=%q revision=%q]: %v", repo.FullName(), revision, err) + return + } + + // Store the data as an string, to make it uniform what data type is returned + // from caches. + _ = cache.Put(cacheKey, string(data), setting.CacheService.TTLSeconds()) + generateLock.Delete(cacheKey) + if genDone != nil { + genDone <- struct{}{} + } +} diff --git a/services/repository/contributors_graph_test.go b/services/repository/contributors_graph_test.go new file mode 100644 index 0000000..8cfe69d --- /dev/null +++ b/services/repository/contributors_graph_test.go @@ -0,0 +1,101 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "slices" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/test" + + "code.forgejo.org/go-chi/cache" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRepository_ContributorsGraph(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) + require.NoError(t, repo.LoadOwner(db.DefaultContext)) + mockCache, err := cache.NewCacher(cache.Options{ + Adapter: "memory", + Interval: 24 * 60, + }) + require.NoError(t, err) + + lc, cleanup := test.NewLogChecker(log.DEFAULT, log.INFO) + lc.StopMark(`getExtendedCommitStats[repo="user2/repo2" revision="404ref"]: object does not exist [id: 404ref, rel_path: ]`) + defer cleanup() + + generateContributorStats(nil, mockCache, "key", repo, "404ref") + assert.False(t, mockCache.IsExist("key")) + _, stopped := lc.Check(100 * time.Millisecond) + assert.True(t, stopped) + + generateContributorStats(nil, mockCache, "key2", repo, "master") + dataString, isData := mockCache.Get("key2").(string) + assert.True(t, isData) + // Verify that JSON is actually stored in the cache. + assert.EqualValues(t, `{"ethantkoenig@gmail.com":{"name":"Ethan Koenig","login":"","avatar_link":"https://secure.gravatar.com/avatar/b42fb195faa8c61b8d88abfefe30e9e3?d=identicon","home_link":"","total_commits":1,"weeks":{"1511654400000":{"week":1511654400000,"additions":3,"deletions":0,"commits":1}}},"jimmy.praet@telenet.be":{"name":"Jimmy Praet","login":"","avatar_link":"https://secure.gravatar.com/avatar/93c49b7c89eb156971d11161c9b52795?d=identicon","home_link":"","total_commits":1,"weeks":{"1624752000000":{"week":1624752000000,"additions":2,"deletions":0,"commits":1}}},"jon@allspice.io":{"name":"Jon","login":"","avatar_link":"https://secure.gravatar.com/avatar/00388ce725e6886f3e07c3733007289b?d=identicon","home_link":"","total_commits":1,"weeks":{"1607817600000":{"week":1607817600000,"additions":10,"deletions":0,"commits":1}}},"total":{"name":"Total","login":"","avatar_link":"","home_link":"","total_commits":3,"weeks":{"1511654400000":{"week":1511654400000,"additions":3,"deletions":0,"commits":1},"1607817600000":{"week":1607817600000,"additions":10,"deletions":0,"commits":1},"1624752000000":{"week":1624752000000,"additions":2,"deletions":0,"commits":1}}}}`, dataString) + + var data map[string]*ContributorData + require.NoError(t, json.Unmarshal([]byte(dataString), &data)) + + var keys []string + for k := range data { + keys = append(keys, k) + } + slices.Sort(keys) + assert.EqualValues(t, []string{ + "ethantkoenig@gmail.com", + "jimmy.praet@telenet.be", + "jon@allspice.io", + "total", // generated summary + }, keys) + + assert.EqualValues(t, &ContributorData{ + Name: "Ethan Koenig", + AvatarLink: "https://secure.gravatar.com/avatar/b42fb195faa8c61b8d88abfefe30e9e3?d=identicon", + TotalCommits: 1, + Weeks: map[int64]*WeekData{ + 1511654400000: { + Week: 1511654400000, // sunday 2017-11-26 + Additions: 3, + Deletions: 0, + Commits: 1, + }, + }, + }, data["ethantkoenig@gmail.com"]) + assert.EqualValues(t, &ContributorData{ + Name: "Total", + AvatarLink: "", + TotalCommits: 3, + Weeks: map[int64]*WeekData{ + 1511654400000: { + Week: 1511654400000, // sunday 2017-11-26 (2017-11-26 20:31:18 -0800) + Additions: 3, + Deletions: 0, + Commits: 1, + }, + 1607817600000: { + Week: 1607817600000, // sunday 2020-12-13 (2020-12-15 15:23:11 -0500) + Additions: 10, + Deletions: 0, + Commits: 1, + }, + 1624752000000: { + Week: 1624752000000, // sunday 2021-06-27 (2021-06-29 21:54:09 +0200) + Additions: 2, + Deletions: 0, + Commits: 1, + }, + }, + }, data["total"]) +} diff --git a/services/repository/create.go b/services/repository/create.go new file mode 100644 index 0000000..d092d02 --- /dev/null +++ b/services/repository/create.go @@ -0,0 +1,318 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/options" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/templates/vars" + "code.gitea.io/gitea/modules/util" +) + +// CreateRepoOptions contains the create repository options +type CreateRepoOptions struct { + Name string + Description string + OriginalURL string + GitServiceType api.GitServiceType + Gitignores string + IssueLabels string + License string + Readme string + DefaultBranch string + IsPrivate bool + IsMirror bool + IsTemplate bool + AutoInit bool + Status repo_model.RepositoryStatus + TrustModel repo_model.TrustModelType + MirrorInterval string + ObjectFormatName string +} + +func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, repoPath string, opts CreateRepoOptions) error { + commitTimeStr := time.Now().Format(time.RFC3339) + authorSig := repo.Owner.NewGitSig() + + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+authorSig.Name, + "GIT_COMMITTER_EMAIL="+authorSig.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + + // Clone to temporary path and do the init commit. + if stdout, _, err := git.NewCommand(ctx, "clone").AddDynamicArguments(repoPath, tmpDir). + SetDescription(fmt.Sprintf("prepareRepoCommit (git clone): %s to %s", repoPath, tmpDir)). + RunStdString(&git.RunOpts{Dir: "", Env: env}); err != nil { + log.Error("Failed to clone from %v into %s: stdout: %s\nError: %v", repo, tmpDir, stdout, err) + return fmt.Errorf("git clone: %w", err) + } + + // README + data, err := options.Readme(opts.Readme) + if err != nil { + return fmt.Errorf("GetRepoInitFile[%s]: %w", opts.Readme, err) + } + + cloneLink := repo.CloneLink() + match := map[string]string{ + "Name": repo.Name, + "Description": repo.Description, + "CloneURL.SSH": cloneLink.SSH, + "CloneURL.HTTPS": cloneLink.HTTPS, + "OwnerName": repo.OwnerName, + } + res, err := vars.Expand(string(data), match) + if err != nil { + // here we could just log the error and continue the rendering + log.Error("unable to expand template vars for repo README: %s, err: %v", opts.Readme, err) + } + if err = os.WriteFile(filepath.Join(tmpDir, "README.md"), + []byte(res), 0o644); err != nil { + return fmt.Errorf("write README.md: %w", err) + } + + // .gitignore + if len(opts.Gitignores) > 0 { + var buf bytes.Buffer + names := strings.Split(opts.Gitignores, ",") + for _, name := range names { + data, err = options.Gitignore(name) + if err != nil { + return fmt.Errorf("GetRepoInitFile[%s]: %w", name, err) + } + buf.WriteString("# ---> " + name + "\n") + buf.Write(data) + buf.WriteString("\n") + } + + if buf.Len() > 0 { + if err = os.WriteFile(filepath.Join(tmpDir, ".gitignore"), buf.Bytes(), 0o644); err != nil { + return fmt.Errorf("write .gitignore: %w", err) + } + } + } + + // LICENSE + if len(opts.License) > 0 { + data, err = repo_module.GetLicense(opts.License, &repo_module.LicenseValues{ + Owner: repo.OwnerName, + Email: authorSig.Email, + Repo: repo.Name, + Year: time.Now().Format("2006"), + }) + if err != nil { + return fmt.Errorf("getLicense[%s]: %w", opts.License, err) + } + + if err = os.WriteFile(filepath.Join(tmpDir, "LICENSE"), data, 0o644); err != nil { + return fmt.Errorf("write LICENSE: %w", err) + } + } + + return nil +} + +// InitRepository initializes README and .gitignore if needed. +func initRepository(ctx context.Context, repoPath string, u *user_model.User, repo *repo_model.Repository, opts CreateRepoOptions) (err error) { + if err = repo_module.CheckInitRepository(ctx, repo.OwnerName, repo.Name, opts.ObjectFormatName); err != nil { + return err + } + + // Initialize repository according to user's choice. + if opts.AutoInit { + tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) + if err != nil { + return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err) + } + defer func() { + if err := util.RemoveAll(tmpDir); err != nil { + log.Warn("Unable to remove temporary directory: %s: Error: %v", tmpDir, err) + } + }() + + if err = prepareRepoCommit(ctx, repo, tmpDir, repoPath, opts); err != nil { + return fmt.Errorf("prepareRepoCommit: %w", err) + } + + // Apply changes and commit. + if err = initRepoCommit(ctx, tmpDir, repo, u, opts.DefaultBranch); err != nil { + return fmt.Errorf("initRepoCommit: %w", err) + } + } + + // Re-fetch the repository from database before updating it (else it would + // override changes that were done earlier with sql) + if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + return fmt.Errorf("getRepositoryByID: %w", err) + } + + if !opts.AutoInit { + repo.IsEmpty = true + } + + repo.DefaultBranch = setting.Repository.DefaultBranch + repo.WikiBranch = setting.Repository.DefaultBranch + + if len(opts.DefaultBranch) > 0 { + repo.DefaultBranch = opts.DefaultBranch + if err = gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + return fmt.Errorf("setDefaultBranch: %w", err) + } + + if !repo.IsEmpty { + if _, err := repo_module.SyncRepoBranches(ctx, repo.ID, u.ID); err != nil { + return fmt.Errorf("SyncRepoBranches: %w", err) + } + } + } + + if err = UpdateRepository(ctx, repo, false); err != nil { + return fmt.Errorf("updateRepository: %w", err) + } + + return nil +} + +// CreateRepositoryDirectly creates a repository for the user/organization. +func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { + if !doer.IsAdmin && !u.CanCreateRepo() { + return nil, repo_model.ErrReachLimitOfRepo{ + Limit: u.MaxRepoCreation, + } + } + + if len(opts.DefaultBranch) == 0 { + opts.DefaultBranch = setting.Repository.DefaultBranch + } + + // Check if label template exist + if len(opts.IssueLabels) > 0 { + if _, err := repo_module.LoadTemplateLabelsByDisplayName(opts.IssueLabels); err != nil { + return nil, err + } + } + + if opts.ObjectFormatName == "" { + opts.ObjectFormatName = git.Sha1ObjectFormat.Name() + } + + repo := &repo_model.Repository{ + OwnerID: u.ID, + Owner: u, + OwnerName: u.Name, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + OriginalURL: opts.OriginalURL, + OriginalServiceType: opts.GitServiceType, + IsPrivate: opts.IsPrivate, + IsFsckEnabled: !opts.IsMirror, + IsTemplate: opts.IsTemplate, + CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, + Status: opts.Status, + IsEmpty: !opts.AutoInit, + TrustModel: opts.TrustModel, + IsMirror: opts.IsMirror, + DefaultBranch: opts.DefaultBranch, + WikiBranch: setting.Repository.DefaultBranch, + ObjectFormatName: opts.ObjectFormatName, + } + + var rollbackRepo *repo_model.Repository + + if err := db.WithTx(ctx, func(ctx context.Context) error { + if err := repo_module.CreateRepositoryByExample(ctx, doer, u, repo, false, false); err != nil { + return err + } + + // No need for init mirror. + if opts.IsMirror { + return nil + } + + repoPath := repo_model.RepoPath(u.Name, repo.Name) + isExist, err := util.IsExist(repoPath) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + return err + } + if isExist { + // repo already exists - We have two or three options. + // 1. We fail stating that the directory exists + // 2. We create the db repository to go with this data and adopt the git repo + // 3. We delete it and start afresh + // + // Previously Gitea would just delete and start afresh - this was naughty. + // So we will now fail and delegate to other functionality to adopt or delete + log.Error("Files already exist in %s and we are not going to adopt or delete.", repoPath) + return repo_model.ErrRepoFilesAlreadyExist{ + Uname: u.Name, + Name: repo.Name, + } + } + + if err = initRepository(ctx, repoPath, doer, repo, opts); err != nil { + if err2 := util.RemoveAll(repoPath); err2 != nil { + log.Error("initRepository: %v", err) + return fmt.Errorf( + "delete repo directory %s/%s failed(2): %v", u.Name, repo.Name, err2) + } + return fmt.Errorf("initRepository: %w", err) + } + + // Initialize Issue Labels if selected + if len(opts.IssueLabels) > 0 { + if err = repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { + rollbackRepo = repo + rollbackRepo.OwnerID = u.ID + return fmt.Errorf("InitializeLabels: %w", err) + } + } + + if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { + return fmt.Errorf("checkDaemonExportOK: %w", err) + } + + if stdout, _, err := git.NewCommand(ctx, "update-server-info"). + SetDescription(fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath)). + RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + rollbackRepo = repo + rollbackRepo.OwnerID = u.ID + return fmt.Errorf("CreateRepository(git update-server-info): %w", err) + } + return nil + }); err != nil { + if rollbackRepo != nil { + if errDelete := DeleteRepositoryDirectly(ctx, doer, rollbackRepo.ID); errDelete != nil { + log.Error("Rollback deleteRepository: %v", errDelete) + } + } + + return nil, err + } + + return repo, nil +} diff --git a/services/repository/create_test.go b/services/repository/create_test.go new file mode 100644 index 0000000..9cde285 --- /dev/null +++ b/services/repository/create_test.go @@ -0,0 +1,149 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "fmt" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/organization" + "code.gitea.io/gitea/models/perm" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIncludesAllRepositoriesTeams(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + testTeamRepositories := func(teamID int64, repoIds []int64) { + team := unittest.AssertExistsAndLoadBean(t, &organization.Team{ID: teamID}) + require.NoError(t, team.LoadRepositories(db.DefaultContext), "%s: GetRepositories", team.Name) + assert.Len(t, team.Repos, team.NumRepos, "%s: len repo", team.Name) + assert.Len(t, team.Repos, len(repoIds), "%s: repo count", team.Name) + for i, rid := range repoIds { + if rid > 0 { + assert.True(t, HasRepository(db.DefaultContext, team, rid), "%s: HasRepository(%d) %d", rid, i) + } + } + } + + // Get an admin user. + user, err := user_model.GetUserByID(db.DefaultContext, 1) + require.NoError(t, err, "GetUserByID") + + // Create org. + org := &organization.Organization{ + Name: "All_repo", + IsActive: true, + Type: user_model.UserTypeOrganization, + Visibility: structs.VisibleTypePublic, + } + require.NoError(t, organization.CreateOrganization(db.DefaultContext, org, user), "CreateOrganization") + + // Check Owner team. + ownerTeam, err := org.GetOwnerTeam(db.DefaultContext) + require.NoError(t, err, "GetOwnerTeam") + assert.True(t, ownerTeam.IncludesAllRepositories, "Owner team includes all repositories") + + // Create repos. + repoIDs := make([]int64, 0) + for i := 0; i < 3; i++ { + r, err := CreateRepositoryDirectly(db.DefaultContext, user, org.AsUser(), CreateRepoOptions{Name: fmt.Sprintf("repo-%d", i)}) + require.NoError(t, err, "CreateRepository %d", i) + if r != nil { + repoIDs = append(repoIDs, r.ID) + } + } + // Get fresh copy of Owner team after creating repos. + ownerTeam, err = org.GetOwnerTeam(db.DefaultContext) + require.NoError(t, err, "GetOwnerTeam") + + // Create teams and check repositories. + teams := []*organization.Team{ + ownerTeam, + { + OrgID: org.ID, + Name: "team one", + AccessMode: perm.AccessModeRead, + IncludesAllRepositories: true, + }, + { + OrgID: org.ID, + Name: "team 2", + AccessMode: perm.AccessModeRead, + IncludesAllRepositories: false, + }, + { + OrgID: org.ID, + Name: "team three", + AccessMode: perm.AccessModeWrite, + IncludesAllRepositories: true, + }, + { + OrgID: org.ID, + Name: "team 4", + AccessMode: perm.AccessModeWrite, + IncludesAllRepositories: false, + }, + } + teamRepos := [][]int64{ + repoIDs, + repoIDs, + {}, + repoIDs, + {}, + } + for i, team := range teams { + if i > 0 { // first team is Owner. + require.NoError(t, models.NewTeam(db.DefaultContext, team), "%s: NewTeam", team.Name) + } + testTeamRepositories(team.ID, teamRepos[i]) + } + + // Update teams and check repositories. + teams[3].IncludesAllRepositories = false + teams[4].IncludesAllRepositories = true + teamRepos[4] = repoIDs + for i, team := range teams { + require.NoError(t, models.UpdateTeam(db.DefaultContext, team, false, true), "%s: UpdateTeam", team.Name) + testTeamRepositories(team.ID, teamRepos[i]) + } + + // Create repo and check teams repositories. + r, err := CreateRepositoryDirectly(db.DefaultContext, user, org.AsUser(), CreateRepoOptions{Name: "repo-last"}) + require.NoError(t, err, "CreateRepository last") + if r != nil { + repoIDs = append(repoIDs, r.ID) + } + teamRepos[0] = repoIDs + teamRepos[1] = repoIDs + teamRepos[4] = repoIDs + for i, team := range teams { + testTeamRepositories(team.ID, teamRepos[i]) + } + + // Remove repo and check teams repositories. + require.NoError(t, DeleteRepositoryDirectly(db.DefaultContext, user, repoIDs[0]), "DeleteRepository") + teamRepos[0] = repoIDs[1:] + teamRepos[1] = repoIDs[1:] + teamRepos[3] = repoIDs[1:3] + teamRepos[4] = repoIDs[1:] + for i, team := range teams { + testTeamRepositories(team.ID, teamRepos[i]) + } + + // Wipe created items. + for i, rid := range repoIDs { + if i > 0 { // first repo already deleted. + require.NoError(t, DeleteRepositoryDirectly(db.DefaultContext, user, rid), "DeleteRepository %d", i) + } + } + require.NoError(t, organization.DeleteOrganization(db.DefaultContext, org), "DeleteOrganization") +} diff --git a/services/repository/delete.go b/services/repository/delete.go new file mode 100644 index 0000000..6e84194 --- /dev/null +++ b/services/repository/delete.go @@ -0,0 +1,471 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/models" + actions_model "code.gitea.io/gitea/models/actions" + activities_model "code.gitea.io/gitea/models/activities" + admin_model "code.gitea.io/gitea/models/admin" + asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/organization" + access_model "code.gitea.io/gitea/models/perm/access" + project_model "code.gitea.io/gitea/models/project" + repo_model "code.gitea.io/gitea/models/repo" + secret_model "code.gitea.io/gitea/models/secret" + system_model "code.gitea.io/gitea/models/system" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/models/webhook" + actions_module "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + + "xorm.io/builder" +) + +// DeleteRepository deletes a repository for a user or organization. +// make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock) +func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID int64, ignoreOrgTeams ...bool) error { + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + sess := db.GetEngine(ctx) + + repo := &repo_model.Repository{} + has, err := sess.ID(repoID).Get(repo) + if err != nil { + return err + } else if !has { + return repo_model.ErrRepoNotExist{ + ID: repoID, + OwnerName: "", + Name: "", + } + } + + // Query the action tasks of this repo, they will be needed after they have been deleted to remove the logs + tasks, err := db.Find[actions_model.ActionTask](ctx, actions_model.FindTaskOptions{RepoID: repoID}) + if err != nil { + return fmt.Errorf("find actions tasks of repo %v: %w", repoID, err) + } + + // Query the artifacts of this repo, they will be needed after they have been deleted to remove artifacts files in ObjectStorage + artifacts, err := db.Find[actions_model.ActionArtifact](ctx, actions_model.FindArtifactsOptions{RepoID: repoID}) + if err != nil { + return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err) + } + + // In case owner is a organization, we have to change repo specific teams + // if ignoreOrgTeams is not true + var org *user_model.User + if len(ignoreOrgTeams) == 0 || !ignoreOrgTeams[0] { + if org, err = user_model.GetUserByID(ctx, repo.OwnerID); err != nil { + return err + } + } + + // Delete Deploy Keys + deployKeys, err := db.Find[asymkey_model.DeployKey](ctx, asymkey_model.ListDeployKeysOptions{RepoID: repoID}) + if err != nil { + return fmt.Errorf("listDeployKeys: %w", err) + } + needRewriteKeysFile := len(deployKeys) > 0 + for _, dKey := range deployKeys { + if err := models.DeleteDeployKey(ctx, doer, dKey.ID); err != nil { + return fmt.Errorf("deleteDeployKeys: %w", err) + } + } + + if cnt, err := sess.ID(repoID).Delete(&repo_model.Repository{}); err != nil { + return err + } else if cnt != 1 { + return repo_model.ErrRepoNotExist{ + ID: repoID, + OwnerName: "", + Name: "", + } + } + + if org != nil && org.IsOrganization() { + teams, err := organization.FindOrgTeams(ctx, org.ID) + if err != nil { + return err + } + for _, t := range teams { + if !organization.HasTeamRepo(ctx, t.OrgID, t.ID, repoID) { + continue + } else if err = removeRepositoryFromTeam(ctx, t, repo, false); err != nil { + return err + } + } + } + + attachments := make([]*repo_model.Attachment, 0, 20) + if err = sess.Join("INNER", "`release`", "`release`.id = `attachment`.release_id"). + Where("`release`.repo_id = ?", repoID). + Find(&attachments); err != nil { + return err + } + releaseAttachments := make([]string, 0, len(attachments)) + for i := 0; i < len(attachments); i++ { + releaseAttachments = append(releaseAttachments, attachments[i].RelativePath()) + } + + if _, err := db.Exec(ctx, "UPDATE `user` SET num_stars=num_stars-1 WHERE id IN (SELECT `uid` FROM `star` WHERE repo_id = ?)", repo.ID); err != nil { + return err + } + + if setting.Database.Type.IsMySQL() { + // mariadb:10 does not use the hook_task KEY when using IN. + // https://codeberg.org/forgejo/forgejo/issues/3678 + // + // Version 11 does support it, but is not available in debian yet. + // Version 11.4 LTS is not available yet (stable should be released mid 2024 https://mariadb.org/mariadb/all-releases/) + + // Sqlite does not support the DELETE *** FROM *** syntax + // https://stackoverflow.com/q/24511153/3207406 + + // in the meantime, use a dedicated query for mysql... + if _, err := db.Exec(ctx, "DELETE `hook_task` FROM `hook_task` INNER JOIN `webhook` ON `webhook`.id = `hook_task`.hook_id WHERE `webhook`.repo_id = ?", repo.ID); err != nil { + return err + } + } else { + if _, err := db.GetEngine(ctx).In("hook_id", builder.Select("id").From("webhook").Where(builder.Eq{"webhook.repo_id": repo.ID})). + Delete(&webhook.HookTask{}); err != nil { + return err + } + } + + if err := db.DeleteBeans(ctx, + &access_model.Access{RepoID: repo.ID}, + &activities_model.Action{RepoID: repo.ID}, + &repo_model.Collaboration{RepoID: repoID}, + &issues_model.Comment{RefRepoID: repoID}, + &git_model.CommitStatus{RepoID: repoID}, + &git_model.Branch{RepoID: repoID}, + &git_model.LFSLock{RepoID: repoID}, + &repo_model.LanguageStat{RepoID: repoID}, + &issues_model.Milestone{RepoID: repoID}, + &repo_model.Mirror{RepoID: repoID}, + &activities_model.Notification{RepoID: repoID}, + &git_model.ProtectedBranch{RepoID: repoID}, + &git_model.ProtectedTag{RepoID: repoID}, + &repo_model.PushMirror{RepoID: repoID}, + &repo_model.Release{RepoID: repoID}, + &repo_model.RepoIndexerStatus{RepoID: repoID}, + &repo_model.Redirect{RedirectRepoID: repoID}, + &repo_model.RepoUnit{RepoID: repoID}, + &repo_model.Star{RepoID: repoID}, + &admin_model.Task{RepoID: repoID}, + &repo_model.Watch{RepoID: repoID}, + &webhook.Webhook{RepoID: repoID}, + &secret_model.Secret{RepoID: repoID}, + &actions_model.ActionTaskStep{RepoID: repoID}, + &actions_model.ActionTask{RepoID: repoID}, + &actions_model.ActionRunJob{RepoID: repoID}, + &actions_model.ActionRun{RepoID: repoID}, + &actions_model.ActionRunner{RepoID: repoID}, + &actions_model.ActionScheduleSpec{RepoID: repoID}, + &actions_model.ActionSchedule{RepoID: repoID}, + &actions_model.ActionArtifact{RepoID: repoID}, + &repo_model.RepoArchiveDownloadCount{RepoID: repoID}, + &actions_model.ActionRunnerToken{RepoID: repoID}, + ); err != nil { + return fmt.Errorf("deleteBeans: %w", err) + } + + // Delete Labels and related objects + if err := issues_model.DeleteLabelsByRepoID(ctx, repoID); err != nil { + return err + } + + // Delete Pulls and related objects + if err := issues_model.DeletePullsByBaseRepoID(ctx, repoID); err != nil { + return err + } + + // Delete Issues and related objects + var attachmentPaths []string + if attachmentPaths, err = issues_model.DeleteIssuesByRepoID(ctx, repoID); err != nil { + return err + } + + // Delete issue index + if err := db.DeleteResourceIndex(ctx, "issue_index", repoID); err != nil { + return err + } + + if repo.IsFork { + if _, err := db.Exec(ctx, "UPDATE `repository` SET num_forks=num_forks-1 WHERE id=?", repo.ForkID); err != nil { + return fmt.Errorf("decrease fork count: %w", err) + } + } + + if _, err := db.Exec(ctx, "UPDATE `user` SET num_repos=num_repos-1 WHERE id=?", repo.OwnerID); err != nil { + return err + } + + if len(repo.Topics) > 0 { + if err := repo_model.RemoveTopicsFromRepo(ctx, repo.ID); err != nil { + return err + } + } + + if err := project_model.DeleteProjectByRepoID(ctx, repoID); err != nil { + return fmt.Errorf("unable to delete projects for repo[%d]: %w", repoID, err) + } + + // Remove LFS objects + var lfsObjects []*git_model.LFSMetaObject + if err = sess.Where("repository_id=?", repoID).Find(&lfsObjects); err != nil { + return err + } + + lfsPaths := make([]string, 0, len(lfsObjects)) + for _, v := range lfsObjects { + count, err := db.CountByBean(ctx, &git_model.LFSMetaObject{Pointer: lfs.Pointer{Oid: v.Oid}}) + if err != nil { + return err + } + if count > 1 { + continue + } + + lfsPaths = append(lfsPaths, v.RelativePath()) + } + + if _, err := db.DeleteByBean(ctx, &git_model.LFSMetaObject{RepositoryID: repoID}); err != nil { + return err + } + + // Remove archives + var archives []*repo_model.RepoArchiver + if err = sess.Where("repo_id=?", repoID).Find(&archives); err != nil { + return err + } + + archivePaths := make([]string, 0, len(archives)) + for _, v := range archives { + archivePaths = append(archivePaths, v.RelativePath()) + } + + if _, err := db.DeleteByBean(ctx, &repo_model.RepoArchiver{RepoID: repoID}); err != nil { + return err + } + + if repo.NumForks > 0 { + if _, err = sess.Exec("UPDATE `repository` SET fork_id=0,is_fork=? WHERE fork_id=?", false, repo.ID); err != nil { + log.Error("reset 'fork_id' and 'is_fork': %v", err) + } + } + + // Get all attachments with both issue_id and release_id are zero + var newAttachments []*repo_model.Attachment + if err := sess.Where(builder.Eq{ + "repo_id": repo.ID, + "issue_id": 0, + "release_id": 0, + }).Find(&newAttachments); err != nil { + return err + } + + newAttachmentPaths := make([]string, 0, len(newAttachments)) + for _, attach := range newAttachments { + newAttachmentPaths = append(newAttachmentPaths, attach.RelativePath()) + } + + if _, err := sess.Where("repo_id=?", repo.ID).Delete(new(repo_model.Attachment)); err != nil { + return err + } + + if err = committer.Commit(); err != nil { + return err + } + + committer.Close() + + if needRewriteKeysFile { + if err := asymkey_model.RewriteAllPublicKeys(ctx); err != nil { + log.Error("RewriteAllPublicKeys failed: %v", err) + } + } + + // We should always delete the files after the database transaction succeed. If + // we delete the file but the database rollback, the repository will be broken. + + // Remove repository files. + repoPath := repo.RepoPath() + system_model.RemoveAllWithNotice(ctx, "Delete repository files", repoPath) + + // Remove wiki files + if repo.HasWiki() { + system_model.RemoveAllWithNotice(ctx, "Delete repository wiki", repo.WikiPath()) + } + + // Remove archives + for _, archive := range archivePaths { + system_model.RemoveStorageWithNotice(ctx, storage.RepoArchives, "Delete repo archive file", archive) + } + + // Remove lfs objects + for _, lfsObj := range lfsPaths { + system_model.RemoveStorageWithNotice(ctx, storage.LFS, "Delete orphaned LFS file", lfsObj) + } + + // Remove issue attachment files. + for _, attachment := range attachmentPaths { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", attachment) + } + + // Remove release attachment files. + for _, releaseAttachment := range releaseAttachments { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete release attachment", releaseAttachment) + } + + // Remove attachment with no issue_id and release_id. + for _, newAttachment := range newAttachmentPaths { + system_model.RemoveStorageWithNotice(ctx, storage.Attachments, "Delete issue attachment", newAttachment) + } + + if len(repo.Avatar) > 0 { + if err := storage.RepoAvatars.Delete(repo.CustomAvatarRelativePath()); err != nil { + return fmt.Errorf("Failed to remove %s: %w", repo.Avatar, err) + } + } + + // Finally, delete action logs after the actions have already been deleted to avoid new log files + for _, task := range tasks { + err := actions_module.RemoveLogs(ctx, task.LogInStorage, task.LogFilename) + if err != nil { + log.Error("remove log file %q: %v", task.LogFilename, err) + // go on + } + } + + // delete actions artifacts in ObjectStorage after the repo have already been deleted + for _, art := range artifacts { + if err := storage.ActionsArtifacts.Delete(art.StoragePath); err != nil { + log.Error("remove artifact file %q: %v", art.StoragePath, err) + // go on + } + } + + return nil +} + +// removeRepositoryFromTeam removes a repository from a team and recalculates access +// Note: Repository shall not be removed from team if it includes all repositories (unless the repository is deleted) +func removeRepositoryFromTeam(ctx context.Context, t *organization.Team, repo *repo_model.Repository, recalculate bool) (err error) { + e := db.GetEngine(ctx) + if err = organization.RemoveTeamRepo(ctx, t.ID, repo.ID); err != nil { + return err + } + + t.NumRepos-- + if _, err = e.ID(t.ID).Cols("num_repos").Update(t); err != nil { + return err + } + + // Don't need to recalculate when delete a repository from organization. + if recalculate { + if err = access_model.RecalculateTeamAccesses(ctx, repo, t.ID); err != nil { + return err + } + } + + teamUsers, err := organization.GetTeamUsersByTeamID(ctx, t.ID) + if err != nil { + return fmt.Errorf("getTeamUsersByTeamID: %w", err) + } + for _, teamUser := range teamUsers { + has, err := access_model.HasAccess(ctx, teamUser.UID, repo) + if err != nil { + return err + } else if has { + continue + } + + if err = repo_model.WatchRepo(ctx, teamUser.UID, repo.ID, false); err != nil { + return err + } + + // Remove all IssueWatches a user has subscribed to in the repositories + if err := issues_model.RemoveIssueWatchersByRepoID(ctx, teamUser.UID, repo.ID); err != nil { + return err + } + } + + return nil +} + +// HasRepository returns true if given repository belong to team. +func HasRepository(ctx context.Context, t *organization.Team, repoID int64) bool { + return organization.HasTeamRepo(ctx, t.OrgID, t.ID, repoID) +} + +// RemoveRepositoryFromTeam removes repository from team of organization. +// If the team shall include all repositories the request is ignored. +func RemoveRepositoryFromTeam(ctx context.Context, t *organization.Team, repoID int64) error { + if !HasRepository(ctx, t, repoID) { + return nil + } + + if t.IncludesAllRepositories { + return nil + } + + repo, err := repo_model.GetRepositoryByID(ctx, repoID) + if err != nil { + return err + } + + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + if err = removeRepositoryFromTeam(ctx, t, repo, true); err != nil { + return err + } + + return committer.Commit() +} + +// DeleteOwnerRepositoriesDirectly calls DeleteRepositoryDirectly for all repos of the given owner +func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User) error { + for { + repos, _, err := repo_model.GetUserRepositories(ctx, &repo_model.SearchRepoOptions{ + ListOptions: db.ListOptions{ + PageSize: repo_model.RepositoryListDefaultPageSize, + Page: 1, + }, + Private: true, + OwnerID: owner.ID, + Actor: owner, + }) + if err != nil { + return fmt.Errorf("GetUserRepositories: %w", err) + } + if len(repos) == 0 { + break + } + for _, repo := range repos { + if err := DeleteRepositoryDirectly(ctx, owner, repo.ID); err != nil { + return fmt.Errorf("unable to delete repository %s for %s[%d]. Error: %w", repo.Name, owner.Name, owner.ID, err) + } + } + } + return nil +} diff --git a/services/repository/files/cherry_pick.go b/services/repository/files/cherry_pick.go new file mode 100644 index 0000000..451a182 --- /dev/null +++ b/services/repository/files/cherry_pick.go @@ -0,0 +1,128 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/services/pull" +) + +// CherryPick cherrypicks or reverts a commit to the given repository +func CherryPick(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, revert bool, opts *ApplyDiffPatchOptions) (*structs.FileResponse, error) { + if err := opts.Validate(ctx, repo, doer); err != nil { + return nil, err + } + message := strings.TrimSpace(opts.Message) + + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + log.Error("NewTemporaryUploadRepository failed: %v", err) + } + defer t.Close() + if err := t.Clone(opts.OldBranch, false); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + if err := t.RefreshIndex(); err != nil { + return nil, err + } + + // Get the commit of the original branch + commit, err := t.GetBranchCommit(opts.OldBranch) + if err != nil { + return nil, err // Couldn't get a commit for the branch + } + + // Assigned LastCommitID in opts if it hasn't been set + if opts.LastCommitID == "" { + opts.LastCommitID = commit.ID.String() + } else { + lastCommitID, err := t.gitRepo.ConvertToGitID(opts.LastCommitID) + if err != nil { + return nil, fmt.Errorf("CherryPick: Invalid last commit ID: %w", err) + } + opts.LastCommitID = lastCommitID.String() + if commit.ID.String() != opts.LastCommitID { + return nil, models.ErrCommitIDDoesNotMatch{ + GivenCommitID: opts.LastCommitID, + CurrentCommitID: opts.LastCommitID, + } + } + } + + commit, err = t.GetCommit(strings.TrimSpace(opts.Content)) + if err != nil { + return nil, err + } + parent, err := commit.ParentID(0) + if err != nil { + parent = git.ObjectFormatFromName(repo.ObjectFormatName).EmptyTree() + } + + base, right := parent.String(), commit.ID.String() + + if revert { + right, base = base, right + } + + description := fmt.Sprintf("CherryPick %s onto %s", right, opts.OldBranch) + conflict, _, err := pull.AttemptThreeWayMerge(ctx, + t.basePath, t.gitRepo, base, opts.LastCommitID, right, description) + if err != nil { + return nil, fmt.Errorf("failed to three-way merge %s onto %s: %w", right, opts.OldBranch, err) + } + + if conflict { + return nil, fmt.Errorf("failed to merge due to conflicts") + } + + treeHash, err := t.WriteTree() + if err != nil { + // likely non-sensical tree due to merge conflicts... + return nil, err + } + + // Now commit the tree + var commitHash string + if opts.Dates != nil { + commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) + } else { + commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff) + } + if err != nil { + return nil, err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return nil, err + } + + commit, err = t.GetCommit(commitHash) + if err != nil { + return nil, err + } + + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, commit) + fileResponse := &structs.FileResponse{ + Commit: fileCommitResponse, + Verification: verification, + } + + return fileResponse, nil +} diff --git a/services/repository/files/commit.go b/services/repository/files/commit.go new file mode 100644 index 0000000..e0dad29 --- /dev/null +++ b/services/repository/files/commit.go @@ -0,0 +1,44 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + + asymkey_model "code.gitea.io/gitea/models/asymkey" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/structs" +) + +// CountDivergingCommits determines how many commits a branch is ahead or behind the repository's base branch +func CountDivergingCommits(ctx context.Context, repo *repo_model.Repository, branch string) (*git.DivergeObject, error) { + divergence, err := git.GetDivergingCommits(ctx, repo.RepoPath(), repo.DefaultBranch, branch) + if err != nil { + return nil, err + } + return &divergence, nil +} + +// GetPayloadCommitVerification returns the verification information of a commit +func GetPayloadCommitVerification(ctx context.Context, commit *git.Commit) *structs.PayloadCommitVerification { + verification := &structs.PayloadCommitVerification{} + commitVerification := asymkey_model.ParseCommitWithSignature(ctx, commit) + if commit.Signature != nil { + verification.Signature = commit.Signature.Signature + verification.Payload = commit.Signature.Payload + } + if commitVerification.SigningUser != nil { + verification.Signer = &structs.PayloadUser{ + Name: commitVerification.SigningUser.Name, + Email: commitVerification.SigningUser.Email, + } + } + verification.Verified = commitVerification.Verified + verification.Reason = commitVerification.Reason + if verification.Reason == "" && !verification.Verified { + verification.Reason = "gpg.error.not_signed_commit" + } + return verification +} diff --git a/services/repository/files/content.go b/services/repository/files/content.go new file mode 100644 index 0000000..8dfd8b8 --- /dev/null +++ b/services/repository/files/content.go @@ -0,0 +1,278 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "net/url" + "path" + "strings" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" +) + +// ContentType repo content type +type ContentType string + +// The string representations of different content types +const ( + // ContentTypeRegular regular content type (file) + ContentTypeRegular ContentType = "file" + // ContentTypeDir dir content type (dir) + ContentTypeDir ContentType = "dir" + // ContentLink link content type (symlink) + ContentTypeLink ContentType = "symlink" + // ContentTag submodule content type (submodule) + ContentTypeSubmodule ContentType = "submodule" +) + +// String gets the string of ContentType +func (ct *ContentType) String() string { + return string(*ct) +} + +// GetContentsOrList gets the meta data of a file's contents (*ContentsResponse) if treePath not a tree +// directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag +func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePath, ref string) (any, error) { + if repo.IsEmpty { + return make([]any, 0), nil + } + if ref == "" { + ref = repo.DefaultBranch + } + origRef := ref + + // Check that the path given in opts.treePath is valid (not a git path) + cleanTreePath := CleanUploadFileName(treePath) + if cleanTreePath == "" && treePath != "" { + return nil, models.ErrFilenameInvalid{ + Path: treePath, + } + } + treePath = cleanTreePath + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return nil, err + } + defer closer.Close() + + // Get the commit object for the ref + commit, err := gitRepo.GetCommit(ref) + if err != nil { + return nil, err + } + + entry, err := commit.GetTreeEntryByPath(treePath) + if err != nil { + return nil, err + } + + if entry.Type() != "tree" { + return GetContents(ctx, repo, treePath, origRef, false) + } + + // We are in a directory, so we return a list of FileContentResponse objects + var fileList []*api.ContentsResponse + + gitTree, err := commit.SubTree(treePath) + if err != nil { + return nil, err + } + entries, err := gitTree.ListEntries() + if err != nil { + return nil, err + } + for _, e := range entries { + subTreePath := path.Join(treePath, e.Name()) + fileContentResponse, err := GetContents(ctx, repo, subTreePath, origRef, true) + if err != nil { + return nil, err + } + fileList = append(fileList, fileContentResponse) + } + return fileList, nil +} + +// GetObjectTypeFromTreeEntry check what content is behind it +func GetObjectTypeFromTreeEntry(entry *git.TreeEntry) ContentType { + switch { + case entry.IsDir(): + return ContentTypeDir + case entry.IsSubModule(): + return ContentTypeSubmodule + case entry.IsExecutable(), entry.IsRegular(): + return ContentTypeRegular + case entry.IsLink(): + return ContentTypeLink + default: + return "" + } +} + +// GetContents gets the meta data on a file's contents. Ref can be a branch, commit or tag +func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref string, forList bool) (*api.ContentsResponse, error) { + if ref == "" { + ref = repo.DefaultBranch + } + origRef := ref + + // Check that the path given in opts.treePath is valid (not a git path) + cleanTreePath := CleanUploadFileName(treePath) + if cleanTreePath == "" && treePath != "" { + return nil, models.ErrFilenameInvalid{ + Path: treePath, + } + } + treePath = cleanTreePath + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return nil, err + } + defer closer.Close() + + // Get the commit object for the ref + commit, err := gitRepo.GetCommit(ref) + if err != nil { + return nil, err + } + commitID := commit.ID.String() + if len(ref) >= 4 && strings.HasPrefix(commitID, ref) { + ref = commit.ID.String() + } + + entry, err := commit.GetTreeEntryByPath(treePath) + if err != nil { + return nil, err + } + + refType := gitRepo.GetRefType(ref) + if refType == "invalid" { + return nil, fmt.Errorf("no commit found for the ref [ref: %s]", ref) + } + + selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(treePath) + "?ref=" + url.QueryEscape(origRef)) + if err != nil { + return nil, err + } + selfURLString := selfURL.String() + + err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(ref, refType != git.ObjectCommit), repo.FullName(), commitID) + if err != nil { + return nil, err + } + + lastCommit, err := commit.GetCommitByPath(treePath) + if err != nil { + return nil, err + } + + // All content types have these fields in populated + contentsResponse := &api.ContentsResponse{ + Name: entry.Name(), + Path: treePath, + SHA: entry.ID.String(), + LastCommitSHA: lastCommit.ID.String(), + Size: entry.Size(), + URL: &selfURLString, + Links: &api.FileLinksResponse{ + Self: &selfURLString, + }, + } + + // Now populate the rest of the ContentsResponse based on entry type + if entry.IsRegular() || entry.IsExecutable() { + contentsResponse.Type = string(ContentTypeRegular) + if blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String()); err != nil { + return nil, err + } else if !forList { + // We don't show the content if we are getting a list of FileContentResponses + contentsResponse.Encoding = &blobResponse.Encoding + contentsResponse.Content = &blobResponse.Content + } + } else if entry.IsDir() { + contentsResponse.Type = string(ContentTypeDir) + } else if entry.IsLink() { + contentsResponse.Type = string(ContentTypeLink) + // The target of a symlink file is the content of the file + targetFromContent, err := entry.Blob().GetBlobContent(1024) + if err != nil { + return nil, err + } + contentsResponse.Target = &targetFromContent + } else if entry.IsSubModule() { + contentsResponse.Type = string(ContentTypeSubmodule) + submodule, err := commit.GetSubModule(treePath) + if err != nil { + return nil, err + } + if submodule != nil && submodule.URL != "" { + contentsResponse.SubmoduleGitURL = &submodule.URL + } + } + // Handle links + if entry.IsRegular() || entry.IsLink() || entry.IsExecutable() { + downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + if err != nil { + return nil, err + } + downloadURLString := downloadURL.String() + contentsResponse.DownloadURL = &downloadURLString + } + if !entry.IsSubModule() { + htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + if err != nil { + return nil, err + } + htmlURLString := htmlURL.String() + contentsResponse.HTMLURL = &htmlURLString + contentsResponse.Links.HTMLURL = &htmlURLString + + gitURL, err := url.Parse(repo.APIURL() + "/git/blobs/" + url.PathEscape(entry.ID.String())) + if err != nil { + return nil, err + } + gitURLString := gitURL.String() + contentsResponse.GitURL = &gitURLString + contentsResponse.Links.GitURL = &gitURLString + } + + return contentsResponse, nil +} + +// GetBlobBySHA get the GitBlobResponse of a repository using a sha hash. +func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) { + gitBlob, err := gitRepo.GetBlob(sha) + if err != nil { + return nil, err + } + content := "" + if gitBlob.Size() <= setting.API.DefaultMaxBlobSize { + content, err = gitBlob.GetBlobContentBase64() + if err != nil { + return nil, err + } + } + return &api.GitBlobResponse{ + SHA: gitBlob.ID.String(), + URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), + Size: gitBlob.Size(), + Encoding: "base64", + Content: content, + }, nil +} + +// TryGetContentLanguage tries to get the (linguist) language of the file content +func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) (string, error) { + attribute, err := gitRepo.GitAttributeFirst(commitID, treePath, "linguist-language", "gitlab-language") + return attribute.Prefix(), err +} diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go new file mode 100644 index 0000000..c22dcd2 --- /dev/null +++ b/services/repository/files/content_test.go @@ -0,0 +1,201 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/gitrepo" + api "code.gitea.io/gitea/modules/structs" + + _ "code.gitea.io/gitea/models/actions" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m) +} + +func getExpectedReadmeContentsResponse() *api.ContentsResponse { + treePath := "README.md" + sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" + encoding := "base64" + content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" + selfURL := "https://try.gitea.io/api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" + htmlURL := "https://try.gitea.io/user2/repo1/src/branch/master/" + treePath + gitURL := "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/" + sha + downloadURL := "https://try.gitea.io/user2/repo1/raw/branch/master/" + treePath + return &api.ContentsResponse{ + Name: treePath, + Path: treePath, + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Type: "file", + Size: 30, + Encoding: &encoding, + Content: &content, + URL: &selfURL, + HTMLURL: &htmlURL, + GitURL: &gitURL, + DownloadURL: &downloadURL, + Links: &api.FileLinksResponse{ + Self: &selfURL, + GitURL: &gitURL, + HTMLURL: &htmlURL, + }, + } +} + +func TestGetContents(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "README.md" + ref := repo.DefaultBranch + + expectedContentsResponse := getExpectedReadmeContentsResponse() + + t.Run("Get README.md contents with GetContents(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContents(db.DefaultContext, repo, treePath, ref, false) + assert.EqualValues(t, expectedContentsResponse, fileContentResponse) + require.NoError(t, err) + }) + + t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContents(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContents(db.DefaultContext, repo, treePath, "", false) + assert.EqualValues(t, expectedContentsResponse, fileContentResponse) + require.NoError(t, err) + }) +} + +func TestGetContentsOrListForDir(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "" // root dir + ref := repo.DefaultBranch + + readmeContentsResponse := getExpectedReadmeContentsResponse() + // because will be in a list, doesn't have encoding and content + readmeContentsResponse.Encoding = nil + readmeContentsResponse.Content = nil + + expectedContentsListResponse := []*api.ContentsResponse{ + readmeContentsResponse, + } + + t.Run("Get root dir contents with GetContentsOrList(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, ref) + assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) + require.NoError(t, err) + }) + + t.Run("Get root dir contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, "") + assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) + require.NoError(t, err) + }) +} + +func TestGetContentsOrListForFile(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "README.md" + ref := repo.DefaultBranch + + expectedContentsResponse := getExpectedReadmeContentsResponse() + + t.Run("Get README.md contents with GetContentsOrList(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, ref) + assert.EqualValues(t, expectedContentsResponse, fileContentResponse) + require.NoError(t, err) + }) + + t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, "") + assert.EqualValues(t, expectedContentsResponse, fileContentResponse) + require.NoError(t, err) + }) +} + +func TestGetContentsErrors(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "README.md" + ref := repo.DefaultBranch + + t.Run("bad treePath", func(t *testing.T) { + badTreePath := "bad/tree.md" + fileContentResponse, err := GetContents(db.DefaultContext, repo, badTreePath, ref, false) + require.EqualError(t, err, "object does not exist [id: , rel_path: bad]") + assert.Nil(t, fileContentResponse) + }) + + t.Run("bad ref", func(t *testing.T) { + badRef := "bad_ref" + fileContentResponse, err := GetContents(db.DefaultContext, repo, treePath, badRef, false) + require.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") + assert.Nil(t, fileContentResponse) + }) +} + +func TestGetContentsOrListErrors(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "README.md" + ref := repo.DefaultBranch + + t.Run("bad treePath", func(t *testing.T) { + badTreePath := "bad/tree.md" + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, badTreePath, ref) + require.EqualError(t, err, "object does not exist [id: , rel_path: bad]") + assert.Nil(t, fileContentResponse) + }) + + t.Run("bad ref", func(t *testing.T) { + badRef := "bad_ref" + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, badRef) + require.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") + assert.Nil(t, fileContentResponse) + }) +} + +func TestGetContentsOrListOfEmptyRepos(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 52}) + + t.Run("empty repo", func(t *testing.T) { + contents, err := GetContentsOrList(db.DefaultContext, repo, "", "") + require.NoError(t, err) + assert.Empty(t, contents) + }) +} + +func TestGetBlobBySHA(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + gitRepo, err := gitrepo.OpenRepository(db.DefaultContext, repo) + require.NoError(t, err) + defer gitRepo.Close() + + gbr, err := GetBlobBySHA(db.DefaultContext, repo, gitRepo, "65f1bf27bc3bf70f64657658635e66094edbcb4d") + expectedGBR := &api.GitBlobResponse{ + Content: "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK", + Encoding: "base64", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Size: 180, + } + require.NoError(t, err) + assert.Equal(t, expectedGBR, gbr) +} diff --git a/services/repository/files/diff.go b/services/repository/files/diff.go new file mode 100644 index 0000000..bf8b938 --- /dev/null +++ b/services/repository/files/diff.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "strings" + + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/services/gitdiff" +) + +// GetDiffPreview produces and returns diff result of a file which is not yet committed. +func GetDiffPreview(ctx context.Context, repo *repo_model.Repository, branch, treePath, content string) (*gitdiff.Diff, error) { + if branch == "" { + branch = repo.DefaultBranch + } + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + return nil, err + } + defer t.Close() + if err := t.Clone(branch, true); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Add the object to the database + objectHash, err := t.HashObject(strings.NewReader(content)) + if err != nil { + return nil, err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil { + return nil, err + } + return t.DiffIndex() +} diff --git a/services/repository/files/diff_test.go b/services/repository/files/diff_test.go new file mode 100644 index 0000000..95de10e --- /dev/null +++ b/services/repository/files/diff_test.go @@ -0,0 +1,166 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/services/contexttest" + "code.gitea.io/gitea/services/gitdiff" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetDiffPreview(t *testing.T) { + unittest.PrepareTestEnv(t) + ctx, _ := contexttest.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + contexttest.LoadRepo(t, ctx, 1) + contexttest.LoadUser(t, ctx, 2) + contexttest.LoadGitRepo(t, ctx) + defer ctx.Repo.GitRepo.Close() + + branch := ctx.Repo.Repository.DefaultBranch + treePath := "README.md" + content := "# repo1\n\nDescription for repo1\nthis is a new line" + + expectedDiff := &gitdiff.Diff{ + TotalAddition: 2, + TotalDeletion: 1, + Files: []*gitdiff.DiffFile{ + { + Name: "README.md", + OldName: "README.md", + NameHash: "8ec9a00bfd09b3190ac6b22251dbb1aa95a0579d", + Index: 1, + Addition: 2, + Deletion: 1, + Type: 2, + IsCreated: false, + IsDeleted: false, + IsBin: false, + IsLFSFile: false, + IsRenamed: false, + IsSubmodule: false, + Sections: []*gitdiff.DiffSection{ + { + FileName: "README.md", + Name: "", + Lines: []*gitdiff.DiffLine{ + { + LeftIdx: 0, + RightIdx: 0, + Type: 4, + Content: "@@ -1,3 +1,4 @@", + Conversations: nil, + SectionInfo: &gitdiff.DiffLineSectionInfo{ + Path: "README.md", + LastLeftIdx: 0, + LastRightIdx: 0, + LeftIdx: 1, + RightIdx: 1, + LeftHunkSize: 3, + RightHunkSize: 4, + }, + }, + { + LeftIdx: 1, + RightIdx: 1, + Type: 1, + Content: " # repo1", + Conversations: nil, + }, + { + LeftIdx: 2, + RightIdx: 2, + Type: 1, + Content: " ", + Conversations: nil, + }, + { + LeftIdx: 3, + RightIdx: 0, + Match: 4, + Type: 3, + Content: "-Description for repo1", + Conversations: nil, + }, + { + LeftIdx: 0, + RightIdx: 3, + Match: 3, + Type: 2, + Content: "+Description for repo1", + Conversations: nil, + }, + { + LeftIdx: 0, + RightIdx: 4, + Match: -1, + Type: 2, + Content: "+this is a new line", + Conversations: nil, + }, + }, + }, + }, + IsIncomplete: false, + }, + }, + IsIncomplete: false, + } + expectedDiff.NumFiles = len(expectedDiff.Files) + + t.Run("with given branch", func(t *testing.T) { + diff, err := GetDiffPreview(ctx, ctx.Repo.Repository, branch, treePath, content) + require.NoError(t, err) + expectedBs, err := json.Marshal(expectedDiff) + require.NoError(t, err) + bs, err := json.Marshal(diff) + require.NoError(t, err) + assert.EqualValues(t, string(expectedBs), string(bs)) + }) + + t.Run("empty branch, same results", func(t *testing.T) { + diff, err := GetDiffPreview(ctx, ctx.Repo.Repository, "", treePath, content) + require.NoError(t, err) + expectedBs, err := json.Marshal(expectedDiff) + require.NoError(t, err) + bs, err := json.Marshal(diff) + require.NoError(t, err) + assert.EqualValues(t, expectedBs, bs) + }) +} + +func TestGetDiffPreviewErrors(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + branch := repo.DefaultBranch + treePath := "README.md" + content := "# repo1\n\nDescription for repo1\nthis is a new line" + + t.Run("empty repo", func(t *testing.T) { + diff, err := GetDiffPreview(db.DefaultContext, &repo_model.Repository{}, branch, treePath, content) + assert.Nil(t, diff) + assert.EqualError(t, err, "repository does not exist [id: 0, uid: 0, owner_name: , name: ]") + }) + + t.Run("bad branch", func(t *testing.T) { + badBranch := "bad_branch" + diff, err := GetDiffPreview(db.DefaultContext, repo, badBranch, treePath, content) + assert.Nil(t, diff) + assert.EqualError(t, err, "branch does not exist [name: "+badBranch+"]") + }) + + t.Run("empty treePath", func(t *testing.T) { + diff, err := GetDiffPreview(db.DefaultContext, repo, branch, "", content) + assert.Nil(t, diff) + assert.EqualError(t, err, "path is invalid [path: ]") + }) +} diff --git a/services/repository/files/file.go b/services/repository/files/file.go new file mode 100644 index 0000000..852cca0 --- /dev/null +++ b/services/repository/files/file.go @@ -0,0 +1,174 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" +) + +func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch string, treeNames []string) (*api.FilesResponse, error) { + files := []*api.ContentsResponse{} + for _, file := range treeNames { + fileContents, _ := GetContents(ctx, repo, file, branch, false) // ok if fails, then will be nil + files = append(files, fileContents) + } + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, commit) + filesResponse := &api.FilesResponse{ + Files: files, + Commit: fileCommitResponse, + Verification: verification, + } + return filesResponse, nil +} + +// GetFileResponseFromCommit Constructs a FileResponse from a Commit object +func GetFileResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch, treeName string) (*api.FileResponse, error) { + fileContents, _ := GetContents(ctx, repo, treeName, branch, false) // ok if fails, then will be nil + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, commit) + fileResponse := &api.FileResponse{ + Content: fileContents, + Commit: fileCommitResponse, + Verification: verification, + } + return fileResponse, nil +} + +// constructs a FileResponse with the file at the index from FilesResponse +func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index int) *api.FileResponse { + content := &api.ContentsResponse{} + if len(filesResponse.Files) > index { + content = filesResponse.Files[index] + } + fileResponse := &api.FileResponse{ + Content: content, + Commit: filesResponse.Commit, + Verification: filesResponse.Verification, + } + return fileResponse +} + +// GetFileCommitResponse Constructs a FileCommitResponse from a Commit object +func GetFileCommitResponse(repo *repo_model.Repository, commit *git.Commit) (*api.FileCommitResponse, error) { + if repo == nil { + return nil, fmt.Errorf("repo cannot be nil") + } + if commit == nil { + return nil, fmt.Errorf("commit cannot be nil") + } + commitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + url.PathEscape(commit.ID.String())) + commitTreeURL, _ := url.Parse(repo.APIURL() + "/git/trees/" + url.PathEscape(commit.Tree.ID.String())) + parents := make([]*api.CommitMeta, commit.ParentCount()) + for i := 0; i <= commit.ParentCount(); i++ { + if parent, err := commit.Parent(i); err == nil && parent != nil { + parentCommitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + url.PathEscape(parent.ID.String())) + parents[i] = &api.CommitMeta{ + SHA: parent.ID.String(), + URL: parentCommitURL.String(), + } + } + } + commitHTMLURL, _ := url.Parse(repo.HTMLURL() + "/commit/" + url.PathEscape(commit.ID.String())) + fileCommit := &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + SHA: commit.ID.String(), + URL: commitURL.String(), + }, + HTMLURL: commitHTMLURL.String(), + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: commit.Author.Name, + Email: commit.Author.Email, + }, + Date: commit.Author.When.UTC().Format(time.RFC3339), + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: commit.Committer.Name, + Email: commit.Committer.Email, + }, + Date: commit.Committer.When.UTC().Format(time.RFC3339), + }, + Message: commit.Message(), + Tree: &api.CommitMeta{ + URL: commitTreeURL.String(), + SHA: commit.Tree.ID.String(), + }, + Parents: parents, + } + return fileCommit, nil +} + +// GetAuthorAndCommitterUsers Gets the author and committer user objects from the IdentityOptions +func GetAuthorAndCommitterUsers(author, committer *IdentityOptions, doer *user_model.User) (authorUser, committerUser *user_model.User) { + // Committer and author are optional. If they are not the doer (not same email address) + // then we use bogus User objects for them to store their FullName and Email. + // If only one of the two are provided, we set both of them to it. + // If neither are provided, both are the doer. + if committer != nil && committer.Email != "" { + if doer != nil && strings.EqualFold(doer.Email, committer.Email) { + committerUser = doer // the committer is the doer, so will use their user object + if committer.Name != "" { + committerUser.FullName = committer.Name + } + // Use the provided email and not revert to placeholder mail. + committerUser.KeepEmailPrivate = false + } else { + committerUser = &user_model.User{ + FullName: committer.Name, + Email: committer.Email, + } + } + } + if author != nil && author.Email != "" { + if doer != nil && strings.EqualFold(doer.Email, author.Email) { + authorUser = doer // the author is the doer, so will use their user object + if authorUser.Name != "" { + authorUser.FullName = author.Name + } + // Use the provided email and not revert to placeholder mail. + authorUser.KeepEmailPrivate = false + } else { + authorUser = &user_model.User{ + FullName: author.Name, + Email: author.Email, + } + } + } + if authorUser == nil { + if committerUser != nil { + authorUser = committerUser // No valid author was given so use the committer + } else if doer != nil { + authorUser = doer // No valid author was given and no valid committer so use the doer + } + } + if committerUser == nil { + committerUser = authorUser // No valid committer so use the author as the committer (was set to a valid user above) + } + return authorUser, committerUser +} + +// CleanUploadFileName Trims a filename and returns empty string if it is a .git directory +func CleanUploadFileName(name string) string { + // Rebase the filename + name = util.PathJoinRel(name) + // Git disallows any filenames to have a .git directory in them. + for _, part := range strings.Split(name, "/") { + if strings.ToLower(part) == ".git" { + return "" + } + } + return name +} diff --git a/services/repository/files/file_test.go b/services/repository/files/file_test.go new file mode 100644 index 0000000..7c387e2 --- /dev/null +++ b/services/repository/files/file_test.go @@ -0,0 +1,115 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCleanUploadFileName(t *testing.T) { + t.Run("Clean regular file", func(t *testing.T) { + name := "this/is/test" + cleanName := CleanUploadFileName(name) + expectedCleanName := name + assert.EqualValues(t, expectedCleanName, cleanName) + }) + + t.Run("Clean a .git path", func(t *testing.T) { + name := "this/is/test/.git" + cleanName := CleanUploadFileName(name) + expectedCleanName := "" + assert.EqualValues(t, expectedCleanName, cleanName) + }) +} + +func getExpectedFileResponse() *api.FileResponse { + treePath := "README.md" + sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" + encoding := "base64" + content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" + selfURL := setting.AppURL + "api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" + htmlURL := setting.AppURL + "user2/repo1/src/branch/master/" + treePath + gitURL := setting.AppURL + "api/v1/repos/user2/repo1/git/blobs/" + sha + downloadURL := setting.AppURL + "user2/repo1/raw/branch/master/" + treePath + return &api.FileResponse{ + Content: &api.ContentsResponse{ + Name: treePath, + Path: treePath, + SHA: sha, + LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Type: "file", + Size: 30, + Encoding: &encoding, + Content: &content, + URL: &selfURL, + HTMLURL: &htmlURL, + GitURL: &gitURL, + DownloadURL: &downloadURL, + Links: &api.FileLinksResponse{ + Self: &selfURL, + GitURL: &gitURL, + HTMLURL: &htmlURL, + }, + }, + Commit: &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + }, + HTMLURL: "https://try.gitea.io/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d", + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: "user1", + Email: "address1@example.com", + }, + Date: "2017-03-19T20:47:59Z", + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: "Ethan Koenig", + Email: "ethantkoenig@gmail.com", + }, + Date: "2017-03-19T20:47:59Z", + }, + Parents: []*api.CommitMeta{}, + Message: "Initial commit\n", + Tree: &api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/2a2f1d4670728a2e10049e345bd7a276468beab6", + SHA: "2a2f1d4670728a2e10049e345bd7a276468beab6", + }, + }, + Verification: &api.PayloadCommitVerification{ + Verified: false, + Reason: "gpg.error.not_signed_commit", + Signature: "", + Payload: "", + }, + } +} + +func TestGetFileResponseFromCommit(t *testing.T) { + unittest.PrepareTestEnv(t) + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + branch := repo.DefaultBranch + treePath := "README.md" + gitRepo, _ := gitrepo.OpenRepository(db.DefaultContext, repo) + defer gitRepo.Close() + commit, _ := gitRepo.GetBranchCommit(branch) + expectedFileResponse := getExpectedFileResponse() + + fileResponse, err := GetFileResponseFromCommit(db.DefaultContext, repo, commit, branch, treePath) + require.NoError(t, err) + assert.EqualValues(t, expectedFileResponse, fileResponse) +} diff --git a/services/repository/files/patch.go b/services/repository/files/patch.go new file mode 100644 index 0000000..e5f7e2a --- /dev/null +++ b/services/repository/files/patch.go @@ -0,0 +1,199 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/structs" + asymkey_service "code.gitea.io/gitea/services/asymkey" +) + +// ApplyDiffPatchOptions holds the repository diff patch update options +type ApplyDiffPatchOptions struct { + LastCommitID string + OldBranch string + NewBranch string + Message string + Content string + SHA string + Author *IdentityOptions + Committer *IdentityOptions + Dates *CommitDateOptions + Signoff bool +} + +// Validate validates the provided options +func (opts *ApplyDiffPatchOptions) Validate(ctx context.Context, repo *repo_model.Repository, doer *user_model.User) error { + // If no branch name is set, assume master + if opts.OldBranch == "" { + opts.OldBranch = repo.DefaultBranch + } + if opts.NewBranch == "" { + opts.NewBranch = opts.OldBranch + } + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return err + } + defer closer.Close() + + // oldBranch must exist for this operation + if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil { + return err + } + // A NewBranch can be specified for the patch to be applied to. + // Check to make sure the branch does not already exist, otherwise we can't proceed. + // If we aren't branching to a new branch, make sure user can commit to the given branch + if opts.NewBranch != opts.OldBranch { + existingBranch, err := gitRepo.GetBranch(opts.NewBranch) + if existingBranch != nil { + return git_model.ErrBranchAlreadyExists{ + BranchName: opts.NewBranch, + } + } + if err != nil && !git.IsErrBranchNotExist(err) { + return err + } + } else { + protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, opts.OldBranch) + if err != nil { + return err + } + if protectedBranch != nil { + protectedBranch.Repo = repo + if !protectedBranch.CanUserPush(ctx, doer) { + return models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + } + if protectedBranch != nil && protectedBranch.RequireSignedCommits { + _, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), opts.OldBranch) + if err != nil { + if !asymkey_service.IsErrWontSign(err) { + return err + } + return models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + } + } + return nil +} + +// ApplyDiffPatch applies a patch to the given repository +func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ApplyDiffPatchOptions) (*structs.FileResponse, error) { + err := repo.MustNotBeArchived() + if err != nil { + return nil, err + } + + if err := opts.Validate(ctx, repo, doer); err != nil { + return nil, err + } + + message := strings.TrimSpace(opts.Message) + + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + log.Error("NewTemporaryUploadRepository failed: %v", err) + } + defer t.Close() + if err := t.Clone(opts.OldBranch, true); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Get the commit of the original branch + commit, err := t.GetBranchCommit(opts.OldBranch) + if err != nil { + return nil, err // Couldn't get a commit for the branch + } + + // Assigned LastCommitID in opts if it hasn't been set + if opts.LastCommitID == "" { + opts.LastCommitID = commit.ID.String() + } else { + lastCommitID, err := t.gitRepo.ConvertToGitID(opts.LastCommitID) + if err != nil { + return nil, fmt.Errorf("ApplyPatch: Invalid last commit ID: %w", err) + } + opts.LastCommitID = lastCommitID.String() + if commit.ID.String() != opts.LastCommitID { + return nil, models.ErrCommitIDDoesNotMatch{ + GivenCommitID: opts.LastCommitID, + CurrentCommitID: opts.LastCommitID, + } + } + } + + stdout := &strings.Builder{} + stderr := &strings.Builder{} + + cmdApply := git.NewCommand(ctx, "apply", "--index", "--recount", "--cached", "--ignore-whitespace", "--whitespace=fix", "--binary") + if git.CheckGitVersionAtLeast("2.32") == nil { + cmdApply.AddArguments("-3") + } + + if err := cmdApply.Run(&git.RunOpts{ + Dir: t.basePath, + Stdout: stdout, + Stderr: stderr, + Stdin: strings.NewReader(opts.Content), + }); err != nil { + return nil, fmt.Errorf("Error: Stdout: %s\nStderr: %s\nErr: %w", stdout.String(), stderr.String(), err) + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return nil, err + } + + // Now commit the tree + var commitHash string + if opts.Dates != nil { + commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) + } else { + commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff) + } + if err != nil { + return nil, err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return nil, err + } + + commit, err = t.GetCommit(commitHash) + if err != nil { + return nil, err + } + + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, commit) + fileResponse := &structs.FileResponse{ + Commit: fileCommitResponse, + Verification: verification, + } + + return fileResponse, nil +} diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go new file mode 100644 index 0000000..50b936c --- /dev/null +++ b/services/repository/files/temp_repo.go @@ -0,0 +1,406 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "regexp" + "strings" + "time" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + asymkey_service "code.gitea.io/gitea/services/asymkey" + "code.gitea.io/gitea/services/gitdiff" +) + +// TemporaryUploadRepository is a type to wrap our upload repositories as a shallow clone +type TemporaryUploadRepository struct { + ctx context.Context + repo *repo_model.Repository + gitRepo *git.Repository + basePath string +} + +// NewTemporaryUploadRepository creates a new temporary upload repository +func NewTemporaryUploadRepository(ctx context.Context, repo *repo_model.Repository) (*TemporaryUploadRepository, error) { + basePath, err := repo_module.CreateTemporaryPath("upload") + if err != nil { + return nil, err + } + t := &TemporaryUploadRepository{ctx: ctx, repo: repo, basePath: basePath} + return t, nil +} + +// Close the repository cleaning up all files +func (t *TemporaryUploadRepository) Close() { + defer t.gitRepo.Close() + if err := repo_module.RemoveTemporaryPath(t.basePath); err != nil { + log.Error("Failed to remove temporary path %s: %v", t.basePath, err) + } +} + +// Clone the base repository to our path and set branch as the HEAD +func (t *TemporaryUploadRepository) Clone(branch string, bare bool) error { + cmd := git.NewCommand(t.ctx, "clone", "-s", "-b").AddDynamicArguments(branch, t.repo.RepoPath(), t.basePath) + if bare { + cmd.AddArguments("--bare") + } + + if _, _, err := cmd.RunStdString(nil); err != nil { + stderr := err.Error() + if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched { + return git.ErrBranchNotExist{ + Name: branch, + } + } else if matched, _ := regexp.MatchString(".* repository .* does not exist.*", stderr); matched { + return repo_model.ErrRepoNotExist{ + ID: t.repo.ID, + UID: t.repo.OwnerID, + OwnerName: t.repo.OwnerName, + Name: t.repo.Name, + } + } + return fmt.Errorf("Clone: %w %s", err, stderr) + } + gitRepo, err := git.OpenRepository(t.ctx, t.basePath) + if err != nil { + return err + } + t.gitRepo = gitRepo + return nil +} + +// Init the repository +func (t *TemporaryUploadRepository) Init(objectFormatName string) error { + if err := git.InitRepository(t.ctx, t.basePath, false, objectFormatName); err != nil { + return err + } + gitRepo, err := git.OpenRepository(t.ctx, t.basePath) + if err != nil { + return err + } + t.gitRepo = gitRepo + return nil +} + +// SetDefaultIndex sets the git index to our HEAD +func (t *TemporaryUploadRepository) SetDefaultIndex() error { + if _, _, err := git.NewCommand(t.ctx, "read-tree", "HEAD").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return fmt.Errorf("SetDefaultIndex: %w", err) + } + return nil +} + +// RefreshIndex looks at the current index and checks to see if merges or updates are needed by checking stat() information. +func (t *TemporaryUploadRepository) RefreshIndex() error { + if _, _, err := git.NewCommand(t.ctx, "update-index", "--refresh").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return fmt.Errorf("RefreshIndex: %w", err) + } + return nil +} + +// LsFiles checks if the given filename arguments are in the index +func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := git.NewCommand(t.ctx, "ls-files", "-z").AddDashesAndList(filenames...). + Run(&git.RunOpts{ + Dir: t.basePath, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + log.Error("Unable to run git ls-files for temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String()) + err = fmt.Errorf("Unable to run git ls-files for temporary repo of: %s Error: %w\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String()) + return nil, err + } + + fileList := make([]string, 0, len(filenames)) + for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + fileList = append(fileList, string(line)) + } + + return fileList, nil +} + +// RemoveFilesFromIndex removes the given files from the index +func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error { + objectFormat, err := t.gitRepo.GetObjectFormat() + if err != nil { + return err + } + + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + stdIn := new(bytes.Buffer) + for _, file := range filenames { + if file != "" { + stdIn.WriteString("0 ") + stdIn.WriteString(objectFormat.EmptyObjectID().String()) + stdIn.WriteByte('\t') + stdIn.WriteString(file) + stdIn.WriteByte('\000') + } + } + + if err := git.NewCommand(t.ctx, "update-index", "--remove", "-z", "--index-info"). + Run(&git.RunOpts{ + Dir: t.basePath, + Stdin: stdIn, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + log.Error("Unable to update-index for temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String()) + return fmt.Errorf("Unable to update-index for temporary repo: %s Error: %w\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String()) + } + return nil +} + +// HashObject writes the provided content to the object db and returns its hash +func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := git.NewCommand(t.ctx, "hash-object", "-w", "--stdin"). + Run(&git.RunOpts{ + Dir: t.basePath, + Stdin: content, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + log.Error("Unable to hash-object to temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String()) + return "", fmt.Errorf("Unable to hash-object to temporary repo: %s Error: %w\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String()) + } + + return strings.TrimSpace(stdOut.String()), nil +} + +// AddObjectToIndex adds the provided object hash to the index with the provided mode and path +func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error { + if _, _, err := git.NewCommand(t.ctx, "update-index", "--add", "--replace", "--cacheinfo").AddDynamicArguments(mode, objectHash, objectPath).RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + stderr := err.Error() + if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched { + return models.ErrFilePathInvalid{ + Message: objectPath, + Path: objectPath, + } + } + log.Error("Unable to add object to index: %s %s %s in temporary repo %s(%s) Error: %v", mode, objectHash, objectPath, t.repo.FullName(), t.basePath, err) + return fmt.Errorf("Unable to add object to index at %s in temporary repo %s Error: %w", objectPath, t.repo.FullName(), err) + } + return nil +} + +// WriteTree writes the current index as a tree to the object db and returns its hash +func (t *TemporaryUploadRepository) WriteTree() (string, error) { + stdout, _, err := git.NewCommand(t.ctx, "write-tree").RunStdString(&git.RunOpts{Dir: t.basePath}) + if err != nil { + log.Error("Unable to write tree in temporary repo: %s(%s): Error: %v", t.repo.FullName(), t.basePath, err) + return "", fmt.Errorf("Unable to write-tree in temporary repo for: %s Error: %w", t.repo.FullName(), err) + } + return strings.TrimSpace(stdout), nil +} + +// GetLastCommit gets the last commit ID SHA of the repo +func (t *TemporaryUploadRepository) GetLastCommit() (string, error) { + return t.GetLastCommitByRef("HEAD") +} + +// GetLastCommitByRef gets the last commit ID SHA of the repo by ref +func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, error) { + if ref == "" { + ref = "HEAD" + } + stdout, _, err := git.NewCommand(t.ctx, "rev-parse").AddDynamicArguments(ref).RunStdString(&git.RunOpts{Dir: t.basePath}) + if err != nil { + log.Error("Unable to get last ref for %s in temporary repo: %s(%s): Error: %v", ref, t.repo.FullName(), t.basePath, err) + return "", fmt.Errorf("Unable to rev-parse %s in temporary repo for: %s Error: %w", ref, t.repo.FullName(), err) + } + return strings.TrimSpace(stdout), nil +} + +// CommitTree creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTree(parent string, author, committer *user_model.User, treeHash, message string, signoff bool) (string, error) { + return t.CommitTreeWithDate(parent, author, committer, treeHash, message, signoff, time.Now(), time.Now()) +} + +// CommitTreeWithDate creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, committer *user_model.User, treeHash, message string, signoff bool, authorDate, committerDate time.Time) (string, error) { + authorSig := author.NewGitSig() + committerSig := committer.NewGitSig() + + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_AUTHOR_DATE="+authorDate.Format(time.RFC3339), + "GIT_COMMITTER_DATE="+committerDate.Format(time.RFC3339), + ) + + messageBytes := new(bytes.Buffer) + _, _ = messageBytes.WriteString(message) + _, _ = messageBytes.WriteString("\n") + + cmdCommitTree := git.NewCommand(t.ctx, "commit-tree").AddDynamicArguments(treeHash) + if parent != "" { + cmdCommitTree.AddOptionValues("-p", parent) + } + + var sign bool + var keyID string + var signer *git.Signature + if parent != "" { + sign, keyID, signer, _ = asymkey_service.SignCRUDAction(t.ctx, t.repo.RepoPath(), author, t.basePath, parent) + } else { + sign, keyID, signer, _ = asymkey_service.SignInitialCommit(t.ctx, t.repo.RepoPath(), author) + } + if sign { + cmdCommitTree.AddOptionFormat("-S%s", keyID) + if t.repo.GetTrustModel() == repo_model.CommitterTrustModel || t.repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { + if committerSig.Name != authorSig.Name || committerSig.Email != authorSig.Email { + // Add trailers + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Co-authored-by: ") + _, _ = messageBytes.WriteString(committerSig.String()) + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Co-committed-by: ") + _, _ = messageBytes.WriteString(committerSig.String()) + _, _ = messageBytes.WriteString("\n") + } + committerSig = signer + } + } else { + cmdCommitTree.AddArguments("--no-gpg-sign") + } + + if signoff { + // Signed-off-by + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Signed-off-by: ") + _, _ = messageBytes.WriteString(committerSig.String()) + } + + env = append(env, + "GIT_COMMITTER_NAME="+committerSig.Name, + "GIT_COMMITTER_EMAIL="+committerSig.Email, + ) + + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + if err := cmdCommitTree. + Run(&git.RunOpts{ + Env: env, + Dir: t.basePath, + Stdin: messageBytes, + Stdout: stdout, + Stderr: stderr, + }); err != nil { + log.Error("Unable to commit-tree in temporary repo: %s (%s) Error: %v\nStdout: %s\nStderr: %s", + t.repo.FullName(), t.basePath, err, stdout, stderr) + return "", fmt.Errorf("Unable to commit-tree in temporary repo: %s Error: %w\nStdout: %s\nStderr: %s", + t.repo.FullName(), err, stdout, stderr) + } + return strings.TrimSpace(stdout.String()), nil +} + +// Push the provided commitHash to the repository branch by the provided user +func (t *TemporaryUploadRepository) Push(doer *user_model.User, commitHash, branch string) error { + // Because calls hooks we need to pass in the environment + env := repo_module.PushingEnvironment(doer, t.repo) + if err := git.Push(t.ctx, t.basePath, git.PushOptions{ + Remote: t.repo.RepoPath(), + Branch: strings.TrimSpace(commitHash) + ":" + git.BranchPrefix + strings.TrimSpace(branch), + Env: env, + }); err != nil { + if git.IsErrPushOutOfDate(err) { + return err + } else if git.IsErrPushRejected(err) { + rejectErr := err.(*git.ErrPushRejected) + log.Info("Unable to push back to repo from temporary repo due to rejection: %s (%s)\nStdout: %s\nStderr: %s\nError: %v", + t.repo.FullName(), t.basePath, rejectErr.StdOut, rejectErr.StdErr, rejectErr.Err) + return err + } + log.Error("Unable to push back to repo from temporary repo: %s (%s)\nError: %v", + t.repo.FullName(), t.basePath, err) + return fmt.Errorf("Unable to push back to repo from temporary repo: %s (%s) Error: %v", + t.repo.FullName(), t.basePath, err) + } + return nil +} + +// DiffIndex returns a Diff of the current index to the head +func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + log.Error("Unable to open stdout pipe: %v", err) + return nil, fmt.Errorf("Unable to open stdout pipe: %w", err) + } + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + stderr := new(bytes.Buffer) + var diff *gitdiff.Diff + var finalErr error + + if err := git.NewCommand(t.ctx, "diff-index", "--src-prefix=\\a/", "--dst-prefix=\\b/", "--cached", "-p", "HEAD"). + Run(&git.RunOpts{ + Timeout: 30 * time.Second, + Dir: t.basePath, + Stdout: stdoutWriter, + Stderr: stderr, + PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { + _ = stdoutWriter.Close() + diff, finalErr = gitdiff.ParsePatch(t.ctx, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader, "") + if finalErr != nil { + log.Error("ParsePatch: %v", finalErr) + cancel() + } + _ = stdoutReader.Close() + return finalErr + }, + }); err != nil { + if finalErr != nil { + log.Error("Unable to ParsePatch in temporary repo %s (%s). Error: %v", t.repo.FullName(), t.basePath, finalErr) + return nil, finalErr + } + log.Error("Unable to run diff-index pipeline in temporary repo %s (%s). Error: %v\nStderr: %s", + t.repo.FullName(), t.basePath, err, stderr) + return nil, fmt.Errorf("Unable to run diff-index pipeline in temporary repo %s. Error: %w\nStderr: %s", + t.repo.FullName(), err, stderr) + } + + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(t.ctx, t.basePath, git.TrustedCmdArgs{"--cached"}, "HEAD") + if err != nil { + return nil, err + } + + return diff, nil +} + +// GetBranchCommit Gets the commit object of the given branch +func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, error) { + if t.gitRepo == nil { + return nil, fmt.Errorf("repository has not been cloned") + } + return t.gitRepo.GetBranchCommit(branch) +} + +// GetCommit Gets the commit object of the given commit ID +func (t *TemporaryUploadRepository) GetCommit(commitID string) (*git.Commit, error) { + if t.gitRepo == nil { + return nil, fmt.Errorf("repository has not been cloned") + } + return t.gitRepo.GetCommit(commitID) +} diff --git a/services/repository/files/temp_repo_test.go b/services/repository/files/temp_repo_test.go new file mode 100644 index 0000000..e7d85ea --- /dev/null +++ b/services/repository/files/temp_repo_test.go @@ -0,0 +1,28 @@ +// Copyright 2024 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/require" +) + +func TestRemoveFilesFromIndexSha256(t *testing.T) { + if git.CheckGitVersionAtLeast("2.42") != nil { + t.Skip("skipping because installed Git version doesn't support SHA256") + } + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + temp, err := NewTemporaryUploadRepository(db.DefaultContext, repo) + require.NoError(t, err) + require.NoError(t, temp.Init("sha256")) + require.NoError(t, temp.RemoveFilesFromIndex("README.md")) +} diff --git a/services/repository/files/tree.go b/services/repository/files/tree.go new file mode 100644 index 0000000..e3a7f3b --- /dev/null +++ b/services/repository/files/tree.go @@ -0,0 +1,101 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "net/url" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" +) + +// GetTreeBySHA get the GitTreeResponse of a repository using a sha hash. +func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string, page, perPage int, recursive bool) (*api.GitTreeResponse, error) { + gitTree, err := gitRepo.GetTree(sha) + if err != nil || gitTree == nil { + return nil, models.ErrSHANotFound{ + SHA: sha, + } + } + tree := new(api.GitTreeResponse) + tree.SHA = gitTree.ResolvedID.String() + tree.URL = repo.APIURL() + "/git/trees/" + url.PathEscape(tree.SHA) + var entries git.Entries + if recursive { + entries, err = gitTree.ListEntriesRecursiveWithSize() + } else { + entries, err = gitTree.ListEntries() + } + if err != nil { + return nil, err + } + apiURL := repo.APIURL() + apiURLLen := len(apiURL) + objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + hashLen := objectFormat.FullLength() + + const gitBlobsPath = "/git/blobs/" + blobURL := make([]byte, apiURLLen+hashLen+len(gitBlobsPath)) + copy(blobURL, apiURL) + copy(blobURL[apiURLLen:], []byte(gitBlobsPath)) + + const gitTreePath = "/git/trees/" + treeURL := make([]byte, apiURLLen+hashLen+len(gitTreePath)) + copy(treeURL, apiURL) + copy(treeURL[apiURLLen:], []byte(gitTreePath)) + + // copyPos is at the start of the hash + copyPos := len(treeURL) - hashLen + + if perPage <= 0 || perPage > setting.API.DefaultGitTreesPerPage { + perPage = setting.API.DefaultGitTreesPerPage + } + if page <= 0 { + page = 1 + } + tree.Page = page + tree.TotalCount = len(entries) + rangeStart := perPage * (page - 1) + if rangeStart >= len(entries) { + return tree, nil + } + var rangeEnd int + if len(entries) > perPage { + tree.Truncated = true + } + if rangeStart+perPage < len(entries) { + rangeEnd = rangeStart + perPage + } else { + rangeEnd = len(entries) + } + tree.Entries = make([]api.GitEntry, rangeEnd-rangeStart) + for e := rangeStart; e < rangeEnd; e++ { + i := e - rangeStart + + tree.Entries[i].Path = entries[e].Name() + tree.Entries[i].Mode = fmt.Sprintf("%06o", entries[e].Mode()) + tree.Entries[i].Type = entries[e].Type() + tree.Entries[i].Size = entries[e].Size() + tree.Entries[i].SHA = entries[e].ID.String() + + if entries[e].IsDir() { + copy(treeURL[copyPos:], entries[e].ID.String()) + tree.Entries[i].URL = string(treeURL) + } else if entries[e].IsSubModule() { + // In Github Rest API Version=2022-11-28, if a tree entry is a submodule, + // its url will be returned as an empty string. + // So the URL will be set to "" here. + tree.Entries[i].URL = "" + } else { + copy(blobURL[copyPos:], entries[e].ID.String()) + tree.Entries[i].URL = string(blobURL) + } + } + return tree, nil +} diff --git a/services/repository/files/tree_test.go b/services/repository/files/tree_test.go new file mode 100644 index 0000000..9e5c5c1 --- /dev/null +++ b/services/repository/files/tree_test.go @@ -0,0 +1,52 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/services/contexttest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetTreeBySHA(t *testing.T) { + unittest.PrepareTestEnv(t) + ctx, _ := contexttest.MockContext(t, "user2/repo1") + contexttest.LoadRepo(t, ctx, 1) + contexttest.LoadUser(t, ctx, 2) + contexttest.LoadGitRepo(t, ctx) + defer ctx.Repo.GitRepo.Close() + + sha := ctx.Repo.Repository.DefaultBranch + page := 1 + perPage := 10 + ctx.SetParams(":id", "1") + ctx.SetParams(":sha", sha) + + tree, err := GetTreeBySHA(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.Params(":sha"), page, perPage, true) + require.NoError(t, err) + expectedTree := &api.GitTreeResponse{ + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/65f1bf27bc3bf70f64657658635e66094edbcb4d", + Entries: []api.GitEntry{ + { + Path: "README.md", + Mode: "100644", + Type: "blob", + Size: 30, + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f", + }, + }, + Truncated: false, + Page: 1, + TotalCount: 1, + } + + assert.EqualValues(t, expectedTree, tree) +} diff --git a/services/repository/files/update.go b/services/repository/files/update.go new file mode 100644 index 0000000..d6025b6 --- /dev/null +++ b/services/repository/files/update.go @@ -0,0 +1,501 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "io" + "path" + "strings" + "time" + + "code.gitea.io/gitea/models" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" + asymkey_service "code.gitea.io/gitea/services/asymkey" +) + +// IdentityOptions for a person's identity like an author or committer +type IdentityOptions struct { + Name string + Email string +} + +// CommitDateOptions store dates for GIT_AUTHOR_DATE and GIT_COMMITTER_DATE +type CommitDateOptions struct { + Author time.Time + Committer time.Time +} + +type ChangeRepoFile struct { + Operation string + TreePath string + FromTreePath string + ContentReader io.ReadSeeker + SHA string + Options *RepoFileOptions +} + +// ChangeRepoFilesOptions holds the repository files update options +type ChangeRepoFilesOptions struct { + LastCommitID string + OldBranch string + NewBranch string + Message string + Files []*ChangeRepoFile + Author *IdentityOptions + Committer *IdentityOptions + Dates *CommitDateOptions + Signoff bool +} + +type RepoFileOptions struct { + treePath string + fromTreePath string + executable bool +} + +// ChangeRepoFiles adds, updates or removes multiple files in the given repository +func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (*structs.FilesResponse, error) { + err := repo.MustNotBeArchived() + if err != nil { + return nil, err + } + + // If no branch name is set, assume default branch + if opts.OldBranch == "" { + opts.OldBranch = repo.DefaultBranch + } + if opts.NewBranch == "" { + opts.NewBranch = opts.OldBranch + } + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return nil, err + } + defer closer.Close() + + // oldBranch must exist for this operation + if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil && !repo.IsEmpty { + return nil, err + } + + var treePaths []string + for _, file := range opts.Files { + // If FromTreePath is not set, set it to the opts.TreePath + if file.TreePath != "" && file.FromTreePath == "" { + file.FromTreePath = file.TreePath + } + + // Check that the path given in opts.treePath is valid (not a git path) + treePath := CleanUploadFileName(file.TreePath) + if treePath == "" { + return nil, models.ErrFilenameInvalid{ + Path: file.TreePath, + } + } + // If there is a fromTreePath (we are copying it), also clean it up + fromTreePath := CleanUploadFileName(file.FromTreePath) + if fromTreePath == "" && file.FromTreePath != "" { + return nil, models.ErrFilenameInvalid{ + Path: file.FromTreePath, + } + } + + file.Options = &RepoFileOptions{ + treePath: treePath, + fromTreePath: fromTreePath, + executable: false, + } + treePaths = append(treePaths, treePath) + } + + // A NewBranch can be specified for the file to be created/updated in a new branch. + // Check to make sure the branch does not already exist, otherwise we can't proceed. + // If we aren't branching to a new branch, make sure user can commit to the given branch + if opts.NewBranch != opts.OldBranch { + existingBranch, err := gitRepo.GetBranch(opts.NewBranch) + if existingBranch != nil { + return nil, git_model.ErrBranchAlreadyExists{ + BranchName: opts.NewBranch, + } + } + if err != nil && !git.IsErrBranchNotExist(err) { + return nil, err + } + } else if err := VerifyBranchProtection(ctx, repo, doer, opts.OldBranch, treePaths); err != nil { + return nil, err + } + + message := strings.TrimSpace(opts.Message) + + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + log.Error("NewTemporaryUploadRepository failed: %v", err) + } + defer t.Close() + hasOldBranch := true + if err := t.Clone(opts.OldBranch, true); err != nil { + for _, file := range opts.Files { + if file.Operation == "delete" { + return nil, err + } + } + if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { + return nil, err + } + if err := t.Init(repo.ObjectFormatName); err != nil { + return nil, err + } + hasOldBranch = false + opts.LastCommitID = "" + } + if hasOldBranch { + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + } + + for _, file := range opts.Files { + if file.Operation == "delete" { + // Get the files in the index + filesInIndex, err := t.LsFiles(file.TreePath) + if err != nil { + return nil, fmt.Errorf("DeleteRepoFile: %w", err) + } + + // Find the file we want to delete in the index + inFilelist := false + for _, indexFile := range filesInIndex { + if indexFile == file.TreePath { + inFilelist = true + break + } + } + if !inFilelist { + return nil, models.ErrRepoFileDoesNotExist{ + Path: file.TreePath, + } + } + } + } + + if hasOldBranch { + // Get the commit of the original branch + commit, err := t.GetBranchCommit(opts.OldBranch) + if err != nil { + return nil, err // Couldn't get a commit for the branch + } + + // Assigned LastCommitID in opts if it hasn't been set + if opts.LastCommitID == "" { + opts.LastCommitID = commit.ID.String() + } else { + lastCommitID, err := t.gitRepo.ConvertToGitID(opts.LastCommitID) + if err != nil { + return nil, fmt.Errorf("ConvertToSHA1: Invalid last commit ID: %w", err) + } + opts.LastCommitID = lastCommitID.String() + } + + for _, file := range opts.Files { + if err := handleCheckErrors(file, commit, opts); err != nil { + return nil, err + } + } + } + + contentStore := lfs.NewContentStore() + for _, file := range opts.Files { + switch file.Operation { + case "create", "update": + if err := CreateOrUpdateFile(ctx, t, file, contentStore, repo.ID, hasOldBranch); err != nil { + return nil, err + } + case "delete": + // Remove the file from the index + if err := t.RemoveFilesFromIndex(file.TreePath); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid file operation: %s %s, supported operations are create, update, delete", file.Operation, file.Options.treePath) + } + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return nil, err + } + + // Now commit the tree + var commitHash string + if opts.Dates != nil { + commitHash, err = t.CommitTreeWithDate(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) + } else { + commitHash, err = t.CommitTree(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff) + } + if err != nil { + return nil, err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + log.Error("%T %v", err, err) + return nil, err + } + + commit, err := t.GetCommit(commitHash) + if err != nil { + return nil, err + } + + filesResponse, err := GetFilesResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePaths) + if err != nil { + return nil, err + } + + if repo.IsEmpty { + if isEmpty, err := gitRepo.IsEmpty(); err == nil && !isEmpty { + _ = repo_model.UpdateRepositoryCols(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch") + } + } + + return filesResponse, nil +} + +// handles the check for various issues for ChangeRepoFiles +func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRepoFilesOptions) error { + if file.Operation == "update" || file.Operation == "delete" { + fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) + if err != nil { + return err + } + if file.SHA != "" { + // If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error + if file.SHA != fromEntry.ID.String() { + return models.ErrSHADoesNotMatch{ + Path: file.Options.treePath, + GivenSHA: file.SHA, + CurrentSHA: fromEntry.ID.String(), + } + } + } else if opts.LastCommitID != "" { + // If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw + // an error, but only if we aren't creating a new branch. + if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch { + if changed, err := commit.FileChangedSinceCommit(file.Options.treePath, opts.LastCommitID); err != nil { + return err + } else if changed { + return models.ErrCommitIDDoesNotMatch{ + GivenCommitID: opts.LastCommitID, + CurrentCommitID: opts.LastCommitID, + } + } + // The file wasn't modified, so we are good to delete it + } + } else { + // When updating a file, a lastCommitID or SHA needs to be given to make sure other commits + // haven't been made. We throw an error if one wasn't provided. + return models.ErrSHAOrCommitIDNotProvided{} + } + file.Options.executable = fromEntry.IsExecutable() + } + if file.Operation == "create" || file.Operation == "update" { + // For the path where this file will be created/updated, we need to make + // sure no parts of the path are existing files or links except for the last + // item in the path which is the file name, and that shouldn't exist IF it is + // a new file OR is being moved to a new path. + treePathParts := strings.Split(file.Options.treePath, "/") + subTreePath := "" + for index, part := range treePathParts { + subTreePath = path.Join(subTreePath, part) + entry, err := commit.GetTreeEntryByPath(subTreePath) + if err != nil { + if git.IsErrNotExist(err) { + // Means there is no item with that name, so we're good + break + } + return err + } + if index < len(treePathParts)-1 { + if !entry.IsDir() { + return models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a file exists where you’re trying to create a subdirectory [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeBlob, + } + } + } else if entry.IsLink() { + return models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a symbolic link exists where you’re trying to create a subdirectory [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeSymlink, + } + } else if entry.IsDir() { + return models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a directory exists where you’re trying to create a file [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeTree, + } + } else if file.Options.fromTreePath != file.Options.treePath || file.Operation == "create" { + // The entry shouldn't exist if we are creating new file or moving to a new path + return models.ErrRepoFileAlreadyExists{ + Path: file.Options.treePath, + } + } + } + } + + return nil +} + +// CreateOrUpdateFile handles creating or updating a file for ChangeRepoFiles +func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64, hasOldBranch bool) error { + // Get the two paths (might be the same if not moving) from the index if they exist + filesInIndex, err := t.LsFiles(file.TreePath, file.FromTreePath) + if err != nil { + return fmt.Errorf("UpdateRepoFile: %w", err) + } + // If is a new file (not updating) then the given path shouldn't exist + if file.Operation == "create" { + for _, indexFile := range filesInIndex { + if indexFile == file.TreePath { + return models.ErrRepoFileAlreadyExists{ + Path: file.TreePath, + } + } + } + } + + // Remove the old path from the tree + if file.Options.fromTreePath != file.Options.treePath && len(filesInIndex) > 0 { + for _, indexFile := range filesInIndex { + if indexFile == file.Options.fromTreePath { + if err := t.RemoveFilesFromIndex(file.FromTreePath); err != nil { + return err + } + } + } + } + + treeObjectContentReader := file.ContentReader + var lfsMetaObject *git_model.LFSMetaObject + if setting.LFS.StartServer && hasOldBranch { + // Check there is no way this can return multiple infos + filterAttribute, err := t.gitRepo.GitAttributeFirst("", file.Options.treePath, "filter") + if err != nil { + return err + } + + if filterAttribute == "lfs" { + // OK so we are supposed to LFS this data! + pointer, err := lfs.GeneratePointer(treeObjectContentReader) + if err != nil { + return err + } + lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repoID} + treeObjectContentReader = strings.NewReader(pointer.StringContent()) + } + } + + // Add the object to the database + objectHash, err := t.HashObject(treeObjectContentReader) + if err != nil { + return err + } + + // Add the object to the index + if file.Options.executable { + if err := t.AddObjectToIndex("100755", objectHash, file.Options.treePath); err != nil { + return err + } + } else { + if err := t.AddObjectToIndex("100644", objectHash, file.Options.treePath); err != nil { + return err + } + } + + if lfsMetaObject != nil { + // We have an LFS object - create it + lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, lfsMetaObject.RepositoryID, lfsMetaObject.Pointer) + if err != nil { + return err + } + exist, err := contentStore.Exists(lfsMetaObject.Pointer) + if err != nil { + return err + } + if !exist { + _, err := file.ContentReader.Seek(0, io.SeekStart) + if err != nil { + return err + } + if err := contentStore.Put(lfsMetaObject.Pointer, file.ContentReader); err != nil { + if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); err2 != nil { + return fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, err2, err) + } + return err + } + } + } + + return nil +} + +// VerifyBranchProtection verify the branch protection for modifying the given treePath on the given branch +func VerifyBranchProtection(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, branchName string, treePaths []string) error { + protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, branchName) + if err != nil { + return err + } + if protectedBranch != nil { + protectedBranch.Repo = repo + globUnprotected := protectedBranch.GetUnprotectedFilePatterns() + globProtected := protectedBranch.GetProtectedFilePatterns() + canUserPush := protectedBranch.CanUserPush(ctx, doer) + for _, treePath := range treePaths { + isUnprotectedFile := false + if len(globUnprotected) != 0 { + isUnprotectedFile = protectedBranch.IsUnprotectedFile(globUnprotected, treePath) + } + if !canUserPush && !isUnprotectedFile { + return models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + if protectedBranch.IsProtectedFile(globProtected, treePath) { + return models.ErrFilePathProtected{ + Path: treePath, + } + } + } + if protectedBranch.RequireSignedCommits { + _, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), branchName) + if err != nil { + if !asymkey_service.IsErrWontSign(err) { + return err + } + return models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + } + } + return nil +} diff --git a/services/repository/files/upload.go b/services/repository/files/upload.go new file mode 100644 index 0000000..1330116 --- /dev/null +++ b/services/repository/files/upload.go @@ -0,0 +1,248 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "os" + "path" + "strings" + + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" +) + +// UploadRepoFileOptions contains the uploaded repository file options +type UploadRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + Message string + Author *IdentityOptions + Committer *IdentityOptions + Files []string // In UUID format. + Signoff bool +} + +type uploadInfo struct { + upload *repo_model.Upload + lfsMetaObject *git_model.LFSMetaObject +} + +func cleanUpAfterFailure(ctx context.Context, infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error { + for _, info := range *infos { + if info.lfsMetaObject == nil { + continue + } + if !info.lfsMetaObject.Existing { + if _, err := git_model.RemoveLFSMetaObjectByOid(ctx, t.repo.ID, info.lfsMetaObject.Oid); err != nil { + original = fmt.Errorf("%w, %v", original, err) // We wrap the original error - as this is the underlying error that required the fallback + } + } + } + return original +} + +// UploadRepoFiles uploads files to the given repository +func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error { + if len(opts.Files) == 0 { + return nil + } + + uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files) + if err != nil { + return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err) + } + + names := make([]string, len(uploads)) + infos := make([]uploadInfo, len(uploads)) + for i, upload := range uploads { + // Check file is not lfs locked, will return nil if lock setting not enabled + filepath := path.Join(opts.TreePath, upload.Name) + lfsLock, err := git_model.GetTreePathLock(ctx, repo.ID, filepath) + if err != nil { + return err + } + if lfsLock != nil && lfsLock.OwnerID != doer.ID { + u, err := user_model.GetUserByID(ctx, lfsLock.OwnerID) + if err != nil { + return err + } + return git_model.ErrLFSFileLocked{RepoID: repo.ID, Path: filepath, UserName: u.Name} + } + + names[i] = upload.Name + infos[i] = uploadInfo{upload: upload} + } + + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + return err + } + defer t.Close() + + hasOldBranch := true + if err = t.Clone(opts.OldBranch, true); err != nil { + if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { + return err + } + if err = t.Init(repo.ObjectFormatName); err != nil { + return err + } + hasOldBranch = false + opts.LastCommitID = "" + } + if hasOldBranch { + if err = t.SetDefaultIndex(); err != nil { + return err + } + } + + // Copy uploaded files into repository. + if err := copyUploadedLFSFilesIntoRepository(infos, t, opts.TreePath); err != nil { + return err + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + + // Now commit the tree + commitHash, err := t.CommitTree(opts.LastCommitID, author, committer, treeHash, opts.Message, opts.Signoff) + if err != nil { + return err + } + + // Now deal with LFS objects + for i := range infos { + if infos[i].lfsMetaObject == nil { + continue + } + infos[i].lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, infos[i].lfsMetaObject.RepositoryID, infos[i].lfsMetaObject.Pointer) + if err != nil { + // OK Now we need to cleanup + return cleanUpAfterFailure(ctx, &infos, t, err) + } + // Don't move the files yet - we need to ensure that + // everything can be inserted first + } + + // OK now we can insert the data into the store - there's no way to clean up the store + // once it's in there, it's in there. + contentStore := lfs.NewContentStore() + for _, info := range infos { + if err := uploadToLFSContentStore(info, contentStore); err != nil { + return cleanUpAfterFailure(ctx, &infos, t, err) + } + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + return repo_model.DeleteUploads(ctx, uploads...) +} + +func copyUploadedLFSFilesIntoRepository(infos []uploadInfo, t *TemporaryUploadRepository, treePath string) error { + var storeInLFSFunc func(string) (bool, error) + + if setting.LFS.StartServer { + checker, err := t.gitRepo.GitAttributeChecker("", "filter") + if err != nil { + return err + } + defer checker.Close() + + storeInLFSFunc = func(name string) (bool, error) { + attrs, err := checker.CheckPath(name) + if err != nil { + return false, fmt.Errorf("could not CheckPath(%s): %w", name, err) + } + return attrs["filter"] == "lfs", nil + } + } + + // Copy uploaded files into repository. + for i, info := range infos { + storeInLFS := false + if storeInLFSFunc != nil { + var err error + storeInLFS, err = storeInLFSFunc(info.upload.Name) + if err != nil { + return err + } + } + + if err := copyUploadedLFSFileIntoRepository(&infos[i], storeInLFS, t, treePath); err != nil { + return err + } + } + return nil +} + +func copyUploadedLFSFileIntoRepository(info *uploadInfo, storeInLFS bool, t *TemporaryUploadRepository, treePath string) error { + file, err := os.Open(info.upload.LocalPath()) + if err != nil { + return err + } + defer file.Close() + + var objectHash string + if storeInLFS { + // Handle LFS + // FIXME: Inefficient! this should probably happen in models.Upload + pointer, err := lfs.GeneratePointer(file) + if err != nil { + return err + } + + info.lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID} + + if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil { + return err + } + } else if objectHash, err = t.HashObject(file); err != nil { + return err + } + + // Add the object to the index + return t.AddObjectToIndex("100644", objectHash, path.Join(treePath, info.upload.Name)) +} + +func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) error { + if info.lfsMetaObject == nil { + return nil + } + exist, err := contentStore.Exists(info.lfsMetaObject.Pointer) + if err != nil { + return err + } + if !exist { + file, err := os.Open(info.upload.LocalPath()) + if err != nil { + return err + } + + defer file.Close() + // FIXME: Put regenerates the hash and copies the file over. + // I guess this strictly ensures the soundness of the store but this is inefficient. + if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil { + // OK Now we need to cleanup + // Can't clean up the store, once uploaded there they're there. + return err + } + } + return nil +} diff --git a/services/repository/fork.go b/services/repository/fork.go new file mode 100644 index 0000000..0378f7b --- /dev/null +++ b/services/repository/fork.go @@ -0,0 +1,248 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" + notify_service "code.gitea.io/gitea/services/notify" +) + +// ErrForkAlreadyExist represents a "ForkAlreadyExist" kind of error. +type ErrForkAlreadyExist struct { + Uname string + RepoName string + ForkName string +} + +// IsErrForkAlreadyExist checks if an error is an ErrForkAlreadyExist. +func IsErrForkAlreadyExist(err error) bool { + _, ok := err.(ErrForkAlreadyExist) + return ok +} + +func (err ErrForkAlreadyExist) Error() string { + return fmt.Sprintf("repository is already forked by user [uname: %s, repo path: %s, fork path: %s]", err.Uname, err.RepoName, err.ForkName) +} + +func (err ErrForkAlreadyExist) Unwrap() error { + return util.ErrAlreadyExist +} + +// ForkRepoOptions contains the fork repository options +type ForkRepoOptions struct { + BaseRepo *repo_model.Repository + Name string + Description string + SingleBranch string +} + +// ForkRepositoryIfNotExists creates a fork of a repository if it does not already exists and fails otherwise +func ForkRepositoryIfNotExists(ctx context.Context, doer, owner *user_model.User, opts ForkRepoOptions) (*repo_model.Repository, error) { + // Fork is prohibited, if user has reached maximum limit of repositories + if !doer.IsAdmin && !owner.CanForkRepo() { + return nil, repo_model.ErrReachLimitOfRepo{ + Limit: owner.MaxRepoCreation, + } + } + + forkedRepo, err := repo_model.GetUserFork(ctx, opts.BaseRepo.ID, owner.ID) + if err != nil { + return nil, err + } + if forkedRepo != nil { + return nil, ErrForkAlreadyExist{ + Uname: owner.Name, + RepoName: opts.BaseRepo.FullName(), + ForkName: forkedRepo.FullName(), + } + } + + defaultBranch := opts.BaseRepo.DefaultBranch + if opts.SingleBranch != "" { + defaultBranch = opts.SingleBranch + } + repo := &repo_model.Repository{ + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + DefaultBranch: defaultBranch, + IsPrivate: opts.BaseRepo.IsPrivate || opts.BaseRepo.Owner.Visibility == structs.VisibleTypePrivate, + IsEmpty: opts.BaseRepo.IsEmpty, + IsFork: true, + ForkID: opts.BaseRepo.ID, + ObjectFormatName: opts.BaseRepo.ObjectFormatName, + } + + oldRepoPath := opts.BaseRepo.RepoPath() + + needsRollback := false + rollbackFn := func() { + if !needsRollback { + return + } + + repoPath := repo_model.RepoPath(owner.Name, repo.Name) + + if exists, _ := util.IsExist(repoPath); !exists { + return + } + + // As the transaction will be failed and hence database changes will be destroyed we only need + // to delete the related repository on the filesystem + if errDelete := util.RemoveAll(repoPath); errDelete != nil { + log.Error("Failed to remove fork repo") + } + } + + needsRollbackInPanic := true + defer func() { + panicErr := recover() + if panicErr == nil { + return + } + + if needsRollbackInPanic { + rollbackFn() + } + panic(panicErr) + }() + + err = db.WithTx(ctx, func(txCtx context.Context) error { + if err = repo_module.CreateRepositoryByExample(txCtx, doer, owner, repo, false, true); err != nil { + return err + } + + if err = repo_model.IncrementRepoForkNum(txCtx, opts.BaseRepo.ID); err != nil { + return err + } + + // copy lfs files failure should not be ignored + if err = git_model.CopyLFS(txCtx, repo, opts.BaseRepo); err != nil { + return err + } + + needsRollback = true + + cloneCmd := git.NewCommand(txCtx, "clone", "--bare") + if opts.SingleBranch != "" { + cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch) + } + repoPath := repo_model.RepoPath(owner.Name, repo.Name) + if stdout, _, err := cloneCmd.AddDynamicArguments(oldRepoPath, repoPath). + SetDescription(fmt.Sprintf("ForkRepositoryIfNotExists(git clone): %s to %s", opts.BaseRepo.FullName(), repo.FullName())). + RunStdBytes(&git.RunOpts{Timeout: 10 * time.Minute}); err != nil { + log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err) + return fmt.Errorf("git clone: %w", err) + } + + if err := repo_module.CheckDaemonExportOK(txCtx, repo); err != nil { + return fmt.Errorf("checkDaemonExportOK: %w", err) + } + + if stdout, _, err := git.NewCommand(txCtx, "update-server-info"). + SetDescription(fmt.Sprintf("ForkRepositoryIfNotExists(git update-server-info): %s", repo.FullName())). + RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + log.Error("Fork Repository (git update-server-info) failed for %v:\nStdout: %s\nError: %v", repo, stdout, err) + return fmt.Errorf("git update-server-info: %w", err) + } + + if err = repo_module.CreateDelegateHooks(repoPath); err != nil { + return fmt.Errorf("createDelegateHooks: %w", err) + } + + gitRepo, err := gitrepo.OpenRepository(txCtx, repo) + if err != nil { + return fmt.Errorf("OpenRepository: %w", err) + } + defer gitRepo.Close() + + _, err = repo_module.SyncRepoBranchesWithRepo(txCtx, repo, gitRepo, doer.ID) + return err + }) + needsRollbackInPanic = false + if err != nil { + rollbackFn() + return nil, err + } + + return repo, nil +} + +// ForkRepositoryAndUpdates forks a repository. On success it updates metadata (size, stats, etc.) and send a notification. +func ForkRepositoryAndUpdates(ctx context.Context, doer, owner *user_model.User, opts ForkRepoOptions) (*repo_model.Repository, error) { + repo, err := ForkRepositoryIfNotExists(ctx, doer, owner, opts) + if err != nil { + return nil, err + } + + // even if below operations failed, it could be ignored. And they will be retried + if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + if err := repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil { + log.Error("Copy language stat from oldRepo failed: %v", err) + } + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + log.Error("Open created git repository failed: %v", err) + } else { + defer gitRepo.Close() + if err := repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { + log.Error("Sync releases from git tags failed: %v", err) + } + } + + notify_service.ForkRepository(ctx, doer, opts.BaseRepo, repo) + + return repo, nil +} + +// ConvertForkToNormalRepository convert the provided repo from a forked repo to normal repo +func ConvertForkToNormalRepository(ctx context.Context, repo *repo_model.Repository) error { + err := db.WithTx(ctx, func(ctx context.Context) error { + repo, err := repo_model.GetRepositoryByID(ctx, repo.ID) + if err != nil { + return err + } + + if !repo.IsFork { + return nil + } + + if err := repo_model.DecrementRepoForkNum(ctx, repo.ForkID); err != nil { + log.Error("Unable to decrement repo fork num for old root repo %d of repository %-v whilst converting from fork. Error: %v", repo.ForkID, repo, err) + return err + } + + repo.IsFork = false + repo.ForkID = 0 + + if err := repo_module.UpdateRepository(ctx, repo, false); err != nil { + log.Error("Unable to update repository %-v whilst converting from fork. Error: %v", repo, err) + return err + } + + return nil + }) + + return err +} diff --git a/services/repository/fork_test.go b/services/repository/fork_test.go new file mode 100644 index 0000000..2e1e72a --- /dev/null +++ b/services/repository/fork_test.go @@ -0,0 +1,49 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "testing" + + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestForkRepository(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + // user 13 has already forked repo10 + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 13}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}) + + fork, err := ForkRepositoryAndUpdates(git.DefaultContext, user, user, ForkRepoOptions{ + BaseRepo: repo, + Name: "test", + Description: "test", + }) + assert.Nil(t, fork) + require.Error(t, err) + assert.True(t, IsErrForkAlreadyExist(err)) + + // user not reached maximum limit of repositories + assert.False(t, repo_model.IsErrReachLimitOfRepo(err)) + + // change AllowForkWithoutMaximumLimit to false for the test + setting.Repository.AllowForkWithoutMaximumLimit = false + // user has reached maximum limit of repositories + user.MaxRepoCreation = 0 + fork2, err := ForkRepositoryAndUpdates(git.DefaultContext, user, user, ForkRepoOptions{ + BaseRepo: repo, + Name: "test", + Description: "test", + }) + assert.Nil(t, fork2) + assert.True(t, repo_model.IsErrReachLimitOfRepo(err)) +} diff --git a/services/repository/generate.go b/services/repository/generate.go new file mode 100644 index 0000000..8bd14ac --- /dev/null +++ b/services/repository/generate.go @@ -0,0 +1,391 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "bufio" + "bytes" + "context" + "fmt" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/util" + + "github.com/gobwas/glob" + "github.com/huandu/xstrings" +) + +type transformer struct { + Name string + Transform func(string) string +} + +type expansion struct { + Name string + Value string + Transformers []transformer +} + +var defaultTransformers = []transformer{ + {Name: "SNAKE", Transform: xstrings.ToSnakeCase}, + {Name: "KEBAB", Transform: xstrings.ToKebabCase}, + // as of xstrings v1.5.0 the CAMEL & PASCAL workarounds are no longer necessary + // and can be removed https://codeberg.org/forgejo/forgejo/pulls/4050 + {Name: "CAMEL", Transform: func(str string) string { + return xstrings.FirstRuneToLower(xstrings.ToCamelCase(str)) + }}, + {Name: "PASCAL", Transform: xstrings.ToCamelCase}, + {Name: "LOWER", Transform: strings.ToLower}, + {Name: "UPPER", Transform: strings.ToUpper}, + {Name: "TITLE", Transform: util.ToTitleCase}, +} + +func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository, sanitizeFileName bool) string { + year, month, day := time.Now().Date() + expansions := []expansion{ + {Name: "YEAR", Value: strconv.Itoa(year), Transformers: nil}, + {Name: "MONTH", Value: fmt.Sprintf("%02d", int(month)), Transformers: nil}, + {Name: "MONTH_ENGLISH", Value: month.String(), Transformers: defaultTransformers}, + {Name: "DAY", Value: fmt.Sprintf("%02d", day), Transformers: nil}, + {Name: "REPO_NAME", Value: generateRepo.Name, Transformers: defaultTransformers}, + {Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: defaultTransformers}, + {Name: "REPO_DESCRIPTION", Value: generateRepo.Description, Transformers: nil}, + {Name: "TEMPLATE_DESCRIPTION", Value: templateRepo.Description, Transformers: nil}, + {Name: "REPO_OWNER", Value: generateRepo.OwnerName, Transformers: defaultTransformers}, + {Name: "TEMPLATE_OWNER", Value: templateRepo.OwnerName, Transformers: defaultTransformers}, + {Name: "REPO_LINK", Value: generateRepo.Link(), Transformers: nil}, + {Name: "TEMPLATE_LINK", Value: templateRepo.Link(), Transformers: nil}, + {Name: "REPO_HTTPS_URL", Value: generateRepo.CloneLink().HTTPS, Transformers: nil}, + {Name: "TEMPLATE_HTTPS_URL", Value: templateRepo.CloneLink().HTTPS, Transformers: nil}, + {Name: "REPO_SSH_URL", Value: generateRepo.CloneLink().SSH, Transformers: nil}, + {Name: "TEMPLATE_SSH_URL", Value: templateRepo.CloneLink().SSH, Transformers: nil}, + } + + expansionMap := make(map[string]string) + for _, e := range expansions { + expansionMap[e.Name] = e.Value + for _, tr := range e.Transformers { + expansionMap[fmt.Sprintf("%s_%s", e.Name, tr.Name)] = tr.Transform(e.Value) + } + } + + return os.Expand(src, func(key string) string { + if expansion, ok := expansionMap[key]; ok { + if sanitizeFileName { + return fileNameSanitize(expansion) + } + return expansion + } + return key + }) +} + +// GiteaTemplate holds information about a .gitea/template file +type GiteaTemplate struct { + Path string + Content []byte + + globs []glob.Glob +} + +// Globs parses the .gitea/template globs or returns them if they were already parsed +func (gt *GiteaTemplate) Globs() []glob.Glob { + if gt.globs != nil { + return gt.globs + } + + gt.globs = make([]glob.Glob, 0) + scanner := bufio.NewScanner(bytes.NewReader(gt.Content)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + g, err := glob.Compile(line, '/') + if err != nil { + log.Info("Invalid glob expression '%s' (skipped): %v", line, err) + continue + } + gt.globs = append(gt.globs, g) + } + return gt.globs +} + +func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) { + gtPath := filepath.Join(tmpDir, ".gitea", "template") + if _, err := os.Stat(gtPath); os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + + content, err := os.ReadFile(gtPath) + if err != nil { + return nil, err + } + + gt := &GiteaTemplate{ + Path: gtPath, + Content: content, + } + + return gt, nil +} + +func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository, tmpDir string) error { + commitTimeStr := time.Now().Format(time.RFC3339) + authorSig := repo.Owner.NewGitSig() + + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+authorSig.Name, + "GIT_COMMITTER_EMAIL="+authorSig.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + + // Clone to temporary path and do the init commit. + templateRepoPath := templateRepo.RepoPath() + if err := git.Clone(ctx, templateRepoPath, tmpDir, git.CloneRepoOptions{ + Depth: 1, + Branch: templateRepo.DefaultBranch, + }); err != nil { + return fmt.Errorf("git clone: %w", err) + } + + if err := util.RemoveAll(path.Join(tmpDir, ".git")); err != nil { + return fmt.Errorf("remove git dir: %w", err) + } + + // Variable expansion + gt, err := checkGiteaTemplate(tmpDir) + if err != nil { + return fmt.Errorf("checkGiteaTemplate: %w", err) + } + + if gt != nil { + if err := util.Remove(gt.Path); err != nil { + return fmt.Errorf("remove .giteatemplate: %w", err) + } + + // Avoid walking tree if there are no globs + if len(gt.Globs()) > 0 { + tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" + if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if d.IsDir() { + return nil + } + + base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash) + for _, g := range gt.Globs() { + if g.Match(base) { + content, err := os.ReadFile(path) + if err != nil { + return err + } + + if err := os.WriteFile(path, + []byte(generateExpansion(string(content), templateRepo, generateRepo, false)), + 0o644); err != nil { + return err + } + + substPath := filepath.FromSlash(filepath.Join(tmpDirSlash, + generateExpansion(base, templateRepo, generateRepo, true))) + + // Create parent subdirectories if needed or continue silently if it exists + if err := os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil { + return err + } + + // Substitute filename variables + if err := os.Rename(path, substPath); err != nil { + return err + } + + break + } + } + return nil + }); err != nil { + return err + } + } + } + + if err := git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil { + return err + } + + repoPath := repo.RepoPath() + if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin").AddDynamicArguments(repoPath). + SetDescription(fmt.Sprintf("generateRepoCommit (git remote add): %s to %s", templateRepoPath, tmpDir)). + RunStdString(&git.RunOpts{Dir: tmpDir, Env: env}); err != nil { + log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err) + return fmt.Errorf("git remote add: %w", err) + } + + // set default branch based on whether it's specified in the newly generated repo or not + defaultBranch := repo.DefaultBranch + if strings.TrimSpace(defaultBranch) == "" { + defaultBranch = templateRepo.DefaultBranch + } + + return initRepoCommit(ctx, tmpDir, repo, repo.Owner, defaultBranch) +} + +func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) { + tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) + if err != nil { + return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err) + } + + defer func() { + if err := util.RemoveAll(tmpDir); err != nil { + log.Error("RemoveAll: %v", err) + } + }() + + if err = generateRepoCommit(ctx, repo, templateRepo, generateRepo, tmpDir); err != nil { + return fmt.Errorf("generateRepoCommit: %w", err) + } + + // re-fetch repo + if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + return fmt.Errorf("getRepositoryByID: %w", err) + } + + // if there was no default branch supplied when generating the repo, use the default one from the template + if strings.TrimSpace(repo.DefaultBranch) == "" { + repo.DefaultBranch = templateRepo.DefaultBranch + } + + if err = gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + return fmt.Errorf("setDefaultBranch: %w", err) + } + if err = UpdateRepository(ctx, repo, false); err != nil { + return fmt.Errorf("updateRepository: %w", err) + } + + return nil +} + +// GenerateGitContent generates git content from a template repository +func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { + if err := generateGitContent(ctx, generateRepo, templateRepo, generateRepo); err != nil { + return err + } + + if err := repo_module.UpdateRepoSize(ctx, generateRepo); err != nil { + return fmt.Errorf("failed to update size for repository: %w", err) + } + + if err := git_model.CopyLFS(ctx, generateRepo, templateRepo); err != nil { + return fmt.Errorf("failed to copy LFS: %w", err) + } + return nil +} + +// GenerateRepoOptions contains the template units to generate +type GenerateRepoOptions struct { + Name string + DefaultBranch string + Description string + Private bool + GitContent bool + Topics bool + GitHooks bool + Webhooks bool + Avatar bool + IssueLabels bool + ProtectedBranch bool +} + +// IsValid checks whether at least one option is chosen for generation +func (gro GenerateRepoOptions) IsValid() bool { + return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || + gro.IssueLabels || gro.ProtectedBranch // or other items as they are added +} + +// generateRepository generates a repository from a template +func generateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) { + generateRepo := &repo_model.Repository{ + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + DefaultBranch: opts.DefaultBranch, + IsPrivate: opts.Private, + IsEmpty: !opts.GitContent || templateRepo.IsEmpty, + IsFsckEnabled: templateRepo.IsFsckEnabled, + TemplateID: templateRepo.ID, + TrustModel: templateRepo.TrustModel, + ObjectFormatName: templateRepo.ObjectFormatName, + } + + if err = repo_module.CreateRepositoryByExample(ctx, doer, owner, generateRepo, false, false); err != nil { + return nil, err + } + + repoPath := generateRepo.RepoPath() + isExist, err := util.IsExist(repoPath) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + return nil, err + } + if isExist { + return nil, repo_model.ErrRepoFilesAlreadyExist{ + Uname: generateRepo.OwnerName, + Name: generateRepo.Name, + } + } + + if err = repo_module.CheckInitRepository(ctx, owner.Name, generateRepo.Name, generateRepo.ObjectFormatName); err != nil { + return generateRepo, err + } + + if err = repo_module.CheckDaemonExportOK(ctx, generateRepo); err != nil { + return generateRepo, fmt.Errorf("checkDaemonExportOK: %w", err) + } + + if stdout, _, err := git.NewCommand(ctx, "update-server-info"). + SetDescription(fmt.Sprintf("GenerateRepository(git update-server-info): %s", repoPath)). + RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + log.Error("GenerateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", generateRepo, stdout, err) + return generateRepo, fmt.Errorf("error in GenerateRepository(git update-server-info): %w", err) + } + + return generateRepo, nil +} + +var fileNameSanitizeRegexp = regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`) + +// Sanitize user input to valid OS filenames +// +// Based on https://github.com/sindresorhus/filename-reserved-regex +// Adds ".." to prevent directory traversal +func fileNameSanitize(s string) string { + return strings.TrimSpace(fileNameSanitizeRegexp.ReplaceAllString(s, "_")) +} diff --git a/services/repository/generate_test.go b/services/repository/generate_test.go new file mode 100644 index 0000000..b0f97d0 --- /dev/null +++ b/services/repository/generate_test.go @@ -0,0 +1,67 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var giteaTemplate = []byte(` +# Header + +# All .go files +**.go + +# All text files in /text/ +text/*.txt + +# All files in modules folders +**/modules/* +`) + +func TestGiteaTemplate(t *testing.T) { + gt := GiteaTemplate{Content: giteaTemplate} + assert.Len(t, gt.Globs(), 3) + + tt := []struct { + Path string + Match bool + }{ + {Path: "main.go", Match: true}, + {Path: "a/b/c/d/e.go", Match: true}, + {Path: "main.txt", Match: false}, + {Path: "a/b.txt", Match: false}, + {Path: "text/a.txt", Match: true}, + {Path: "text/b.txt", Match: true}, + {Path: "text/c.json", Match: false}, + {Path: "a/b/c/modules/README.md", Match: true}, + {Path: "a/b/c/modules/d/README.md", Match: false}, + } + + for _, tc := range tt { + t.Run(tc.Path, func(t *testing.T) { + match := false + for _, g := range gt.Globs() { + if g.Match(tc.Path) { + match = true + break + } + } + assert.Equal(t, tc.Match, match) + }) + } +} + +func TestFileNameSanitize(t *testing.T) { + assert.Equal(t, "test_CON", fileNameSanitize("test_CON")) + assert.Equal(t, "test CON", fileNameSanitize("test CON ")) + assert.Equal(t, "__traverse__", fileNameSanitize("../traverse/..")) + assert.Equal(t, "http___localhost_3003_user_test.git", fileNameSanitize("http://localhost:3003/user/test.git")) + assert.Equal(t, "_", fileNameSanitize("CON")) + assert.Equal(t, "_", fileNameSanitize("con")) + assert.Equal(t, "_", fileNameSanitize("\u0000")) + assert.Equal(t, "目标", fileNameSanitize("目标")) +} diff --git a/services/repository/hooks.go b/services/repository/hooks.go new file mode 100644 index 0000000..97e9e29 --- /dev/null +++ b/services/repository/hooks.go @@ -0,0 +1,110 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + + "xorm.io/builder" +) + +// SyncRepositoryHooks rewrites all repositories' pre-receive, update and post-receive hooks +// to make sure the binary and custom conf path are up-to-date. +func SyncRepositoryHooks(ctx context.Context) error { + log.Trace("Doing: SyncRepositoryHooks") + + if err := db.Iterate( + ctx, + builder.Gt{"id": 0}, + func(ctx context.Context, repo *repo_model.Repository) error { + select { + case <-ctx.Done(): + return db.ErrCancelledf("before sync repository hooks for %s", repo.FullName()) + default: + } + + if err := repo_module.CreateDelegateHooks(repo.RepoPath()); err != nil { + return fmt.Errorf("SyncRepositoryHook: %w", err) + } + if repo.HasWiki() { + if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { + return fmt.Errorf("SyncRepositoryHook: %w", err) + } + } + return nil + }, + ); err != nil { + return err + } + + log.Trace("Finished: SyncRepositoryHooks") + return nil +} + +// GenerateGitHooks generates git hooks from a template repository +func GenerateGitHooks(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { + generateGitRepo, err := gitrepo.OpenRepository(ctx, generateRepo) + if err != nil { + return err + } + defer generateGitRepo.Close() + + templateGitRepo, err := gitrepo.OpenRepository(ctx, templateRepo) + if err != nil { + return err + } + defer templateGitRepo.Close() + + templateHooks, err := templateGitRepo.Hooks() + if err != nil { + return err + } + + for _, templateHook := range templateHooks { + generateHook, err := generateGitRepo.GetHook(templateHook.Name()) + if err != nil { + return err + } + + generateHook.Content = templateHook.Content + if err := generateHook.Update(); err != nil { + return err + } + } + return nil +} + +// GenerateWebhooks generates webhooks from a template repository +func GenerateWebhooks(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { + templateWebhooks, err := db.Find[webhook.Webhook](ctx, webhook.ListWebhookOptions{RepoID: templateRepo.ID}) + if err != nil { + return err + } + + ws := make([]*webhook.Webhook, 0, len(templateWebhooks)) + for _, templateWebhook := range templateWebhooks { + ws = append(ws, &webhook.Webhook{ + RepoID: generateRepo.ID, + URL: templateWebhook.URL, + HTTPMethod: templateWebhook.HTTPMethod, + ContentType: templateWebhook.ContentType, + Secret: templateWebhook.Secret, + HookEvent: templateWebhook.HookEvent, + IsActive: templateWebhook.IsActive, + Type: templateWebhook.Type, + OwnerID: templateWebhook.OwnerID, + Events: templateWebhook.Events, + Meta: templateWebhook.Meta, + }) + } + return webhook.CreateWebhooks(ctx, ws) +} diff --git a/services/repository/init.go b/services/repository/init.go new file mode 100644 index 0000000..817fa4a --- /dev/null +++ b/services/repository/init.go @@ -0,0 +1,83 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "os" + "time" + + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + asymkey_service "code.gitea.io/gitea/services/asymkey" +) + +// initRepoCommit temporarily changes with work directory. +func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Repository, u *user_model.User, defaultBranch string) (err error) { + commitTimeStr := time.Now().Format(time.RFC3339) + + sig := u.NewGitSig() + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+sig.Name, + "GIT_AUTHOR_EMAIL="+sig.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + committerName := sig.Name + committerEmail := sig.Email + + if stdout, _, err := git.NewCommand(ctx, "add", "--all"). + SetDescription(fmt.Sprintf("initRepoCommit (git add): %s", tmpPath)). + RunStdString(&git.RunOpts{Dir: tmpPath}); err != nil { + log.Error("git add --all failed: Stdout: %s\nError: %v", stdout, err) + return fmt.Errorf("git add --all: %w", err) + } + + cmd := git.NewCommand(ctx, "commit", "--message=Initial commit"). + AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email) + + sign, keyID, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) + if sign { + cmd.AddOptionFormat("-S%s", keyID) + + if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { + // need to set the committer to the KeyID owner + committerName = signer.Name + committerEmail = signer.Email + } + } else { + cmd.AddArguments("--no-gpg-sign") + } + + env = append(env, + "GIT_COMMITTER_NAME="+committerName, + "GIT_COMMITTER_EMAIL="+committerEmail, + ) + + if stdout, _, err := cmd. + SetDescription(fmt.Sprintf("initRepoCommit (git commit): %s", tmpPath)). + RunStdString(&git.RunOpts{Dir: tmpPath, Env: env}); err != nil { + log.Error("Failed to commit: %v: Stdout: %s\nError: %v", cmd.String(), stdout, err) + return fmt.Errorf("git commit: %w", err) + } + + if len(defaultBranch) == 0 { + defaultBranch = setting.Repository.DefaultBranch + } + + if stdout, _, err := git.NewCommand(ctx, "push", "origin").AddDynamicArguments("HEAD:" + defaultBranch). + SetDescription(fmt.Sprintf("initRepoCommit (git push): %s", tmpPath)). + RunStdString(&git.RunOpts{Dir: tmpPath, Env: repo_module.InternalPushingEnvironment(u, repo)}); err != nil { + log.Error("Failed to push back to HEAD: Stdout: %s\nError: %v", stdout, err) + return fmt.Errorf("git push: %w", err) + } + + return nil +} diff --git a/services/repository/lfs.go b/services/repository/lfs.go new file mode 100644 index 0000000..4cd1110 --- /dev/null +++ b/services/repository/lfs.go @@ -0,0 +1,123 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "time" + + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" +) + +// GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function +type GarbageCollectLFSMetaObjectsOptions struct { + LogDetail func(format string, v ...any) + AutoFix bool + OlderThan time.Time + UpdatedLessRecentlyThan time.Time +} + +// GarbageCollectLFSMetaObjects garbage collects LFS objects for all repositories +func GarbageCollectLFSMetaObjects(ctx context.Context, opts GarbageCollectLFSMetaObjectsOptions) error { + log.Trace("Doing: GarbageCollectLFSMetaObjects") + defer log.Trace("Finished: GarbageCollectLFSMetaObjects") + + if opts.LogDetail == nil { + opts.LogDetail = log.Debug + } + + if !setting.LFS.StartServer { + opts.LogDetail("LFS support is disabled") + return nil + } + + return git_model.IterateRepositoryIDsWithLFSMetaObjects(ctx, func(ctx context.Context, repoID, count int64) error { + repo, err := repo_model.GetRepositoryByID(ctx, repoID) + if err != nil { + return err + } + + return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, opts) + }) +} + +// GarbageCollectLFSMetaObjectsForRepo garbage collects LFS objects for a specific repository +func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, opts GarbageCollectLFSMetaObjectsOptions) error { + opts.LogDetail("Checking %s", repo.FullName()) + total, orphaned, collected, deleted := int64(0), 0, 0, 0 + defer func() { + if orphaned == 0 { + opts.LogDetail("Found %d total LFSMetaObjects in %s", total, repo.FullName()) + } else if !opts.AutoFix { + opts.LogDetail("Found %d/%d orphaned LFSMetaObjects in %s", orphaned, total, repo.FullName()) + } else { + opts.LogDetail("Collected %d/%d orphaned/%d total LFSMetaObjects in %s. %d removed from storage.", collected, orphaned, total, repo.FullName(), deleted) + } + }() + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + log.Error("Unable to open git repository %s: %v", repo.FullName(), err) + return err + } + defer gitRepo.Close() + + store := lfs.NewContentStore() + objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + + err = git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject) error { + total++ + pointerSha := git.ComputeBlobHash(objectFormat, []byte(metaObject.Pointer.StringContent())) + + if gitRepo.IsObjectExist(pointerSha.String()) { + return git_model.MarkLFSMetaObject(ctx, metaObject.ID) + } + orphaned++ + + if !opts.AutoFix { + return nil + } + // Non-existent pointer file + _, err = git_model.RemoveLFSMetaObjectByOidFn(ctx, repo.ID, metaObject.Oid, func(count int64) error { + if count > 0 { + return nil + } + + if err := store.Delete(metaObject.RelativePath()); err != nil { + log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err) + } + deleted++ + return nil + }) + if err != nil { + return fmt.Errorf("unable to remove meta-object %s in %s: %w", metaObject.Oid, repo.FullName(), err) + } + collected++ + + return nil + }, &git_model.IterateLFSMetaObjectsForRepoOptions{ + // Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload + // and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby + // an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid + // changes in new branches that might lead to lfs objects becoming temporarily unassociated with git + // objects. + // + // It is likely that a week is potentially excessive but it should definitely be enough that any + // unassociated LFS object is genuinely unassociated. + OlderThan: timeutil.TimeStamp(opts.OlderThan.Unix()), + UpdatedLessRecentlyThan: timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()), + }) + if err != nil { + return err + } + return nil +} diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go new file mode 100644 index 0000000..a0c01df --- /dev/null +++ b/services/repository/lfs_test.go @@ -0,0 +1,75 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository_test + +import ( + "bytes" + "context" + "testing" + "time" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + repo_service "code.gitea.io/gitea/services/repository" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGarbageCollectLFSMetaObjects(t *testing.T) { + unittest.PrepareTestEnv(t) + + setting.LFS.StartServer = true + err := storage.Init() + require.NoError(t, err) + + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, "user2", "lfs") + require.NoError(t, err) + + validLFSObjects, err := db.GetEngine(db.DefaultContext).Count(git_model.LFSMetaObject{RepositoryID: repo.ID}) + require.NoError(t, err) + assert.Greater(t, validLFSObjects, int64(1)) + + // add lfs object + lfsContent := []byte("gitea1") + lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent) + + // gc + err = repo_service.GarbageCollectLFSMetaObjects(context.Background(), repo_service.GarbageCollectLFSMetaObjectsOptions{ + AutoFix: true, + OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour), + UpdatedLessRecentlyThan: time.Time{}, // ensure that the models/fixtures/lfs_meta_object.yml objects are considered as well + LogDetail: t.Logf, + }) + require.NoError(t, err) + + // lfs meta has been deleted + _, err = git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, lfsOid) + require.ErrorIs(t, err, git_model.ErrLFSObjectNotExist) + + remainingLFSObjects, err := db.GetEngine(db.DefaultContext).Count(git_model.LFSMetaObject{RepositoryID: repo.ID}) + require.NoError(t, err) + assert.Equal(t, validLFSObjects-1, remainingLFSObjects) +} + +func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string { + pointer, err := lfs.GeneratePointer(bytes.NewReader(*content)) + require.NoError(t, err) + + _, err = git_model.NewLFSMetaObject(db.DefaultContext, repositoryID, pointer) + require.NoError(t, err) + contentStore := lfs.NewContentStore() + exist, err := contentStore.Exists(pointer) + require.NoError(t, err) + if !exist { + err := contentStore.Put(pointer, bytes.NewReader(*content)) + require.NoError(t, err) + } + return pointer.Oid +} diff --git a/services/repository/main_test.go b/services/repository/main_test.go new file mode 100644 index 0000000..7ad1540 --- /dev/null +++ b/services/repository/main_test.go @@ -0,0 +1,14 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m) +} diff --git a/services/repository/migrate.go b/services/repository/migrate.go new file mode 100644 index 0000000..39ced04 --- /dev/null +++ b/services/repository/migrate.go @@ -0,0 +1,289 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/organization" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migration" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +// MigrateRepositoryGitData starts migrating git related data after created migrating repository +func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, + repo *repo_model.Repository, opts migration.MigrateOptions, + httpTransport *http.Transport, +) (*repo_model.Repository, error) { + repoPath := repo_model.RepoPath(u.Name, opts.RepoName) + + if u.IsOrganization() { + t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx) + if err != nil { + return nil, err + } + repo.NumWatches = t.NumMembers + } else { + repo.NumWatches = 1 + } + + migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + + var err error + if err = util.RemoveAll(repoPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", repoPath, err) + } + + if err = git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + SkipTLSVerify: setting.Migrations.SkipTLSVerify, + }); err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return repo, fmt.Errorf("Clone timed out. Consider increasing [git.timeout] MIGRATE in app.ini. Underlying Error: %w", err) + } + return repo, fmt.Errorf("Clone: %w", err) + } + + if err := git.WriteCommitGraph(ctx, repoPath); err != nil { + return repo, err + } + + if opts.Wiki { + wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) + wikiRemotePath := repo_module.WikiRemoteURL(ctx, opts.CloneAddr) + if len(wikiRemotePath) > 0 { + if err := util.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + } + + if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + SkipTLSVerify: setting.Migrations.SkipTLSVerify, + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := util.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + } + } else { + // Figure out the branch of the wiki we just cloned. We assume + // that the default branch is to be used, and we'll use the same + // name as the source. + gitRepo, err := git.OpenRepository(ctx, wikiPath) + if err != nil { + log.Warn("Failed to open wiki repository during migration: %v", err) + if err := util.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + } + return repo, err + } + defer gitRepo.Close() + + branch, err := gitrepo.GetDefaultBranch(ctx, repo) + if err != nil { + log.Warn("Failed to get the default branch of a migrated wiki repo: %v", err) + if err := util.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + } + + return repo, err + } + repo.WikiBranch = branch + + if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { + return repo, err + } + } + } + } + + if repo.OwnerID == u.ID { + repo.Owner = u + } + + if err = repo_module.CheckDaemonExportOK(ctx, repo); err != nil { + return repo, fmt.Errorf("checkDaemonExportOK: %w", err) + } + + if stdout, _, err := git.NewCommand(ctx, "update-server-info"). + SetDescription(fmt.Sprintf("MigrateRepositoryGitData(git update-server-info): %s", repoPath)). + RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + log.Error("MigrateRepositoryGitData(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return repo, fmt.Errorf("error in MigrateRepositoryGitData(git update-server-info): %w", err) + } + + gitRepo, err := git.OpenRepository(ctx, repoPath) + if err != nil { + return repo, fmt.Errorf("OpenRepository: %w", err) + } + defer gitRepo.Close() + + repo.IsEmpty, err = gitRepo.IsEmpty() + if err != nil { + return repo, fmt.Errorf("git.IsEmpty: %w", err) + } + + if !repo.IsEmpty { + if len(repo.DefaultBranch) == 0 { + // Try to get HEAD branch and set it as default branch. + headBranch, err := gitRepo.GetHEADBranch() + if err != nil { + return repo, fmt.Errorf("GetHEADBranch: %w", err) + } + if headBranch != nil { + repo.DefaultBranch = headBranch.Name + } + } + + if _, err := repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil { + return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err) + } + + if !opts.Releases { + // note: this will greatly improve release (tag) sync + // for pull-mirrors with many tags + repo.IsMirror = opts.Mirror + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { + log.Error("Failed to synchronize tags to releases for repository: %v", err) + } + } + + if opts.LFS { + endpoint := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) + lfsClient := lfs.NewClient(endpoint, httpTransport) + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil { + log.Error("Failed to store missing LFS objects for repository: %v", err) + return repo, fmt.Errorf("StoreMissingLfsObjectsInRepository: %w", err) + } + } + } + + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return nil, err + } + defer committer.Close() + + if opts.Mirror { + remoteAddress, err := util.SanitizeURL(opts.CloneAddr) + if err != nil { + return repo, err + } + mirrorModel := repo_model.Mirror{ + RepoID: repo.ID, + Interval: setting.Mirror.DefaultInterval, + EnablePrune: true, + NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), + LFS: opts.LFS, + RemoteAddress: remoteAddress, + } + if opts.LFS { + mirrorModel.LFSEndpoint = opts.LFSEndpoint + } + + if opts.MirrorInterval != "" { + parsedInterval, err := time.ParseDuration(opts.MirrorInterval) + if err != nil { + log.Error("Failed to set Interval: %v", err) + return repo, err + } + if parsedInterval == 0 { + mirrorModel.Interval = 0 + mirrorModel.NextUpdateUnix = 0 + } else if parsedInterval < setting.Mirror.MinInterval { + err := fmt.Errorf("interval %s is set below Minimum Interval of %s", parsedInterval, setting.Mirror.MinInterval) + log.Error("Interval: %s is too frequent", opts.MirrorInterval) + return repo, err + } else { + mirrorModel.Interval = parsedInterval + mirrorModel.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(parsedInterval) + } + } + + if err = repo_model.InsertMirror(ctx, &mirrorModel); err != nil { + return repo, fmt.Errorf("InsertOne: %w", err) + } + + repo.IsMirror = true + if err = UpdateRepository(ctx, repo, false); err != nil { + return nil, err + } + + // this is necessary for sync local tags from remote + configName := fmt.Sprintf("remote.%s.fetch", mirrorModel.GetRemoteName()) + if stdout, _, err := git.NewCommand(ctx, "config"). + AddOptionValues("--add", configName, `+refs/tags/*:refs/tags/*`). + RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + log.Error("MigrateRepositoryGitData(git config --add +refs/tags/*:refs/tags/*) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return repo, fmt.Errorf("error in MigrateRepositoryGitData(git config --add +refs/tags/*:refs/tags/*): %w", err) + } + } else { + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + if repo, err = CleanUpMigrateInfo(ctx, repo); err != nil { + return nil, err + } + } + + return repo, committer.Commit() +} + +// cleanUpMigrateGitConfig removes mirror info which prevents "push --all". +// This also removes possible user credentials. +func cleanUpMigrateGitConfig(ctx context.Context, repoPath string) error { + cmd := git.NewCommand(ctx, "remote", "rm", "origin") + // if the origin does not exist + _, stderr, err := cmd.RunStdString(&git.RunOpts{ + Dir: repoPath, + }) + if err != nil && !strings.HasPrefix(stderr, "fatal: No such remote") { + return err + } + return nil +} + +// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. +func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo_model.Repository, error) { + repoPath := repo.RepoPath() + if err := repo_module.CreateDelegateHooks(repoPath); err != nil { + return repo, fmt.Errorf("createDelegateHooks: %w", err) + } + if repo.HasWiki() { + if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { + return repo, fmt.Errorf("createDelegateHooks.(wiki): %w", err) + } + } + + _, _, err := git.NewCommand(ctx, "remote", "rm", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return repo, fmt.Errorf("CleanUpMigrateInfo: %w", err) + } + + if repo.HasWiki() { + if err := cleanUpMigrateGitConfig(ctx, repo.WikiPath()); err != nil { + return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %w", err) + } + } + + return repo, UpdateRepository(ctx, repo, false) +} diff --git a/services/repository/push.go b/services/repository/push.go new file mode 100644 index 0000000..afd6308 --- /dev/null +++ b/services/repository/push.go @@ -0,0 +1,420 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/queue" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + issue_service "code.gitea.io/gitea/services/issue" + notify_service "code.gitea.io/gitea/services/notify" + pull_service "code.gitea.io/gitea/services/pull" +) + +// pushQueue represents a queue to handle update pull request tests +var pushQueue *queue.WorkerPoolQueue[[]*repo_module.PushUpdateOptions] + +// handle passed PR IDs and test the PRs +func handler(items ...[]*repo_module.PushUpdateOptions) [][]*repo_module.PushUpdateOptions { + for _, opts := range items { + if err := pushUpdates(opts); err != nil { + // Username and repository stays the same between items in opts. + pushUpdate := opts[0] + log.Error("pushUpdate[%s/%s] failed: %v", pushUpdate.RepoUserName, pushUpdate.RepoName, err) + } + } + return nil +} + +func initPushQueue() error { + pushQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "push_update", handler) + if pushQueue == nil { + return errors.New("unable to create push_update queue") + } + go graceful.GetManager().RunWithCancel(pushQueue) + return nil +} + +// PushUpdate is an alias of PushUpdates for single push update options +func PushUpdate(opts *repo_module.PushUpdateOptions) error { + return PushUpdates([]*repo_module.PushUpdateOptions{opts}) +} + +// PushUpdates adds a push update to push queue +func PushUpdates(opts []*repo_module.PushUpdateOptions) error { + if len(opts) == 0 { + return nil + } + + for _, opt := range opts { + if opt.IsNewRef() && opt.IsDelRef() { + return fmt.Errorf("Old and new revisions are both NULL") + } + } + + return pushQueue.Push(opts) +} + +// pushUpdates generates push action history feeds for push updating multiple refs +func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { + if len(optsList) == 0 { + return nil + } + + ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("PushUpdates: %s/%s", optsList[0].RepoUserName, optsList[0].RepoName)) + defer finished() + + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, optsList[0].RepoUserName, optsList[0].RepoName) + if err != nil { + return fmt.Errorf("GetRepositoryByOwnerAndName failed: %w", err) + } + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + return fmt.Errorf("OpenRepository[%s]: %w", repo.FullName(), err) + } + defer gitRepo.Close() + + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + return fmt.Errorf("Failed to update size for repository: %v", err) + } + + addTags := make([]string, 0, len(optsList)) + delTags := make([]string, 0, len(optsList)) + var pusher *user_model.User + objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + + for _, opts := range optsList { + log.Trace("pushUpdates: %-v %s %s %s", repo, opts.OldCommitID, opts.NewCommitID, opts.RefFullName) + + if opts.IsNewRef() && opts.IsDelRef() { + return fmt.Errorf("old and new revisions are both %s", objectFormat.EmptyObjectID()) + } + if opts.RefFullName.IsTag() { + if pusher == nil || pusher.ID != opts.PusherID { + if opts.PusherID == user_model.ActionsUserID { + pusher = user_model.NewActionsUser() + } else { + var err error + if pusher, err = user_model.GetUserByID(ctx, opts.PusherID); err != nil { + return err + } + } + } + tagName := opts.RefFullName.TagName() + if opts.IsDelRef() { + notify_service.PushCommits( + ctx, pusher, repo, + &repo_module.PushUpdateOptions{ + RefFullName: git.RefNameFromTag(tagName), + OldCommitID: opts.OldCommitID, + NewCommitID: objectFormat.EmptyObjectID().String(), + }, repo_module.NewPushCommits()) + + delTags = append(delTags, tagName) + notify_service.DeleteRef(ctx, pusher, repo, opts.RefFullName) + } else { // is new tag + newCommit, err := gitRepo.GetCommit(opts.NewCommitID) + if err != nil { + return fmt.Errorf("gitRepo.GetCommit(%s) in %s/%s[%d]: %w", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err) + } + + commits := repo_module.NewPushCommits() + commits.HeadCommit = repo_module.CommitToPushCommit(newCommit) + commits.CompareURL = repo.ComposeCompareURL(objectFormat.EmptyObjectID().String(), opts.NewCommitID) + + notify_service.PushCommits( + ctx, pusher, repo, + &repo_module.PushUpdateOptions{ + RefFullName: opts.RefFullName, + OldCommitID: objectFormat.EmptyObjectID().String(), + NewCommitID: opts.NewCommitID, + }, commits) + + addTags = append(addTags, tagName) + notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) + } + } else if opts.RefFullName.IsBranch() { + if pusher == nil || pusher.ID != opts.PusherID { + if opts.PusherID == user_model.ActionsUserID { + pusher = user_model.NewActionsUser() + } else { + var err error + if pusher, err = user_model.GetUserByID(ctx, opts.PusherID); err != nil { + return err + } + } + } + + branch := opts.RefFullName.BranchName() + if !opts.IsDelRef() { + log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) + pull_service.AddTestPullRequestTask(ctx, pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID, opts.TimeNano) + + newCommit, err := gitRepo.GetCommit(opts.NewCommitID) + if err != nil { + return fmt.Errorf("gitRepo.GetCommit(%s) in %s/%s[%d]: %w", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err) + } + + refName := opts.RefName() + + // Push new branch. + var l []*git.Commit + if opts.IsNewRef() { + if repo.IsEmpty { // Change default branch and empty status only if pushed ref is non-empty branch. + repo.DefaultBranch = refName + repo.IsEmpty = false + if repo.DefaultBranch != setting.Repository.DefaultBranch { + if err := gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + if !git.IsErrUnsupportedVersion(err) { + return err + } + } + } + // Update the is empty and default_branch columns + if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_branch", "is_empty"); err != nil { + return fmt.Errorf("UpdateRepositoryCols: %w", err) + } + } + + l, err = newCommit.CommitsBeforeLimit(10) + if err != nil { + return fmt.Errorf("newCommit.CommitsBeforeLimit: %w", err) + } + notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) + } else { + l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) + if err != nil { + return fmt.Errorf("newCommit.CommitsBeforeUntil: %w", err) + } + + isForcePush, err := newCommit.IsForcePush(opts.OldCommitID) + if err != nil { + log.Error("IsForcePush %s:%s failed: %v", repo.FullName(), branch, err) + } + + if isForcePush { + log.Trace("Push %s is a force push", opts.NewCommitID) + + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) + } else { + // TODO: increment update the commit count cache but not remove + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) + } + } + + commits := repo_module.GitToPushCommits(l) + commits.HeadCommit = repo_module.CommitToPushCommit(newCommit) + + if err := issue_service.UpdateIssuesCommit(ctx, pusher, repo, commits.Commits, refName); err != nil { + log.Error("updateIssuesCommit: %v", err) + } + + oldCommitID := opts.OldCommitID + if oldCommitID == objectFormat.EmptyObjectID().String() && len(commits.Commits) > 0 { + oldCommit, err := gitRepo.GetCommit(commits.Commits[len(commits.Commits)-1].Sha1) + if err != nil && !git.IsErrNotExist(err) { + log.Error("unable to GetCommit %s from %-v: %v", oldCommitID, repo, err) + } + if oldCommit != nil { + for i := 0; i < oldCommit.ParentCount(); i++ { + commitID, _ := oldCommit.ParentID(i) + if !commitID.IsZero() { + oldCommitID = commitID.String() + break + } + } + } + } + + if oldCommitID == objectFormat.EmptyObjectID().String() && repo.DefaultBranch != branch { + oldCommitID = repo.DefaultBranch + } + + if oldCommitID != objectFormat.EmptyObjectID().String() { + commits.CompareURL = repo.ComposeCompareURL(oldCommitID, opts.NewCommitID) + } else { + commits.CompareURL = "" + } + + if len(commits.Commits) > setting.UI.FeedMaxCommitNum { + commits.Commits = commits.Commits[:setting.UI.FeedMaxCommitNum] + } + + notify_service.PushCommits(ctx, pusher, repo, opts, commits) + + // Cache for big repository + if err := CacheRef(graceful.GetManager().HammerContext(), repo, gitRepo, opts.RefFullName); err != nil { + log.Error("repo_module.CacheRef %s/%s failed: %v", repo.ID, branch, err) + } + } else { + notify_service.DeleteRef(ctx, pusher, repo, opts.RefFullName) + if err = pull_service.CloseBranchPulls(ctx, pusher, repo.ID, branch); err != nil { + // close all related pulls + log.Error("close related pull request failed: %v", err) + } + } + + // Even if user delete a branch on a repository which he didn't watch, he will be watch that. + if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil { + log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) + } + } else { + log.Trace("Non-tag and non-branch commits pushed.") + } + } + if err := PushUpdateAddDeleteTags(ctx, repo, gitRepo, addTags, delTags); err != nil { + return fmt.Errorf("PushUpdateAddDeleteTags: %w", err) + } + + // Change repository last updated time. + if err := repo_model.UpdateRepositoryUpdatedTime(ctx, repo.ID, time.Now()); err != nil { + return fmt.Errorf("UpdateRepositoryUpdatedTime: %w", err) + } + + return nil +} + +// PushUpdateAddDeleteTags updates a number of added and delete tags +func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, addTags, delTags []string) error { + return db.WithTx(ctx, func(ctx context.Context) error { + if err := repo_model.PushUpdateDeleteTagsContext(ctx, repo, delTags); err != nil { + return err + } + return pushUpdateAddTags(ctx, repo, gitRepo, addTags) + }) +} + +// pushUpdateAddTags updates a number of add tags +func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tags []string) error { + if len(tags) == 0 { + return nil + } + + releases, err := db.Find[repo_model.Release](ctx, repo_model.FindReleasesOptions{ + RepoID: repo.ID, + TagNames: tags, + IncludeTags: true, + }) + if err != nil { + return fmt.Errorf("db.Find[repo_model.Release]: %w", err) + } + relMap := make(map[string]*repo_model.Release) + for _, rel := range releases { + relMap[rel.LowerTagName] = rel + } + + lowerTags := make([]string, 0, len(tags)) + for _, tag := range tags { + lowerTags = append(lowerTags, strings.ToLower(tag)) + } + + newReleases := make([]*repo_model.Release, 0, len(lowerTags)-len(relMap)) + + emailToUser := make(map[string]*user_model.User) + + for i, lowerTag := range lowerTags { + tag, err := gitRepo.GetTag(tags[i]) + if err != nil { + return fmt.Errorf("GetTag: %w", err) + } + commit, err := tag.Commit(gitRepo) + if err != nil { + return fmt.Errorf("Commit: %w", err) + } + + sig := tag.Tagger + if sig == nil { + sig = commit.Author + } + if sig == nil { + sig = commit.Committer + } + var author *user_model.User + createdAt := time.Unix(1, 0) + + if sig != nil { + var ok bool + author, ok = emailToUser[sig.Email] + if !ok { + author, err = user_model.GetUserByEmail(ctx, sig.Email) + if err != nil && !user_model.IsErrUserNotExist(err) { + return fmt.Errorf("GetUserByEmail: %w", err) + } + if author != nil { + emailToUser[sig.Email] = author + } + } + createdAt = sig.When + } + + commitsCount, err := commit.CommitsCount() + if err != nil { + return fmt.Errorf("CommitsCount: %w", err) + } + + parts := strings.SplitN(tag.Message, "\n", 2) + note := "" + if len(parts) > 1 { + note = parts[1] + } + + if rel, has := relMap[lowerTag]; !has { + rel = &repo_model.Release{ + RepoID: repo.ID, + Title: parts[0], + TagName: tags[i], + LowerTagName: lowerTag, + Target: "", + Sha1: commit.ID.String(), + NumCommits: commitsCount, + Note: note, + IsDraft: false, + IsPrerelease: false, + IsTag: true, + CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), + } + if author != nil { + rel.PublisherID = author.ID + } + newReleases = append(newReleases, rel) + } else { + rel.Title = parts[0] + rel.Note = note + rel.Sha1 = commit.ID.String() + rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix()) + rel.NumCommits = commitsCount + if rel.IsTag && author != nil { + rel.PublisherID = author.ID + } + if err = repo_model.UpdateRelease(ctx, rel); err != nil { + return fmt.Errorf("Update: %w", err) + } + } + } + + if len(newReleases) > 0 { + if err = db.Insert(ctx, newReleases); err != nil { + return fmt.Errorf("Insert: %w", err) + } + } + + return nil +} diff --git a/services/repository/repository.go b/services/repository/repository.go new file mode 100644 index 0000000..116e241 --- /dev/null +++ b/services/repository/repository.go @@ -0,0 +1,153 @@ +// Copyright 2024 The Forgejo Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/git" + issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/organization" + packages_model "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + system_model "code.gitea.io/gitea/models/system" + "code.gitea.io/gitea/models/unit" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" + federation_service "code.gitea.io/gitea/services/federation" + notify_service "code.gitea.io/gitea/services/notify" + pull_service "code.gitea.io/gitea/services/pull" +) + +// WebSearchRepository represents a repository returned by web search +type WebSearchRepository struct { + Repository *structs.Repository `json:"repository"` + LatestCommitStatus *git.CommitStatus `json:"latest_commit_status"` + LocaleLatestCommitStatus string `json:"locale_latest_commit_status"` +} + +// WebSearchResults results of a successful web search +type WebSearchResults struct { + OK bool `json:"ok"` + Data []*WebSearchRepository `json:"data"` +} + +// CreateRepository creates a repository for the user/organization. +func CreateRepository(ctx context.Context, doer, owner *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { + repo, err := CreateRepositoryDirectly(ctx, doer, owner, opts) + if err != nil { + // No need to rollback here we should do this in CreateRepository... + return nil, err + } + + notify_service.CreateRepository(ctx, doer, owner, repo) + + return repo, nil +} + +// DeleteRepository deletes a repository for a user or organization. +func DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, notify bool) error { + if err := pull_service.CloseRepoBranchesPulls(ctx, doer, repo); err != nil { + log.Error("CloseRepoBranchesPulls failed: %v", err) + } + + if notify { + // If the repo itself has webhooks, we need to trigger them before deleting it... + notify_service.DeleteRepository(ctx, doer, repo) + } + + if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { + return err + } + + if err := federation_service.DeleteFollowingRepos(ctx, repo.ID); err != nil { + return err + } + + return packages_model.UnlinkRepositoryFromAllPackages(ctx, repo.ID) +} + +// PushCreateRepo creates a repository when a new repository is pushed to an appropriate namespace +func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoName string) (*repo_model.Repository, error) { + if !authUser.IsAdmin { + if owner.IsOrganization() { + if ok, err := organization.CanCreateOrgRepo(ctx, owner.ID, authUser.ID); err != nil { + return nil, err + } else if !ok { + return nil, fmt.Errorf("cannot push-create repository for org") + } + } else if authUser.ID != owner.ID { + return nil, fmt.Errorf("cannot push-create repository for another user") + } + } + + repo, err := CreateRepository(ctx, authUser, owner, CreateRepoOptions{ + Name: repoName, + IsPrivate: setting.Repository.DefaultPushCreatePrivate || setting.Repository.ForcePrivate, + }) + if err != nil { + return nil, err + } + + return repo, nil +} + +// Init start repository service +func Init(ctx context.Context) error { + if err := repo_module.LoadRepoConfig(); err != nil { + return err + } + system_model.RemoveAllWithNotice(ctx, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath) + system_model.RemoveAllWithNotice(ctx, "Clean up temporary repositories", repo_module.LocalCopyPath()) + if err := initPushQueue(); err != nil { + return err + } + return initBranchSyncQueue(graceful.GetManager().ShutdownContext()) +} + +// UpdateRepository updates a repository +func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + if err = repo_module.UpdateRepository(ctx, repo, visibilityChanged); err != nil { + return fmt.Errorf("updateRepository: %w", err) + } + + return committer.Commit() +} + +// LinkedRepository returns the linked repo if any +func LinkedRepository(ctx context.Context, a *repo_model.Attachment) (*repo_model.Repository, unit.Type, error) { + if a.IssueID != 0 { + iss, err := issues_model.GetIssueByID(ctx, a.IssueID) + if err != nil { + return nil, unit.TypeIssues, err + } + repo, err := repo_model.GetRepositoryByID(ctx, iss.RepoID) + unitType := unit.TypeIssues + if iss.IsPull { + unitType = unit.TypePullRequests + } + return repo, unitType, err + } else if a.ReleaseID != 0 { + rel, err := repo_model.GetReleaseByID(ctx, a.ReleaseID) + if err != nil { + return nil, unit.TypeReleases, err + } + repo, err := repo_model.GetRepositoryByID(ctx, rel.RepoID) + return repo, unit.TypeReleases, err + } + return nil, -1, nil +} diff --git a/services/repository/repository_test.go b/services/repository/repository_test.go new file mode 100644 index 0000000..a5c0b3e --- /dev/null +++ b/services/repository/repository_test.go @@ -0,0 +1,43 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unit" + "code.gitea.io/gitea/models/unittest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLinkedRepository(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + testCases := []struct { + name string + attachID int64 + expectedRepo *repo_model.Repository + expectedUnitType unit.Type + }{ + {"LinkedIssue", 1, &repo_model.Repository{ID: 1}, unit.TypeIssues}, + {"LinkedComment", 3, &repo_model.Repository{ID: 1}, unit.TypePullRequests}, + {"LinkedRelease", 9, &repo_model.Repository{ID: 1}, unit.TypeReleases}, + {"Notlinked", 10, nil, -1}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + attach, err := repo_model.GetAttachmentByID(db.DefaultContext, tc.attachID) + require.NoError(t, err) + repo, unitType, err := LinkedRepository(db.DefaultContext, attach) + require.NoError(t, err) + if tc.expectedRepo != nil { + assert.Equal(t, tc.expectedRepo.ID, repo.ID) + } + assert.Equal(t, tc.expectedUnitType, unitType) + }) + } +} diff --git a/services/repository/review.go b/services/repository/review.go new file mode 100644 index 0000000..40513e6 --- /dev/null +++ b/services/repository/review.go @@ -0,0 +1,24 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + + "code.gitea.io/gitea/models/organization" + "code.gitea.io/gitea/models/perm" + repo_model "code.gitea.io/gitea/models/repo" +) + +// GetReviewerTeams get all teams can be requested to review +func GetReviewerTeams(ctx context.Context, repo *repo_model.Repository) ([]*organization.Team, error) { + if err := repo.LoadOwner(ctx); err != nil { + return nil, err + } + if !repo.Owner.IsOrganization() { + return nil, nil + } + + return organization.GetTeamsWithAccessToRepo(ctx, repo.OwnerID, repo.ID, perm.AccessModeRead) +} diff --git a/services/repository/review_test.go b/services/repository/review_test.go new file mode 100644 index 0000000..eb1712c --- /dev/null +++ b/services/repository/review_test.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRepoGetReviewerTeams(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + repo2 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) + teams, err := GetReviewerTeams(db.DefaultContext, repo2) + require.NoError(t, err) + assert.Empty(t, teams) + + repo3 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) + teams, err = GetReviewerTeams(db.DefaultContext, repo3) + require.NoError(t, err) + assert.Len(t, teams, 2) +} diff --git a/services/repository/setting.go b/services/repository/setting.go new file mode 100644 index 0000000..33b00cc --- /dev/null +++ b/services/repository/setting.go @@ -0,0 +1,57 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "slices" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unit" + "code.gitea.io/gitea/modules/log" + actions_service "code.gitea.io/gitea/services/actions" +) + +// UpdateRepositoryUnits updates a repository's units +func UpdateRepositoryUnits(ctx context.Context, repo *repo_model.Repository, units []repo_model.RepoUnit, deleteUnitTypes []unit.Type) (err error) { + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + // Delete existing settings of units before adding again + for _, u := range units { + deleteUnitTypes = append(deleteUnitTypes, u.Type) + } + + if slices.Contains(deleteUnitTypes, unit.TypeActions) { + if err := actions_model.CleanRepoScheduleTasks(ctx, repo, true); err != nil { + log.Error("CleanRepoScheduleTasks: %v", err) + } + } + + for _, u := range units { + if u.Type == unit.TypeActions { + if err := actions_service.DetectAndHandleSchedules(ctx, repo); err != nil { + log.Error("DetectAndHandleSchedules: %v", err) + } + break + } + } + + if _, err = db.GetEngine(ctx).Where("repo_id = ?", repo.ID).In("type", deleteUnitTypes).Delete(new(repo_model.RepoUnit)); err != nil { + return err + } + + if len(units) > 0 { + if err = db.Insert(ctx, units); err != nil { + return err + } + } + + return committer.Commit() +} diff --git a/services/repository/star.go b/services/repository/star.go new file mode 100644 index 0000000..505da0f --- /dev/null +++ b/services/repository/star.go @@ -0,0 +1,27 @@ +// Copyright 2024 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + + "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/federation" +) + +func StarRepoAndSendLikeActivities(ctx context.Context, doer user.User, repoID int64, star bool) error { + if err := repo.StarRepo(ctx, doer.ID, repoID, star); err != nil { + return err + } + + if star && setting.Federation.Enabled { + if err := federation.SendLikeActivities(ctx, doer, repoID); err != nil { + return err + } + } + + return nil +} diff --git a/services/repository/template.go b/services/repository/template.go new file mode 100644 index 0000000..36a680c --- /dev/null +++ b/services/repository/template.go @@ -0,0 +1,135 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + notify_service "code.gitea.io/gitea/services/notify" +) + +// GenerateIssueLabels generates issue labels from a template repository +func GenerateIssueLabels(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { + templateLabels, err := issues_model.GetLabelsByRepoID(ctx, templateRepo.ID, "", db.ListOptions{}) + if err != nil { + return err + } + // Prevent insert being called with an empty slice which would result in + // err "no element on slice when insert". + if len(templateLabels) == 0 { + return nil + } + + newLabels := make([]*issues_model.Label, 0, len(templateLabels)) + for _, templateLabel := range templateLabels { + newLabels = append(newLabels, &issues_model.Label{ + RepoID: generateRepo.ID, + Name: templateLabel.Name, + Exclusive: templateLabel.Exclusive, + Description: templateLabel.Description, + Color: templateLabel.Color, + }) + } + return db.Insert(ctx, newLabels) +} + +func GenerateProtectedBranch(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { + templateBranches, err := git_model.FindRepoProtectedBranchRules(ctx, templateRepo.ID) + if err != nil { + return err + } + // Prevent insert being called with an empty slice which would result in + // err "no element on slice when insert". + if len(templateBranches) == 0 { + return nil + } + + newBranches := make([]*git_model.ProtectedBranch, 0, len(templateBranches)) + for _, templateBranch := range templateBranches { + templateBranch.ID = 0 + templateBranch.RepoID = generateRepo.ID + templateBranch.UpdatedUnix = 0 + templateBranch.CreatedUnix = 0 + newBranches = append(newBranches, templateBranch) + } + return db.Insert(ctx, newBranches) +} + +// GenerateRepository generates a repository from a template +func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) { + if !doer.IsAdmin && !owner.CanCreateRepo() { + return nil, repo_model.ErrReachLimitOfRepo{ + Limit: owner.MaxRepoCreation, + } + } + + var generateRepo *repo_model.Repository + if err = db.WithTx(ctx, func(ctx context.Context) error { + generateRepo, err = generateRepository(ctx, doer, owner, templateRepo, opts) + if err != nil { + return err + } + + // Git Content + if opts.GitContent && !templateRepo.IsEmpty { + if err = GenerateGitContent(ctx, templateRepo, generateRepo); err != nil { + return err + } + } + + // Topics + if opts.Topics { + if err = repo_model.GenerateTopics(ctx, templateRepo, generateRepo); err != nil { + return err + } + } + + // Git Hooks + if opts.GitHooks { + if err = GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil { + return err + } + } + + // Webhooks + if opts.Webhooks { + if err = GenerateWebhooks(ctx, templateRepo, generateRepo); err != nil { + return err + } + } + + // Avatar + if opts.Avatar && len(templateRepo.Avatar) > 0 { + if err = generateAvatar(ctx, templateRepo, generateRepo); err != nil { + return err + } + } + + // Issue Labels + if opts.IssueLabels { + if err = GenerateIssueLabels(ctx, templateRepo, generateRepo); err != nil { + return err + } + } + + if opts.ProtectedBranch { + if err = GenerateProtectedBranch(ctx, templateRepo, generateRepo); err != nil { + return err + } + } + + return nil + }); err != nil { + return nil, err + } + + notify_service.CreateRepository(ctx, doer, owner, generateRepo) + + return generateRepo, nil +} diff --git a/services/repository/transfer.go b/services/repository/transfer.go new file mode 100644 index 0000000..467c85e --- /dev/null +++ b/services/repository/transfer.go @@ -0,0 +1,434 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "os" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/db" + issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/organization" + "code.gitea.io/gitea/models/perm" + access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/sync" + "code.gitea.io/gitea/modules/util" + notify_service "code.gitea.io/gitea/services/notify" +) + +// repoWorkingPool represents a working pool to order the parallel changes to the same repository +// TODO: use clustered lock (unique queue? or *abuse* cache) +var repoWorkingPool = sync.NewExclusivePool() + +// TransferOwnership transfers all corresponding setting from old user to new one. +func TransferOwnership(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository, teams []*organization.Team) error { + if err := repo.LoadOwner(ctx); err != nil { + return err + } + for _, team := range teams { + if newOwner.ID != team.OrgID { + return fmt.Errorf("team %d does not belong to organization", team.ID) + } + } + + oldOwner := repo.Owner + + repoWorkingPool.CheckIn(fmt.Sprint(repo.ID)) + if err := transferOwnership(ctx, doer, newOwner.Name, repo); err != nil { + repoWorkingPool.CheckOut(fmt.Sprint(repo.ID)) + return err + } + repoWorkingPool.CheckOut(fmt.Sprint(repo.ID)) + + newRepo, err := repo_model.GetRepositoryByID(ctx, repo.ID) + if err != nil { + return err + } + + for _, team := range teams { + if err := models.AddRepository(ctx, team, newRepo); err != nil { + return err + } + } + + notify_service.TransferRepository(ctx, doer, repo, oldOwner.Name) + + return nil +} + +// transferOwnership transfers all corresponding repository items from old user to new one. +func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName string, repo *repo_model.Repository) (err error) { + repoRenamed := false + wikiRenamed := false + oldOwnerName := doer.Name + + defer func() { + if !repoRenamed && !wikiRenamed { + return + } + + recoverErr := recover() + if err == nil && recoverErr == nil { + return + } + + if repoRenamed { + if err := util.Rename(repo_model.RepoPath(newOwnerName, repo.Name), repo_model.RepoPath(oldOwnerName, repo.Name)); err != nil { + log.Critical("Unable to move repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, + repo_model.RepoPath(newOwnerName, repo.Name), repo_model.RepoPath(oldOwnerName, repo.Name), err) + } + } + + if wikiRenamed { + if err := util.Rename(repo_model.WikiPath(newOwnerName, repo.Name), repo_model.WikiPath(oldOwnerName, repo.Name)); err != nil { + log.Critical("Unable to move wiki for repository %s/%s directory from %s back to correct place %s: %v", oldOwnerName, repo.Name, + repo_model.WikiPath(newOwnerName, repo.Name), repo_model.WikiPath(oldOwnerName, repo.Name), err) + } + } + + if recoverErr != nil { + log.Error("Panic within TransferOwnership: %v\n%s", recoverErr, log.Stack(2)) + panic(recoverErr) + } + }() + + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + sess := db.GetEngine(ctx) + + newOwner, err := user_model.GetUserByName(ctx, newOwnerName) + if err != nil { + return fmt.Errorf("get new owner '%s': %w", newOwnerName, err) + } + newOwnerName = newOwner.Name // ensure capitalisation matches + + // Check if new owner has repository with same name. + if has, err := repo_model.IsRepositoryModelOrDirExist(ctx, newOwner, repo.Name); err != nil { + return fmt.Errorf("IsRepositoryExist: %w", err) + } else if has { + return repo_model.ErrRepoAlreadyExist{ + Uname: newOwnerName, + Name: repo.Name, + } + } + + oldOwner := repo.Owner + oldOwnerName = oldOwner.Name + + // Note: we have to set value here to make sure recalculate accesses is based on + // new owner. + repo.OwnerID = newOwner.ID + repo.Owner = newOwner + repo.OwnerName = newOwner.Name + + // Update repository. + if _, err := sess.ID(repo.ID).Update(repo); err != nil { + return fmt.Errorf("update owner: %w", err) + } + + // Remove redundant collaborators. + collaborators, err := repo_model.GetCollaborators(ctx, repo.ID, db.ListOptions{}) + if err != nil { + return fmt.Errorf("getCollaborators: %w", err) + } + + // Dummy object. + collaboration := &repo_model.Collaboration{RepoID: repo.ID} + for _, c := range collaborators { + if c.IsGhost() { + collaboration.ID = c.Collaboration.ID + if _, err := sess.Delete(collaboration); err != nil { + return fmt.Errorf("remove collaborator '%d': %w", c.ID, err) + } + collaboration.ID = 0 + } + + if c.ID != newOwner.ID { + isMember, err := organization.IsOrganizationMember(ctx, newOwner.ID, c.ID) + if err != nil { + return fmt.Errorf("IsOrgMember: %w", err) + } else if !isMember { + continue + } + } + collaboration.UserID = c.ID + if _, err := sess.Delete(collaboration); err != nil { + return fmt.Errorf("remove collaborator '%d': %w", c.ID, err) + } + collaboration.UserID = 0 + } + + // Remove old team-repository relations. + if oldOwner.IsOrganization() { + if err := organization.RemoveOrgRepo(ctx, oldOwner.ID, repo.ID); err != nil { + return fmt.Errorf("removeOrgRepo: %w", err) + } + } + + if newOwner.IsOrganization() { + teams, err := organization.FindOrgTeams(ctx, newOwner.ID) + if err != nil { + return fmt.Errorf("LoadTeams: %w", err) + } + for _, t := range teams { + if t.IncludesAllRepositories { + if err := models.AddRepository(ctx, t, repo); err != nil { + return fmt.Errorf("AddRepository: %w", err) + } + } + } + } else if err := access_model.RecalculateAccesses(ctx, repo); err != nil { + // Organization called this in addRepository method. + return fmt.Errorf("recalculateAccesses: %w", err) + } + + // Update repository count. + if _, err := sess.Exec("UPDATE `user` SET num_repos=num_repos+1 WHERE id=?", newOwner.ID); err != nil { + return fmt.Errorf("increase new owner repository count: %w", err) + } else if _, err := sess.Exec("UPDATE `user` SET num_repos=num_repos-1 WHERE id=?", oldOwner.ID); err != nil { + return fmt.Errorf("decrease old owner repository count: %w", err) + } + + if err := repo_model.WatchRepo(ctx, doer.ID, repo.ID, true); err != nil { + return fmt.Errorf("watchRepo: %w", err) + } + + // Remove watch for organization. + if oldOwner.IsOrganization() { + if err := repo_model.WatchRepo(ctx, oldOwner.ID, repo.ID, false); err != nil { + return fmt.Errorf("watchRepo [false]: %w", err) + } + } + + // Delete labels that belong to the old organization and comments that added these labels + if oldOwner.IsOrganization() { + if _, err := sess.Exec(`DELETE FROM issue_label WHERE issue_label.id IN ( + SELECT il_too.id FROM ( + SELECT il_too_too.id + FROM issue_label AS il_too_too + INNER JOIN label ON il_too_too.label_id = label.id + INNER JOIN issue on issue.id = il_too_too.issue_id + WHERE + issue.repo_id = ? AND ((label.org_id = 0 AND issue.repo_id != label.repo_id) OR (label.repo_id = 0 AND label.org_id != ?)) + ) AS il_too )`, repo.ID, newOwner.ID); err != nil { + return fmt.Errorf("Unable to remove old org labels: %w", err) + } + + if _, err := sess.Exec(`DELETE FROM comment WHERE comment.id IN ( + SELECT il_too.id FROM ( + SELECT com.id + FROM comment AS com + INNER JOIN label ON com.label_id = label.id + INNER JOIN issue ON issue.id = com.issue_id + WHERE + com.type = ? AND issue.repo_id = ? AND ((label.org_id = 0 AND issue.repo_id != label.repo_id) OR (label.repo_id = 0 AND label.org_id != ?)) + ) AS il_too)`, issues_model.CommentTypeLabel, repo.ID, newOwner.ID); err != nil { + return fmt.Errorf("Unable to remove old org label comments: %w", err) + } + } + + // Rename remote repository to new path and delete local copy. + dir := user_model.UserPath(newOwner.Name) + + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("Failed to create dir %s: %w", dir, err) + } + + if err := util.Rename(repo_model.RepoPath(oldOwner.Name, repo.Name), repo_model.RepoPath(newOwner.Name, repo.Name)); err != nil { + return fmt.Errorf("rename repository directory: %w", err) + } + repoRenamed = true + + // Rename remote wiki repository to new path and delete local copy. + wikiPath := repo_model.WikiPath(oldOwner.Name, repo.Name) + + if isExist, err := util.IsExist(wikiPath); err != nil { + log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) + return err + } else if isExist { + if err := util.Rename(wikiPath, repo_model.WikiPath(newOwner.Name, repo.Name)); err != nil { + return fmt.Errorf("rename repository wiki: %w", err) + } + wikiRenamed = true + } + + if err := models.DeleteRepositoryTransfer(ctx, repo.ID); err != nil { + return fmt.Errorf("deleteRepositoryTransfer: %w", err) + } + repo.Status = repo_model.RepositoryReady + if err := repo_model.UpdateRepositoryCols(ctx, repo, "status"); err != nil { + return err + } + + // If there was previously a redirect at this location, remove it. + if err := repo_model.DeleteRedirect(ctx, newOwner.ID, repo.Name); err != nil { + return fmt.Errorf("delete repo redirect: %w", err) + } + + if err := repo_model.NewRedirect(ctx, oldOwner.ID, repo.ID, repo.Name, repo.Name); err != nil { + return fmt.Errorf("repo_model.NewRedirect: %w", err) + } + + return committer.Commit() +} + +// changeRepositoryName changes all corresponding setting from old repository name to new one. +func changeRepositoryName(ctx context.Context, repo *repo_model.Repository, newRepoName string) (err error) { + oldRepoName := repo.Name + newRepoName = strings.ToLower(newRepoName) + if err = repo_model.IsUsableRepoName(newRepoName); err != nil { + return err + } + + if err := repo.LoadOwner(ctx); err != nil { + return err + } + + has, err := repo_model.IsRepositoryModelOrDirExist(ctx, repo.Owner, newRepoName) + if err != nil { + return fmt.Errorf("IsRepositoryExist: %w", err) + } else if has { + return repo_model.ErrRepoAlreadyExist{ + Uname: repo.Owner.Name, + Name: newRepoName, + } + } + + newRepoPath := repo_model.RepoPath(repo.Owner.Name, newRepoName) + if err = util.Rename(repo.RepoPath(), newRepoPath); err != nil { + return fmt.Errorf("rename repository directory: %w", err) + } + + wikiPath := repo.WikiPath() + isExist, err := util.IsExist(wikiPath) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) + return err + } + if isExist { + if err = util.Rename(wikiPath, repo_model.WikiPath(repo.Owner.Name, newRepoName)); err != nil { + return fmt.Errorf("rename repository wiki: %w", err) + } + } + + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + if err := repo_model.NewRedirect(ctx, repo.Owner.ID, repo.ID, oldRepoName, newRepoName); err != nil { + return err + } + + return committer.Commit() +} + +// ChangeRepositoryName changes all corresponding setting from old repository name to new one. +func ChangeRepositoryName(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, newRepoName string) error { + log.Trace("ChangeRepositoryName: %s/%s -> %s", doer.Name, repo.Name, newRepoName) + + oldRepoName := repo.Name + + // Change repository directory name. We must lock the local copy of the + // repo so that we can atomically rename the repo path and updates the + // local copy's origin accordingly. + + repoWorkingPool.CheckIn(fmt.Sprint(repo.ID)) + if err := changeRepositoryName(ctx, repo, newRepoName); err != nil { + repoWorkingPool.CheckOut(fmt.Sprint(repo.ID)) + return err + } + repoWorkingPool.CheckOut(fmt.Sprint(repo.ID)) + + repo.Name = newRepoName + notify_service.RenameRepository(ctx, doer, repo, oldRepoName) + + return nil +} + +// StartRepositoryTransfer transfer a repo from one owner to a new one. +// it make repository into pending transfer state, if doer can not create repo for new owner. +func StartRepositoryTransfer(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository, teams []*organization.Team) error { + if user_model.IsBlocked(ctx, newOwner.ID, doer.ID) { + return user_model.ErrBlockedByUser + } + + if err := models.TestRepositoryReadyForTransfer(repo.Status); err != nil { + return err + } + + // Admin is always allowed to transfer || user transfer repo back to his account + if doer.IsAdmin || doer.ID == newOwner.ID { + return TransferOwnership(ctx, doer, newOwner, repo, teams) + } + + // If new owner is an org and user can create repos he can transfer directly too + if newOwner.IsOrganization() { + allowed, err := organization.CanCreateOrgRepo(ctx, newOwner.ID, doer.ID) + if err != nil { + return err + } + if allowed { + return TransferOwnership(ctx, doer, newOwner, repo, teams) + } + } + + // In case the new owner would not have sufficient access to the repo, give access rights for read + hasAccess, err := access_model.HasAccess(ctx, newOwner.ID, repo) + if err != nil { + return err + } + if !hasAccess { + if err := repo_module.AddCollaborator(ctx, repo, newOwner); err != nil { + return err + } + if err := repo_model.ChangeCollaborationAccessMode(ctx, repo, newOwner.ID, perm.AccessModeRead); err != nil { + return err + } + } + + // Make repo as pending for transfer + repo.Status = repo_model.RepositoryPendingTransfer + if err := models.CreatePendingRepositoryTransfer(ctx, doer, newOwner, repo.ID, teams); err != nil { + return err + } + + // notify users who are able to accept / reject transfer + notify_service.RepoPendingTransfer(ctx, doer, newOwner, repo) + + return nil +} + +// CancelRepositoryTransfer marks the repository as ready and remove pending transfer entry, +// thus cancel the transfer process. +func CancelRepositoryTransfer(ctx context.Context, repo *repo_model.Repository) error { + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + + repo.Status = repo_model.RepositoryReady + if err := repo_model.UpdateRepositoryCols(ctx, repo, "status"); err != nil { + return err + } + + if err := models.DeleteRepositoryTransfer(ctx, repo.ID); err != nil { + return err + } + + return committer.Commit() +} diff --git a/services/repository/transfer_test.go b/services/repository/transfer_test.go new file mode 100644 index 0000000..cc51a05 --- /dev/null +++ b/services/repository/transfer_test.go @@ -0,0 +1,124 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "sync" + "testing" + + "code.gitea.io/gitea/models" + activities_model "code.gitea.io/gitea/models/activities" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/organization" + access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/feed" + notify_service "code.gitea.io/gitea/services/notify" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var notifySync sync.Once + +func registerNotifier() { + notifySync.Do(func() { + notify_service.RegisterNotifier(feed.NewNotifier()) + }) +} + +func TestTransferOwnership(t *testing.T) { + registerNotifier() + + require.NoError(t, unittest.PrepareTestDatabase()) + + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) + repo.Owner = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}) + require.NoError(t, TransferOwnership(db.DefaultContext, doer, doer, repo, nil)) + + transferredRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) + assert.EqualValues(t, 2, transferredRepo.OwnerID) + + exist, err := util.IsExist(repo_model.RepoPath("org3", "repo3")) + require.NoError(t, err) + assert.False(t, exist) + exist, err = util.IsExist(repo_model.RepoPath("user2", "repo3")) + require.NoError(t, err) + assert.True(t, exist) + unittest.AssertExistsAndLoadBean(t, &activities_model.Action{ + OpType: activities_model.ActionTransferRepo, + ActUserID: 2, + RepoID: 3, + Content: "org3/repo3", + }) + + unittest.CheckConsistencyFor(t, &repo_model.Repository{}, &user_model.User{}, &organization.Team{}) +} + +func TestStartRepositoryTransferSetPermission(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) + recipient := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 5}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 5}) + repo.Owner = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}) + + hasAccess, err := access_model.HasAccess(db.DefaultContext, recipient.ID, repo) + require.NoError(t, err) + assert.False(t, hasAccess) + + require.NoError(t, StartRepositoryTransfer(db.DefaultContext, doer, recipient, repo, nil)) + + hasAccess, err = access_model.HasAccess(db.DefaultContext, recipient.ID, repo) + require.NoError(t, err) + assert.True(t, hasAccess) + + unittest.CheckConsistencyFor(t, &repo_model.Repository{}, &user_model.User{}, &organization.Team{}) +} + +func TestRepositoryTransfer(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) + + transfer, err := models.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.NoError(t, err) + assert.NotNil(t, transfer) + + // Cancel transfer + require.NoError(t, CancelRepositoryTransfer(db.DefaultContext, repo)) + + transfer, err = models.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.Error(t, err) + assert.Nil(t, transfer) + assert.True(t, models.IsErrNoPendingTransfer(err)) + + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + require.NoError(t, models.CreatePendingRepositoryTransfer(db.DefaultContext, doer, user2, repo.ID, nil)) + + transfer, err = models.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.NoError(t, err) + require.NoError(t, transfer.LoadAttributes(db.DefaultContext)) + assert.Equal(t, "user2", transfer.Recipient.Name) + + org6 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + // Only transfer can be started at any given time + err = models.CreatePendingRepositoryTransfer(db.DefaultContext, doer, org6, repo.ID, nil) + require.Error(t, err) + assert.True(t, models.IsErrRepoTransferInProgress(err)) + + // Unknown user + err = models.CreatePendingRepositoryTransfer(db.DefaultContext, doer, &user_model.User{ID: 1000, LowerName: "user1000"}, repo.ID, nil) + require.Error(t, err) + + // Cancel transfer + require.NoError(t, CancelRepositoryTransfer(db.DefaultContext, repo)) +} -- cgit v1.2.3