From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- services/task/migrate.go | 154 ++++++++++++++++++++++++++++++++++++++++++ services/task/task.go | 169 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 323 insertions(+) create mode 100644 services/task/migrate.go create mode 100644 services/task/task.go (limited to 'services/task') diff --git a/services/task/migrate.go b/services/task/migrate.go new file mode 100644 index 0000000..9cef77a --- /dev/null +++ b/services/task/migrate.go @@ -0,0 +1,154 @@ +// Copyright 2019 Gitea. All rights reserved. +// SPDX-License-Identifier: MIT + +package task + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + admin_model "code.gitea.io/gitea/models/admin" + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migration" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/migrations" + notify_service "code.gitea.io/gitea/services/notify" +) + +func handleCreateError(owner *user_model.User, err error) error { + switch { + case repo_model.IsErrReachLimitOfRepo(err): + return fmt.Errorf("you have already reached your limit of %d repositories", owner.MaxCreationLimit()) + case repo_model.IsErrRepoAlreadyExist(err): + return errors.New("the repository name is already used") + case db.IsErrNameReserved(err): + return fmt.Errorf("the repository name '%s' is reserved", err.(db.ErrNameReserved).Name) + case db.IsErrNamePatternNotAllowed(err): + return fmt.Errorf("the pattern '%s' is not allowed in a repository name", err.(db.ErrNamePatternNotAllowed).Pattern) + default: + return err + } +} + +func runMigrateTask(ctx context.Context, t *admin_model.Task) (err error) { + defer func(ctx context.Context) { + if e := recover(); e != nil { + err = fmt.Errorf("PANIC whilst trying to do migrate task: %v", e) + log.Critical("PANIC during runMigrateTask[%d] by DoerID[%d] to RepoID[%d] for OwnerID[%d]: %v\nStacktrace: %v", t.ID, t.DoerID, t.RepoID, t.OwnerID, e, log.Stack(2)) + } + if err == nil { + err = admin_model.FinishMigrateTask(ctx, t) + if err == nil { + notify_service.MigrateRepository(ctx, t.Doer, t.Owner, t.Repo) + return + } + + log.Error("FinishMigrateTask[%d] by DoerID[%d] to RepoID[%d] for OwnerID[%d] failed: %v", t.ID, t.DoerID, t.RepoID, t.OwnerID, err) + } + + log.Error("runMigrateTask[%d] by DoerID[%d] to RepoID[%d] for OwnerID[%d] failed: %v", t.ID, t.DoerID, t.RepoID, t.OwnerID, err) + + t.EndTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusFailed + t.Message = err.Error() + if err := t.UpdateCols(ctx, "status", "message", "end_time"); err != nil { + log.Error("Task UpdateCols failed: %v", err) + } + + // then, do not delete the repository, otherwise the users won't be able to see the last error + }(graceful.GetManager().ShutdownContext()) // even if the parent ctx is canceled, this defer-function still needs to update the task record in database + + if err = t.LoadRepo(ctx); err != nil { + return err + } + + // if repository is ready, then just finish the task + if t.Repo.Status == repo_model.RepositoryReady { + return nil + } + + if err = t.LoadDoer(ctx); err != nil { + return err + } + if err = t.LoadOwner(ctx); err != nil { + return err + } + + var opts *migration.MigrateOptions + opts, err = t.MigrateConfig() + if err != nil { + return err + } + + opts.MigrateToRepoID = t.RepoID + + pm := process.GetManager() + ctx, cancel, finished := pm.AddContext(graceful.GetManager().ShutdownContext(), fmt.Sprintf("MigrateTask: %s/%s", t.Owner.Name, opts.RepoName)) + defer finished() + + t.StartTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusRunning + if err = t.UpdateCols(ctx, "start_time", "status"); err != nil { + return err + } + + // check whether the task should be canceled, this goroutine is also managed by process manager + go func() { + for { + select { + case <-time.After(2 * time.Second): + case <-ctx.Done(): + return + } + task, _ := admin_model.GetMigratingTask(ctx, t.RepoID) + if task != nil && task.Status != structs.TaskStatusRunning { + log.Debug("MigrateTask[%d] by DoerID[%d] to RepoID[%d] for OwnerID[%d] is canceled due to status is not 'running'", t.ID, t.DoerID, t.RepoID, t.OwnerID) + cancel() + return + } + } + }() + + t.Repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts, func(format string, args ...any) { + message := admin_model.TranslatableMessage{ + Format: format, + Args: args, + } + bs, _ := json.Marshal(message) + t.Message = string(bs) + _ = t.UpdateCols(ctx, "message") + }) + + if err == nil { + log.Trace("Repository migrated [%d]: %s/%s", t.Repo.ID, t.Owner.Name, t.Repo.Name) + return nil + } + + if repo_model.IsErrRepoAlreadyExist(err) { + return errors.New("the repository name is already used") + } + + // remoteAddr may contain credentials, so we sanitize it + err = util.SanitizeErrorCredentialURLs(err) + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "could not read Username") { + return fmt.Errorf("authentication failed: %w", err) + } else if strings.Contains(err.Error(), "fatal:") { + return fmt.Errorf("migration failed: %w", err) + } + + // do not be tempted to coalesce this line with the return + err = handleCreateError(t.Owner, err) + return err +} diff --git a/services/task/task.go b/services/task/task.go new file mode 100644 index 0000000..ac659ac --- /dev/null +++ b/services/task/task.go @@ -0,0 +1,169 @@ +// Copyright 2019 Gitea. All rights reserved. +// SPDX-License-Identifier: MIT + +package task + +import ( + "context" + "fmt" + + admin_model "code.gitea.io/gitea/models/admin" + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + base "code.gitea.io/gitea/modules/migration" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/secret" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + repo_service "code.gitea.io/gitea/services/repository" +) + +// taskQueue is a global queue of tasks +var taskQueue *queue.WorkerPoolQueue[*admin_model.Task] + +// Run a task +func Run(ctx context.Context, t *admin_model.Task) error { + switch t.Type { + case structs.TaskTypeMigrateRepo: + return runMigrateTask(ctx, t) + default: + return fmt.Errorf("Unknown task type: %d", t.Type) + } +} + +// Init will start the service to get all unfinished tasks and run them +func Init() error { + taskQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "task", handler) + if taskQueue == nil { + return fmt.Errorf("unable to create task queue") + } + go graceful.GetManager().RunWithCancel(taskQueue) + return nil +} + +func handler(items ...*admin_model.Task) []*admin_model.Task { + for _, task := range items { + if err := Run(db.DefaultContext, task); err != nil { + log.Error("Run task failed: %v", err) + } + } + return nil +} + +// MigrateRepository add migration repository to task +func MigrateRepository(ctx context.Context, doer, u *user_model.User, opts base.MigrateOptions) error { + task, err := CreateMigrateTask(ctx, doer, u, opts) + if err != nil { + return err + } + + return taskQueue.Push(task) +} + +// CreateMigrateTask creates a migrate task +func CreateMigrateTask(ctx context.Context, doer, u *user_model.User, opts base.MigrateOptions) (*admin_model.Task, error) { + // encrypt credentials for persistence + var err error + opts.CloneAddrEncrypted, err = secret.EncryptSecret(setting.SecretKey, opts.CloneAddr) + if err != nil { + return nil, err + } + opts.CloneAddr = util.SanitizeCredentialURLs(opts.CloneAddr) + opts.AuthPasswordEncrypted, err = secret.EncryptSecret(setting.SecretKey, opts.AuthPassword) + if err != nil { + return nil, err + } + opts.AuthPassword = "" + opts.AuthTokenEncrypted, err = secret.EncryptSecret(setting.SecretKey, opts.AuthToken) + if err != nil { + return nil, err + } + opts.AuthToken = "" + bs, err := json.Marshal(&opts) + if err != nil { + return nil, err + } + + task := &admin_model.Task{ + DoerID: doer.ID, + OwnerID: u.ID, + Type: structs.TaskTypeMigrateRepo, + Status: structs.TaskStatusQueued, + PayloadContent: string(bs), + } + + if err := admin_model.CreateTask(ctx, task); err != nil { + return nil, err + } + + repo, err := repo_service.CreateRepositoryDirectly(ctx, doer, u, repo_service.CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + OriginalURL: opts.OriginalURL, + GitServiceType: opts.GitServiceType, + IsPrivate: opts.Private || setting.Repository.ForcePrivate, + IsMirror: opts.Mirror, + Status: repo_model.RepositoryBeingMigrated, + }) + if err != nil { + task.EndTime = timeutil.TimeStampNow() + task.Status = structs.TaskStatusFailed + err2 := task.UpdateCols(ctx, "end_time", "status") + if err2 != nil { + log.Error("UpdateCols Failed: %v", err2.Error()) + } + return nil, err + } + + task.RepoID = repo.ID + if err = task.UpdateCols(ctx, "repo_id"); err != nil { + return nil, err + } + + return task, nil +} + +// RetryMigrateTask retry a migrate task +func RetryMigrateTask(ctx context.Context, repoID int64) error { + migratingTask, err := admin_model.GetMigratingTask(ctx, repoID) + if err != nil { + log.Error("GetMigratingTask: %v", err) + return err + } + if migratingTask.Status == structs.TaskStatusQueued || migratingTask.Status == structs.TaskStatusRunning { + return nil + } + + // TODO Need to removing the storage/database garbage brought by the failed task + + // Reset task status and messages + migratingTask.Status = structs.TaskStatusQueued + migratingTask.Message = "" + if err = migratingTask.UpdateCols(ctx, "status", "message"); err != nil { + log.Error("task.UpdateCols failed: %v", err) + return err + } + + return taskQueue.Push(migratingTask) +} + +func SetMigrateTaskMessage(ctx context.Context, repoID int64, message string) error { + migratingTask, err := admin_model.GetMigratingTask(ctx, repoID) + if err != nil { + log.Error("GetMigratingTask: %v", err) + return err + } + + migratingTask.Message = message + if err = migratingTask.UpdateCols(ctx, "message"); err != nil { + log.Error("task.UpdateCols failed: %v", err) + return err + } + return nil +} -- cgit v1.2.3