From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- services/cron/cron.go | 130 +++++++++++++++++++++ services/cron/setting.go | 86 ++++++++++++++ services/cron/tasks.go | 230 +++++++++++++++++++++++++++++++++++++ services/cron/tasks_actions.go | 76 +++++++++++++ services/cron/tasks_basic.go | 175 +++++++++++++++++++++++++++++ services/cron/tasks_extended.go | 243 ++++++++++++++++++++++++++++++++++++++++ services/cron/tasks_test.go | 68 +++++++++++ 7 files changed, 1008 insertions(+) create mode 100644 services/cron/cron.go create mode 100644 services/cron/setting.go create mode 100644 services/cron/tasks.go create mode 100644 services/cron/tasks_actions.go create mode 100644 services/cron/tasks_basic.go create mode 100644 services/cron/tasks_extended.go create mode 100644 services/cron/tasks_test.go (limited to 'services/cron') diff --git a/services/cron/cron.go b/services/cron/cron.go new file mode 100644 index 0000000..3c5737e --- /dev/null +++ b/services/cron/cron.go @@ -0,0 +1,130 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cron + +import ( + "context" + "runtime/pprof" + "time" + + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/sync" + "code.gitea.io/gitea/modules/translation" + + "github.com/go-co-op/gocron" +) + +var scheduler = gocron.NewScheduler(time.Local) + +// Prevent duplicate running tasks. +var taskStatusTable = sync.NewStatusTable() + +// NewContext begins cron tasks +// Each cron task is run within the shutdown context as a running server +// AtShutdown the cron server is stopped +func NewContext(original context.Context) { + defer pprof.SetGoroutineLabels(original) + _, _, finished := process.GetManager().AddTypedContext(graceful.GetManager().ShutdownContext(), "Service: Cron", process.SystemProcessType, true) + initBasicTasks() + initExtendedTasks() + initActionsTasks() + + lock.Lock() + for _, task := range tasks { + if task.IsEnabled() && task.DoRunAtStart() { + go task.Run() + } + } + + scheduler.StartAsync() + started = true + lock.Unlock() + graceful.GetManager().RunAtShutdown(context.Background(), func() { + scheduler.Stop() + lock.Lock() + started = false + lock.Unlock() + finished() + }) +} + +// TaskTableRow represents a task row in the tasks table +type TaskTableRow struct { + Name string + Spec string + Next time.Time + Prev time.Time + Status string + LastMessage string + LastDoer string + ExecTimes int64 + task *Task +} + +func (t *TaskTableRow) FormatLastMessage(locale translation.Locale) string { + if t.Status == "finished" { + return t.task.GetConfig().FormatMessage(locale, t.Name, t.Status, t.LastDoer) + } + + return t.task.GetConfig().FormatMessage(locale, t.Name, t.Status, t.LastDoer, t.LastMessage) +} + +// TaskTable represents a table of tasks +type TaskTable []*TaskTableRow + +// ListTasks returns all running cron tasks. +func ListTasks() TaskTable { + jobs := scheduler.Jobs() + jobMap := map[string]*gocron.Job{} + for _, job := range jobs { + // the first tag is the task name + tags := job.Tags() + if len(tags) == 0 { // should never happen + continue + } + jobMap[job.Tags()[0]] = job + } + + lock.Lock() + defer lock.Unlock() + + tTable := make([]*TaskTableRow, 0, len(tasks)) + for _, task := range tasks { + spec := "-" + var ( + next time.Time + prev time.Time + ) + if e, ok := jobMap[task.Name]; ok { + tags := e.Tags() + if len(tags) > 1 { + spec = tags[1] // the second tag is the task spec + } + next = e.NextRun() + prev = e.PreviousRun() + } + + task.lock.Lock() + // If the manual run is after the cron run, use that instead. + if prev.Before(task.LastRun) { + prev = task.LastRun + } + tTable = append(tTable, &TaskTableRow{ + Name: task.Name, + Spec: spec, + Next: next, + Prev: prev, + ExecTimes: task.ExecTimes, + LastMessage: task.LastMessage, + Status: task.Status, + LastDoer: task.LastDoer, + task: task, + }) + task.lock.Unlock() + } + + return tTable +} diff --git a/services/cron/setting.go b/services/cron/setting.go new file mode 100644 index 0000000..6dad888 --- /dev/null +++ b/services/cron/setting.go @@ -0,0 +1,86 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cron + +import ( + "time" + + "code.gitea.io/gitea/modules/translation" +) + +// Config represents a basic configuration interface that cron task +type Config interface { + IsEnabled() bool + DoRunAtStart() bool + GetSchedule() string + FormatMessage(locale translation.Locale, name, status, doer string, args ...any) string + DoNoticeOnSuccess() bool +} + +// BaseConfig represents the basic config for a Cron task +type BaseConfig struct { + Enabled bool + RunAtStart bool + Schedule string + NoticeOnSuccess bool +} + +// OlderThanConfig represents a cron task with OlderThan setting +type OlderThanConfig struct { + BaseConfig + OlderThan time.Duration +} + +// UpdateExistingConfig represents a cron task with UpdateExisting setting +type UpdateExistingConfig struct { + BaseConfig + UpdateExisting bool +} + +// CleanupHookTaskConfig represents a cron task with settings to cleanup hook_task +type CleanupHookTaskConfig struct { + BaseConfig + CleanupType string + OlderThan time.Duration + NumberToKeep int +} + +// GetSchedule returns the schedule for the base config +func (b *BaseConfig) GetSchedule() string { + return b.Schedule +} + +// IsEnabled returns the enabled status for the config +func (b *BaseConfig) IsEnabled() bool { + return b.Enabled +} + +// DoRunAtStart returns whether the task should be run at the start +func (b *BaseConfig) DoRunAtStart() bool { + return b.RunAtStart +} + +// DoNoticeOnSuccess returns whether a success notice should be posted +func (b *BaseConfig) DoNoticeOnSuccess() bool { + return b.NoticeOnSuccess +} + +// FormatMessage returns a message for the task +// Please note the `status` string will be concatenated with `admin.dashboard.cron.` and `admin.dashboard.task.` to provide locale messages. Similarly `name` will be composed with `admin.dashboard.` to provide the locale name for the task. +func (b *BaseConfig) FormatMessage(locale translation.Locale, name, status, doer string, args ...any) string { + realArgs := make([]any, 0, len(args)+2) + realArgs = append(realArgs, locale.TrString("admin.dashboard."+name)) + if doer == "" { + realArgs = append(realArgs, "(Cron)") + } else { + realArgs = append(realArgs, doer) + } + if len(args) > 0 { + realArgs = append(realArgs, args...) + } + if doer == "" { + return locale.TrString("admin.dashboard.cron."+status, realArgs...) + } + return locale.TrString("admin.dashboard.task."+status, realArgs...) +} diff --git a/services/cron/tasks.go b/services/cron/tasks.go new file mode 100644 index 0000000..f8a7444 --- /dev/null +++ b/services/cron/tasks.go @@ -0,0 +1,230 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cron + +import ( + "context" + "fmt" + "reflect" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/models/db" + system_model "code.gitea.io/gitea/models/system" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/translation" +) + +var ( + lock = sync.Mutex{} + started = false + tasks = []*Task{} + tasksMap = map[string]*Task{} +) + +// Task represents a Cron task +type Task struct { + lock sync.Mutex + Name string + config Config + fun func(context.Context, *user_model.User, Config) error + Status string + LastMessage string + LastDoer string + ExecTimes int64 + // This stores the time of the last manual run of this task. + LastRun time.Time +} + +// DoRunAtStart returns if this task should run at the start +func (t *Task) DoRunAtStart() bool { + return t.config.DoRunAtStart() +} + +// IsEnabled returns if this task is enabled as cron task +func (t *Task) IsEnabled() bool { + return t.config.IsEnabled() +} + +// GetConfig will return a copy of the task's config +func (t *Task) GetConfig() Config { + if reflect.TypeOf(t.config).Kind() == reflect.Ptr { + // Pointer: + return reflect.New(reflect.ValueOf(t.config).Elem().Type()).Interface().(Config) + } + // Not pointer: + return reflect.New(reflect.TypeOf(t.config)).Elem().Interface().(Config) +} + +// Run will run the task incrementing the cron counter with no user defined +func (t *Task) Run() { + t.RunWithUser(&user_model.User{ + ID: -1, + Name: "(Cron)", + LowerName: "(cron)", + }, t.config) +} + +// RunWithUser will run the task incrementing the cron counter at the time with User +func (t *Task) RunWithUser(doer *user_model.User, config Config) { + if !taskStatusTable.StartIfNotRunning(t.Name) { + return + } + t.lock.Lock() + if config == nil { + config = t.config + } + t.ExecTimes++ + t.lock.Unlock() + defer func() { + taskStatusTable.Stop(t.Name) + }() + graceful.GetManager().RunWithShutdownContext(func(baseCtx context.Context) { + defer func() { + if err := recover(); err != nil { + // Recover a panic within the execution of the task. + combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) + log.Error("PANIC whilst running task: %s Value: %v", t.Name, combinedErr) + } + }() + // Store the time of this run, before the function is executed, so it + // matches the behavior of what the cron library does. + t.lock.Lock() + t.LastRun = time.Now() + t.lock.Unlock() + + pm := process.GetManager() + doerName := "" + if doer != nil && doer.ID != -1 { + doerName = doer.Name + } + + ctx, _, finished := pm.AddContext(baseCtx, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "process", doerName)) + defer finished() + + if err := t.fun(ctx, doer, config); err != nil { + var message string + var status string + if db.IsErrCancelled(err) { + status = "cancelled" + message = err.(db.ErrCancelled).Message + } else { + status = "error" + message = err.Error() + } + + t.lock.Lock() + t.LastMessage = message + t.Status = status + t.LastDoer = doerName + t.lock.Unlock() + + if err := system_model.CreateNotice(ctx, system_model.NoticeTask, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "cancelled", doerName, message)); err != nil { + log.Error("CreateNotice: %v", err) + } + return + } + + t.lock.Lock() + t.Status = "finished" + t.LastMessage = "" + t.LastDoer = doerName + t.lock.Unlock() + + if config.DoNoticeOnSuccess() { + if err := system_model.CreateNotice(ctx, system_model.NoticeTask, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "finished", doerName)); err != nil { + log.Error("CreateNotice: %v", err) + } + } + }) +} + +// GetTask gets the named task +func GetTask(name string) *Task { + lock.Lock() + defer lock.Unlock() + log.Info("Getting %s in %v", name, tasksMap[name]) + + return tasksMap[name] +} + +// RegisterTask allows a task to be registered with the cron service +func RegisterTask(name string, config Config, fun func(context.Context, *user_model.User, Config) error) error { + log.Debug("Registering task: %s", name) + + i18nKey := "admin.dashboard." + name + if value := translation.NewLocale("en-US").TrString(i18nKey); value == i18nKey { + return fmt.Errorf("translation is missing for task %q, please add translation for %q", name, i18nKey) + } + + _, err := setting.GetCronSettings(name, config) + if err != nil { + log.Error("Unable to register cron task with name: %s Error: %v", name, err) + return err + } + + task := &Task{ + Name: name, + config: config, + fun: fun, + } + lock.Lock() + locked := true + defer func() { + if locked { + lock.Unlock() + } + }() + if _, has := tasksMap[task.Name]; has { + log.Error("A task with this name: %s has already been registered", name) + return fmt.Errorf("duplicate task with name: %s", task.Name) + } + + if config.IsEnabled() { + // We cannot use the entry return as there is no way to lock it + if err := addTaskToScheduler(task); err != nil { + return err + } + } + + tasks = append(tasks, task) + tasksMap[task.Name] = task + if started && config.IsEnabled() && config.DoRunAtStart() { + lock.Unlock() + locked = false + task.Run() + } + + return nil +} + +// RegisterTaskFatal will register a task but if there is an error log.Fatal +func RegisterTaskFatal(name string, config Config, fun func(context.Context, *user_model.User, Config) error) { + if err := RegisterTask(name, config, fun); err != nil { + log.Fatal("Unable to register cron task %s Error: %v", name, err) + } +} + +func addTaskToScheduler(task *Task) error { + tags := []string{task.Name, task.config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag + if scheduleHasSeconds(task.config.GetSchedule()) { + scheduler = scheduler.CronWithSeconds(task.config.GetSchedule()) + } else { + scheduler = scheduler.Cron(task.config.GetSchedule()) + } + if _, err := scheduler.Tag(tags...).Do(task.Run); err != nil { + log.Error("Unable to register cron task with name: %s Error: %v", task.Name, err) + return err + } + return nil +} + +func scheduleHasSeconds(schedule string) bool { + return len(strings.Fields(schedule)) >= 6 +} diff --git a/services/cron/tasks_actions.go b/services/cron/tasks_actions.go new file mode 100644 index 0000000..59cfe36 --- /dev/null +++ b/services/cron/tasks_actions.go @@ -0,0 +1,76 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cron + +import ( + "context" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + actions_service "code.gitea.io/gitea/services/actions" +) + +func initActionsTasks() { + if !setting.Actions.Enabled { + return + } + registerStopZombieTasks() + registerStopEndlessTasks() + registerCancelAbandonedJobs() + registerScheduleTasks() + registerActionsCleanup() +} + +func registerStopZombieTasks() { + RegisterTaskFatal("stop_zombie_tasks", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@every 5m", + }, func(ctx context.Context, _ *user_model.User, cfg Config) error { + return actions_service.StopZombieTasks(ctx) + }) +} + +func registerStopEndlessTasks() { + RegisterTaskFatal("stop_endless_tasks", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@every 30m", + }, func(ctx context.Context, _ *user_model.User, cfg Config) error { + return actions_service.StopEndlessTasks(ctx) + }) +} + +func registerCancelAbandonedJobs() { + RegisterTaskFatal("cancel_abandoned_jobs", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@every 6h", + }, func(ctx context.Context, _ *user_model.User, cfg Config) error { + return actions_service.CancelAbandonedJobs(ctx) + }) +} + +// registerScheduleTasks registers a scheduled task that runs every minute to start any due schedule tasks. +func registerScheduleTasks() { + // Register the task with a unique name, enabled status, and schedule for every minute. + RegisterTaskFatal("start_schedule_tasks", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 1m", + }, func(ctx context.Context, _ *user_model.User, cfg Config) error { + // Call the function to start schedule tasks and pass the context. + return actions_service.StartScheduleTasks(ctx) + }) +} + +func registerActionsCleanup() { + RegisterTaskFatal("cleanup_actions", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@midnight", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return actions_service.Cleanup(ctx) + }) +} diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go new file mode 100644 index 0000000..2a213ae --- /dev/null +++ b/services/cron/tasks_basic.go @@ -0,0 +1,175 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cron + +import ( + "context" + "time" + + "code.gitea.io/gitea/models" + git_model "code.gitea.io/gitea/models/git" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/models/webhook" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/auth" + "code.gitea.io/gitea/services/migrations" + mirror_service "code.gitea.io/gitea/services/mirror" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" + repo_service "code.gitea.io/gitea/services/repository" + archiver_service "code.gitea.io/gitea/services/repository/archiver" +) + +func registerUpdateMirrorTask() { + type UpdateMirrorTaskConfig struct { + BaseConfig + PullLimit int + PushLimit int + } + + RegisterTaskFatal("update_mirrors", &UpdateMirrorTaskConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 10m", + }, + PullLimit: 50, + PushLimit: 50, + }, func(ctx context.Context, _ *user_model.User, cfg Config) error { + umtc := cfg.(*UpdateMirrorTaskConfig) + return mirror_service.Update(ctx, umtc.PullLimit, umtc.PushLimit) + }) +} + +func registerRepoHealthCheck() { + type RepoHealthCheckConfig struct { + BaseConfig + Timeout time.Duration + Args []string `delim:" "` + } + RegisterTaskFatal("repo_health_check", &RepoHealthCheckConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@midnight", + }, + Timeout: 60 * time.Second, + Args: []string{}, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + rhcConfig := config.(*RepoHealthCheckConfig) + // the git args are set by config, they can be safe to be trusted + return repo_service.GitFsckRepos(ctx, rhcConfig.Timeout, git.ToTrustedCmdArgs(rhcConfig.Args)) + }) +} + +func registerCheckRepoStats() { + RegisterTaskFatal("check_repo_stats", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return models.CheckRepoStats(ctx) + }) +} + +func registerArchiveCleanup() { + RegisterTaskFatal("archive_cleanup", &OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", + }, + OlderThan: 24 * time.Hour, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + acConfig := config.(*OlderThanConfig) + return archiver_service.DeleteOldRepositoryArchives(ctx, acConfig.OlderThan) + }) +} + +func registerSyncExternalUsers() { + RegisterTaskFatal("sync_external_users", &UpdateExistingConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@midnight", + }, + UpdateExisting: true, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + realConfig := config.(*UpdateExistingConfig) + return auth.SyncExternalUsers(ctx, realConfig.UpdateExisting) + }) +} + +func registerDeletedBranchesCleanup() { + RegisterTaskFatal("deleted_branches_cleanup", &OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", + }, + OlderThan: 24 * time.Hour, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + realConfig := config.(*OlderThanConfig) + git_model.RemoveOldDeletedBranches(ctx, realConfig.OlderThan) + return nil + }) +} + +func registerUpdateMigrationPosterID() { + RegisterTaskFatal("update_migration_poster_id", &BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return migrations.UpdateMigrationPosterID(ctx) + }) +} + +func registerCleanupHookTaskTable() { + RegisterTaskFatal("cleanup_hook_task_table", &CleanupHookTaskConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@midnight", + }, + CleanupType: "OlderThan", + OlderThan: 168 * time.Hour, + NumberToKeep: 10, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + realConfig := config.(*CleanupHookTaskConfig) + return webhook.CleanupHookTaskTable(ctx, webhook.ToHookTaskCleanupType(realConfig.CleanupType), realConfig.OlderThan, realConfig.NumberToKeep) + }) +} + +func registerCleanupPackages() { + RegisterTaskFatal("cleanup_packages", &OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: true, + Schedule: "@midnight", + }, + OlderThan: 24 * time.Hour, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + realConfig := config.(*OlderThanConfig) + return packages_cleanup_service.CleanupTask(ctx, realConfig.OlderThan) + }) +} + +func initBasicTasks() { + if setting.Mirror.Enabled { + registerUpdateMirrorTask() + } + registerRepoHealthCheck() + registerCheckRepoStats() + registerArchiveCleanup() + registerSyncExternalUsers() + registerDeletedBranchesCleanup() + if !setting.Repository.DisableMigrations { + registerUpdateMigrationPosterID() + } + registerCleanupHookTaskTable() + if setting.Packages.Enabled { + registerCleanupPackages() + } +} diff --git a/services/cron/tasks_extended.go b/services/cron/tasks_extended.go new file mode 100644 index 0000000..e1ba527 --- /dev/null +++ b/services/cron/tasks_extended.go @@ -0,0 +1,243 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cron + +import ( + "context" + "time" + + activities_model "code.gitea.io/gitea/models/activities" + asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/system" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + issue_indexer "code.gitea.io/gitea/modules/indexer/issues" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/updatechecker" + repo_service "code.gitea.io/gitea/services/repository" + archiver_service "code.gitea.io/gitea/services/repository/archiver" + user_service "code.gitea.io/gitea/services/user" +) + +func registerDeleteInactiveUsers() { + RegisterTaskFatal("delete_inactive_accounts", &OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@annually", + }, + OlderThan: time.Minute * time.Duration(setting.Service.ActiveCodeLives), + }, func(ctx context.Context, _ *user_model.User, config Config) error { + olderThanConfig := config.(*OlderThanConfig) + return user_service.DeleteInactiveUsers(ctx, olderThanConfig.OlderThan) + }) +} + +func registerDeleteRepositoryArchives() { + RegisterTaskFatal("delete_repo_archives", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@annually", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return archiver_service.DeleteRepositoryArchives(ctx) + }) +} + +func registerGarbageCollectRepositories() { + type RepoHealthCheckConfig struct { + BaseConfig + Timeout time.Duration + Args []string `delim:" "` + } + RegisterTaskFatal("git_gc_repos", &RepoHealthCheckConfig{ + BaseConfig: BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 72h", + }, + Timeout: time.Duration(setting.Git.Timeout.GC) * time.Second, + Args: setting.Git.GCArgs, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + rhcConfig := config.(*RepoHealthCheckConfig) + // the git args are set by config, they can be safe to be trusted + return repo_service.GitGcRepos(ctx, rhcConfig.Timeout, git.ToTrustedCmdArgs(rhcConfig.Args)) + }) +} + +func registerRewriteAllPublicKeys() { + RegisterTaskFatal("resync_all_sshkeys", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 72h", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return asymkey_model.RewriteAllPublicKeys(ctx) + }) +} + +func registerRewriteAllPrincipalKeys() { + RegisterTaskFatal("resync_all_sshprincipals", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 72h", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return asymkey_model.RewriteAllPrincipalKeys(ctx) + }) +} + +func registerRepositoryUpdateHook() { + RegisterTaskFatal("resync_all_hooks", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 72h", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return repo_service.SyncRepositoryHooks(ctx) + }) +} + +func registerReinitMissingRepositories() { + RegisterTaskFatal("reinit_missing_repos", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 72h", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return repo_service.ReinitMissingRepositories(ctx) + }) +} + +func registerDeleteMissingRepositories() { + RegisterTaskFatal("delete_missing_repos", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 72h", + }, func(ctx context.Context, user *user_model.User, _ Config) error { + return repo_service.DeleteMissingRepositories(ctx, user) + }) +} + +func registerRemoveRandomAvatars() { + RegisterTaskFatal("delete_generated_repository_avatars", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 72h", + }, func(ctx context.Context, _ *user_model.User, _ Config) error { + return repo_service.RemoveRandomAvatars(ctx) + }) +} + +func registerDeleteOldActions() { + RegisterTaskFatal("delete_old_actions", &OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 168h", + }, + OlderThan: 365 * 24 * time.Hour, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + olderThanConfig := config.(*OlderThanConfig) + return activities_model.DeleteOldActions(ctx, olderThanConfig.OlderThan) + }) +} + +func registerUpdateGiteaChecker() { + type UpdateCheckerConfig struct { + BaseConfig + HTTPEndpoint string + DomainEndpoint string + } + RegisterTaskFatal("update_checker", &UpdateCheckerConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 168h", + }, + HTTPEndpoint: "https://dl.gitea.com/gitea/version.json", + DomainEndpoint: "release.forgejo.org", + }, func(ctx context.Context, _ *user_model.User, config Config) error { + updateCheckerConfig := config.(*UpdateCheckerConfig) + return updatechecker.GiteaUpdateChecker(updateCheckerConfig.HTTPEndpoint, updateCheckerConfig.DomainEndpoint) + }) +} + +func registerDeleteOldSystemNotices() { + RegisterTaskFatal("delete_old_system_notices", &OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 168h", + }, + OlderThan: 365 * 24 * time.Hour, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + olderThanConfig := config.(*OlderThanConfig) + return system.DeleteOldSystemNotices(ctx, olderThanConfig.OlderThan) + }) +} + +func registerGCLFS() { + if !setting.LFS.StartServer { + return + } + type GCLFSConfig struct { + OlderThanConfig + LastUpdatedMoreThanAgo time.Duration + NumberToCheckPerRepo int64 + ProportionToCheckPerRepo float64 + } + + RegisterTaskFatal("gc_lfs", &GCLFSConfig{ + OlderThanConfig: OlderThanConfig{ + BaseConfig: BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@every 24h", + }, + // Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload + // and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby + // an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid + // changes in new branches that might lead to lfs objects becoming temporarily unassociated with git + // objects. + // + // It is likely that a week is potentially excessive but it should definitely be enough that any + // unassociated LFS object is genuinely unassociated. + OlderThan: 24 * time.Hour * 7, + }, + // Only GC things that haven't been looked at in the past 3 days + LastUpdatedMoreThanAgo: 24 * time.Hour * 3, + NumberToCheckPerRepo: 100, + ProportionToCheckPerRepo: 0.6, + }, func(ctx context.Context, _ *user_model.User, config Config) error { + gcLFSConfig := config.(*GCLFSConfig) + return repo_service.GarbageCollectLFSMetaObjects(ctx, repo_service.GarbageCollectLFSMetaObjectsOptions{ + AutoFix: true, + OlderThan: time.Now().Add(-gcLFSConfig.OlderThan), + UpdatedLessRecentlyThan: time.Now().Add(-gcLFSConfig.LastUpdatedMoreThanAgo), + }) + }) +} + +func registerRebuildIssueIndexer() { + RegisterTaskFatal("rebuild_issue_indexer", &BaseConfig{ + Enabled: false, + RunAtStart: false, + Schedule: "@annually", + }, func(ctx context.Context, _ *user_model.User, config Config) error { + return issue_indexer.PopulateIssueIndexer(ctx) + }) +} + +func initExtendedTasks() { + registerDeleteInactiveUsers() + registerDeleteRepositoryArchives() + registerGarbageCollectRepositories() + registerRewriteAllPublicKeys() + registerRewriteAllPrincipalKeys() + registerRepositoryUpdateHook() + registerReinitMissingRepositories() + registerDeleteMissingRepositories() + registerRemoveRandomAvatars() + registerDeleteOldActions() + registerUpdateGiteaChecker() + registerDeleteOldSystemNotices() + registerGCLFS() + registerRebuildIssueIndexer() +} diff --git a/services/cron/tasks_test.go b/services/cron/tasks_test.go new file mode 100644 index 0000000..9b969a6 --- /dev/null +++ b/services/cron/tasks_test.go @@ -0,0 +1,68 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cron + +import ( + "sort" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddTaskToScheduler(t *testing.T) { + assert.Empty(t, scheduler.Jobs()) + defer scheduler.Clear() + + // no seconds + err := addTaskToScheduler(&Task{ + Name: "task 1", + config: &BaseConfig{ + Schedule: "5 4 * * *", + }, + }) + require.NoError(t, err) + jobs := scheduler.Jobs() + assert.Len(t, jobs, 1) + assert.Equal(t, "task 1", jobs[0].Tags()[0]) + assert.Equal(t, "5 4 * * *", jobs[0].Tags()[1]) + + // with seconds + err = addTaskToScheduler(&Task{ + Name: "task 2", + config: &BaseConfig{ + Schedule: "30 5 4 * * *", + }, + }) + require.NoError(t, err) + jobs = scheduler.Jobs() // the item order is not guaranteed, so we need to sort it before "assert" + sort.Slice(jobs, func(i, j int) bool { + return jobs[i].Tags()[0] < jobs[j].Tags()[0] + }) + assert.Len(t, jobs, 2) + assert.Equal(t, "task 2", jobs[1].Tags()[0]) + assert.Equal(t, "30 5 4 * * *", jobs[1].Tags()[1]) +} + +func TestScheduleHasSeconds(t *testing.T) { + tests := []struct { + schedule string + hasSecond bool + }{ + {"* * * * * *", true}, + {"* * * * *", false}, + {"5 4 * * *", false}, + {"5 4 * * *", false}, + {"5,8 4 * * *", false}, + {"* * * * * *", true}, + {"5,8 4 * * *", false}, + } + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + assert.Equal(t, test.hasSecond, scheduleHasSeconds(test.schedule)) + }) + } +} -- cgit v1.2.3