From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- cmd/migrate_storage.go | 267 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 cmd/migrate_storage.go (limited to 'cmd/migrate_storage.go') diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go new file mode 100644 index 0000000..3a69b55 --- /dev/null +++ b/cmd/migrate_storage.go @@ -0,0 +1,267 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "context" + "errors" + "fmt" + "io/fs" + "strings" + + actions_model "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + "code.gitea.io/gitea/models/migrations" + packages_model "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + + "github.com/urfave/cli/v2" +) + +// CmdMigrateStorage represents the available migrate storage sub-command. +var CmdMigrateStorage = &cli.Command{ + Name: "migrate-storage", + Usage: "Migrate the storage", + Description: "Copies stored files from storage configured in app.ini to parameter-configured storage", + Action: runMigrateStorage, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "type", + Aliases: []string{"t"}, + Value: "", + Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages', 'actions-log', 'actions-artifacts'", + }, + &cli.StringFlag{ + Name: "storage", + Aliases: []string{"s"}, + Value: "", + Usage: "New storage type: local (default) or minio", + }, + &cli.StringFlag{ + Name: "path", + Aliases: []string{"p"}, + Value: "", + Usage: "New storage placement if store is local (leave blank for default)", + }, + &cli.StringFlag{ + Name: "minio-endpoint", + Value: "", + Usage: "Minio storage endpoint", + }, + &cli.StringFlag{ + Name: "minio-access-key-id", + Value: "", + Usage: "Minio storage accessKeyID", + }, + &cli.StringFlag{ + Name: "minio-secret-access-key", + Value: "", + Usage: "Minio storage secretAccessKey", + }, + &cli.StringFlag{ + Name: "minio-bucket", + Value: "", + Usage: "Minio storage bucket", + }, + &cli.StringFlag{ + Name: "minio-location", + Value: "", + Usage: "Minio storage location to create bucket", + }, + &cli.StringFlag{ + Name: "minio-base-path", + Value: "", + Usage: "Minio storage base path on the bucket", + }, + &cli.BoolFlag{ + Name: "minio-use-ssl", + Usage: "Enable SSL for minio", + }, + &cli.BoolFlag{ + Name: "minio-insecure-skip-verify", + Usage: "Skip SSL verification", + }, + &cli.StringFlag{ + Name: "minio-checksum-algorithm", + Value: "", + Usage: "Minio checksum algorithm (default/md5)", + }, + }, +} + +func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.Iterate(ctx, nil, func(ctx context.Context, attach *repo_model.Attachment) error { + _, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath()) + return err + }) +} + +func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.Iterate(ctx, nil, func(ctx context.Context, mo *git_model.LFSMetaObject) error { + _, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath()) + return err + }) +} + +func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.Iterate(ctx, nil, func(ctx context.Context, user *user_model.User) error { + if user.CustomAvatarRelativePath() == "" { + return nil + } + _, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath()) + return err + }) +} + +func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.Iterate(ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error { + if repo.CustomAvatarRelativePath() == "" { + return nil + } + _, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath()) + return err + }) +} + +func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.Iterate(ctx, nil, func(ctx context.Context, archiver *repo_model.RepoArchiver) error { + p := archiver.RelativePath() + _, err := storage.Copy(dstStorage, p, storage.RepoArchives, p) + return err + }) +} + +func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.Iterate(ctx, nil, func(ctx context.Context, pb *packages_model.PackageBlob) error { + p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256)) + _, err := storage.Copy(dstStorage, p, storage.Packages, p) + return err + }) +} + +func migrateActionsLog(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.Iterate(ctx, nil, func(ctx context.Context, task *actions_model.ActionTask) error { + if task.LogExpired { + // the log has been cleared + return nil + } + if !task.LogInStorage { + // running tasks store logs in DBFS + return nil + } + p := task.LogFilename + _, err := storage.Copy(dstStorage, p, storage.Actions, p) + return err + }) +} + +func migrateActionsArtifacts(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.Iterate(ctx, nil, func(ctx context.Context, artifact *actions_model.ActionArtifact) error { + if artifact.Status == int64(actions_model.ArtifactStatusExpired) { + return nil + } + + _, err := storage.Copy(dstStorage, artifact.StoragePath, storage.ActionsArtifacts, artifact.StoragePath) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + log.Warn("ignored: actions artifact %s exists in the database but not in storage", artifact.StoragePath) + return nil + } + return err + } + + return nil + }) +} + +func runMigrateStorage(ctx *cli.Context) error { + stdCtx, cancel := installSignals() + defer cancel() + + if err := initDB(stdCtx); err != nil { + return err + } + + log.Info("AppPath: %s", setting.AppPath) + log.Info("AppWorkPath: %s", setting.AppWorkPath) + log.Info("Custom path: %s", setting.CustomPath) + log.Info("Log path: %s", setting.Log.RootPath) + log.Info("Configuration file: %s", setting.CustomConf) + + if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil { + log.Fatal("Failed to initialize ORM engine: %v", err) + return err + } + + if err := storage.Init(); err != nil { + return err + } + + var dstStorage storage.ObjectStorage + var err error + switch strings.ToLower(ctx.String("storage")) { + case "": + fallthrough + case string(setting.LocalStorageType): + p := ctx.String("path") + if p == "" { + log.Fatal("Path must be given when storage is local") + return nil + } + dstStorage, err = storage.NewLocalStorage( + stdCtx, + &setting.Storage{ + Path: p, + }) + case string(setting.MinioStorageType): + dstStorage, err = storage.NewMinioStorage( + stdCtx, + &setting.Storage{ + MinioConfig: setting.MinioStorageConfig{ + Endpoint: ctx.String("minio-endpoint"), + AccessKeyID: ctx.String("minio-access-key-id"), + SecretAccessKey: ctx.String("minio-secret-access-key"), + Bucket: ctx.String("minio-bucket"), + Location: ctx.String("minio-location"), + BasePath: ctx.String("minio-base-path"), + UseSSL: ctx.Bool("minio-use-ssl"), + InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"), + ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"), + }, + }) + default: + return fmt.Errorf("unsupported storage type: %s", ctx.String("storage")) + } + if err != nil { + return err + } + + migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{ + "attachments": migrateAttachments, + "lfs": migrateLFS, + "avatars": migrateAvatars, + "repo-avatars": migrateRepoAvatars, + "repo-archivers": migrateRepoArchivers, + "packages": migratePackages, + "actions-log": migrateActionsLog, + "actions-artifacts": migrateActionsArtifacts, + } + + tp := strings.ToLower(ctx.String("type")) + if m, ok := migratedMethods[tp]; ok { + if err := m(stdCtx, dstStorage); err != nil { + return err + } + log.Info("%s files have successfully been copied to the new storage.", tp) + return nil + } + + return fmt.Errorf("unsupported storage: %s", ctx.String("type")) +} -- cgit v1.2.3