diff options
Diffstat (limited to '')
| -rw-r--r-- | services/migrations/migrate.go | 510 |
1 files changed, 510 insertions, 0 deletions
diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go new file mode 100644 index 0000000..29b815d --- /dev/null +++ b/services/migrations/migrate.go @@ -0,0 +1,510 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "context" + "fmt" + "net" + "net/url" + "path/filepath" + "strings" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + system_model "code.gitea.io/gitea/models/system" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/hostmatcher" + "code.gitea.io/gitea/modules/log" + base "code.gitea.io/gitea/modules/migration" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" +) + +// MigrateOptions is equal to base.MigrateOptions +type MigrateOptions = base.MigrateOptions + +var ( + factories []base.DownloaderFactory + + allowList *hostmatcher.HostMatchList + blockList *hostmatcher.HostMatchList +) + +// RegisterDownloaderFactory registers a downloader factory +func RegisterDownloaderFactory(factory base.DownloaderFactory) { + factories = append(factories, factory) +} + +// IsMigrateURLAllowed checks if an URL is allowed to be migrated from +func IsMigrateURLAllowed(remoteURL string, doer *user_model.User) error { + // Remote address can be HTTP/HTTPS/Git URL or local path. + u, err := url.Parse(remoteURL) + if err != nil { + return &models.ErrInvalidCloneAddr{IsURLError: true, Host: remoteURL} + } + + if u.Scheme == "file" || u.Scheme == "" { + if !doer.CanImportLocal() { + return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true} + } + isAbs := filepath.IsAbs(u.Host + u.Path) + if !isAbs { + return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true} + } + isDir, err := util.IsDir(u.Host + u.Path) + if err != nil { + log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err) + return err + } + if !isDir { + return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true} + } + + return nil + } + + if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) { + return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true} + } + + if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" && u.Scheme != "ssh" { + return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true} + } + + hostName, _, err := net.SplitHostPort(u.Host) + if err != nil { + // u.Host can be "host" or "host:port" + err = nil //nolint + hostName = u.Host + } + + // some users only use proxy, there is no DNS resolver. it's safe to ignore the LookupIP error + addrList, _ := net.LookupIP(hostName) + return checkByAllowBlockList(hostName, addrList) +} + +func checkByAllowBlockList(hostName string, addrList []net.IP) error { + var ipAllowed bool + var ipBlocked bool + for _, addr := range addrList { + ipAllowed = ipAllowed || allowList.MatchIPAddr(addr) + ipBlocked = ipBlocked || blockList.MatchIPAddr(addr) + } + var blockedError error + if blockList.MatchHostName(hostName) || ipBlocked { + blockedError = &models.ErrInvalidCloneAddr{Host: hostName, IsPermissionDenied: true} + } + // if we have an allow-list, check the allow-list before return to get the more accurate error + if !allowList.IsEmpty() { + if !allowList.MatchHostName(hostName) && !ipAllowed { + return &models.ErrInvalidCloneAddr{Host: hostName, IsPermissionDenied: true} + } + } + // otherwise, we always follow the blocked list + return blockedError +} + +// MigrateRepository migrate repository according MigrateOptions +func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*repo_model.Repository, error) { + err := IsMigrateURLAllowed(opts.CloneAddr, doer) + if err != nil { + return nil, err + } + if opts.LFS && len(opts.LFSEndpoint) > 0 { + err := IsMigrateURLAllowed(opts.LFSEndpoint, doer) + if err != nil { + return nil, err + } + } + downloader, err := newDownloader(ctx, ownerName, opts) + if err != nil { + return nil, err + } + + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) + uploader.gitServiceType = opts.GitServiceType + + if err := migrateRepository(ctx, doer, downloader, uploader, opts, messenger); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + if err2 := system_model.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { + log.Error("create respotiry notice failed: ", err2) + } + return nil, err + } + return uploader.repo, nil +} + +func getFactoryFromServiceType(serviceType structs.GitServiceType) base.DownloaderFactory { + for _, factory := range factories { + if factory.GitServiceType() == serviceType { + return factory + } + } + return nil +} + +func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) { + var ( + downloader base.Downloader + err error + ) + + if factory := getFactoryFromServiceType(opts.GitServiceType); factory != nil { + downloader, err = factory.New(ctx, opts) + if err != nil { + return nil, err + } + } + + if downloader == nil { + opts.Wiki = true + opts.Milestones = false + opts.Labels = false + opts.Releases = false + opts.Comments = false + opts.Issues = false + opts.PullRequests = false + downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) + log.Trace("Will migrate from git: %s", opts.OriginalURL) + } + + if setting.Migrations.MaxAttempts > 1 { + downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) + } + return downloader, nil +} + +// migrateRepository will download information and then upload it to Uploader, this is a simple +// process for small repository. For a big repository, save all the data to disk +// before upload is better +func migrateRepository(_ context.Context, doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { + if messenger == nil { + messenger = base.NilMessenger + } + + repo, err := downloader.GetRepoInfo() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Info("migrating repo infos is not supported, ignored") + } + repo.IsPrivate = opts.Private + repo.IsMirror = opts.Mirror + if opts.Description != "" { + repo.Description = opts.Description + } + if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil { + return err + } + + // SECURITY: If the downloader is not a RepositoryRestorer then we need to recheck the CloneURL + if _, ok := downloader.(*RepositoryRestorer); !ok { + // Now the clone URL can be rewritten by the downloader so we must recheck + if err := IsMigrateURLAllowed(repo.CloneURL, doer); err != nil { + return err + } + + // SECURITY: Ensure that we haven't been redirected from an external to a local filesystem + // Now we know all of these must parse + cloneAddrURL, _ := url.Parse(opts.CloneAddr) + cloneURL, _ := url.Parse(repo.CloneURL) + + if cloneURL.Scheme == "file" || cloneURL.Scheme == "" { + if cloneAddrURL.Scheme != "file" && cloneAddrURL.Scheme != "" { + return fmt.Errorf("repo info has changed from external to local filesystem") + } + } + + // We don't actually need to check the OriginalURL as it isn't used anywhere + } + + log.Trace("migrating git data from %s", repo.CloneURL) + messenger("repo.migrate.migrating_git") + if err = uploader.CreateRepo(repo, opts); err != nil { + return err + } + defer uploader.Close() + + log.Trace("migrating topics") + messenger("repo.migrate.migrating_topics") + topics, err := downloader.GetTopics() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating topics is not supported, ignored") + } + if len(topics) != 0 { + if err = uploader.CreateTopics(topics...); err != nil { + return err + } + } + + if opts.Milestones { + log.Trace("migrating milestones") + messenger("repo.migrate.migrating_milestones") + milestones, err := downloader.GetMilestones() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating milestones is not supported, ignored") + } + msBatchSize := uploader.MaxBatchInsertSize("milestone") + for len(milestones) > 0 { + if len(milestones) < msBatchSize { + msBatchSize = len(milestones) + } + + if err := uploader.CreateMilestones(milestones[:msBatchSize]...); err != nil { + return err + } + milestones = milestones[msBatchSize:] + } + } + + if opts.Labels { + log.Trace("migrating labels") + messenger("repo.migrate.migrating_labels") + labels, err := downloader.GetLabels() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating labels is not supported, ignored") + } + + lbBatchSize := uploader.MaxBatchInsertSize("label") + for len(labels) > 0 { + if len(labels) < lbBatchSize { + lbBatchSize = len(labels) + } + + if err := uploader.CreateLabels(labels[:lbBatchSize]...); err != nil { + return err + } + labels = labels[lbBatchSize:] + } + } + + if opts.Releases { + log.Trace("migrating releases") + messenger("repo.migrate.migrating_releases") + releases, err := downloader.GetReleases() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating releases is not supported, ignored") + } + + relBatchSize := uploader.MaxBatchInsertSize("release") + for len(releases) > 0 { + if len(releases) < relBatchSize { + relBatchSize = len(releases) + } + + if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil { + return err + } + releases = releases[relBatchSize:] + } + + // Once all releases (if any) are inserted, sync any remaining non-release tags + if err = uploader.SyncTags(); err != nil { + return err + } + } + + var ( + commentBatchSize = uploader.MaxBatchInsertSize("comment") + reviewBatchSize = uploader.MaxBatchInsertSize("review") + ) + + supportAllComments := downloader.SupportGetRepoComments() + + if opts.Issues { + log.Trace("migrating issues and comments") + messenger("repo.migrate.migrating_issues") + issueBatchSize := uploader.MaxBatchInsertSize("issue") + + for i := 1; ; i++ { + issues, isEnd, err := downloader.GetIssues(i, issueBatchSize) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating issues is not supported, ignored") + break + } + + if err := uploader.CreateIssues(issues...); err != nil { + return err + } + + if opts.Comments && !supportAllComments { + allComments := make([]*base.Comment, 0, commentBatchSize) + for _, issue := range issues { + log.Trace("migrating issue %d's comments", issue.Number) + comments, _, err := downloader.GetComments(issue) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating comments is not supported, ignored") + } + + allComments = append(allComments, comments...) + + if len(allComments) >= commentBatchSize { + if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + return err + } + + allComments = allComments[commentBatchSize:] + } + } + + if len(allComments) > 0 { + if err = uploader.CreateComments(allComments...); err != nil { + return err + } + } + } + + if isEnd { + break + } + } + } + + if opts.PullRequests { + log.Trace("migrating pull requests and comments") + messenger("repo.migrate.migrating_pulls") + prBatchSize := uploader.MaxBatchInsertSize("pullrequest") + for i := 1; ; i++ { + prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating pull requests is not supported, ignored") + break + } + + if err := uploader.CreatePullRequests(prs...); err != nil { + return err + } + + if opts.Comments { + if !supportAllComments { + // plain comments + allComments := make([]*base.Comment, 0, commentBatchSize) + for _, pr := range prs { + log.Trace("migrating pull request %d's comments", pr.Number) + comments, _, err := downloader.GetComments(pr) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating comments is not supported, ignored") + } + + allComments = append(allComments, comments...) + + if len(allComments) >= commentBatchSize { + if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + return err + } + allComments = allComments[commentBatchSize:] + } + } + if len(allComments) > 0 { + if err = uploader.CreateComments(allComments...); err != nil { + return err + } + } + } + + // migrate reviews + allReviews := make([]*base.Review, 0, reviewBatchSize) + for _, pr := range prs { + reviews, err := downloader.GetReviews(pr) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating reviews is not supported, ignored") + break + } + + allReviews = append(allReviews, reviews...) + + if len(allReviews) >= reviewBatchSize { + if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { + return err + } + allReviews = allReviews[reviewBatchSize:] + } + } + if len(allReviews) > 0 { + if err = uploader.CreateReviews(allReviews...); err != nil { + return err + } + } + } + + if isEnd { + break + } + } + } + + if opts.Comments && supportAllComments { + log.Trace("migrating comments") + for i := 1; ; i++ { + comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize) + if err != nil { + return err + } + + if err := uploader.CreateComments(comments...); err != nil { + return err + } + + if isEnd { + break + } + } + } + + return uploader.Finish() +} + +// Init migrations service +func Init() error { + // TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead + + blockList = hostmatcher.ParseSimpleMatchList("migrations.BLOCKED_DOMAINS", setting.Migrations.BlockedDomains) + + allowList = hostmatcher.ParseSimpleMatchList("migrations.ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS", setting.Migrations.AllowedDomains) + if allowList.IsEmpty() { + // the default policy is that migration module can access external hosts + allowList.AppendBuiltin(hostmatcher.MatchBuiltinExternal) + } + if setting.Migrations.AllowLocalNetworks { + allowList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate) + allowList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback) + } + // TODO: at the moment, if ALLOW_LOCALNETWORKS=false, ALLOWED_DOMAINS=domain.com, and domain.com has IP 127.0.0.1, then it's still allowed. + // if we want to block such case, the private&loopback should be added to the blockList when ALLOW_LOCALNETWORKS=false + + return nil +} |