diff options
Diffstat (limited to 'modules/migration')
29 files changed, 1266 insertions, 0 deletions
diff --git a/modules/migration/comment.go b/modules/migration/comment.go new file mode 100644 index 0000000..e041758 --- /dev/null +++ b/modules/migration/comment.go @@ -0,0 +1,34 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import "time" + +// Commentable can be commented upon +type Commentable interface { + Reviewable + GetContext() DownloaderContext +} + +// Comment is a standard comment information +type Comment struct { + IssueIndex int64 `yaml:"issue_index"` + Index int64 + CommentType string `yaml:"comment_type"` // see `commentStrings` in models/issues/comment.go + PosterID int64 `yaml:"poster_id"` + PosterName string `yaml:"poster_name"` + PosterEmail string `yaml:"poster_email"` + Created time.Time + Updated time.Time + Content string + Reactions []*Reaction + Meta map[string]any `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct +} + +// GetExternalName ExternalUserMigrated interface +func (c *Comment) GetExternalName() string { return c.PosterName } + +// ExternalID ExternalUserMigrated interface +func (c *Comment) GetExternalID() int64 { return c.PosterID } diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go new file mode 100644 index 0000000..08dbbc2 --- /dev/null +++ b/modules/migration/downloader.go @@ -0,0 +1,37 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import ( + "context" + + "code.gitea.io/gitea/modules/structs" +) + +// Downloader downloads the site repo information +type Downloader interface { + SetContext(context.Context) + GetRepoInfo() (*Repository, error) + GetTopics() ([]string, error) + GetMilestones() ([]*Milestone, error) + GetReleases() ([]*Release, error) + GetLabels() ([]*Label, error) + GetIssues(page, perPage int) ([]*Issue, bool, error) + GetComments(commentable Commentable) ([]*Comment, bool, error) + GetAllComments(page, perPage int) ([]*Comment, bool, error) + SupportGetRepoComments() bool + GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) + GetReviews(reviewable Reviewable) ([]*Review, error) + FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) +} + +// DownloaderFactory defines an interface to match a downloader implementation and create a downloader +type DownloaderFactory interface { + New(ctx context.Context, opts MigrateOptions) (Downloader, error) + GitServiceType() structs.GitServiceType +} + +// DownloaderContext has opaque information only relevant to a given downloader +type DownloaderContext any diff --git a/modules/migration/error.go b/modules/migration/error.go new file mode 100644 index 0000000..64cda9d --- /dev/null +++ b/modules/migration/error.go @@ -0,0 +1,25 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import "fmt" + +// ErrNotSupported represents status if a downloader do not supported something. +type ErrNotSupported struct { + Entity string +} + +// IsErrNotSupported checks if an error is an ErrNotSupported +func IsErrNotSupported(err error) bool { + _, ok := err.(ErrNotSupported) + return ok +} + +// Error return error message +func (err ErrNotSupported) Error() string { + if len(err.Entity) != 0 { + return fmt.Sprintf("'%s' not supported", err.Entity) + } + return "not supported" +} diff --git a/modules/migration/file_format.go b/modules/migration/file_format.go new file mode 100644 index 0000000..d29d24d --- /dev/null +++ b/modules/migration/file_format.go @@ -0,0 +1,110 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import ( + "fmt" + "os" + "strings" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + + "github.com/santhosh-tekuri/jsonschema/v6" + "gopkg.in/yaml.v3" +) + +// Load project data from file, with optional validation +func Load(filename string, data any, validation bool) error { + isJSON := strings.HasSuffix(filename, ".json") + + bs, err := os.ReadFile(filename) + if err != nil { + return err + } + + if validation { + err := validate(bs, data, isJSON) + if err != nil { + return err + } + } + return unmarshal(bs, data, isJSON) +} + +func unmarshal(bs []byte, data any, isJSON bool) error { + if isJSON { + return json.Unmarshal(bs, data) + } + return yaml.Unmarshal(bs, data) +} + +func getSchema(filename string) (*jsonschema.Schema, error) { + c := jsonschema.NewCompiler() + c.UseLoader(&SchemaLoader{}) + return c.Compile(filename) +} + +func validate(bs []byte, datatype any, isJSON bool) error { + var v any + err := unmarshal(bs, &v, isJSON) + if err != nil { + return err + } + if !isJSON { + v, err = toStringKeys(v) + if err != nil { + return err + } + } + + var schemaFilename string + switch datatype := datatype.(type) { + case *[]*Issue: + schemaFilename = "issue.json" + case *[]*Milestone: + schemaFilename = "milestone.json" + default: + return fmt.Errorf("file_format:validate: %T has not a validation implemented", datatype) + } + + sch, err := getSchema(schemaFilename) + if err != nil { + return err + } + err = sch.Validate(v) + if err != nil { + log.Error("migration validation with %s failed:\n%#v", schemaFilename, err) + } + return err +} + +func toStringKeys(val any) (any, error) { + var err error + switch val := val.(type) { + case map[string]any: + m := make(map[string]any) + for k, v := range val { + m[k], err = toStringKeys(v) + if err != nil { + return nil, err + } + } + return m, nil + case []any: + l := make([]any, len(val)) + for i, v := range val { + l[i], err = toStringKeys(v) + if err != nil { + return nil, err + } + } + return l, nil + case time.Time: + return val.Format(time.RFC3339), nil + default: + return val, nil + } +} diff --git a/modules/migration/file_format_test.go b/modules/migration/file_format_test.go new file mode 100644 index 0000000..f6651cd --- /dev/null +++ b/modules/migration/file_format_test.go @@ -0,0 +1,39 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import ( + "strings" + "testing" + + "github.com/santhosh-tekuri/jsonschema/v6" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrationJSON_IssueOK(t *testing.T) { + issues := make([]*Issue, 0, 10) + err := Load("file_format_testdata/issue_a.json", &issues, true) + require.NoError(t, err) + err = Load("file_format_testdata/issue_a.yml", &issues, true) + require.NoError(t, err) +} + +func TestMigrationJSON_IssueFail(t *testing.T) { + issues := make([]*Issue, 0, 10) + err := Load("file_format_testdata/issue_b.json", &issues, true) + if _, ok := err.(*jsonschema.ValidationError); ok { + errors := strings.Split(err.(*jsonschema.ValidationError).GoString(), "\n") + assert.Contains(t, errors[1], "missing properties") + assert.Contains(t, errors[1], "poster_id") + } else { + t.Fatalf("got: type %T with value %s, want: *jsonschema.ValidationError", err, err) + } +} + +func TestMigrationJSON_MilestoneOK(t *testing.T) { + milestones := make([]*Milestone, 0, 10) + err := Load("file_format_testdata/milestones.json", &milestones, true) + require.NoError(t, err) +} diff --git a/modules/migration/file_format_testdata/issue_a.json b/modules/migration/file_format_testdata/issue_a.json new file mode 100644 index 0000000..33d7759 --- /dev/null +++ b/modules/migration/file_format_testdata/issue_a.json @@ -0,0 +1,14 @@ +[ + { + "number": 1, + "poster_id": 1, + "poster_name": "name_a", + "title": "title_a", + "content": "content_a", + "state": "closed", + "is_locked": false, + "created": "1985-04-12T23:20:50.52Z", + "updated": "1986-04-12T23:20:50.52Z", + "closed": "1987-04-12T23:20:50.52Z" + } +] diff --git a/modules/migration/file_format_testdata/issue_a.yml b/modules/migration/file_format_testdata/issue_a.yml new file mode 100644 index 0000000..d03bfb3 --- /dev/null +++ b/modules/migration/file_format_testdata/issue_a.yml @@ -0,0 +1,10 @@ +- number: 1 + poster_id: 1 + poster_name: name_a + title: title_a + content: content_a + state: closed + is_locked: false + created: 2021-05-27T15:24:13+02:00 + updated: 2021-11-11T10:52:45+01:00 + closed: 2021-11-11T10:52:45+01:00 diff --git a/modules/migration/file_format_testdata/issue_b.json b/modules/migration/file_format_testdata/issue_b.json new file mode 100644 index 0000000..2a824d4 --- /dev/null +++ b/modules/migration/file_format_testdata/issue_b.json @@ -0,0 +1,5 @@ +[ + { + "number": 1 + } +] diff --git a/modules/migration/file_format_testdata/milestones.json b/modules/migration/file_format_testdata/milestones.json new file mode 100644 index 0000000..8fb770d --- /dev/null +++ b/modules/migration/file_format_testdata/milestones.json @@ -0,0 +1,20 @@ +[ + { + "title": "title_a", + "description": "description_a", + "deadline": "1988-04-12T23:20:50.52Z", + "created": "1985-04-12T23:20:50.52Z", + "updated": "1986-04-12T23:20:50.52Z", + "closed": "1987-04-12T23:20:50.52Z", + "state": "closed" + }, + { + "title": "title_b", + "description": "description_b", + "deadline": "1998-04-12T23:20:50.52Z", + "created": "1995-04-12T23:20:50.52Z", + "updated": "1996-04-12T23:20:50.52Z", + "closed": null, + "state": "open" + } +] diff --git a/modules/migration/issue.go b/modules/migration/issue.go new file mode 100644 index 0000000..3d1d1b4 --- /dev/null +++ b/modules/migration/issue.go @@ -0,0 +1,48 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import "time" + +// Issue is a standard issue information +type Issue struct { + Number int64 `json:"number"` + PosterID int64 `yaml:"poster_id" json:"poster_id"` + PosterName string `yaml:"poster_name" json:"poster_name"` + PosterEmail string `yaml:"poster_email" json:"poster_email"` + Title string `json:"title"` + Content string `json:"content"` + Ref string `json:"ref"` + Milestone string `json:"milestone"` + State string `json:"state"` // closed, open + IsLocked bool `yaml:"is_locked" json:"is_locked"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Closed *time.Time `json:"closed"` + Labels []*Label `json:"labels"` + Reactions []*Reaction `json:"reactions"` + Assignees []string `json:"assignees"` + ForeignIndex int64 `json:"foreign_id"` + Context DownloaderContext `yaml:"-"` +} + +// GetExternalName ExternalUserMigrated interface +func (issue *Issue) GetExternalName() string { return issue.PosterName } + +// GetExternalID ExternalUserMigrated interface +func (issue *Issue) GetExternalID() int64 { return issue.PosterID } + +func (issue *Issue) GetLocalIndex() int64 { return issue.Number } + +func (issue *Issue) GetForeignIndex() int64 { + // see the comment of Reviewable.GetForeignIndex + // if there is no ForeignIndex, then use LocalIndex + if issue.ForeignIndex == 0 { + return issue.Number + } + return issue.ForeignIndex +} + +func (issue *Issue) GetContext() DownloaderContext { return issue.Context } diff --git a/modules/migration/label.go b/modules/migration/label.go new file mode 100644 index 0000000..4927be3 --- /dev/null +++ b/modules/migration/label.go @@ -0,0 +1,13 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +// Label defines a standard label information +type Label struct { + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` + Exclusive bool `json:"exclusive"` +} diff --git a/modules/migration/messenger.go b/modules/migration/messenger.go new file mode 100644 index 0000000..6f9cad3 --- /dev/null +++ b/modules/migration/messenger.go @@ -0,0 +1,10 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +// Messenger is a formatting function similar to i18n.TrString +type Messenger func(key string, args ...any) + +// NilMessenger represents an empty formatting function +func NilMessenger(string, ...any) {} diff --git a/modules/migration/milestone.go b/modules/migration/milestone.go new file mode 100644 index 0000000..34355b8 --- /dev/null +++ b/modules/migration/milestone.go @@ -0,0 +1,18 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import "time" + +// Milestone defines a standard milestone +type Milestone struct { + Title string `json:"title"` + Description string `json:"description"` + Deadline *time.Time `json:"deadline"` + Created time.Time `json:"created"` + Updated *time.Time `json:"updated"` + Closed *time.Time `json:"closed"` + State string `json:"state"` // open, closed +} diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go new file mode 100644 index 0000000..e5b6933 --- /dev/null +++ b/modules/migration/null_downloader.go @@ -0,0 +1,88 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import ( + "context" + "net/url" +) + +// NullDownloader implements a blank downloader +type NullDownloader struct{} + +var _ Downloader = &NullDownloader{} + +// SetContext set context +func (n NullDownloader) SetContext(_ context.Context) {} + +// GetRepoInfo returns a repository information +func (n NullDownloader) GetRepoInfo() (*Repository, error) { + return nil, ErrNotSupported{Entity: "RepoInfo"} +} + +// GetTopics return repository topics +func (n NullDownloader) GetTopics() ([]string, error) { + return nil, ErrNotSupported{Entity: "Topics"} +} + +// GetMilestones returns milestones +func (n NullDownloader) GetMilestones() ([]*Milestone, error) { + return nil, ErrNotSupported{Entity: "Milestones"} +} + +// GetReleases returns releases +func (n NullDownloader) GetReleases() ([]*Release, error) { + return nil, ErrNotSupported{Entity: "Releases"} +} + +// GetLabels returns labels +func (n NullDownloader) GetLabels() ([]*Label, error) { + return nil, ErrNotSupported{Entity: "Labels"} +} + +// GetIssues returns issues according start and limit +func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { + return nil, false, ErrNotSupported{Entity: "Issues"} +} + +// GetComments returns comments of an issue or PR +func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { + return nil, false, ErrNotSupported{Entity: "Comments"} +} + +// GetAllComments returns paginated comments +func (n NullDownloader) GetAllComments(page, perPage int) ([]*Comment, bool, error) { + return nil, false, ErrNotSupported{Entity: "AllComments"} +} + +// GetPullRequests returns pull requests according page and perPage +func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { + return nil, false, ErrNotSupported{Entity: "PullRequests"} +} + +// GetReviews returns pull requests review +func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { + return nil, ErrNotSupported{Entity: "Reviews"} +} + +// FormatCloneURL add authentication into remote URLs +func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) { + if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { + u, err := url.Parse(remoteAddr) + if err != nil { + return "", err + } + u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) + if len(opts.AuthToken) > 0 { + u.User = url.UserPassword("oauth2", opts.AuthToken) + } + return u.String(), nil + } + return remoteAddr, nil +} + +// SupportGetRepoComments return true if it supports get repo comments +func (n NullDownloader) SupportGetRepoComments() bool { + return false +} diff --git a/modules/migration/options.go b/modules/migration/options.go new file mode 100644 index 0000000..234e72c --- /dev/null +++ b/modules/migration/options.go @@ -0,0 +1,41 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import "code.gitea.io/gitea/modules/structs" + +// MigrateOptions defines the way a repository gets migrated +// this is for internal usage by migrations module and func who interact with it +type MigrateOptions struct { + // required: true + CloneAddr string `json:"clone_addr" binding:"Required"` + CloneAddrEncrypted string `json:"clone_addr_encrypted,omitempty"` + AuthUsername string `json:"auth_username"` + AuthPassword string `json:"-"` + AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"` + AuthToken string `json:"-"` + AuthTokenEncrypted string `json:"auth_token_encrypted,omitempty"` + // required: true + UID int `json:"uid" binding:"Required"` + // required: true + RepoName string `json:"repo_name" binding:"Required"` + Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` + LFSEndpoint string `json:"lfs_endpoint"` + Private bool `json:"private"` + Description string `json:"description"` + OriginalURL string + GitServiceType structs.GitServiceType + Wiki bool + Issues bool + Milestones bool + Labels bool + Releases bool + Comments bool + PullRequests bool + ReleaseAssets bool + MigrateToRepoID int64 + MirrorInterval string `json:"mirror_interval"` +} diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go new file mode 100644 index 0000000..1435991 --- /dev/null +++ b/modules/migration/pullrequest.go @@ -0,0 +1,74 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import ( + "fmt" + "time" + + "code.gitea.io/gitea/modules/git" +) + +// PullRequest defines a standard pull request information +type PullRequest struct { + Number int64 + Title string + PosterName string `yaml:"poster_name"` + PosterID int64 `yaml:"poster_id"` + PosterEmail string `yaml:"poster_email"` + Content string + Milestone string + State string + Created time.Time + Updated time.Time + Closed *time.Time + Labels []*Label + PatchURL string `yaml:"patch_url"` // SECURITY: This must be safe to download directly from + Merged bool + MergedTime *time.Time `yaml:"merged_time"` + MergeCommitSHA string `yaml:"merge_commit_sha"` + Head PullRequestBranch + Base PullRequestBranch + Assignees []string + IsLocked bool `yaml:"is_locked"` + Reactions []*Reaction + ForeignIndex int64 + Context DownloaderContext `yaml:"-"` + EnsuredSafe bool `yaml:"ensured_safe"` +} + +func (p *PullRequest) GetLocalIndex() int64 { return p.Number } +func (p *PullRequest) GetForeignIndex() int64 { return p.ForeignIndex } +func (p *PullRequest) GetContext() DownloaderContext { return p.Context } + +// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository +func (p *PullRequest) IsForkPullRequest() bool { + return p.Head.RepoFullName() != p.Base.RepoFullName() +} + +// GetGitRefName returns pull request relative path to head +func (p PullRequest) GetGitRefName() string { + return fmt.Sprintf("%s%d/head", git.PullPrefix, p.Number) +} + +// PullRequestBranch represents a pull request branch +type PullRequestBranch struct { + CloneURL string `yaml:"clone_url"` // SECURITY: This must be safe to download from + Ref string // SECURITY: this must be a git.IsValidRefPattern + SHA string // SECURITY: this must be a git.IsValidSHAPattern + RepoName string `yaml:"repo_name"` + OwnerName string `yaml:"owner_name"` +} + +// RepoFullName returns pull request repo full name +func (p PullRequestBranch) RepoFullName() string { + return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName) +} + +// GetExternalName ExternalUserMigrated interface +func (p *PullRequest) GetExternalName() string { return p.PosterName } + +// ExternalID ExternalUserMigrated interface +func (p *PullRequest) GetExternalID() int64 { return p.PosterID } diff --git a/modules/migration/reaction.go b/modules/migration/reaction.go new file mode 100644 index 0000000..ca1df6c --- /dev/null +++ b/modules/migration/reaction.go @@ -0,0 +1,17 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +// Reaction represents a reaction to an issue/pr/comment. +type Reaction struct { + UserID int64 `yaml:"user_id" json:"user_id"` + UserName string `yaml:"user_name" json:"user_name"` + Content string `json:"content"` +} + +// GetExternalName ExternalUserMigrated interface +func (r *Reaction) GetExternalName() string { return r.UserName } + +// GetExternalID ExternalUserMigrated interface +func (r *Reaction) GetExternalID() int64 { return r.UserID } diff --git a/modules/migration/release.go b/modules/migration/release.go new file mode 100644 index 0000000..f92cf25 --- /dev/null +++ b/modules/migration/release.go @@ -0,0 +1,46 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import ( + "io" + "time" +) + +// ReleaseAsset represents a release asset +type ReleaseAsset struct { + ID int64 + Name string + ContentType *string `yaml:"content_type"` + Size *int + DownloadCount *int `yaml:"download_count"` + Created time.Time + Updated time.Time + + DownloadURL *string `yaml:"download_url"` // SECURITY: It is the responsibility of downloader to make sure this is safe + // if DownloadURL is nil, the function should be invoked + DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` // SECURITY: It is the responsibility of downloader to make sure this is safe +} + +// Release represents a release +type Release struct { + TagName string `yaml:"tag_name"` // SECURITY: This must pass git.IsValidRefPattern + TargetCommitish string `yaml:"target_commitish"` // SECURITY: This must pass git.IsValidRefPattern + Name string + Body string + Draft bool + Prerelease bool + PublisherID int64 `yaml:"publisher_id"` + PublisherName string `yaml:"publisher_name"` + PublisherEmail string `yaml:"publisher_email"` + Assets []*ReleaseAsset + Created time.Time + Published time.Time +} + +// GetExternalName ExternalUserMigrated interface +func (r *Release) GetExternalName() string { return r.PublisherName } + +// GetExternalID ExternalUserMigrated interface +func (r *Release) GetExternalID() int64 { return r.PublisherID } diff --git a/modules/migration/repo.go b/modules/migration/repo.go new file mode 100644 index 0000000..22c2cf6 --- /dev/null +++ b/modules/migration/repo.go @@ -0,0 +1,17 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +// Repository defines a standard repository information +type Repository struct { + Name string + Owner string + IsPrivate bool `yaml:"is_private"` + IsMirror bool `yaml:"is_mirror"` + Description string + CloneURL string `yaml:"clone_url"` // SECURITY: This must be checked to ensure that is safe to be used + OriginalURL string `yaml:"original_url"` + DefaultBranch string +} diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go new file mode 100644 index 0000000..1cacf5f --- /dev/null +++ b/modules/migration/retry_downloader.go @@ -0,0 +1,194 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import ( + "context" + "time" +) + +var _ Downloader = &RetryDownloader{} + +// RetryDownloader retry the downloads +type RetryDownloader struct { + Downloader + ctx context.Context + RetryTimes int // the total execute times + RetryDelay int // time to delay seconds +} + +// NewRetryDownloader creates a retry downloader +func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader { + return &RetryDownloader{ + Downloader: downloader, + ctx: ctx, + RetryTimes: retryTimes, + RetryDelay: retryDelay, + } +} + +func (d *RetryDownloader) retry(work func() error) error { + var ( + times = d.RetryTimes + err error + ) + for ; times > 0; times-- { + if err = work(); err == nil { + return nil + } + if IsErrNotSupported(err) { + return err + } + select { + case <-d.ctx.Done(): + return d.ctx.Err() + case <-time.After(time.Second * time.Duration(d.RetryDelay)): + } + } + return err +} + +// SetContext set context +func (d *RetryDownloader) SetContext(ctx context.Context) { + d.ctx = ctx + d.Downloader.SetContext(ctx) +} + +// GetRepoInfo returns a repository information with retry +func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { + var ( + repo *Repository + err error + ) + + err = d.retry(func() error { + repo, err = d.Downloader.GetRepoInfo() + return err + }) + + return repo, err +} + +// GetTopics returns a repository's topics with retry +func (d *RetryDownloader) GetTopics() ([]string, error) { + var ( + topics []string + err error + ) + + err = d.retry(func() error { + topics, err = d.Downloader.GetTopics() + return err + }) + + return topics, err +} + +// GetMilestones returns a repository's milestones with retry +func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) { + var ( + milestones []*Milestone + err error + ) + + err = d.retry(func() error { + milestones, err = d.Downloader.GetMilestones() + return err + }) + + return milestones, err +} + +// GetReleases returns a repository's releases with retry +func (d *RetryDownloader) GetReleases() ([]*Release, error) { + var ( + releases []*Release + err error + ) + + err = d.retry(func() error { + releases, err = d.Downloader.GetReleases() + return err + }) + + return releases, err +} + +// GetLabels returns a repository's labels with retry +func (d *RetryDownloader) GetLabels() ([]*Label, error) { + var ( + labels []*Label + err error + ) + + err = d.retry(func() error { + labels, err = d.Downloader.GetLabels() + return err + }) + + return labels, err +} + +// GetIssues returns a repository's issues with retry +func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { + var ( + issues []*Issue + isEnd bool + err error + ) + + err = d.retry(func() error { + issues, isEnd, err = d.Downloader.GetIssues(page, perPage) + return err + }) + + return issues, isEnd, err +} + +// GetComments returns a repository's comments with retry +func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) { + var ( + comments []*Comment + isEnd bool + err error + ) + + err = d.retry(func() error { + comments, isEnd, err = d.Downloader.GetComments(commentable) + return err + }) + + return comments, isEnd, err +} + +// GetPullRequests returns a repository's pull requests with retry +func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { + var ( + prs []*PullRequest + err error + isEnd bool + ) + + err = d.retry(func() error { + prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage) + return err + }) + + return prs, isEnd, err +} + +// GetReviews returns pull requests reviews +func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) { + var ( + reviews []*Review + err error + ) + + err = d.retry(func() error { + reviews, err = d.Downloader.GetReviews(reviewable) + return err + }) + + return reviews, err +} diff --git a/modules/migration/review.go b/modules/migration/review.go new file mode 100644 index 0000000..79e821b --- /dev/null +++ b/modules/migration/review.go @@ -0,0 +1,67 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +import "time" + +// Reviewable can be reviewed +type Reviewable interface { + GetLocalIndex() int64 + + // GetForeignIndex presents the foreign index, which could be misused: + // For example, if there are 2 Gitea sites: site-A exports a dataset, then site-B imports it: + // * if site-A exports files by using its LocalIndex + // * from site-A's view, LocalIndex is site-A's IssueIndex while ForeignIndex is site-B's IssueIndex + // * but from site-B's view, LocalIndex is site-B's IssueIndex while ForeignIndex is site-A's IssueIndex + // + // So the exporting/importing must be paired, but the meaning of them looks confusing then: + // * either site-A and site-B both use LocalIndex during dumping/restoring + // * or site-A and site-B both use ForeignIndex + GetForeignIndex() int64 +} + +// enumerate all review states +const ( + ReviewStatePending = "PENDING" + ReviewStateApproved = "APPROVED" + ReviewStateChangesRequested = "CHANGES_REQUESTED" + ReviewStateCommented = "COMMENTED" + ReviewStateRequestReview = "REQUEST_REVIEW" +) + +// Review is a standard review information +type Review struct { + ID int64 + IssueIndex int64 `yaml:"issue_index"` + ReviewerID int64 `yaml:"reviewer_id"` + ReviewerName string `yaml:"reviewer_name"` + Official bool + CommitID string `yaml:"commit_id"` + Content string + CreatedAt time.Time `yaml:"created_at"` + State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT + Comments []*ReviewComment +} + +// GetExternalName ExternalUserMigrated interface +func (r *Review) GetExternalName() string { return r.ReviewerName } + +// GetExternalID ExternalUserMigrated interface +func (r *Review) GetExternalID() int64 { return r.ReviewerID } + +// ReviewComment represents a review comment +type ReviewComment struct { + ID int64 + InReplyTo int64 `yaml:"in_reply_to"` + Content string + TreePath string `yaml:"tree_path"` + DiffHunk string `yaml:"diff_hunk"` + Position int + Line int + CommitID string `yaml:"commit_id"` + PosterID int64 `yaml:"poster_id"` + Reactions []*Reaction + CreatedAt time.Time `yaml:"created_at"` + UpdatedAt time.Time `yaml:"updated_at"` +} diff --git a/modules/migration/schemas/issue.json b/modules/migration/schemas/issue.json new file mode 100644 index 0000000..25753c3 --- /dev/null +++ b/modules/migration/schemas/issue.json @@ -0,0 +1,114 @@ +{ + "title": "Issue", + "description": "Issues associated to a repository within a forge (Gitea, GitLab, etc.).", + + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "number": { + "description": "Unique identifier, relative to the repository.", + "type": "number" + }, + "poster_id": { + "description": "Unique identifier of the user who authored the issue.", + "type": "number" + }, + "poster_name": { + "description": "Name of the user who authored the issue.", + "type": "string" + }, + "poster_email": { + "description": "Email of the user who authored the issue.", + "type": "string" + }, + "title": { + "description": "Short description displayed as the title.", + "type": "string" + }, + "content": { + "description": "Long, multiline, description.", + "type": "string" + }, + "ref": { + "description": "Target branch in the repository.", + "type": "string" + }, + "milestone": { + "description": "Name of the milestone.", + "type": "string" + }, + "state": { + "description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.", + "enum": [ + "closed", + "open" + ] + }, + "is_locked": { + "description": "A locked issue can only be modified by privileged users.", + "type": "boolean" + }, + "created": { + "description": "Creation time.", + "type": "string", + "format": "date-time" + }, + "updated": { + "description": "Last update time.", + "type": "string", + "format": "date-time" + }, + "closed": { + "description": "The last time 'state' changed to 'closed'.", + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "labels": { + "description": "List of labels.", + "type": "array", + "items": { + "$ref": "label.json" + } + }, + "reactions": { + "description": "List of reactions.", + "type": "array", + "items": { + "$ref": "reaction.json" + } + }, + "assignees": { + "description": "List of assignees.", + "type": "array", + "items": { + "description": "Name of a user assigned to the issue.", + "type": "string" + } + } + }, + "required": [ + "number", + "poster_id", + "poster_name", + "title", + "content", + "state", + "is_locked", + "created", + "updated" + ] + }, + + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "http://example.com/issue.json", + "$$target": "issue.json" +} diff --git a/modules/migration/schemas/label.json b/modules/migration/schemas/label.json new file mode 100644 index 0000000..561a2e3 --- /dev/null +++ b/modules/migration/schemas/label.json @@ -0,0 +1,28 @@ +{ + "title": "Label", + "description": "Label associated to an issue.", + + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "Name of the label, unique within the repository.", + "type": "string" + }, + "color": { + "description": "Color code of the label.", + "type": "string" + }, + "description": { + "description": "Long, multiline, description.", + "type": "string" + } + }, + "required": [ + "name" + ], + + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "label.json", + "$$target": "label.json" +} diff --git a/modules/migration/schemas/milestone.json b/modules/migration/schemas/milestone.json new file mode 100644 index 0000000..7024ef4 --- /dev/null +++ b/modules/migration/schemas/milestone.json @@ -0,0 +1,67 @@ +{ + "title": "Milestone", + "description": "Milestone associated to a repository within a forge.", + + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "title": { + "description": "Short description.", + "type": "string" + }, + "description": { + "description": "Long, multiline, description.", + "type": "string" + }, + "deadline": { + "description": "Deadline after which the milestone is overdue.", + "type": "string", + "format": "date-time" + }, + "created": { + "description": "Creation time.", + "type": "string", + "format": "date-time" + }, + "updated": { + "description": "Last update time.", + "type": "string", + "format": "date-time" + }, + "closed": { + "description": "The last time 'state' changed to 'closed'.", + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "state": { + "description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.", + "enum": [ + "closed", + "open" + ] + } + }, + "required": [ + "title", + "description", + "deadline", + "created", + "updated", + "closed", + "state" + ] + }, + + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "http://example.com/milestone.json", + "$$target": "milestone.json" +} diff --git a/modules/migration/schemas/reaction.json b/modules/migration/schemas/reaction.json new file mode 100644 index 0000000..2565251 --- /dev/null +++ b/modules/migration/schemas/reaction.json @@ -0,0 +1,29 @@ +{ + "title": "Reaction", + "description": "Reaction associated to an issue or a comment.", + + "type": "object", + "additionalProperties": false, + "properties": { + "user_id": { + "description": "Unique identifier of the user who authored the reaction.", + "type": "number" + }, + "user_name": { + "description": "Name of the user who authored the reaction.", + "type": "string" + }, + "content": { + "description": "Representation of the reaction", + "type": "string" + } + }, + "required": [ + "user_id", + "content" + ], + + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "http://example.com/reaction.json", + "$$target": "reaction.json" +} diff --git a/modules/migration/schemas_bindata.go b/modules/migration/schemas_bindata.go new file mode 100644 index 0000000..c5db3b3 --- /dev/null +++ b/modules/migration/schemas_bindata.go @@ -0,0 +1,8 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build bindata + +package migration + +//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go diff --git a/modules/migration/schemas_dynamic.go b/modules/migration/schemas_dynamic.go new file mode 100644 index 0000000..3741691 --- /dev/null +++ b/modules/migration/schemas_dynamic.go @@ -0,0 +1,47 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build !bindata + +package migration + +import ( + "net/url" + "os" + "path" + "path/filepath" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +type SchemaLoader struct{} + +func (*SchemaLoader) Load(s string) (any, error) { + u, err := url.Parse(s) + if err != nil { + return nil, err + } + basename := path.Base(u.Path) + filename := basename + // + // Schema reference each other within the schemas directory but + // the tests run in the parent directory. + // + if _, err := os.Stat(filename); os.IsNotExist(err) { + filename = filepath.Join("schemas", basename) + // + // Integration tests run from the git root directory, not the + // directory in which the test source is located. + // + if _, err := os.Stat(filename); os.IsNotExist(err) { + filename = filepath.Join("modules/migration/schemas", basename) + } + } + + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + return jsonschema.UnmarshalJSON(f) +} diff --git a/modules/migration/schemas_static.go b/modules/migration/schemas_static.go new file mode 100644 index 0000000..832dfd8 --- /dev/null +++ b/modules/migration/schemas_static.go @@ -0,0 +1,23 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +//go:build bindata + +package migration + +import ( + "path" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +type SchemaLoader struct{} + +func (*SchemaLoader) Load(filename string) (any, error) { + f, err := Assets.Open(path.Base(filename)) + if err != nil { + return nil, err + } + defer f.Close() + return jsonschema.UnmarshalJSON(f) +} diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go new file mode 100644 index 0000000..ff642aa --- /dev/null +++ b/modules/migration/uploader.go @@ -0,0 +1,23 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// SPDX-License-Identifier: MIT + +package migration + +// Uploader uploads all the information of one repository +type Uploader interface { + MaxBatchInsertSize(tp string) int + CreateRepo(repo *Repository, opts MigrateOptions) error + CreateTopics(topic ...string) error + CreateMilestones(milestones ...*Milestone) error + CreateReleases(releases ...*Release) error + SyncTags() error + CreateLabels(labels ...*Label) error + CreateIssues(issues ...*Issue) error + CreateComments(comments ...*Comment) error + CreatePullRequests(prs ...*PullRequest) error + CreateReviews(reviews ...*Review) error + Rollback() error + Finish() error + Close() +} |