summaryrefslogtreecommitdiffstats
path: root/modules/migration
diff options
context:
space:
mode:
Diffstat (limited to 'modules/migration')
-rw-r--r--modules/migration/comment.go34
-rw-r--r--modules/migration/downloader.go37
-rw-r--r--modules/migration/error.go25
-rw-r--r--modules/migration/file_format.go110
-rw-r--r--modules/migration/file_format_test.go39
-rw-r--r--modules/migration/file_format_testdata/issue_a.json14
-rw-r--r--modules/migration/file_format_testdata/issue_a.yml10
-rw-r--r--modules/migration/file_format_testdata/issue_b.json5
-rw-r--r--modules/migration/file_format_testdata/milestones.json20
-rw-r--r--modules/migration/issue.go48
-rw-r--r--modules/migration/label.go13
-rw-r--r--modules/migration/messenger.go10
-rw-r--r--modules/migration/milestone.go18
-rw-r--r--modules/migration/null_downloader.go88
-rw-r--r--modules/migration/options.go41
-rw-r--r--modules/migration/pullrequest.go74
-rw-r--r--modules/migration/reaction.go17
-rw-r--r--modules/migration/release.go46
-rw-r--r--modules/migration/repo.go17
-rw-r--r--modules/migration/retry_downloader.go194
-rw-r--r--modules/migration/review.go67
-rw-r--r--modules/migration/schemas/issue.json114
-rw-r--r--modules/migration/schemas/label.json28
-rw-r--r--modules/migration/schemas/milestone.json67
-rw-r--r--modules/migration/schemas/reaction.json29
-rw-r--r--modules/migration/schemas_bindata.go8
-rw-r--r--modules/migration/schemas_dynamic.go47
-rw-r--r--modules/migration/schemas_static.go23
-rw-r--r--modules/migration/uploader.go23
29 files changed, 1266 insertions, 0 deletions
diff --git a/modules/migration/comment.go b/modules/migration/comment.go
new file mode 100644
index 0000000..e041758
--- /dev/null
+++ b/modules/migration/comment.go
@@ -0,0 +1,34 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "time"
+
+// Commentable can be commented upon
+type Commentable interface {
+ Reviewable
+ GetContext() DownloaderContext
+}
+
+// Comment is a standard comment information
+type Comment struct {
+ IssueIndex int64 `yaml:"issue_index"`
+ Index int64
+ CommentType string `yaml:"comment_type"` // see `commentStrings` in models/issues/comment.go
+ PosterID int64 `yaml:"poster_id"`
+ PosterName string `yaml:"poster_name"`
+ PosterEmail string `yaml:"poster_email"`
+ Created time.Time
+ Updated time.Time
+ Content string
+ Reactions []*Reaction
+ Meta map[string]any `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (c *Comment) GetExternalName() string { return c.PosterName }
+
+// ExternalID ExternalUserMigrated interface
+func (c *Comment) GetExternalID() int64 { return c.PosterID }
diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go
new file mode 100644
index 0000000..08dbbc2
--- /dev/null
+++ b/modules/migration/downloader.go
@@ -0,0 +1,37 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+ "context"
+
+ "code.gitea.io/gitea/modules/structs"
+)
+
+// Downloader downloads the site repo information
+type Downloader interface {
+ SetContext(context.Context)
+ GetRepoInfo() (*Repository, error)
+ GetTopics() ([]string, error)
+ GetMilestones() ([]*Milestone, error)
+ GetReleases() ([]*Release, error)
+ GetLabels() ([]*Label, error)
+ GetIssues(page, perPage int) ([]*Issue, bool, error)
+ GetComments(commentable Commentable) ([]*Comment, bool, error)
+ GetAllComments(page, perPage int) ([]*Comment, bool, error)
+ SupportGetRepoComments() bool
+ GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
+ GetReviews(reviewable Reviewable) ([]*Review, error)
+ FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
+}
+
+// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
+type DownloaderFactory interface {
+ New(ctx context.Context, opts MigrateOptions) (Downloader, error)
+ GitServiceType() structs.GitServiceType
+}
+
+// DownloaderContext has opaque information only relevant to a given downloader
+type DownloaderContext any
diff --git a/modules/migration/error.go b/modules/migration/error.go
new file mode 100644
index 0000000..64cda9d
--- /dev/null
+++ b/modules/migration/error.go
@@ -0,0 +1,25 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "fmt"
+
+// ErrNotSupported represents status if a downloader do not supported something.
+type ErrNotSupported struct {
+ Entity string
+}
+
+// IsErrNotSupported checks if an error is an ErrNotSupported
+func IsErrNotSupported(err error) bool {
+ _, ok := err.(ErrNotSupported)
+ return ok
+}
+
+// Error return error message
+func (err ErrNotSupported) Error() string {
+ if len(err.Entity) != 0 {
+ return fmt.Sprintf("'%s' not supported", err.Entity)
+ }
+ return "not supported"
+}
diff --git a/modules/migration/file_format.go b/modules/migration/file_format.go
new file mode 100644
index 0000000..d29d24d
--- /dev/null
+++ b/modules/migration/file_format.go
@@ -0,0 +1,110 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+ "fmt"
+ "os"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/modules/json"
+ "code.gitea.io/gitea/modules/log"
+
+ "github.com/santhosh-tekuri/jsonschema/v6"
+ "gopkg.in/yaml.v3"
+)
+
+// Load project data from file, with optional validation
+func Load(filename string, data any, validation bool) error {
+ isJSON := strings.HasSuffix(filename, ".json")
+
+ bs, err := os.ReadFile(filename)
+ if err != nil {
+ return err
+ }
+
+ if validation {
+ err := validate(bs, data, isJSON)
+ if err != nil {
+ return err
+ }
+ }
+ return unmarshal(bs, data, isJSON)
+}
+
+func unmarshal(bs []byte, data any, isJSON bool) error {
+ if isJSON {
+ return json.Unmarshal(bs, data)
+ }
+ return yaml.Unmarshal(bs, data)
+}
+
+func getSchema(filename string) (*jsonschema.Schema, error) {
+ c := jsonschema.NewCompiler()
+ c.UseLoader(&SchemaLoader{})
+ return c.Compile(filename)
+}
+
+func validate(bs []byte, datatype any, isJSON bool) error {
+ var v any
+ err := unmarshal(bs, &v, isJSON)
+ if err != nil {
+ return err
+ }
+ if !isJSON {
+ v, err = toStringKeys(v)
+ if err != nil {
+ return err
+ }
+ }
+
+ var schemaFilename string
+ switch datatype := datatype.(type) {
+ case *[]*Issue:
+ schemaFilename = "issue.json"
+ case *[]*Milestone:
+ schemaFilename = "milestone.json"
+ default:
+ return fmt.Errorf("file_format:validate: %T has not a validation implemented", datatype)
+ }
+
+ sch, err := getSchema(schemaFilename)
+ if err != nil {
+ return err
+ }
+ err = sch.Validate(v)
+ if err != nil {
+ log.Error("migration validation with %s failed:\n%#v", schemaFilename, err)
+ }
+ return err
+}
+
+func toStringKeys(val any) (any, error) {
+ var err error
+ switch val := val.(type) {
+ case map[string]any:
+ m := make(map[string]any)
+ for k, v := range val {
+ m[k], err = toStringKeys(v)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return m, nil
+ case []any:
+ l := make([]any, len(val))
+ for i, v := range val {
+ l[i], err = toStringKeys(v)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return l, nil
+ case time.Time:
+ return val.Format(time.RFC3339), nil
+ default:
+ return val, nil
+ }
+}
diff --git a/modules/migration/file_format_test.go b/modules/migration/file_format_test.go
new file mode 100644
index 0000000..f6651cd
--- /dev/null
+++ b/modules/migration/file_format_test.go
@@ -0,0 +1,39 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/santhosh-tekuri/jsonschema/v6"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestMigrationJSON_IssueOK(t *testing.T) {
+ issues := make([]*Issue, 0, 10)
+ err := Load("file_format_testdata/issue_a.json", &issues, true)
+ require.NoError(t, err)
+ err = Load("file_format_testdata/issue_a.yml", &issues, true)
+ require.NoError(t, err)
+}
+
+func TestMigrationJSON_IssueFail(t *testing.T) {
+ issues := make([]*Issue, 0, 10)
+ err := Load("file_format_testdata/issue_b.json", &issues, true)
+ if _, ok := err.(*jsonschema.ValidationError); ok {
+ errors := strings.Split(err.(*jsonschema.ValidationError).GoString(), "\n")
+ assert.Contains(t, errors[1], "missing properties")
+ assert.Contains(t, errors[1], "poster_id")
+ } else {
+ t.Fatalf("got: type %T with value %s, want: *jsonschema.ValidationError", err, err)
+ }
+}
+
+func TestMigrationJSON_MilestoneOK(t *testing.T) {
+ milestones := make([]*Milestone, 0, 10)
+ err := Load("file_format_testdata/milestones.json", &milestones, true)
+ require.NoError(t, err)
+}
diff --git a/modules/migration/file_format_testdata/issue_a.json b/modules/migration/file_format_testdata/issue_a.json
new file mode 100644
index 0000000..33d7759
--- /dev/null
+++ b/modules/migration/file_format_testdata/issue_a.json
@@ -0,0 +1,14 @@
+[
+ {
+ "number": 1,
+ "poster_id": 1,
+ "poster_name": "name_a",
+ "title": "title_a",
+ "content": "content_a",
+ "state": "closed",
+ "is_locked": false,
+ "created": "1985-04-12T23:20:50.52Z",
+ "updated": "1986-04-12T23:20:50.52Z",
+ "closed": "1987-04-12T23:20:50.52Z"
+ }
+]
diff --git a/modules/migration/file_format_testdata/issue_a.yml b/modules/migration/file_format_testdata/issue_a.yml
new file mode 100644
index 0000000..d03bfb3
--- /dev/null
+++ b/modules/migration/file_format_testdata/issue_a.yml
@@ -0,0 +1,10 @@
+- number: 1
+ poster_id: 1
+ poster_name: name_a
+ title: title_a
+ content: content_a
+ state: closed
+ is_locked: false
+ created: 2021-05-27T15:24:13+02:00
+ updated: 2021-11-11T10:52:45+01:00
+ closed: 2021-11-11T10:52:45+01:00
diff --git a/modules/migration/file_format_testdata/issue_b.json b/modules/migration/file_format_testdata/issue_b.json
new file mode 100644
index 0000000..2a824d4
--- /dev/null
+++ b/modules/migration/file_format_testdata/issue_b.json
@@ -0,0 +1,5 @@
+[
+ {
+ "number": 1
+ }
+]
diff --git a/modules/migration/file_format_testdata/milestones.json b/modules/migration/file_format_testdata/milestones.json
new file mode 100644
index 0000000..8fb770d
--- /dev/null
+++ b/modules/migration/file_format_testdata/milestones.json
@@ -0,0 +1,20 @@
+[
+ {
+ "title": "title_a",
+ "description": "description_a",
+ "deadline": "1988-04-12T23:20:50.52Z",
+ "created": "1985-04-12T23:20:50.52Z",
+ "updated": "1986-04-12T23:20:50.52Z",
+ "closed": "1987-04-12T23:20:50.52Z",
+ "state": "closed"
+ },
+ {
+ "title": "title_b",
+ "description": "description_b",
+ "deadline": "1998-04-12T23:20:50.52Z",
+ "created": "1995-04-12T23:20:50.52Z",
+ "updated": "1996-04-12T23:20:50.52Z",
+ "closed": null,
+ "state": "open"
+ }
+]
diff --git a/modules/migration/issue.go b/modules/migration/issue.go
new file mode 100644
index 0000000..3d1d1b4
--- /dev/null
+++ b/modules/migration/issue.go
@@ -0,0 +1,48 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "time"
+
+// Issue is a standard issue information
+type Issue struct {
+ Number int64 `json:"number"`
+ PosterID int64 `yaml:"poster_id" json:"poster_id"`
+ PosterName string `yaml:"poster_name" json:"poster_name"`
+ PosterEmail string `yaml:"poster_email" json:"poster_email"`
+ Title string `json:"title"`
+ Content string `json:"content"`
+ Ref string `json:"ref"`
+ Milestone string `json:"milestone"`
+ State string `json:"state"` // closed, open
+ IsLocked bool `yaml:"is_locked" json:"is_locked"`
+ Created time.Time `json:"created"`
+ Updated time.Time `json:"updated"`
+ Closed *time.Time `json:"closed"`
+ Labels []*Label `json:"labels"`
+ Reactions []*Reaction `json:"reactions"`
+ Assignees []string `json:"assignees"`
+ ForeignIndex int64 `json:"foreign_id"`
+ Context DownloaderContext `yaml:"-"`
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (issue *Issue) GetExternalName() string { return issue.PosterName }
+
+// GetExternalID ExternalUserMigrated interface
+func (issue *Issue) GetExternalID() int64 { return issue.PosterID }
+
+func (issue *Issue) GetLocalIndex() int64 { return issue.Number }
+
+func (issue *Issue) GetForeignIndex() int64 {
+ // see the comment of Reviewable.GetForeignIndex
+ // if there is no ForeignIndex, then use LocalIndex
+ if issue.ForeignIndex == 0 {
+ return issue.Number
+ }
+ return issue.ForeignIndex
+}
+
+func (issue *Issue) GetContext() DownloaderContext { return issue.Context }
diff --git a/modules/migration/label.go b/modules/migration/label.go
new file mode 100644
index 0000000..4927be3
--- /dev/null
+++ b/modules/migration/label.go
@@ -0,0 +1,13 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Label defines a standard label information
+type Label struct {
+ Name string `json:"name"`
+ Color string `json:"color"`
+ Description string `json:"description"`
+ Exclusive bool `json:"exclusive"`
+}
diff --git a/modules/migration/messenger.go b/modules/migration/messenger.go
new file mode 100644
index 0000000..6f9cad3
--- /dev/null
+++ b/modules/migration/messenger.go
@@ -0,0 +1,10 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Messenger is a formatting function similar to i18n.TrString
+type Messenger func(key string, args ...any)
+
+// NilMessenger represents an empty formatting function
+func NilMessenger(string, ...any) {}
diff --git a/modules/migration/milestone.go b/modules/migration/milestone.go
new file mode 100644
index 0000000..34355b8
--- /dev/null
+++ b/modules/migration/milestone.go
@@ -0,0 +1,18 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "time"
+
+// Milestone defines a standard milestone
+type Milestone struct {
+ Title string `json:"title"`
+ Description string `json:"description"`
+ Deadline *time.Time `json:"deadline"`
+ Created time.Time `json:"created"`
+ Updated *time.Time `json:"updated"`
+ Closed *time.Time `json:"closed"`
+ State string `json:"state"` // open, closed
+}
diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go
new file mode 100644
index 0000000..e5b6933
--- /dev/null
+++ b/modules/migration/null_downloader.go
@@ -0,0 +1,88 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+ "context"
+ "net/url"
+)
+
+// NullDownloader implements a blank downloader
+type NullDownloader struct{}
+
+var _ Downloader = &NullDownloader{}
+
+// SetContext set context
+func (n NullDownloader) SetContext(_ context.Context) {}
+
+// GetRepoInfo returns a repository information
+func (n NullDownloader) GetRepoInfo() (*Repository, error) {
+ return nil, ErrNotSupported{Entity: "RepoInfo"}
+}
+
+// GetTopics return repository topics
+func (n NullDownloader) GetTopics() ([]string, error) {
+ return nil, ErrNotSupported{Entity: "Topics"}
+}
+
+// GetMilestones returns milestones
+func (n NullDownloader) GetMilestones() ([]*Milestone, error) {
+ return nil, ErrNotSupported{Entity: "Milestones"}
+}
+
+// GetReleases returns releases
+func (n NullDownloader) GetReleases() ([]*Release, error) {
+ return nil, ErrNotSupported{Entity: "Releases"}
+}
+
+// GetLabels returns labels
+func (n NullDownloader) GetLabels() ([]*Label, error) {
+ return nil, ErrNotSupported{Entity: "Labels"}
+}
+
+// GetIssues returns issues according start and limit
+func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
+ return nil, false, ErrNotSupported{Entity: "Issues"}
+}
+
+// GetComments returns comments of an issue or PR
+func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) {
+ return nil, false, ErrNotSupported{Entity: "Comments"}
+}
+
+// GetAllComments returns paginated comments
+func (n NullDownloader) GetAllComments(page, perPage int) ([]*Comment, bool, error) {
+ return nil, false, ErrNotSupported{Entity: "AllComments"}
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
+ return nil, false, ErrNotSupported{Entity: "PullRequests"}
+}
+
+// GetReviews returns pull requests review
+func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) {
+ return nil, ErrNotSupported{Entity: "Reviews"}
+}
+
+// FormatCloneURL add authentication into remote URLs
+func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
+ if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
+ u, err := url.Parse(remoteAddr)
+ if err != nil {
+ return "", err
+ }
+ u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+ if len(opts.AuthToken) > 0 {
+ u.User = url.UserPassword("oauth2", opts.AuthToken)
+ }
+ return u.String(), nil
+ }
+ return remoteAddr, nil
+}
+
+// SupportGetRepoComments return true if it supports get repo comments
+func (n NullDownloader) SupportGetRepoComments() bool {
+ return false
+}
diff --git a/modules/migration/options.go b/modules/migration/options.go
new file mode 100644
index 0000000..234e72c
--- /dev/null
+++ b/modules/migration/options.go
@@ -0,0 +1,41 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "code.gitea.io/gitea/modules/structs"
+
+// MigrateOptions defines the way a repository gets migrated
+// this is for internal usage by migrations module and func who interact with it
+type MigrateOptions struct {
+ // required: true
+ CloneAddr string `json:"clone_addr" binding:"Required"`
+ CloneAddrEncrypted string `json:"clone_addr_encrypted,omitempty"`
+ AuthUsername string `json:"auth_username"`
+ AuthPassword string `json:"-"`
+ AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
+ AuthToken string `json:"-"`
+ AuthTokenEncrypted string `json:"auth_token_encrypted,omitempty"`
+ // required: true
+ UID int `json:"uid" binding:"Required"`
+ // required: true
+ RepoName string `json:"repo_name" binding:"Required"`
+ Mirror bool `json:"mirror"`
+ LFS bool `json:"lfs"`
+ LFSEndpoint string `json:"lfs_endpoint"`
+ Private bool `json:"private"`
+ Description string `json:"description"`
+ OriginalURL string
+ GitServiceType structs.GitServiceType
+ Wiki bool
+ Issues bool
+ Milestones bool
+ Labels bool
+ Releases bool
+ Comments bool
+ PullRequests bool
+ ReleaseAssets bool
+ MigrateToRepoID int64
+ MirrorInterval string `json:"mirror_interval"`
+}
diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go
new file mode 100644
index 0000000..1435991
--- /dev/null
+++ b/modules/migration/pullrequest.go
@@ -0,0 +1,74 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+ "fmt"
+ "time"
+
+ "code.gitea.io/gitea/modules/git"
+)
+
+// PullRequest defines a standard pull request information
+type PullRequest struct {
+ Number int64
+ Title string
+ PosterName string `yaml:"poster_name"`
+ PosterID int64 `yaml:"poster_id"`
+ PosterEmail string `yaml:"poster_email"`
+ Content string
+ Milestone string
+ State string
+ Created time.Time
+ Updated time.Time
+ Closed *time.Time
+ Labels []*Label
+ PatchURL string `yaml:"patch_url"` // SECURITY: This must be safe to download directly from
+ Merged bool
+ MergedTime *time.Time `yaml:"merged_time"`
+ MergeCommitSHA string `yaml:"merge_commit_sha"`
+ Head PullRequestBranch
+ Base PullRequestBranch
+ Assignees []string
+ IsLocked bool `yaml:"is_locked"`
+ Reactions []*Reaction
+ ForeignIndex int64
+ Context DownloaderContext `yaml:"-"`
+ EnsuredSafe bool `yaml:"ensured_safe"`
+}
+
+func (p *PullRequest) GetLocalIndex() int64 { return p.Number }
+func (p *PullRequest) GetForeignIndex() int64 { return p.ForeignIndex }
+func (p *PullRequest) GetContext() DownloaderContext { return p.Context }
+
+// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
+func (p *PullRequest) IsForkPullRequest() bool {
+ return p.Head.RepoFullName() != p.Base.RepoFullName()
+}
+
+// GetGitRefName returns pull request relative path to head
+func (p PullRequest) GetGitRefName() string {
+ return fmt.Sprintf("%s%d/head", git.PullPrefix, p.Number)
+}
+
+// PullRequestBranch represents a pull request branch
+type PullRequestBranch struct {
+ CloneURL string `yaml:"clone_url"` // SECURITY: This must be safe to download from
+ Ref string // SECURITY: this must be a git.IsValidRefPattern
+ SHA string // SECURITY: this must be a git.IsValidSHAPattern
+ RepoName string `yaml:"repo_name"`
+ OwnerName string `yaml:"owner_name"`
+}
+
+// RepoFullName returns pull request repo full name
+func (p PullRequestBranch) RepoFullName() string {
+ return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (p *PullRequest) GetExternalName() string { return p.PosterName }
+
+// ExternalID ExternalUserMigrated interface
+func (p *PullRequest) GetExternalID() int64 { return p.PosterID }
diff --git a/modules/migration/reaction.go b/modules/migration/reaction.go
new file mode 100644
index 0000000..ca1df6c
--- /dev/null
+++ b/modules/migration/reaction.go
@@ -0,0 +1,17 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Reaction represents a reaction to an issue/pr/comment.
+type Reaction struct {
+ UserID int64 `yaml:"user_id" json:"user_id"`
+ UserName string `yaml:"user_name" json:"user_name"`
+ Content string `json:"content"`
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (r *Reaction) GetExternalName() string { return r.UserName }
+
+// GetExternalID ExternalUserMigrated interface
+func (r *Reaction) GetExternalID() int64 { return r.UserID }
diff --git a/modules/migration/release.go b/modules/migration/release.go
new file mode 100644
index 0000000..f92cf25
--- /dev/null
+++ b/modules/migration/release.go
@@ -0,0 +1,46 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+ "io"
+ "time"
+)
+
+// ReleaseAsset represents a release asset
+type ReleaseAsset struct {
+ ID int64
+ Name string
+ ContentType *string `yaml:"content_type"`
+ Size *int
+ DownloadCount *int `yaml:"download_count"`
+ Created time.Time
+ Updated time.Time
+
+ DownloadURL *string `yaml:"download_url"` // SECURITY: It is the responsibility of downloader to make sure this is safe
+ // if DownloadURL is nil, the function should be invoked
+ DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` // SECURITY: It is the responsibility of downloader to make sure this is safe
+}
+
+// Release represents a release
+type Release struct {
+ TagName string `yaml:"tag_name"` // SECURITY: This must pass git.IsValidRefPattern
+ TargetCommitish string `yaml:"target_commitish"` // SECURITY: This must pass git.IsValidRefPattern
+ Name string
+ Body string
+ Draft bool
+ Prerelease bool
+ PublisherID int64 `yaml:"publisher_id"`
+ PublisherName string `yaml:"publisher_name"`
+ PublisherEmail string `yaml:"publisher_email"`
+ Assets []*ReleaseAsset
+ Created time.Time
+ Published time.Time
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (r *Release) GetExternalName() string { return r.PublisherName }
+
+// GetExternalID ExternalUserMigrated interface
+func (r *Release) GetExternalID() int64 { return r.PublisherID }
diff --git a/modules/migration/repo.go b/modules/migration/repo.go
new file mode 100644
index 0000000..22c2cf6
--- /dev/null
+++ b/modules/migration/repo.go
@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Repository defines a standard repository information
+type Repository struct {
+ Name string
+ Owner string
+ IsPrivate bool `yaml:"is_private"`
+ IsMirror bool `yaml:"is_mirror"`
+ Description string
+ CloneURL string `yaml:"clone_url"` // SECURITY: This must be checked to ensure that is safe to be used
+ OriginalURL string `yaml:"original_url"`
+ DefaultBranch string
+}
diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go
new file mode 100644
index 0000000..1cacf5f
--- /dev/null
+++ b/modules/migration/retry_downloader.go
@@ -0,0 +1,194 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+ "context"
+ "time"
+)
+
+var _ Downloader = &RetryDownloader{}
+
+// RetryDownloader retry the downloads
+type RetryDownloader struct {
+ Downloader
+ ctx context.Context
+ RetryTimes int // the total execute times
+ RetryDelay int // time to delay seconds
+}
+
+// NewRetryDownloader creates a retry downloader
+func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
+ return &RetryDownloader{
+ Downloader: downloader,
+ ctx: ctx,
+ RetryTimes: retryTimes,
+ RetryDelay: retryDelay,
+ }
+}
+
+func (d *RetryDownloader) retry(work func() error) error {
+ var (
+ times = d.RetryTimes
+ err error
+ )
+ for ; times > 0; times-- {
+ if err = work(); err == nil {
+ return nil
+ }
+ if IsErrNotSupported(err) {
+ return err
+ }
+ select {
+ case <-d.ctx.Done():
+ return d.ctx.Err()
+ case <-time.After(time.Second * time.Duration(d.RetryDelay)):
+ }
+ }
+ return err
+}
+
+// SetContext set context
+func (d *RetryDownloader) SetContext(ctx context.Context) {
+ d.ctx = ctx
+ d.Downloader.SetContext(ctx)
+}
+
+// GetRepoInfo returns a repository information with retry
+func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
+ var (
+ repo *Repository
+ err error
+ )
+
+ err = d.retry(func() error {
+ repo, err = d.Downloader.GetRepoInfo()
+ return err
+ })
+
+ return repo, err
+}
+
+// GetTopics returns a repository's topics with retry
+func (d *RetryDownloader) GetTopics() ([]string, error) {
+ var (
+ topics []string
+ err error
+ )
+
+ err = d.retry(func() error {
+ topics, err = d.Downloader.GetTopics()
+ return err
+ })
+
+ return topics, err
+}
+
+// GetMilestones returns a repository's milestones with retry
+func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
+ var (
+ milestones []*Milestone
+ err error
+ )
+
+ err = d.retry(func() error {
+ milestones, err = d.Downloader.GetMilestones()
+ return err
+ })
+
+ return milestones, err
+}
+
+// GetReleases returns a repository's releases with retry
+func (d *RetryDownloader) GetReleases() ([]*Release, error) {
+ var (
+ releases []*Release
+ err error
+ )
+
+ err = d.retry(func() error {
+ releases, err = d.Downloader.GetReleases()
+ return err
+ })
+
+ return releases, err
+}
+
+// GetLabels returns a repository's labels with retry
+func (d *RetryDownloader) GetLabels() ([]*Label, error) {
+ var (
+ labels []*Label
+ err error
+ )
+
+ err = d.retry(func() error {
+ labels, err = d.Downloader.GetLabels()
+ return err
+ })
+
+ return labels, err
+}
+
+// GetIssues returns a repository's issues with retry
+func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
+ var (
+ issues []*Issue
+ isEnd bool
+ err error
+ )
+
+ err = d.retry(func() error {
+ issues, isEnd, err = d.Downloader.GetIssues(page, perPage)
+ return err
+ })
+
+ return issues, isEnd, err
+}
+
+// GetComments returns a repository's comments with retry
+func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) {
+ var (
+ comments []*Comment
+ isEnd bool
+ err error
+ )
+
+ err = d.retry(func() error {
+ comments, isEnd, err = d.Downloader.GetComments(commentable)
+ return err
+ })
+
+ return comments, isEnd, err
+}
+
+// GetPullRequests returns a repository's pull requests with retry
+func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
+ var (
+ prs []*PullRequest
+ err error
+ isEnd bool
+ )
+
+ err = d.retry(func() error {
+ prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage)
+ return err
+ })
+
+ return prs, isEnd, err
+}
+
+// GetReviews returns pull requests reviews
+func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) {
+ var (
+ reviews []*Review
+ err error
+ )
+
+ err = d.retry(func() error {
+ reviews, err = d.Downloader.GetReviews(reviewable)
+ return err
+ })
+
+ return reviews, err
+}
diff --git a/modules/migration/review.go b/modules/migration/review.go
new file mode 100644
index 0000000..79e821b
--- /dev/null
+++ b/modules/migration/review.go
@@ -0,0 +1,67 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "time"
+
+// Reviewable can be reviewed
+type Reviewable interface {
+ GetLocalIndex() int64
+
+ // GetForeignIndex presents the foreign index, which could be misused:
+ // For example, if there are 2 Gitea sites: site-A exports a dataset, then site-B imports it:
+ // * if site-A exports files by using its LocalIndex
+ // * from site-A's view, LocalIndex is site-A's IssueIndex while ForeignIndex is site-B's IssueIndex
+ // * but from site-B's view, LocalIndex is site-B's IssueIndex while ForeignIndex is site-A's IssueIndex
+ //
+ // So the exporting/importing must be paired, but the meaning of them looks confusing then:
+ // * either site-A and site-B both use LocalIndex during dumping/restoring
+ // * or site-A and site-B both use ForeignIndex
+ GetForeignIndex() int64
+}
+
+// enumerate all review states
+const (
+ ReviewStatePending = "PENDING"
+ ReviewStateApproved = "APPROVED"
+ ReviewStateChangesRequested = "CHANGES_REQUESTED"
+ ReviewStateCommented = "COMMENTED"
+ ReviewStateRequestReview = "REQUEST_REVIEW"
+)
+
+// Review is a standard review information
+type Review struct {
+ ID int64
+ IssueIndex int64 `yaml:"issue_index"`
+ ReviewerID int64 `yaml:"reviewer_id"`
+ ReviewerName string `yaml:"reviewer_name"`
+ Official bool
+ CommitID string `yaml:"commit_id"`
+ Content string
+ CreatedAt time.Time `yaml:"created_at"`
+ State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
+ Comments []*ReviewComment
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (r *Review) GetExternalName() string { return r.ReviewerName }
+
+// GetExternalID ExternalUserMigrated interface
+func (r *Review) GetExternalID() int64 { return r.ReviewerID }
+
+// ReviewComment represents a review comment
+type ReviewComment struct {
+ ID int64
+ InReplyTo int64 `yaml:"in_reply_to"`
+ Content string
+ TreePath string `yaml:"tree_path"`
+ DiffHunk string `yaml:"diff_hunk"`
+ Position int
+ Line int
+ CommitID string `yaml:"commit_id"`
+ PosterID int64 `yaml:"poster_id"`
+ Reactions []*Reaction
+ CreatedAt time.Time `yaml:"created_at"`
+ UpdatedAt time.Time `yaml:"updated_at"`
+}
diff --git a/modules/migration/schemas/issue.json b/modules/migration/schemas/issue.json
new file mode 100644
index 0000000..25753c3
--- /dev/null
+++ b/modules/migration/schemas/issue.json
@@ -0,0 +1,114 @@
+{
+ "title": "Issue",
+ "description": "Issues associated to a repository within a forge (Gitea, GitLab, etc.).",
+
+ "type": "array",
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "number": {
+ "description": "Unique identifier, relative to the repository.",
+ "type": "number"
+ },
+ "poster_id": {
+ "description": "Unique identifier of the user who authored the issue.",
+ "type": "number"
+ },
+ "poster_name": {
+ "description": "Name of the user who authored the issue.",
+ "type": "string"
+ },
+ "poster_email": {
+ "description": "Email of the user who authored the issue.",
+ "type": "string"
+ },
+ "title": {
+ "description": "Short description displayed as the title.",
+ "type": "string"
+ },
+ "content": {
+ "description": "Long, multiline, description.",
+ "type": "string"
+ },
+ "ref": {
+ "description": "Target branch in the repository.",
+ "type": "string"
+ },
+ "milestone": {
+ "description": "Name of the milestone.",
+ "type": "string"
+ },
+ "state": {
+ "description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.",
+ "enum": [
+ "closed",
+ "open"
+ ]
+ },
+ "is_locked": {
+ "description": "A locked issue can only be modified by privileged users.",
+ "type": "boolean"
+ },
+ "created": {
+ "description": "Creation time.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "updated": {
+ "description": "Last update time.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "closed": {
+ "description": "The last time 'state' changed to 'closed'.",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date-time"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "labels": {
+ "description": "List of labels.",
+ "type": "array",
+ "items": {
+ "$ref": "label.json"
+ }
+ },
+ "reactions": {
+ "description": "List of reactions.",
+ "type": "array",
+ "items": {
+ "$ref": "reaction.json"
+ }
+ },
+ "assignees": {
+ "description": "List of assignees.",
+ "type": "array",
+ "items": {
+ "description": "Name of a user assigned to the issue.",
+ "type": "string"
+ }
+ }
+ },
+ "required": [
+ "number",
+ "poster_id",
+ "poster_name",
+ "title",
+ "content",
+ "state",
+ "is_locked",
+ "created",
+ "updated"
+ ]
+ },
+
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "$id": "http://example.com/issue.json",
+ "$$target": "issue.json"
+}
diff --git a/modules/migration/schemas/label.json b/modules/migration/schemas/label.json
new file mode 100644
index 0000000..561a2e3
--- /dev/null
+++ b/modules/migration/schemas/label.json
@@ -0,0 +1,28 @@
+{
+ "title": "Label",
+ "description": "Label associated to an issue.",
+
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "name": {
+ "description": "Name of the label, unique within the repository.",
+ "type": "string"
+ },
+ "color": {
+ "description": "Color code of the label.",
+ "type": "string"
+ },
+ "description": {
+ "description": "Long, multiline, description.",
+ "type": "string"
+ }
+ },
+ "required": [
+ "name"
+ ],
+
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "$id": "label.json",
+ "$$target": "label.json"
+}
diff --git a/modules/migration/schemas/milestone.json b/modules/migration/schemas/milestone.json
new file mode 100644
index 0000000..7024ef4
--- /dev/null
+++ b/modules/migration/schemas/milestone.json
@@ -0,0 +1,67 @@
+{
+ "title": "Milestone",
+ "description": "Milestone associated to a repository within a forge.",
+
+ "type": "array",
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "title": {
+ "description": "Short description.",
+ "type": "string"
+ },
+ "description": {
+ "description": "Long, multiline, description.",
+ "type": "string"
+ },
+ "deadline": {
+ "description": "Deadline after which the milestone is overdue.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "created": {
+ "description": "Creation time.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "updated": {
+ "description": "Last update time.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "closed": {
+ "description": "The last time 'state' changed to 'closed'.",
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date-time"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "state": {
+ "description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.",
+ "enum": [
+ "closed",
+ "open"
+ ]
+ }
+ },
+ "required": [
+ "title",
+ "description",
+ "deadline",
+ "created",
+ "updated",
+ "closed",
+ "state"
+ ]
+ },
+
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "$id": "http://example.com/milestone.json",
+ "$$target": "milestone.json"
+}
diff --git a/modules/migration/schemas/reaction.json b/modules/migration/schemas/reaction.json
new file mode 100644
index 0000000..2565251
--- /dev/null
+++ b/modules/migration/schemas/reaction.json
@@ -0,0 +1,29 @@
+{
+ "title": "Reaction",
+ "description": "Reaction associated to an issue or a comment.",
+
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "user_id": {
+ "description": "Unique identifier of the user who authored the reaction.",
+ "type": "number"
+ },
+ "user_name": {
+ "description": "Name of the user who authored the reaction.",
+ "type": "string"
+ },
+ "content": {
+ "description": "Representation of the reaction",
+ "type": "string"
+ }
+ },
+ "required": [
+ "user_id",
+ "content"
+ ],
+
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "$id": "http://example.com/reaction.json",
+ "$$target": "reaction.json"
+}
diff --git a/modules/migration/schemas_bindata.go b/modules/migration/schemas_bindata.go
new file mode 100644
index 0000000..c5db3b3
--- /dev/null
+++ b/modules/migration/schemas_bindata.go
@@ -0,0 +1,8 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build bindata
+
+package migration
+
+//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go
diff --git a/modules/migration/schemas_dynamic.go b/modules/migration/schemas_dynamic.go
new file mode 100644
index 0000000..3741691
--- /dev/null
+++ b/modules/migration/schemas_dynamic.go
@@ -0,0 +1,47 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build !bindata
+
+package migration
+
+import (
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+
+ "github.com/santhosh-tekuri/jsonschema/v6"
+)
+
+type SchemaLoader struct{}
+
+func (*SchemaLoader) Load(s string) (any, error) {
+ u, err := url.Parse(s)
+ if err != nil {
+ return nil, err
+ }
+ basename := path.Base(u.Path)
+ filename := basename
+ //
+ // Schema reference each other within the schemas directory but
+ // the tests run in the parent directory.
+ //
+ if _, err := os.Stat(filename); os.IsNotExist(err) {
+ filename = filepath.Join("schemas", basename)
+ //
+ // Integration tests run from the git root directory, not the
+ // directory in which the test source is located.
+ //
+ if _, err := os.Stat(filename); os.IsNotExist(err) {
+ filename = filepath.Join("modules/migration/schemas", basename)
+ }
+ }
+
+ f, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return jsonschema.UnmarshalJSON(f)
+}
diff --git a/modules/migration/schemas_static.go b/modules/migration/schemas_static.go
new file mode 100644
index 0000000..832dfd8
--- /dev/null
+++ b/modules/migration/schemas_static.go
@@ -0,0 +1,23 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build bindata
+
+package migration
+
+import (
+ "path"
+
+ "github.com/santhosh-tekuri/jsonschema/v6"
+)
+
+type SchemaLoader struct{}
+
+func (*SchemaLoader) Load(filename string) (any, error) {
+ f, err := Assets.Open(path.Base(filename))
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return jsonschema.UnmarshalJSON(f)
+}
diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go
new file mode 100644
index 0000000..ff642aa
--- /dev/null
+++ b/modules/migration/uploader.go
@@ -0,0 +1,23 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Uploader uploads all the information of one repository
+type Uploader interface {
+ MaxBatchInsertSize(tp string) int
+ CreateRepo(repo *Repository, opts MigrateOptions) error
+ CreateTopics(topic ...string) error
+ CreateMilestones(milestones ...*Milestone) error
+ CreateReleases(releases ...*Release) error
+ SyncTags() error
+ CreateLabels(labels ...*Label) error
+ CreateIssues(issues ...*Issue) error
+ CreateComments(comments ...*Comment) error
+ CreatePullRequests(prs ...*PullRequest) error
+ CreateReviews(reviews ...*Review) error
+ Rollback() error
+ Finish() error
+ Close()
+}