diff options
Diffstat (limited to 'services/migrations/codebase.go')
| -rw-r--r-- | services/migrations/codebase.go | 651 |
1 files changed, 651 insertions, 0 deletions
diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go new file mode 100644 index 0000000..492fc90 --- /dev/null +++ b/services/migrations/codebase.go @@ -0,0 +1,651 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/modules/log" + base "code.gitea.io/gitea/modules/migration" + "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/structs" +) + +var ( + _ base.Downloader = &CodebaseDownloader{} + _ base.DownloaderFactory = &CodebaseDownloaderFactory{} +) + +func init() { + RegisterDownloaderFactory(&CodebaseDownloaderFactory{}) +} + +// CodebaseDownloaderFactory defines a downloader factory +type CodebaseDownloaderFactory struct{} + +// New returns a downloader related to this factory according MigrateOptions +func (f *CodebaseDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { + u, err := url.Parse(opts.CloneAddr) + if err != nil { + return nil, err + } + u.User = nil + + fields := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(fields) != 2 { + return nil, fmt.Errorf("invalid path: %s", u.Path) + } + project := fields[0] + repoName := strings.TrimSuffix(fields[1], ".git") + + log.Trace("Create Codebase downloader. BaseURL: %v RepoName: %s", u, repoName) + + return NewCodebaseDownloader(ctx, u, project, repoName, opts.AuthUsername, opts.AuthPassword), nil +} + +// GitServiceType returns the type of git service +func (f *CodebaseDownloaderFactory) GitServiceType() structs.GitServiceType { + return structs.CodebaseService +} + +type codebaseUser struct { + ID int64 `json:"id"` + Name string `json:"name"` + Email string `json:"email"` +} + +// CodebaseDownloader implements a Downloader interface to get repository information +// from Codebase +type CodebaseDownloader struct { + base.NullDownloader + ctx context.Context + client *http.Client + baseURL *url.URL + projectURL *url.URL + project string + repoName string + maxIssueIndex int64 + userMap map[int64]*codebaseUser + commitMap map[string]string +} + +// SetContext set context +func (d *CodebaseDownloader) SetContext(ctx context.Context) { + d.ctx = ctx +} + +// NewCodebaseDownloader creates a new downloader +func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader { + baseURL, _ := url.Parse("https://api3.codebasehq.com") + + downloader := &CodebaseDownloader{ + ctx: ctx, + baseURL: baseURL, + projectURL: projectURL, + project: project, + repoName: repoName, + client: &http.Client{ + Transport: &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + if len(username) > 0 && len(password) > 0 { + req.SetBasicAuth(username, password) + } + return proxy.Proxy()(req) + }, + }, + }, + userMap: make(map[int64]*codebaseUser), + commitMap: make(map[string]string), + } + + log.Trace("Create Codebase downloader. BaseURL: %s Project: %s RepoName: %s", baseURL, project, repoName) + return downloader +} + +// String implements Stringer +func (d *CodebaseDownloader) String() string { + return fmt.Sprintf("migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName) +} + +func (d *CodebaseDownloader) LogString() string { + if d == nil { + return "<CodebaseDownloader nil>" + } + return fmt.Sprintf("<CodebaseDownloader %s %s/%s>", d.baseURL, d.project, d.repoName) +} + +// FormatCloneURL add authentication into remote URLs +func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr string) (string, error) { + return opts.CloneAddr, nil +} + +func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]string, result any) error { + u, err := d.baseURL.Parse(endpoint) + if err != nil { + return err + } + + if parameter != nil { + query := u.Query() + for k, v := range parameter { + query.Set(k, v) + } + u.RawQuery = query.Encode() + } + + req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) + if err != nil { + return err + } + req.Header.Add("Accept", "application/xml") + + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + return xml.NewDecoder(resp.Body).Decode(&result) +} + +// GetRepoInfo returns repository information +// https://support.codebasehq.com/kb/projects +func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { + var rawRepository struct { + XMLName xml.Name `xml:"repository"` + Name string `xml:"name"` + Description string `xml:"description"` + Permalink string `xml:"permalink"` + CloneURL string `xml:"clone-url"` + Source string `xml:"source"` + } + + err := d.callAPI( + fmt.Sprintf("/%s/%s", d.project, d.repoName), + nil, + &rawRepository, + ) + if err != nil { + return nil, err + } + + return &base.Repository{ + Name: rawRepository.Name, + Description: rawRepository.Description, + CloneURL: rawRepository.CloneURL, + OriginalURL: d.projectURL.String(), + }, nil +} + +// GetMilestones returns milestones +// https://support.codebasehq.com/kb/tickets-and-milestones/milestones +func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { + var rawMilestones struct { + XMLName xml.Name `xml:"ticketing-milestone"` + Type string `xml:"type,attr"` + TicketingMilestone []struct { + Text string `xml:",chardata"` + ID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"id"` + Identifier string `xml:"identifier"` + Name string `xml:"name"` + Deadline struct { + Value string `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"deadline"` + Description string `xml:"description"` + Status string `xml:"status"` + } `xml:"ticketing-milestone"` + } + + err := d.callAPI( + fmt.Sprintf("/%s/milestones", d.project), + nil, + &rawMilestones, + ) + if err != nil { + return nil, err + } + + milestones := make([]*base.Milestone, 0, len(rawMilestones.TicketingMilestone)) + for _, milestone := range rawMilestones.TicketingMilestone { + var deadline *time.Time + if len(milestone.Deadline.Value) > 0 { + if val, err := time.Parse("2006-01-02", milestone.Deadline.Value); err == nil { + deadline = &val + } + } + + closed := deadline + state := "closed" + if milestone.Status == "active" { + closed = nil + state = "" + } + + milestones = append(milestones, &base.Milestone{ + Title: milestone.Name, + Deadline: deadline, + Closed: closed, + State: state, + }) + } + return milestones, nil +} + +// GetLabels returns labels +// https://support.codebasehq.com/kb/tickets-and-milestones/statuses-priorities-and-categories +func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) { + var rawTypes struct { + XMLName xml.Name `xml:"ticketing-types"` + Type string `xml:"type,attr"` + TicketingType []struct { + ID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"id"` + Name string `xml:"name"` + } `xml:"ticketing-type"` + } + + err := d.callAPI( + fmt.Sprintf("/%s/tickets/types", d.project), + nil, + &rawTypes, + ) + if err != nil { + return nil, err + } + + labels := make([]*base.Label, 0, len(rawTypes.TicketingType)) + for _, label := range rawTypes.TicketingType { + labels = append(labels, &base.Label{ + Name: label.Name, + Color: "ffffff", + }) + } + return labels, nil +} + +type codebaseIssueContext struct { + Comments []*base.Comment +} + +// GetIssues returns issues, limits are not supported +// https://support.codebasehq.com/kb/tickets-and-milestones +// https://support.codebasehq.com/kb/tickets-and-milestones/updating-tickets +func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + var rawIssues struct { + XMLName xml.Name `xml:"tickets"` + Type string `xml:"type,attr"` + Ticket []struct { + TicketID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"ticket-id"` + Summary string `xml:"summary"` + TicketType string `xml:"ticket-type"` + ReporterID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"reporter-id"` + Reporter string `xml:"reporter"` + Type struct { + Name string `xml:"name"` + } `xml:"type"` + Status struct { + TreatAsClosed struct { + Value bool `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"treat-as-closed"` + } `xml:"status"` + Milestone struct { + Name string `xml:"name"` + } `xml:"milestone"` + UpdatedAt struct { + Value time.Time `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"updated-at"` + CreatedAt struct { + Value time.Time `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"created-at"` + } `xml:"ticket"` + } + + err := d.callAPI( + fmt.Sprintf("/%s/tickets", d.project), + nil, + &rawIssues, + ) + if err != nil { + return nil, false, err + } + + issues := make([]*base.Issue, 0, len(rawIssues.Ticket)) + for _, issue := range rawIssues.Ticket { + var notes struct { + XMLName xml.Name `xml:"ticket-notes"` + Type string `xml:"type,attr"` + TicketNote []struct { + Content string `xml:"content"` + CreatedAt struct { + Value time.Time `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"created-at"` + UpdatedAt struct { + Value time.Time `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"updated-at"` + ID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"id"` + UserID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"user-id"` + } `xml:"ticket-note"` + } + err := d.callAPI( + fmt.Sprintf("/%s/tickets/%d/notes", d.project, issue.TicketID.Value), + nil, + ¬es, + ) + if err != nil { + return nil, false, err + } + comments := make([]*base.Comment, 0, len(notes.TicketNote)) + for _, note := range notes.TicketNote { + if len(note.Content) == 0 { + continue + } + poster := d.tryGetUser(note.UserID.Value) + comments = append(comments, &base.Comment{ + IssueIndex: issue.TicketID.Value, + Index: note.ID.Value, + PosterID: poster.ID, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: note.Content, + Created: note.CreatedAt.Value, + Updated: note.UpdatedAt.Value, + }) + } + if len(comments) == 0 { + comments = append(comments, &base.Comment{}) + } + + state := "open" + if issue.Status.TreatAsClosed.Value { + state = "closed" + } + poster := d.tryGetUser(issue.ReporterID.Value) + issues = append(issues, &base.Issue{ + Title: issue.Summary, + Number: issue.TicketID.Value, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: comments[0].Content, + Milestone: issue.Milestone.Name, + State: state, + Created: issue.CreatedAt.Value, + Updated: issue.UpdatedAt.Value, + Labels: []*base.Label{ + {Name: issue.Type.Name}, + }, + ForeignIndex: issue.TicketID.Value, + Context: codebaseIssueContext{ + Comments: comments[1:], + }, + }) + + if d.maxIssueIndex < issue.TicketID.Value { + d.maxIssueIndex = issue.TicketID.Value + } + } + + return issues, true, nil +} + +// GetComments returns comments +func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { + context, ok := commentable.GetContext().(codebaseIssueContext) + if !ok { + return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) + } + + return context.Comments, true, nil +} + +// GetPullRequests returns pull requests +// https://support.codebasehq.com/kb/repositories/merge-requests +func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + var rawMergeRequests struct { + XMLName xml.Name `xml:"merge-requests"` + Type string `xml:"type,attr"` + MergeRequest []struct { + ID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"id"` + } `xml:"merge-request"` + } + + err := d.callAPI( + fmt.Sprintf("/%s/%s/merge_requests", d.project, d.repoName), + map[string]string{ + "query": `"Target Project" is "` + d.repoName + `"`, + "offset": strconv.Itoa((page - 1) * perPage), + "count": strconv.Itoa(perPage), + }, + &rawMergeRequests, + ) + if err != nil { + return nil, false, err + } + + pullRequests := make([]*base.PullRequest, 0, len(rawMergeRequests.MergeRequest)) + for i, mr := range rawMergeRequests.MergeRequest { + var rawMergeRequest struct { + XMLName xml.Name `xml:"merge-request"` + ID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"id"` + SourceRef string `xml:"source-ref"` // NOTE: from the documentation these are actually just branches NOT full refs + TargetRef string `xml:"target-ref"` // NOTE: from the documentation these are actually just branches NOT full refs + Subject string `xml:"subject"` + Status string `xml:"status"` + UserID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"user-id"` + CreatedAt struct { + Value time.Time `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"created-at"` + UpdatedAt struct { + Value time.Time `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"updated-at"` + Comments struct { + Type string `xml:"type,attr"` + Comment []struct { + Content string `xml:"content"` + ID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"id"` + UserID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"user-id"` + Action struct { + Value string `xml:",chardata"` + Nil string `xml:"nil,attr"` + } `xml:"action"` + CreatedAt struct { + Value time.Time `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"created-at"` + } `xml:"comment"` + } `xml:"comments"` + } + err := d.callAPI( + fmt.Sprintf("/%s/%s/merge_requests/%d", d.project, d.repoName, mr.ID.Value), + nil, + &rawMergeRequest, + ) + if err != nil { + return nil, false, err + } + + number := d.maxIssueIndex + int64(i) + 1 + + state := "open" + merged := false + var closeTime *time.Time + var mergedTime *time.Time + if rawMergeRequest.Status != "new" { + state = "closed" + closeTime = &rawMergeRequest.UpdatedAt.Value + } + + comments := make([]*base.Comment, 0, len(rawMergeRequest.Comments.Comment)) + for _, comment := range rawMergeRequest.Comments.Comment { + if len(comment.Content) == 0 { + if comment.Action.Value == "merging" { + merged = true + mergedTime = &comment.CreatedAt.Value + } + continue + } + poster := d.tryGetUser(comment.UserID.Value) + comments = append(comments, &base.Comment{ + IssueIndex: number, + Index: comment.ID.Value, + PosterID: poster.ID, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: comment.Content, + Created: comment.CreatedAt.Value, + Updated: comment.CreatedAt.Value, + }) + } + if len(comments) == 0 { + comments = append(comments, &base.Comment{}) + } + + poster := d.tryGetUser(rawMergeRequest.UserID.Value) + + pullRequests = append(pullRequests, &base.PullRequest{ + Title: rawMergeRequest.Subject, + Number: number, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: comments[0].Content, + State: state, + Created: rawMergeRequest.CreatedAt.Value, + Updated: rawMergeRequest.UpdatedAt.Value, + Closed: closeTime, + Merged: merged, + MergedTime: mergedTime, + Head: base.PullRequestBranch{ + Ref: rawMergeRequest.SourceRef, + SHA: d.getHeadCommit(rawMergeRequest.SourceRef), + RepoName: d.repoName, + }, + Base: base.PullRequestBranch{ + Ref: rawMergeRequest.TargetRef, + SHA: d.getHeadCommit(rawMergeRequest.TargetRef), + RepoName: d.repoName, + }, + ForeignIndex: rawMergeRequest.ID.Value, + Context: codebaseIssueContext{ + Comments: comments[1:], + }, + }) + + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d) + } + + return pullRequests, true, nil +} + +func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { + if len(d.userMap) == 0 { + var rawUsers struct { + XMLName xml.Name `xml:"users"` + Type string `xml:"type,attr"` + User []struct { + EmailAddress string `xml:"email-address"` + ID struct { + Value int64 `xml:",chardata"` + Type string `xml:"type,attr"` + } `xml:"id"` + LastName string `xml:"last-name"` + FirstName string `xml:"first-name"` + Username string `xml:"username"` + } `xml:"user"` + } + + err := d.callAPI( + "/users", + nil, + &rawUsers, + ) + if err == nil { + for _, user := range rawUsers.User { + d.userMap[user.ID.Value] = &codebaseUser{ + Name: user.Username, + Email: user.EmailAddress, + } + } + } + } + + user, ok := d.userMap[userID] + if !ok { + user = &codebaseUser{ + Name: fmt.Sprintf("User %d", userID), + } + d.userMap[userID] = user + } + + return user +} + +func (d *CodebaseDownloader) getHeadCommit(ref string) string { + commitRef, ok := d.commitMap[ref] + if !ok { + var rawCommits struct { + XMLName xml.Name `xml:"commits"` + Type string `xml:"type,attr"` + Commit []struct { + Ref string `xml:"ref"` + } `xml:"commit"` + } + err := d.callAPI( + fmt.Sprintf("/%s/%s/commits/%s", d.project, d.repoName, ref), + nil, + &rawCommits, + ) + if err == nil && len(rawCommits.Commit) > 0 { + commitRef = rawCommits.Commit[0].Ref + d.commitMap[ref] = commitRef + } + } + return commitRef +} |