From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- services/migrations/onedev.go | 634 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 634 insertions(+) create mode 100644 services/migrations/onedev.go (limited to 'services/migrations/onedev.go') diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go new file mode 100644 index 0000000..e2f7b77 --- /dev/null +++ b/services/migrations/onedev.go @@ -0,0 +1,634 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + base "code.gitea.io/gitea/modules/migration" + "code.gitea.io/gitea/modules/structs" +) + +var ( + _ base.Downloader = &OneDevDownloader{} + _ base.DownloaderFactory = &OneDevDownloaderFactory{} +) + +func init() { + RegisterDownloaderFactory(&OneDevDownloaderFactory{}) +} + +// OneDevDownloaderFactory defines a downloader factory +type OneDevDownloaderFactory struct{} + +// New returns a downloader related to this factory according MigrateOptions +func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { + u, err := url.Parse(opts.CloneAddr) + if err != nil { + return nil, err + } + + var repoName string + + fields := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(fields) == 2 && fields[0] == "projects" { + repoName = fields[1] + } else if len(fields) == 1 { + repoName = fields[0] + } else { + return nil, fmt.Errorf("invalid path: %s", u.Path) + } + + u.Path = "" + u.Fragment = "" + + log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName) + + return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil +} + +// GitServiceType returns the type of git service +func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType { + return structs.OneDevService +} + +type onedevUser struct { + ID int64 `json:"id"` + Name string `json:"name"` + Email string `json:"email"` +} + +// OneDevDownloader implements a Downloader interface to get repository information +// from OneDev +type OneDevDownloader struct { + base.NullDownloader + ctx context.Context + client *http.Client + baseURL *url.URL + repoName string + repoID int64 + maxIssueIndex int64 + userMap map[int64]*onedevUser + milestoneMap map[int64]string +} + +// SetContext set context +func (d *OneDevDownloader) SetContext(ctx context.Context) { + d.ctx = ctx +} + +// NewOneDevDownloader creates a new downloader +func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { + downloader := &OneDevDownloader{ + ctx: ctx, + baseURL: baseURL, + repoName: repoName, + client: &http.Client{ + Transport: &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + if len(username) > 0 && len(password) > 0 { + req.SetBasicAuth(username, password) + } + return nil, nil + }, + }, + }, + userMap: make(map[int64]*onedevUser), + milestoneMap: make(map[int64]string), + } + + return downloader +} + +// String implements Stringer +func (d *OneDevDownloader) String() string { + return fmt.Sprintf("migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName) +} + +func (d *OneDevDownloader) LogString() string { + if d == nil { + return "" + } + return fmt.Sprintf("", d.baseURL, d.repoID, d.repoName) +} + +func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result any) error { + u, err := d.baseURL.Parse(endpoint) + if err != nil { + return err + } + + if parameter != nil { + query := u.Query() + for k, v := range parameter { + query.Set(k, v) + } + u.RawQuery = query.Encode() + } + + req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) + if err != nil { + return err + } + + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + decoder := json.NewDecoder(resp.Body) + return decoder.Decode(&result) +} + +// GetRepoInfo returns repository information +func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { + info := make([]struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + }, 0, 1) + + err := d.callAPI( + "/api/projects", + map[string]string{ + "query": `"Name" is "` + d.repoName + `"`, + "offset": "0", + "count": "1", + }, + &info, + ) + if err != nil { + return nil, err + } + if len(info) != 1 { + return nil, fmt.Errorf("Project %s not found", d.repoName) + } + + d.repoID = info[0].ID + + cloneURL, err := d.baseURL.Parse(info[0].Name) + if err != nil { + return nil, err + } + originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name) + if err != nil { + return nil, err + } + + return &base.Repository{ + Name: info[0].Name, + Description: info[0].Description, + CloneURL: cloneURL.String(), + OriginalURL: originalURL.String(), + }, nil +} + +// GetMilestones returns milestones +func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { + rawMilestones := make([]struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DueDate *time.Time `json:"dueDate"` + Closed bool `json:"closed"` + }, 0, 100) + + endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID) + + milestones := make([]*base.Milestone, 0, 100) + offset := 0 + for { + err := d.callAPI( + endpoint, + map[string]string{ + "offset": strconv.Itoa(offset), + "count": "100", + }, + &rawMilestones, + ) + if err != nil { + return nil, err + } + if len(rawMilestones) == 0 { + break + } + offset += 100 + + for _, milestone := range rawMilestones { + d.milestoneMap[milestone.ID] = milestone.Name + closed := milestone.DueDate + if !milestone.Closed { + closed = nil + } + + milestones = append(milestones, &base.Milestone{ + Title: milestone.Name, + Description: milestone.Description, + Deadline: milestone.DueDate, + Closed: closed, + }) + } + } + return milestones, nil +} + +// GetLabels returns labels +func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { + return []*base.Label{ + { + Name: "Bug", + Color: "f64e60", + }, + { + Name: "Build Failure", + Color: "f64e60", + }, + { + Name: "Discussion", + Color: "8950fc", + }, + { + Name: "Improvement", + Color: "1bc5bd", + }, + { + Name: "New Feature", + Color: "1bc5bd", + }, + { + Name: "Support Request", + Color: "8950fc", + }, + }, nil +} + +type onedevIssueContext struct { + IsPullRequest bool +} + +// GetIssues returns issues +func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + rawIssues := make([]struct { + ID int64 `json:"id"` + Number int64 `json:"number"` + State string `json:"state"` + Title string `json:"title"` + Description string `json:"description"` + SubmitterID int64 `json:"submitterId"` + SubmitDate time.Time `json:"submitDate"` + }, 0, perPage) + + err := d.callAPI( + "/api/issues", + map[string]string{ + "query": `"Project" is "` + d.repoName + `"`, + "offset": strconv.Itoa((page - 1) * perPage), + "count": strconv.Itoa(perPage), + }, + &rawIssues, + ) + if err != nil { + return nil, false, err + } + + issues := make([]*base.Issue, 0, len(rawIssues)) + for _, issue := range rawIssues { + fields := make([]struct { + Name string `json:"name"` + Value string `json:"value"` + }, 0, 10) + err := d.callAPI( + fmt.Sprintf("/api/issues/%d/fields", issue.ID), + nil, + &fields, + ) + if err != nil { + return nil, false, err + } + + var label *base.Label + for _, field := range fields { + if field.Name == "Type" { + label = &base.Label{Name: field.Value} + break + } + } + + milestones := make([]struct { + ID int64 `json:"id"` + Name string `json:"name"` + }, 0, 10) + err = d.callAPI( + fmt.Sprintf("/api/issues/%d/milestones", issue.ID), + nil, + &milestones, + ) + if err != nil { + return nil, false, err + } + milestoneID := int64(0) + if len(milestones) > 0 { + milestoneID = milestones[0].ID + } + + state := strings.ToLower(issue.State) + if state == "released" { + state = "closed" + } + poster := d.tryGetUser(issue.SubmitterID) + issues = append(issues, &base.Issue{ + Title: issue.Title, + Number: issue.Number, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: issue.Description, + Milestone: d.milestoneMap[milestoneID], + State: state, + Created: issue.SubmitDate, + Updated: issue.SubmitDate, + Labels: []*base.Label{label}, + ForeignIndex: issue.ID, + Context: onedevIssueContext{IsPullRequest: false}, + }) + + if d.maxIssueIndex < issue.Number { + d.maxIssueIndex = issue.Number + } + } + + return issues, len(issues) == 0, nil +} + +// GetComments returns comments +func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { + context, ok := commentable.GetContext().(onedevIssueContext) + if !ok { + return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) + } + + rawComments := make([]struct { + ID int64 `json:"id"` + Date time.Time `json:"date"` + UserID int64 `json:"userId"` + Content string `json:"content"` + }, 0, 100) + + var endpoint string + if context.IsPullRequest { + endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", commentable.GetForeignIndex()) + } else { + endpoint = fmt.Sprintf("/api/issues/%d/comments", commentable.GetForeignIndex()) + } + + err := d.callAPI( + endpoint, + nil, + &rawComments, + ) + if err != nil { + return nil, false, err + } + + rawChanges := make([]struct { + Date time.Time `json:"date"` + UserID int64 `json:"userId"` + Data map[string]any `json:"data"` + }, 0, 100) + + if context.IsPullRequest { + endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", commentable.GetForeignIndex()) + } else { + endpoint = fmt.Sprintf("/api/issues/%d/changes", commentable.GetForeignIndex()) + } + + err = d.callAPI( + endpoint, + nil, + &rawChanges, + ) + if err != nil { + return nil, false, err + } + + comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges)) + for _, comment := range rawComments { + if len(comment.Content) == 0 { + continue + } + poster := d.tryGetUser(comment.UserID) + comments = append(comments, &base.Comment{ + IssueIndex: commentable.GetLocalIndex(), + Index: comment.ID, + PosterID: poster.ID, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: comment.Content, + Created: comment.Date, + Updated: comment.Date, + }) + } + for _, change := range rawChanges { + contentV, ok := change.Data["content"] + if !ok { + contentV, ok = change.Data["comment"] + if !ok { + continue + } + } + content, ok := contentV.(string) + if !ok || len(content) == 0 { + continue + } + + poster := d.tryGetUser(change.UserID) + comments = append(comments, &base.Comment{ + IssueIndex: commentable.GetLocalIndex(), + PosterID: poster.ID, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: content, + Created: change.Date, + Updated: change.Date, + }) + } + + return comments, true, nil +} + +// GetPullRequests returns pull requests +func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + rawPullRequests := make([]struct { + ID int64 `json:"id"` + Number int64 `json:"number"` + Title string `json:"title"` + SubmitterID int64 `json:"submitterId"` + SubmitDate time.Time `json:"submitDate"` + Description string `json:"description"` + TargetBranch string `json:"targetBranch"` + SourceBranch string `json:"sourceBranch"` + BaseCommitHash string `json:"baseCommitHash"` + CloseInfo *struct { + Date *time.Time `json:"date"` + Status string `json:"status"` + } + }, 0, perPage) + + err := d.callAPI( + "/api/pull-requests", + map[string]string{ + "query": `"Target Project" is "` + d.repoName + `"`, + "offset": strconv.Itoa((page - 1) * perPage), + "count": strconv.Itoa(perPage), + }, + &rawPullRequests, + ) + if err != nil { + return nil, false, err + } + + pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests)) + for _, pr := range rawPullRequests { + var mergePreview struct { + TargetHeadCommitHash string `json:"targetHeadCommitHash"` + HeadCommitHash string `json:"headCommitHash"` + MergeStrategy string `json:"mergeStrategy"` + MergeCommitHash string `json:"mergeCommitHash"` + } + err := d.callAPI( + fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID), + nil, + &mergePreview, + ) + if err != nil { + return nil, false, err + } + + state := "open" + merged := false + var closeTime *time.Time + var mergedTime *time.Time + if pr.CloseInfo != nil { + state = "closed" + closeTime = pr.CloseInfo.Date + if pr.CloseInfo.Status == "MERGED" { // "DISCARDED" + merged = true + mergedTime = pr.CloseInfo.Date + } + } + poster := d.tryGetUser(pr.SubmitterID) + + number := pr.Number + d.maxIssueIndex + pullRequests = append(pullRequests, &base.PullRequest{ + Title: pr.Title, + Number: number, + PosterName: poster.Name, + PosterID: poster.ID, + Content: pr.Description, + State: state, + Created: pr.SubmitDate, + Updated: pr.SubmitDate, + Closed: closeTime, + Merged: merged, + MergedTime: mergedTime, + Head: base.PullRequestBranch{ + Ref: pr.SourceBranch, + SHA: mergePreview.HeadCommitHash, + RepoName: d.repoName, + }, + Base: base.PullRequestBranch{ + Ref: pr.TargetBranch, + SHA: mergePreview.TargetHeadCommitHash, + RepoName: d.repoName, + }, + ForeignIndex: pr.ID, + Context: onedevIssueContext{IsPullRequest: true}, + }) + + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d) + } + + return pullRequests, len(pullRequests) == 0, nil +} + +// GetReviews returns pull requests reviews +func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { + rawReviews := make([]struct { + ID int64 `json:"id"` + UserID int64 `json:"userId"` + Result *struct { + Commit string `json:"commit"` + Approved bool `json:"approved"` + Comment string `json:"comment"` + } + }, 0, 100) + + err := d.callAPI( + fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()), + nil, + &rawReviews, + ) + if err != nil { + return nil, err + } + + reviews := make([]*base.Review, 0, len(rawReviews)) + for _, review := range rawReviews { + state := base.ReviewStatePending + content := "" + if review.Result != nil { + if len(review.Result.Comment) > 0 { + state = base.ReviewStateCommented + content = review.Result.Comment + } + if review.Result.Approved { + state = base.ReviewStateApproved + } + } + + poster := d.tryGetUser(review.UserID) + reviews = append(reviews, &base.Review{ + IssueIndex: reviewable.GetLocalIndex(), + ReviewerID: poster.ID, + ReviewerName: poster.Name, + Content: content, + State: state, + }) + } + + return reviews, nil +} + +// GetTopics return repository topics +func (d *OneDevDownloader) GetTopics() ([]string, error) { + return []string{}, nil +} + +func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser { + user, ok := d.userMap[userID] + if !ok { + err := d.callAPI( + fmt.Sprintf("/api/users/%d", userID), + nil, + &user, + ) + if err != nil { + user = &onedevUser{ + Name: fmt.Sprintf("User %d", userID), + } + } + d.userMap[userID] = user + } + + return user +} -- cgit v1.2.3