mirror of https://github.com/go-gitea/gitea
637 lines
16 KiB
Go
637 lines
16 KiB
Go
// Copyright 2021 The Gitea Authors. All rights reserved.
|
|
// Use of this source code is governed by a MIT-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package migrations
|
|
|
|
import (
|
|
"context"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"code.gitea.io/gitea/modules/log"
|
|
base "code.gitea.io/gitea/modules/migration"
|
|
"code.gitea.io/gitea/modules/proxy"
|
|
"code.gitea.io/gitea/modules/structs"
|
|
)
|
|
|
|
var (
|
|
_ base.Downloader = &CodebaseDownloader{}
|
|
_ base.DownloaderFactory = &CodebaseDownloaderFactory{}
|
|
)
|
|
|
|
func init() {
|
|
RegisterDownloaderFactory(&CodebaseDownloaderFactory{})
|
|
}
|
|
|
|
// CodebaseDownloaderFactory defines a downloader factory
|
|
type CodebaseDownloaderFactory struct{}
|
|
|
|
// New returns a downloader related to this factory according MigrateOptions
|
|
func (f *CodebaseDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
|
|
u, err := url.Parse(opts.CloneAddr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
u.User = nil
|
|
|
|
fields := strings.Split(strings.Trim(u.Path, "/"), "/")
|
|
if len(fields) != 2 {
|
|
return nil, fmt.Errorf("invalid path: %s", u.Path)
|
|
}
|
|
project := fields[0]
|
|
repoName := strings.TrimSuffix(fields[1], ".git")
|
|
|
|
log.Trace("Create Codebase downloader. BaseURL: %v RepoName: %s", u, repoName)
|
|
|
|
return NewCodebaseDownloader(ctx, u, project, repoName, opts.AuthUsername, opts.AuthPassword), nil
|
|
}
|
|
|
|
// GitServiceType returns the type of git service
|
|
func (f *CodebaseDownloaderFactory) GitServiceType() structs.GitServiceType {
|
|
return structs.CodebaseService
|
|
}
|
|
|
|
type codebaseUser struct {
|
|
ID int64 `json:"id"`
|
|
Name string `json:"name"`
|
|
Email string `json:"email"`
|
|
}
|
|
|
|
// CodebaseDownloader implements a Downloader interface to get repository information
|
|
// from Codebase
|
|
type CodebaseDownloader struct {
|
|
base.NullDownloader
|
|
ctx context.Context
|
|
client *http.Client
|
|
baseURL *url.URL
|
|
projectURL *url.URL
|
|
project string
|
|
repoName string
|
|
maxIssueIndex int64
|
|
userMap map[int64]*codebaseUser
|
|
commitMap map[string]string
|
|
}
|
|
|
|
// SetContext set context
|
|
func (d *CodebaseDownloader) SetContext(ctx context.Context) {
|
|
d.ctx = ctx
|
|
}
|
|
|
|
// NewCodebaseDownloader creates a new downloader
|
|
func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader {
|
|
baseURL, _ := url.Parse("https://api3.codebasehq.com")
|
|
|
|
downloader := &CodebaseDownloader{
|
|
ctx: ctx,
|
|
baseURL: baseURL,
|
|
projectURL: projectURL,
|
|
project: project,
|
|
repoName: repoName,
|
|
client: &http.Client{
|
|
Transport: &http.Transport{
|
|
Proxy: func(req *http.Request) (*url.URL, error) {
|
|
if len(username) > 0 && len(password) > 0 {
|
|
req.SetBasicAuth(username, password)
|
|
}
|
|
return proxy.Proxy()(req)
|
|
},
|
|
},
|
|
},
|
|
userMap: make(map[int64]*codebaseUser),
|
|
commitMap: make(map[string]string),
|
|
}
|
|
|
|
return downloader
|
|
}
|
|
|
|
// FormatCloneURL add authentication into remote URLs
|
|
func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
|
|
return opts.CloneAddr, nil
|
|
}
|
|
|
|
func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error {
|
|
u, err := d.baseURL.Parse(endpoint)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if parameter != nil {
|
|
query := u.Query()
|
|
for k, v := range parameter {
|
|
query.Set(k, v)
|
|
}
|
|
u.RawQuery = query.Encode()
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Add("Accept", "application/xml")
|
|
|
|
resp, err := d.client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
return xml.NewDecoder(resp.Body).Decode(&result)
|
|
}
|
|
|
|
// GetRepoInfo returns repository information
|
|
// https://support.codebasehq.com/kb/projects
|
|
func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) {
|
|
var rawRepository struct {
|
|
XMLName xml.Name `xml:"repository"`
|
|
Name string `xml:"name"`
|
|
Description string `xml:"description"`
|
|
Permalink string `xml:"permalink"`
|
|
CloneURL string `xml:"clone-url"`
|
|
Source string `xml:"source"`
|
|
}
|
|
|
|
err := d.callAPI(
|
|
fmt.Sprintf("/%s/%s", d.project, d.repoName),
|
|
nil,
|
|
&rawRepository,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &base.Repository{
|
|
Name: rawRepository.Name,
|
|
Description: rawRepository.Description,
|
|
CloneURL: rawRepository.CloneURL,
|
|
OriginalURL: d.projectURL.String(),
|
|
}, nil
|
|
}
|
|
|
|
// GetMilestones returns milestones
|
|
// https://support.codebasehq.com/kb/tickets-and-milestones/milestones
|
|
func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) {
|
|
var rawMilestones struct {
|
|
XMLName xml.Name `xml:"ticketing-milestone"`
|
|
Type string `xml:"type,attr"`
|
|
TicketingMilestone []struct {
|
|
Text string `xml:",chardata"`
|
|
ID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"id"`
|
|
Identifier string `xml:"identifier"`
|
|
Name string `xml:"name"`
|
|
Deadline struct {
|
|
Value string `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"deadline"`
|
|
Description string `xml:"description"`
|
|
Status string `xml:"status"`
|
|
} `xml:"ticketing-milestone"`
|
|
}
|
|
|
|
err := d.callAPI(
|
|
fmt.Sprintf("/%s/milestones", d.project),
|
|
nil,
|
|
&rawMilestones,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
milestones := make([]*base.Milestone, 0, len(rawMilestones.TicketingMilestone))
|
|
for _, milestone := range rawMilestones.TicketingMilestone {
|
|
var deadline *time.Time
|
|
if len(milestone.Deadline.Value) > 0 {
|
|
if val, err := time.Parse("2006-01-02", milestone.Deadline.Value); err == nil {
|
|
deadline = &val
|
|
}
|
|
}
|
|
|
|
closed := deadline
|
|
state := "closed"
|
|
if milestone.Status == "active" {
|
|
closed = nil
|
|
state = ""
|
|
}
|
|
|
|
milestones = append(milestones, &base.Milestone{
|
|
Title: milestone.Name,
|
|
Deadline: deadline,
|
|
Closed: closed,
|
|
State: state,
|
|
})
|
|
}
|
|
return milestones, nil
|
|
}
|
|
|
|
// GetLabels returns labels
|
|
// https://support.codebasehq.com/kb/tickets-and-milestones/statuses-priorities-and-categories
|
|
func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) {
|
|
var rawTypes struct {
|
|
XMLName xml.Name `xml:"ticketing-types"`
|
|
Type string `xml:"type,attr"`
|
|
TicketingType []struct {
|
|
ID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"id"`
|
|
Name string `xml:"name"`
|
|
} `xml:"ticketing-type"`
|
|
}
|
|
|
|
err := d.callAPI(
|
|
fmt.Sprintf("/%s/tickets/types", d.project),
|
|
nil,
|
|
&rawTypes,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
labels := make([]*base.Label, 0, len(rawTypes.TicketingType))
|
|
for _, label := range rawTypes.TicketingType {
|
|
labels = append(labels, &base.Label{
|
|
Name: label.Name,
|
|
Color: "ffffff",
|
|
})
|
|
}
|
|
return labels, nil
|
|
}
|
|
|
|
type codebaseIssueContext struct {
|
|
Comments []*base.Comment
|
|
}
|
|
|
|
// GetIssues returns issues, limits are not supported
|
|
// https://support.codebasehq.com/kb/tickets-and-milestones
|
|
// https://support.codebasehq.com/kb/tickets-and-milestones/updating-tickets
|
|
func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
|
|
var rawIssues struct {
|
|
XMLName xml.Name `xml:"tickets"`
|
|
Type string `xml:"type,attr"`
|
|
Ticket []struct {
|
|
TicketID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"ticket-id"`
|
|
Summary string `xml:"summary"`
|
|
TicketType string `xml:"ticket-type"`
|
|
ReporterID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"reporter-id"`
|
|
Reporter string `xml:"reporter"`
|
|
Type struct {
|
|
Name string `xml:"name"`
|
|
} `xml:"type"`
|
|
Status struct {
|
|
TreatAsClosed struct {
|
|
Value bool `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"treat-as-closed"`
|
|
} `xml:"status"`
|
|
Milestone struct {
|
|
Name string `xml:"name"`
|
|
} `xml:"milestone"`
|
|
UpdatedAt struct {
|
|
Value time.Time `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"updated-at"`
|
|
CreatedAt struct {
|
|
Value time.Time `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"created-at"`
|
|
} `xml:"ticket"`
|
|
}
|
|
|
|
err := d.callAPI(
|
|
fmt.Sprintf("/%s/tickets", d.project),
|
|
nil,
|
|
&rawIssues,
|
|
)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
issues := make([]*base.Issue, 0, len(rawIssues.Ticket))
|
|
for _, issue := range rawIssues.Ticket {
|
|
var notes struct {
|
|
XMLName xml.Name `xml:"ticket-notes"`
|
|
Type string `xml:"type,attr"`
|
|
TicketNote []struct {
|
|
Content string `xml:"content"`
|
|
CreatedAt struct {
|
|
Value time.Time `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"created-at"`
|
|
UpdatedAt struct {
|
|
Value time.Time `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"updated-at"`
|
|
ID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"id"`
|
|
UserID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"user-id"`
|
|
} `xml:"ticket-note"`
|
|
}
|
|
err := d.callAPI(
|
|
fmt.Sprintf("/%s/tickets/%d/notes", d.project, issue.TicketID.Value),
|
|
nil,
|
|
¬es,
|
|
)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
comments := make([]*base.Comment, 0, len(notes.TicketNote))
|
|
for _, note := range notes.TicketNote {
|
|
if len(note.Content) == 0 {
|
|
continue
|
|
}
|
|
poster := d.tryGetUser(note.UserID.Value)
|
|
comments = append(comments, &base.Comment{
|
|
IssueIndex: issue.TicketID.Value,
|
|
Index: note.ID.Value,
|
|
PosterID: poster.ID,
|
|
PosterName: poster.Name,
|
|
PosterEmail: poster.Email,
|
|
Content: note.Content,
|
|
Created: note.CreatedAt.Value,
|
|
Updated: note.UpdatedAt.Value,
|
|
})
|
|
}
|
|
if len(comments) == 0 {
|
|
comments = append(comments, &base.Comment{})
|
|
}
|
|
|
|
state := "open"
|
|
if issue.Status.TreatAsClosed.Value {
|
|
state = "closed"
|
|
}
|
|
poster := d.tryGetUser(issue.ReporterID.Value)
|
|
issues = append(issues, &base.Issue{
|
|
Title: issue.Summary,
|
|
Number: issue.TicketID.Value,
|
|
PosterName: poster.Name,
|
|
PosterEmail: poster.Email,
|
|
Content: comments[0].Content,
|
|
Milestone: issue.Milestone.Name,
|
|
State: state,
|
|
Created: issue.CreatedAt.Value,
|
|
Updated: issue.UpdatedAt.Value,
|
|
Labels: []*base.Label{
|
|
{Name: issue.Type.Name},
|
|
},
|
|
ForeignIndex: issue.TicketID.Value,
|
|
Context: codebaseIssueContext{
|
|
Comments: comments[1:],
|
|
},
|
|
})
|
|
|
|
if d.maxIssueIndex < issue.TicketID.Value {
|
|
d.maxIssueIndex = issue.TicketID.Value
|
|
}
|
|
}
|
|
|
|
return issues, true, nil
|
|
}
|
|
|
|
// GetComments returns comments
|
|
func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
|
|
context, ok := commentable.GetContext().(codebaseIssueContext)
|
|
if !ok {
|
|
return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())
|
|
}
|
|
|
|
return context.Comments, true, nil
|
|
}
|
|
|
|
// GetPullRequests returns pull requests
|
|
// https://support.codebasehq.com/kb/repositories/merge-requests
|
|
func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
|
|
var rawMergeRequests struct {
|
|
XMLName xml.Name `xml:"merge-requests"`
|
|
Type string `xml:"type,attr"`
|
|
MergeRequest []struct {
|
|
ID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"id"`
|
|
} `xml:"merge-request"`
|
|
}
|
|
|
|
err := d.callAPI(
|
|
fmt.Sprintf("/%s/%s/merge_requests", d.project, d.repoName),
|
|
map[string]string{
|
|
"query": `"Target Project" is "` + d.repoName + `"`,
|
|
"offset": strconv.Itoa((page - 1) * perPage),
|
|
"count": strconv.Itoa(perPage),
|
|
},
|
|
&rawMergeRequests,
|
|
)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
pullRequests := make([]*base.PullRequest, 0, len(rawMergeRequests.MergeRequest))
|
|
for i, mr := range rawMergeRequests.MergeRequest {
|
|
var rawMergeRequest struct {
|
|
XMLName xml.Name `xml:"merge-request"`
|
|
ID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"id"`
|
|
SourceRef string `xml:"source-ref"`
|
|
TargetRef string `xml:"target-ref"`
|
|
Subject string `xml:"subject"`
|
|
Status string `xml:"status"`
|
|
UserID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"user-id"`
|
|
CreatedAt struct {
|
|
Value time.Time `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"created-at"`
|
|
UpdatedAt struct {
|
|
Value time.Time `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"updated-at"`
|
|
Comments struct {
|
|
Type string `xml:"type,attr"`
|
|
Comment []struct {
|
|
Content string `xml:"content"`
|
|
ID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"id"`
|
|
UserID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"user-id"`
|
|
Action struct {
|
|
Value string `xml:",chardata"`
|
|
Nil string `xml:"nil,attr"`
|
|
} `xml:"action"`
|
|
CreatedAt struct {
|
|
Value time.Time `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"created-at"`
|
|
} `xml:"comment"`
|
|
} `xml:"comments"`
|
|
}
|
|
err := d.callAPI(
|
|
fmt.Sprintf("/%s/%s/merge_requests/%d", d.project, d.repoName, mr.ID.Value),
|
|
nil,
|
|
&rawMergeRequest,
|
|
)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
number := d.maxIssueIndex + int64(i) + 1
|
|
|
|
state := "open"
|
|
merged := false
|
|
var closeTime *time.Time
|
|
var mergedTime *time.Time
|
|
if rawMergeRequest.Status != "new" {
|
|
state = "closed"
|
|
closeTime = &rawMergeRequest.UpdatedAt.Value
|
|
}
|
|
|
|
comments := make([]*base.Comment, 0, len(rawMergeRequest.Comments.Comment))
|
|
for _, comment := range rawMergeRequest.Comments.Comment {
|
|
if len(comment.Content) == 0 {
|
|
if comment.Action.Value == "merging" {
|
|
merged = true
|
|
mergedTime = &comment.CreatedAt.Value
|
|
}
|
|
continue
|
|
}
|
|
poster := d.tryGetUser(comment.UserID.Value)
|
|
comments = append(comments, &base.Comment{
|
|
IssueIndex: number,
|
|
Index: comment.ID.Value,
|
|
PosterID: poster.ID,
|
|
PosterName: poster.Name,
|
|
PosterEmail: poster.Email,
|
|
Content: comment.Content,
|
|
Created: comment.CreatedAt.Value,
|
|
Updated: comment.CreatedAt.Value,
|
|
})
|
|
}
|
|
if len(comments) == 0 {
|
|
comments = append(comments, &base.Comment{})
|
|
}
|
|
|
|
poster := d.tryGetUser(rawMergeRequest.UserID.Value)
|
|
|
|
pullRequests = append(pullRequests, &base.PullRequest{
|
|
Title: rawMergeRequest.Subject,
|
|
Number: number,
|
|
PosterName: poster.Name,
|
|
PosterEmail: poster.Email,
|
|
Content: comments[0].Content,
|
|
State: state,
|
|
Created: rawMergeRequest.CreatedAt.Value,
|
|
Updated: rawMergeRequest.UpdatedAt.Value,
|
|
Closed: closeTime,
|
|
Merged: merged,
|
|
MergedTime: mergedTime,
|
|
Head: base.PullRequestBranch{
|
|
Ref: rawMergeRequest.SourceRef,
|
|
SHA: d.getHeadCommit(rawMergeRequest.SourceRef),
|
|
RepoName: d.repoName,
|
|
},
|
|
Base: base.PullRequestBranch{
|
|
Ref: rawMergeRequest.TargetRef,
|
|
SHA: d.getHeadCommit(rawMergeRequest.TargetRef),
|
|
RepoName: d.repoName,
|
|
},
|
|
ForeignIndex: rawMergeRequest.ID.Value,
|
|
Context: codebaseIssueContext{
|
|
Comments: comments[1:],
|
|
},
|
|
})
|
|
}
|
|
|
|
return pullRequests, true, nil
|
|
}
|
|
|
|
func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser {
|
|
if len(d.userMap) == 0 {
|
|
var rawUsers struct {
|
|
XMLName xml.Name `xml:"users"`
|
|
Type string `xml:"type,attr"`
|
|
User []struct {
|
|
EmailAddress string `xml:"email-address"`
|
|
ID struct {
|
|
Value int64 `xml:",chardata"`
|
|
Type string `xml:"type,attr"`
|
|
} `xml:"id"`
|
|
LastName string `xml:"last-name"`
|
|
FirstName string `xml:"first-name"`
|
|
Username string `xml:"username"`
|
|
} `xml:"user"`
|
|
}
|
|
|
|
err := d.callAPI(
|
|
"/users",
|
|
nil,
|
|
&rawUsers,
|
|
)
|
|
if err == nil {
|
|
for _, user := range rawUsers.User {
|
|
d.userMap[user.ID.Value] = &codebaseUser{
|
|
Name: user.Username,
|
|
Email: user.EmailAddress,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
user, ok := d.userMap[userID]
|
|
if !ok {
|
|
user = &codebaseUser{
|
|
Name: fmt.Sprintf("User %d", userID),
|
|
}
|
|
d.userMap[userID] = user
|
|
}
|
|
|
|
return user
|
|
}
|
|
|
|
func (d *CodebaseDownloader) getHeadCommit(ref string) string {
|
|
commitRef, ok := d.commitMap[ref]
|
|
if !ok {
|
|
var rawCommits struct {
|
|
XMLName xml.Name `xml:"commits"`
|
|
Type string `xml:"type,attr"`
|
|
Commit []struct {
|
|
Ref string `xml:"ref"`
|
|
} `xml:"commit"`
|
|
}
|
|
err := d.callAPI(
|
|
fmt.Sprintf("/%s/%s/commits/%s", d.project, d.repoName, ref),
|
|
nil,
|
|
&rawCommits,
|
|
)
|
|
if err == nil && len(rawCommits.Commit) > 0 {
|
|
commitRef = rawCommits.Commit[0].Ref
|
|
d.commitMap[ref] = commitRef
|
|
}
|
|
}
|
|
return commitRef
|
|
}
|