1
1
mirror of https://github.com/go-gitea/gitea synced 2024-09-21 11:16:03 +00:00
gitea/services/migrations/codecommit.go
Zettat123 def1c9670b
Support migration from AWS CodeCommit (#31981)
This PR adds support for migrating repos from [AWS
CodeCommit](https://docs.aws.amazon.com/codecommit/latest/userguide/welcome.html).

The access key ID and secret access key are required to get repository
information and pull requests. And [HTTPS Git
credentials](https://docs.aws.amazon.com/codecommit/latest/userguide/setting-up-gc.html)
are required to clone the repository.

<img
src="https://github.com/user-attachments/assets/82ecb2d0-8d43-42b0-b5af-f5347a13b9d0"
width="680" />

The AWS CodeCommit icon is from [AWS Architecture
Icons](https://aws.amazon.com/architecture/icons/).

<img
src="https://github.com/user-attachments/assets/3c44d21f-d753-40f5-9eae-5d3589e0d50d"
width="320" />
2024-09-11 07:49:42 +08:00

270 lines
7.4 KiB
Go

// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"context"
"fmt"
"net/url"
"strconv"
"strings"
git_module "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/structs"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/codecommit"
"github.com/aws/aws-sdk-go-v2/service/codecommit/types"
"github.com/aws/aws-sdk-go/aws"
)
var (
_ base.Downloader = &CodeCommitDownloader{}
_ base.DownloaderFactory = &CodeCommitDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&CodeCommitDownloaderFactory{})
}
// CodeCommitDownloaderFactory defines a codecommit downloader factory
type CodeCommitDownloaderFactory struct{}
// New returns a Downloader related to this factory according MigrateOptions
func (c *CodeCommitDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
hostElems := strings.Split(u.Host, ".")
if len(hostElems) != 4 {
return nil, fmt.Errorf("cannot get the region from clone URL")
}
region := hostElems[1]
pathElems := strings.Split(u.Path, "/")
if len(pathElems) == 0 {
return nil, fmt.Errorf("cannot get the repo name from clone URL")
}
repoName := pathElems[len(pathElems)-1]
baseURL := u.Scheme + "://" + u.Host
return NewCodeCommitDownloader(ctx, repoName, baseURL, opts.AWSAccessKeyID, opts.AWSSecretAccessKey, region), nil
}
// GitServiceType returns the type of git service
func (c *CodeCommitDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.CodeCommitService
}
func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader {
downloader := CodeCommitDownloader{
ctx: ctx,
repoName: repoName,
baseURL: baseURL,
codeCommitClient: codecommit.New(codecommit.Options{
Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""),
Region: region,
}),
}
return &downloader
}
// CodeCommitDownloader implements a downloader for AWS CodeCommit
type CodeCommitDownloader struct {
base.NullDownloader
ctx context.Context
codeCommitClient *codecommit.Client
repoName string
baseURL string
allPullRequestIDs []string
}
// SetContext set context
func (c *CodeCommitDownloader) SetContext(ctx context.Context) {
c.ctx = ctx
}
// GetRepoInfo returns a repository information
func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) {
output, err := c.codeCommitClient.GetRepository(c.ctx, &codecommit.GetRepositoryInput{
RepositoryName: aws.String(c.repoName),
})
if err != nil {
return nil, err
}
repoMeta := output.RepositoryMetadata
repo := &base.Repository{
Name: *repoMeta.RepositoryName,
Owner: *repoMeta.AccountId,
IsPrivate: true, // CodeCommit repos are always private
CloneURL: *repoMeta.CloneUrlHttp,
}
if repoMeta.DefaultBranch != nil {
repo.DefaultBranch = *repoMeta.DefaultBranch
}
if repoMeta.RepositoryDescription != nil {
repo.DefaultBranch = *repoMeta.RepositoryDescription
}
return repo, nil
}
// GetComments returns comments of an issue or PR
func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
var (
nextToken *string
comments []*base.Comment
)
for {
resp, err := c.codeCommitClient.GetCommentsForPullRequest(c.ctx, &codecommit.GetCommentsForPullRequestInput{
NextToken: nextToken,
PullRequestId: aws.String(strconv.FormatInt(commentable.GetForeignIndex(), 10)),
})
if err != nil {
return nil, false, err
}
for _, prComment := range resp.CommentsForPullRequestData {
for _, ccComment := range prComment.Comments {
comment := &base.Comment{
IssueIndex: commentable.GetForeignIndex(),
PosterName: c.getUsernameFromARN(*ccComment.AuthorArn),
Content: *ccComment.Content,
Created: *ccComment.CreationDate,
Updated: *ccComment.LastModifiedDate,
}
comments = append(comments, comment)
}
}
nextToken = resp.NextToken
if nextToken == nil {
break
}
}
return comments, true, nil
}
// GetPullRequests returns pull requests according page and perPage
func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
allPullRequestIDs, err := c.getAllPullRequestIDs()
if err != nil {
return nil, false, err
}
startIndex := (page - 1) * perPage
endIndex := page * perPage
if endIndex > len(allPullRequestIDs) {
endIndex = len(allPullRequestIDs)
}
batch := allPullRequestIDs[startIndex:endIndex]
prs := make([]*base.PullRequest, 0, len(batch))
for _, id := range batch {
output, err := c.codeCommitClient.GetPullRequest(c.ctx, &codecommit.GetPullRequestInput{
PullRequestId: aws.String(id),
})
if err != nil {
return nil, false, err
}
orig := output.PullRequest
number, err := strconv.ParseInt(*orig.PullRequestId, 10, 64)
if err != nil {
log.Error("CodeCommit pull request id is not a number: %s", *orig.PullRequestId)
continue
}
if len(orig.PullRequestTargets) == 0 {
log.Error("CodeCommit pull request does not contain targets", *orig.PullRequestId)
continue
}
target := orig.PullRequestTargets[0]
pr := &base.PullRequest{
Number: number,
Title: *orig.Title,
PosterName: c.getUsernameFromARN(*orig.AuthorArn),
Content: *orig.Description,
State: "open",
Created: *orig.CreationDate,
Updated: *orig.LastActivityDate,
Merged: target.MergeMetadata.IsMerged,
Head: base.PullRequestBranch{
Ref: strings.TrimPrefix(*target.SourceReference, git_module.BranchPrefix),
SHA: *target.SourceCommit,
RepoName: c.repoName,
},
Base: base.PullRequestBranch{
Ref: strings.TrimPrefix(*target.DestinationReference, git_module.BranchPrefix),
SHA: *target.DestinationCommit,
RepoName: c.repoName,
},
ForeignIndex: number,
}
if orig.PullRequestStatus == types.PullRequestStatusEnumClosed {
pr.State = "closed"
pr.Closed = orig.LastActivityDate
}
_ = CheckAndEnsureSafePR(pr, c.baseURL, c)
prs = append(prs, pr)
}
return prs, len(prs) < perPage, nil
}
// FormatCloneURL add authentication into remote URLs
func (c *CodeCommitDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
return u.String(), nil
}
func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) {
if len(c.allPullRequestIDs) > 0 {
return c.allPullRequestIDs, nil
}
var (
nextToken *string
prIDs []string
)
for {
output, err := c.codeCommitClient.ListPullRequests(c.ctx, &codecommit.ListPullRequestsInput{
RepositoryName: aws.String(c.repoName),
NextToken: nextToken,
})
if err != nil {
return nil, err
}
prIDs = append(prIDs, output.PullRequestIds...)
nextToken = output.NextToken
if nextToken == nil {
break
}
}
c.allPullRequestIDs = prIDs
return c.allPullRequestIDs, nil
}
func (c *CodeCommitDownloader) getUsernameFromARN(arn string) string {
parts := strings.Split(arn, "/")
if len(parts) > 0 {
return parts[len(parts)-1]
}
return ""
}