Improve migrations to support migrating milestones/labels/issues/comments/pullrequests (#6290)

* add migrations

* fix package dependency

* fix lints

* implements migrations except pull requests

* add releases

* migrating releases

* fix bug

* fix lint

* fix migrate releases

* fix tests

* add rollback

* pull request migtations

* fix import

* fix go module vendor

* add tests for upload to gitea

* more migrate options

* fix swagger-check

* fix misspell

* add options on migration UI

* fix log error

* improve UI options on migrating

* add support for username password when migrating from github

* fix tests

* remove comments and fix migrate limitation

* improve error handles

* migrate API will also support migrate milestones/labels/issues/pulls/releases

* fix tests and remove unused codes

* add DownloaderFactory and docs about how to create a new Downloader

* fix misspell

* fix migration docs

* Add hints about migrate options on migration page

* fix tests

modules/migrations/base/comment.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// Comment is a standard comment information
+type Comment struct {
+	PosterName  string
+	PosterEmail string
+	Created     time.Time
+	Content     string
+	Reactions   *Reactions
+}

modules/migrations/base/downloader.go

@@ -0,0 +1,23 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Downloader downloads the site repo informations
+type Downloader interface {
+	GetRepoInfo() (*Repository, error)
+	GetMilestones() ([]*Milestone, error)
+	GetReleases() ([]*Release, error)
+	GetLabels() ([]*Label, error)
+	GetIssues(start, limit int) ([]*Issue, error)
+	GetComments(issueNumber int64) ([]*Comment, error)
+	GetPullRequests(start, limit int) ([]*PullRequest, error)
+}
+
+// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
+type DownloaderFactory interface {
+	Match(opts MigrateOptions) (bool, error)
+	New(opts MigrateOptions) (Downloader, error)
+}

modules/migrations/base/issue.go

@@ -0,0 +1,24 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// Issue is a standard issue information
+type Issue struct {
+	Number      int64
+	PosterName  string
+	PosterEmail string
+	Title       string
+	Content     string
+	Milestone   string
+	State       string // closed, open
+	IsLocked    bool
+	Created     time.Time
+	Closed      *time.Time
+	Labels      []*Label
+	Reactions   *Reactions
+}

modules/migrations/base/label.go

@@ -0,0 +1,13 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Label defines a standard label informations
+type Label struct {
+	Name        string
+	Color       string
+	Description string
+}

modules/migrations/base/milestone.go

@@ -0,0 +1,19 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// Milestone defines a standard milestone
+type Milestone struct {
+	Title       string
+	Description string
+	Deadline    *time.Time
+	Created     time.Time
+	Updated     *time.Time
+	Closed      *time.Time
+	State       string
+}

modules/migrations/base/options.go

@@ -0,0 +1,26 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// MigrateOptions defines the way a repository gets migrated
+type MigrateOptions struct {
+	RemoteURL    string
+	AuthUsername string
+	AuthPassword string
+	Name         string
+	Description  string
+
+	Wiki              bool
+	Issues            bool
+	Milestones        bool
+	Labels            bool
+	Releases          bool
+	Comments          bool
+	PullRequests      bool
+	Private           bool
+	Mirror            bool
+	IgnoreIssueAuthor bool // if true will not add original author information before issues or comments content.
+}

modules/migrations/base/pullrequest.go

@@ -0,0 +1,53 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import (
+	"fmt"
+	"time"
+)
+
+// PullRequest defines a standard pull request information
+type PullRequest struct {
+	Number         int64
+	Title          string
+	PosterName     string
+	PosterEmail    string
+	Content        string
+	Milestone      string
+	State          string
+	Created        time.Time
+	Closed         *time.Time
+	Labels         []*Label
+	PatchURL       string
+	Merged         bool
+	MergedTime     *time.Time
+	MergeCommitSHA string
+	Head           PullRequestBranch
+	Base           PullRequestBranch
+	Assignee       string
+	Assignees      []string
+	IsLocked       bool
+}
+
+// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
+func (p *PullRequest) IsForkPullRequest() bool {
+	return p.Head.RepoPath() != p.Base.RepoPath()
+}
+
+// PullRequestBranch represents a pull request branch
+type PullRequestBranch struct {
+	CloneURL  string
+	Ref       string
+	SHA       string
+	RepoName  string
+	OwnerName string
+}
+
+// RepoPath returns pull request repo path
+func (p PullRequestBranch) RepoPath() string {
+	return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
+}

modules/migrations/base/reaction.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Reactions represents a summary of reactions.
+type Reactions struct {
+	TotalCount int
+	PlusOne    int
+	MinusOne   int
+	Laugh      int
+	Confused   int
+	Heart      int
+	Hooray     int
+}

modules/migrations/base/release.go

@@ -0,0 +1,31 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+import "time"
+
+// ReleaseAsset represents a release asset
+type ReleaseAsset struct {
+	URL           string
+	Name          string
+	ContentType   *string
+	Size          *int
+	DownloadCount *int
+	Created       time.Time
+	Updated       time.Time
+}
+
+// Release represents a release
+type Release struct {
+	TagName         string
+	TargetCommitish string
+	Name            string
+	Body            string
+	Draft           bool
+	Prerelease      bool
+	Assets          []ReleaseAsset
+	Created         time.Time
+	Published       time.Time
+}

modules/migrations/base/repo.go

@@ -0,0 +1,18 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Repository defines a standard repository information
+type Repository struct {
+	Name         string
+	Owner        string
+	IsPrivate    bool
+	IsMirror     bool
+	Description  string
+	AuthUsername string
+	AuthPassword string
+	CloneURL     string
+}

modules/migrations/base/uploader.go

@@ -0,0 +1,18 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Uploader uploads all the informations
+type Uploader interface {
+	CreateRepo(repo *Repository, includeWiki bool) error
+	CreateMilestone(milestone *Milestone) error
+	CreateRelease(release *Release) error
+	CreateLabel(label *Label) error
+	CreateIssue(issue *Issue) error
+	CreateComment(issueNumber int64, comment *Comment) error
+	CreatePullRequest(pr *PullRequest) error
+	Rollback() error
+}