Improve migrations to support migrating milestones/labels/issues/comments/pullrequests (#6290)

* add migrations

* fix package dependency

* fix lints

* implements migrations except pull requests

* add releases

* migrating releases

* fix bug

* fix lint

* fix migrate releases

* fix tests

* add rollback

* pull request migtations

* fix import

* fix go module vendor

* add tests for upload to gitea

* more migrate options

* fix swagger-check

* fix misspell

* add options on migration UI

* fix log error

* improve UI options on migrating

* add support for username password when migrating from github

* fix tests

* remove comments and fix migrate limitation

* improve error handles

* migrate API will also support migrate milestones/labels/issues/pulls/releases

* fix tests and remove unused codes

* add DownloaderFactory and docs about how to create a new Downloader

* fix misspell

* fix migration docs

* Add hints about migrate options on migration page

* fix tests

models/migrate.go

@@ -0,0 +1,145 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import "github.com/go-xorm/xorm"
+
+// InsertIssue insert one issue to database
+func InsertIssue(issue *Issue, labelIDs []int64) error {
+	sess := x.NewSession()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := insertIssue(sess, issue, labelIDs); err != nil {
+		return err
+	}
+	return sess.Commit()
+}
+
+func insertIssue(sess *xorm.Session, issue *Issue, labelIDs []int64) error {
+	if issue.MilestoneID > 0 {
+		sess.Incr("num_issues")
+		if issue.IsClosed {
+			sess.Incr("num_closed_issues")
+		}
+		if _, err := sess.ID(issue.MilestoneID).NoAutoTime().Update(new(Milestone)); err != nil {
+			return err
+		}
+	}
+	if _, err := sess.NoAutoTime().Insert(issue); err != nil {
+		return err
+	}
+	var issueLabels = make([]IssueLabel, 0, len(labelIDs))
+	for _, labelID := range labelIDs {
+		issueLabels = append(issueLabels, IssueLabel{
+			IssueID: issue.ID,
+			LabelID: labelID,
+		})
+	}
+	if _, err := sess.Insert(issueLabels); err != nil {
+		return err
+	}
+	if !issue.IsPull {
+		sess.ID(issue.RepoID).Incr("num_issues")
+		if issue.IsClosed {
+			sess.Incr("num_closed_issues")
+		}
+	} else {
+		sess.ID(issue.RepoID).Incr("num_pulls")
+		if issue.IsClosed {
+			sess.Incr("num_closed_pulls")
+		}
+	}
+	if _, err := sess.NoAutoTime().Update(issue.Repo); err != nil {
+		return err
+	}
+
+	sess.Incr("num_issues")
+	if issue.IsClosed {
+		sess.Incr("num_closed_issues")
+	}
+	if _, err := sess.In("id", labelIDs).Update(new(Label)); err != nil {
+		return err
+	}
+
+	if issue.MilestoneID > 0 {
+		if _, err := sess.ID(issue.MilestoneID).SetExpr("completeness", "num_closed_issues * 100 / num_issues").Update(new(Milestone)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// InsertComment inserted a comment
+func InsertComment(comment *Comment) error {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+	if _, err := sess.NoAutoTime().Insert(comment); err != nil {
+		return err
+	}
+	if _, err := sess.ID(comment.IssueID).Incr("num_comments").Update(new(Issue)); err != nil {
+		return err
+	}
+	return sess.Commit()
+}
+
+// InsertPullRequest inserted a pull request
+func InsertPullRequest(pr *PullRequest, labelIDs []int64) error {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+	if err := insertIssue(sess, pr.Issue, labelIDs); err != nil {
+		return err
+	}
+	pr.IssueID = pr.Issue.ID
+	if _, err := sess.NoAutoTime().Insert(pr); err != nil {
+		return err
+	}
+	return sess.Commit()
+}
+
+// MigrateRelease migrates release
+func MigrateRelease(rel *Release) error {
+	sess := x.NewSession()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	var oriRel = Release{
+		RepoID:  rel.RepoID,
+		TagName: rel.TagName,
+	}
+	exist, err := sess.Get(&oriRel)
+	if err != nil {
+		return err
+	}
+	if !exist {
+		if _, err := sess.NoAutoTime().Insert(rel); err != nil {
+			return err
+		}
+	} else {
+		rel.ID = oriRel.ID
+		if _, err := sess.ID(rel.ID).Cols("target, title, note, is_tag, num_commits").Update(rel); err != nil {
+			return err
+		}
+	}
+
+	for i := 0; i < len(rel.Attachments); i++ {
+		rel.Attachments[i].ReleaseID = rel.ID
+	}
+
+	if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}

models/release_test.go

@@ -107,6 +107,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 		IsPrivate:   false,
 		IsMirror:    true,
 		RemoteAddr:  repoPath,
+		Wiki:        true,
 	}
 	mirror, err := MigrateRepository(user, user, migrationOptions)
 	assert.NoError(t, err)

models/repo.go

@@ -896,6 +896,7 @@ type MigrateRepoOptions struct {
 	IsPrivate   bool
 	IsMirror    bool
 	RemoteAddr  string
+	Wiki        bool // include wiki repository
 }
 
 /*
@@ -917,7 +918,7 @@ func wikiRemoteURL(remote string) string {
 	return ""
 }
 
-// MigrateRepository migrates a existing repository from other project hosting.
+// MigrateRepository migrates an existing repository from other project hosting.
 func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) {
 	repo, err := CreateRepository(doer, u, CreateRepoOptions{
 		Name:        opts.Name,
@@ -930,7 +931,6 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
 	}
 
 	repoPath := RepoPath(u.Name, opts.Name)
-	wikiPath := WikiPath(u.Name, opts.Name)
 
 	if u.IsOrganization() {
 		t, err := u.GetOwnerTeam()
@@ -956,22 +956,25 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
 		return repo, fmt.Errorf("Clone: %v", err)
 	}
 
-	wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
-	if len(wikiRemotePath) > 0 {
-		if err := os.RemoveAll(wikiPath); err != nil {
-			return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
-		}
-
-		if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
-			Mirror:  true,
-			Quiet:   true,
-			Timeout: migrateTimeout,
-			Branch:  "master",
-		}); err != nil {
-			log.Warn("Clone wiki: %v", err)
+	if opts.Wiki {
+		wikiPath := WikiPath(u.Name, opts.Name)
+		wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
+		if len(wikiRemotePath) > 0 {
 			if err := os.RemoveAll(wikiPath); err != nil {
 				return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
 			}
+
+			if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
+				Mirror:  true,
+				Quiet:   true,
+				Timeout: migrateTimeout,
+				Branch:  "master",
+			}); err != nil {
+				log.Warn("Clone wiki: %v", err)
+				if err := os.RemoveAll(wikiPath); err != nil {
+					return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+				}
+			}
 		}
 	}