Merge remote-tracking branch 'upstream/master' into team-grant-all-repos

modules/auth/auth.go

@@ -224,6 +224,9 @@ func SignedInUser(ctx *macaron.Context, sess session.Store) (*models.User, bool)
 			}
 
 			if u == nil {
+				if !setting.Service.EnableBasicAuth {
+					return nil, false
+				}
 				u, err = models.UserSignIn(uname, passwd)
 				if err != nil {
 					if !models.IsErrUserNotExist(err) {

modules/auth/oauth2/oauth2.go

@@ -11,7 +11,6 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
-	"github.com/go-xorm/xorm"
 	"github.com/lafriks/xormstore"
 	"github.com/markbates/goth"
 	"github.com/markbates/goth/gothic"
@@ -26,6 +25,7 @@ import (
 	"github.com/markbates/goth/providers/openidConnect"
 	"github.com/markbates/goth/providers/twitter"
 	"github.com/satori/go.uuid"
+	"xorm.io/xorm"
 )
 
 var (

modules/auth/org.go

@@ -22,8 +22,9 @@ import (
 
 // CreateOrgForm form for creating organization
 type CreateOrgForm struct {
-	OrgName    string `binding:"Required;AlphaDashDot;MaxSize(40)" locale:"org.org_name_holder"`
-	Visibility structs.VisibleType
+	OrgName                   string `binding:"Required;AlphaDashDot;MaxSize(40)" locale:"org.org_name_holder"`
+	Visibility                structs.VisibleType
+	RepoAdminChangeTeamAccess bool
 }
 
 // Validate validates the fields

modules/auth/repo_form.go

@@ -152,6 +152,7 @@ type ProtectBranchForm struct {
 	EnableWhitelist         bool
 	WhitelistUsers          string
 	WhitelistTeams          string
+	WhitelistDeployKeys     bool
 	EnableMergeWhitelist    bool
 	MergeWhitelistUsers     string
 	MergeWhitelistTeams     string
@@ -557,7 +558,7 @@ func (f *NewWikiForm) Validate(ctx *macaron.Context, errs binding.Errors) bindin
 // EditRepoFileForm form for changing repository file
 type EditRepoFileForm struct {
 	TreePath      string `binding:"Required;MaxSize(500)"`
-	Content       string `binding:"Required"`
+	Content       string
 	CommitSummary string `binding:"MaxSize(100)"`
 	CommitMessage string
 	CommitChoice  string `binding:"Required;MaxSize(50)"`

modules/charset/charset_test.go

@@ -179,7 +179,8 @@ func TestToUTF8DropErrors(t *testing.T) {
 
 	// "Hola, así cómo ños"
 	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
-	assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res)
+	assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8])
+	assert.Equal(t, []byte{0x73}, res[len(res)-1:])
 
 	// "Hola, así cómo "
 	minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}

modules/context/org.go

@@ -63,7 +63,7 @@ func HandleOrgAssignment(ctx *Context, args ...bool) {
 
 	// Force redirection when username is actually a user.
 	if !org.IsOrganization() {
-		ctx.Redirect("/" + org.Name)
+		ctx.Redirect(setting.AppSubURL + "/" + org.Name)
 		return
 	}
 

modules/context/repo.go

@@ -19,8 +19,8 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 
 	"gitea.com/macaron/macaron"
+	"github.com/editorconfig/editorconfig-core-go/v2"
 	"github.com/unknwon/com"
-	"gopkg.in/editorconfig/editorconfig-core-go.v1"
 )
 
 // PullRequest contains informations to make a pull request
@@ -146,6 +146,9 @@ func (r *Repository) FileExists(path string, branch string) (bool, error) {
 // GetEditorconfig returns the .editorconfig definition if found in the
 // HEAD of the default repo branch.
 func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) {
+	if r.GitRepo == nil {
+		return nil, nil
+	}
 	commit, err := r.GitRepo.GetBranchCommit(r.Repository.DefaultBranch)
 	if err != nil {
 		return nil, err
@@ -233,7 +236,7 @@ func RedirectToRepo(ctx *Context, redirectRepoID int64) {
 	if ctx.Req.URL.RawQuery != "" {
 		redirectPath += "?" + ctx.Req.URL.RawQuery
 	}
-	ctx.Redirect(redirectPath)
+	ctx.Redirect(path.Join(setting.AppSubURL, redirectPath))
 }
 
 func repoAssignment(ctx *Context, repo *models.Repository) {
@@ -358,12 +361,6 @@ func RepoAssignment() macaron.Handler {
 			return
 		}
 
-		gitRepo, err := git.OpenRepository(models.RepoPath(userName, repoName))
-		if err != nil {
-			ctx.ServerError("RepoAssignment Invalid repo "+models.RepoPath(userName, repoName), err)
-			return
-		}
-		ctx.Repo.GitRepo = gitRepo
 		ctx.Repo.RepoLink = repo.Link()
 		ctx.Data["RepoLink"] = ctx.Repo.RepoLink
 		ctx.Data["RepoRelPath"] = ctx.Repo.Owner.Name + "/" + ctx.Repo.Repository.Name
@@ -373,13 +370,6 @@ func RepoAssignment() macaron.Handler {
 			ctx.Data["RepoExternalIssuesLink"] = unit.ExternalTrackerConfig().ExternalTrackerURL
 		}
 
-		tags, err := ctx.Repo.GitRepo.GetTags()
-		if err != nil {
-			ctx.ServerError("GetTags", err)
-			return
-		}
-		ctx.Data["Tags"] = tags
-
 		count, err := models.GetReleaseCountByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{
 			IncludeDrafts: false,
 			IncludeTags:   true,
@@ -424,13 +414,32 @@ func RepoAssignment() macaron.Handler {
 			}
 		}
 
-		// repo is empty and display enable
+		// Disable everything when the repo is being created
+		if ctx.Repo.Repository.IsBeingCreated() {
+			ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch
+			return
+		}
+
+		gitRepo, err := git.OpenRepository(models.RepoPath(userName, repoName))
+		if err != nil {
+			ctx.ServerError("RepoAssignment Invalid repo "+models.RepoPath(userName, repoName), err)
+			return
+		}
+		ctx.Repo.GitRepo = gitRepo
+
+		// Stop at this point when the repo is empty.
 		if ctx.Repo.Repository.IsEmpty {
 			ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch
 			return
 		}
 
-		ctx.Data["TagName"] = ctx.Repo.TagName
+		tags, err := ctx.Repo.GitRepo.GetTags()
+		if err != nil {
+			ctx.ServerError("GetTags", err)
+			return
+		}
+		ctx.Data["Tags"] = tags
+
 		brs, err := ctx.Repo.GitRepo.GetBranches()
 		if err != nil {
 			ctx.ServerError("GetBranches", err)
@@ -439,6 +448,8 @@ func RepoAssignment() macaron.Handler {
 		ctx.Data["Branches"] = brs
 		ctx.Data["BranchesCount"] = len(brs)
 
+		ctx.Data["TagName"] = ctx.Repo.TagName
+
 		// If not branch selected, try default one.
 		// If default branch doesn't exists, fall back to some other branch.
 		if len(ctx.Repo.BranchName) == 0 {

modules/cron/cron.go

@@ -10,6 +10,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/sync"
 	mirror_service "code.gitea.io/gitea/services/mirror"
@@ -18,12 +19,13 @@ import (
 )
 
 const (
-	mirrorUpdate           = "mirror_update"
-	gitFsck                = "git_fsck"
-	checkRepos             = "check_repos"
-	archiveCleanup         = "archive_cleanup"
-	syncExternalUsers      = "sync_external_users"
-	deletedBranchesCleanup = "deleted_branches_cleanup"
+	mirrorUpdate            = "mirror_update"
+	gitFsck                 = "git_fsck"
+	checkRepos              = "check_repos"
+	archiveCleanup          = "archive_cleanup"
+	syncExternalUsers       = "sync_external_users"
+	deletedBranchesCleanup  = "deleted_branches_cleanup"
+	updateMigrationPosterID = "update_migration_post_id"
 )
 
 var c = cron.New()
@@ -117,6 +119,15 @@ func NewContext() {
 			go WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches)()
 		}
 	}
+
+	entry, err = c.AddFunc("Update migrated repositories' issues and comments' posterid", setting.Cron.UpdateMigrationPosterID.Schedule, WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID))
+	if err != nil {
+		log.Fatal("Cron[Update migrated repositories]: %v", err)
+	}
+	entry.Prev = time.Now()
+	entry.ExecTimes++
+	go WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)()
+
 	c.Start()
 }
 

modules/git/commit.go

@@ -498,3 +498,11 @@ func GetFullCommitID(repoPath, shortID string) (string, error) {
 	}
 	return strings.TrimSpace(commitID), nil
 }
+
+// GetRepositoryDefaultPublicGPGKey returns the default public key for this commit
+func (c *Commit) GetRepositoryDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, error) {
+	if c.repo == nil {
+		return nil, nil
+	}
+	return c.repo.GetDefaultPublicGPGKey(forceUpdate)
+}

modules/git/commit_info.go

@@ -72,6 +72,7 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom
 		treeCommit = commit
 	} else if rev, ok := revs[""]; ok {
 		treeCommit = convertCommit(rev)
+		treeCommit.repo = commit.repo
 	}
 	return commitsInfo, treeCommit, nil
 }

modules/git/repo.go

@@ -32,6 +32,16 @@ type Repository struct {
 
 	gogitRepo    *gogit.Repository
 	gogitStorage *filesystem.Storage
+	gpgSettings  *GPGSettings
+}
+
+// GPGSettings represents the default GPG settings for this repository
+type GPGSettings struct {
+	Sign             bool
+	KeyID            string
+	Email            string
+	Name             string
+	PublicKeyContent string
 }
 
 const prettyLogFormat = `--pretty=format:%H`

modules/git/repo_branch.go

@@ -28,8 +28,14 @@ func IsBranchExist(repoPath, name string) bool {
 
 // IsBranchExist returns true if given branch exists in current repository.
 func (repo *Repository) IsBranchExist(name string) bool {
-	_, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true)
-	return err == nil
+	if name == "" {
+		return false
+	}
+	reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true)
+	if err != nil {
+		return false
+	}
+	return reference.Type() != plumbing.InvalidReference
 }
 
 // Branch represents a Git branch.
@@ -165,7 +171,7 @@ func (repo *Repository) AddRemote(name, url string, fetch bool) error {
 
 // RemoveRemote removes a remote from repository.
 func (repo *Repository) RemoveRemote(name string) error {
-	_, err := NewCommand("remote", "remove", name).RunInDir(repo.Path)
+	_, err := NewCommand("remote", "rm", name).RunInDir(repo.Path)
 	return err
 }
 

modules/git/repo_gpg.go

@@ -0,0 +1,59 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package git
+
+import (
+	"fmt"
+	"strings"
+
+	"code.gitea.io/gitea/modules/process"
+)
+
+// LoadPublicKeyContent will load the key from gpg
+func (gpgSettings *GPGSettings) LoadPublicKeyContent() error {
+	content, stderr, err := process.GetManager().Exec(
+		"gpg -a --export",
+		"gpg", "-a", "--export", gpgSettings.KeyID)
+	if err != nil {
+		return fmt.Errorf("Unable to get default signing key: %s, %s, %v", gpgSettings.KeyID, stderr, err)
+	}
+	gpgSettings.PublicKeyContent = content
+	return nil
+}
+
+// GetDefaultPublicGPGKey will return and cache the default public GPG settings for this repository
+func (repo *Repository) GetDefaultPublicGPGKey(forceUpdate bool) (*GPGSettings, error) {
+	if repo.gpgSettings != nil && !forceUpdate {
+		return repo.gpgSettings, nil
+	}
+
+	gpgSettings := &GPGSettings{
+		Sign: true,
+	}
+
+	value, _ := NewCommand("config", "--get", "commit.gpgsign").RunInDir(repo.Path)
+	sign, valid := ParseBool(strings.TrimSpace(value))
+	if !sign || !valid {
+		gpgSettings.Sign = false
+		repo.gpgSettings = gpgSettings
+		return gpgSettings, nil
+	}
+
+	signingKey, _ := NewCommand("config", "--get", "user.signingkey").RunInDir(repo.Path)
+	gpgSettings.KeyID = strings.TrimSpace(signingKey)
+
+	defaultEmail, _ := NewCommand("config", "--get", "user.email").RunInDir(repo.Path)
+	gpgSettings.Email = strings.TrimSpace(defaultEmail)
+
+	defaultName, _ := NewCommand("config", "--get", "user.name").RunInDir(repo.Path)
+	gpgSettings.Name = strings.TrimSpace(defaultName)
+
+	if err := gpgSettings.LoadPublicKeyContent(); err != nil {
+		return nil, err
+	}
+	repo.gpgSettings = gpgSettings
+	return repo.gpgSettings, nil
+}

modules/git/repo_tree.go

@@ -6,10 +6,13 @@
 package git
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"strings"
 	"time"
+
+	"github.com/mcuadros/go-version"
 )
 
 func (repo *Repository) getTree(id SHA1) (*Tree, error) {
@@ -53,14 +56,20 @@ func (repo *Repository) GetTree(idStr string) (*Tree, error) {
 
 // CommitTreeOpts represents the possible options to CommitTree
 type CommitTreeOpts struct {
-	Parents   []string
-	Message   string
-	KeyID     string
-	NoGPGSign bool
+	Parents    []string
+	Message    string
+	KeyID      string
+	NoGPGSign  bool
+	AlwaysSign bool
 }
 
 // CommitTree creates a commit from a given tree id for the user with provided message
 func (repo *Repository) CommitTree(sig *Signature, tree *Tree, opts CommitTreeOpts) (SHA1, error) {
+	binVersion, err := BinVersion()
+	if err != nil {
+		return SHA1{}, err
+	}
+
 	commitTimeStr := time.Now().Format(time.RFC3339)
 
 	// Because this may call hooks we should pass in the environment
@@ -78,20 +87,24 @@ func (repo *Repository) CommitTree(sig *Signature, tree *Tree, opts CommitTreeOp
 		cmd.AddArguments("-p", parent)
 	}
 
-	cmd.AddArguments("-m", opts.Message)
+	messageBytes := new(bytes.Buffer)
+	_, _ = messageBytes.WriteString(opts.Message)
+	_, _ = messageBytes.WriteString("\n")
 
-	if opts.KeyID != "" {
+	if version.Compare(binVersion, "1.7.9", ">=") && (opts.KeyID != "" || opts.AlwaysSign) {
 		cmd.AddArguments(fmt.Sprintf("-S%s", opts.KeyID))
 	}
 
-	if opts.NoGPGSign {
+	if version.Compare(binVersion, "2.0.0", ">=") && opts.NoGPGSign {
 		cmd.AddArguments("--no-gpg-sign")
 	}
 
-	res, err := cmd.RunInDirWithEnv(repo.Path, env)
+	stdout := new(bytes.Buffer)
+	stderr := new(bytes.Buffer)
+	err = cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdout, stderr, messageBytes)
 
 	if err != nil {
-		return SHA1{}, err
+		return SHA1{}, concatenateError(err, stderr.String())
 	}
-	return NewIDFromString(strings.TrimSpace(res))
+	return NewIDFromString(strings.TrimSpace(stdout.String()))
 }

modules/git/utils.go

@@ -7,6 +7,7 @@ package git
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
 )
@@ -86,3 +87,30 @@ func RefEndName(refStr string) string {
 
 	return refStr
 }
+
+// ParseBool returns the boolean value represented by the string as per git's git_config_bool
+// true will be returned for the result if the string is empty, but valid will be false.
+// "true", "yes", "on" are all true, true
+// "false", "no", "off" are all false, true
+// 0 is false, true
+// Any other integer is true, true
+// Anything else will return false, false
+func ParseBool(value string) (result bool, valid bool) {
+	// Empty strings are true but invalid
+	if len(value) == 0 {
+		return true, false
+	}
+	// These are the git expected true and false values
+	if strings.EqualFold(value, "true") || strings.EqualFold(value, "yes") || strings.EqualFold(value, "on") {
+		return true, true
+	}
+	if strings.EqualFold(value, "false") || strings.EqualFold(value, "no") || strings.EqualFold(value, "off") {
+		return false, true
+	}
+	// Try a number
+	intValue, err := strconv.ParseInt(value, 10, 32)
+	if err != nil {
+		return false, false
+	}
+	return intValue != 0, true
+}

modules/graceful/cleanup.go

@@ -0,0 +1,40 @@
+// +build !windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package graceful
+
+import "sync"
+
+var cleanupWaitGroup sync.WaitGroup
+
+func init() {
+	cleanupWaitGroup = sync.WaitGroup{}
+
+	// There are three places that could inherit sockets:
+	//
+	// * HTTP or HTTPS main listener
+	// * HTTP redirection fallback
+	// * SSH
+	//
+	// If you add an additional place you must increment this number
+	// and add a function to call InformCleanup if it's not going to be used
+	cleanupWaitGroup.Add(3)
+
+	// Wait till we're done getting all of the listeners and then close
+	// the unused ones
+	go func() {
+		cleanupWaitGroup.Wait()
+		// Ignore the error here there's not much we can do with it
+		// They're logged in the CloseProvidedListeners function
+		_ = CloseProvidedListeners()
+	}()
+}
+
+// InformCleanup tells the cleanup wait group that we have either taken a listener
+// or will not be taking a listener
+func InformCleanup() {
+	cleanupWaitGroup.Done()
+}

modules/graceful/graceful_windows.go

@@ -0,0 +1,16 @@
+// +build windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+// This code is heavily inspired by the archived gofacebook/gracenet/net.go handler
+
+package graceful
+
+// This file contains shims for windows builds
+const IsChild = false
+
+// WaitForServers waits for all running servers to finish
+func WaitForServers() {
+
+}

modules/graceful/net.go

@@ -0,0 +1,211 @@
+// +build !windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+// This code is heavily inspired by the archived gofacebook/gracenet/net.go handler
+
+package graceful
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+const (
+	listenFDs = "LISTEN_FDS"
+	startFD   = 3
+)
+
+// In order to keep the working directory the same as when we started we record
+// it at startup.
+var originalWD, _ = os.Getwd()
+
+var (
+	once  = sync.Once{}
+	mutex = sync.Mutex{}
+
+	providedListeners = []net.Listener{}
+	activeListeners   = []net.Listener{}
+)
+
+func getProvidedFDs() (savedErr error) {
+	// Only inherit the provided FDS once but we will save the error so that repeated calls to this function will return the same error
+	once.Do(func() {
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		numFDs := os.Getenv(listenFDs)
+		if numFDs == "" {
+			return
+		}
+		n, err := strconv.Atoi(numFDs)
+		if err != nil {
+			savedErr = fmt.Errorf("%s is not a number: %s. Err: %v", listenFDs, numFDs, err)
+			return
+		}
+
+		for i := startFD; i < n+startFD; i++ {
+			file := os.NewFile(uintptr(i), fmt.Sprintf("listener_FD%d", i))
+
+			l, err := net.FileListener(file)
+			if err == nil {
+				// Close the inherited file if it's a listener
+				if err = file.Close(); err != nil {
+					savedErr = fmt.Errorf("error closing provided socket fd %d: %s", i, err)
+					return
+				}
+				providedListeners = append(providedListeners, l)
+				continue
+			}
+
+			// If needed we can handle packetconns here.
+			savedErr = fmt.Errorf("Error getting provided socket fd %d: %v", i, err)
+			return
+		}
+	})
+	return savedErr
+}
+
+// CloseProvidedListeners closes all unused provided listeners.
+func CloseProvidedListeners() error {
+	mutex.Lock()
+	defer mutex.Unlock()
+	var returnableError error
+	for _, l := range providedListeners {
+		err := l.Close()
+		if err != nil {
+			log.Error("Error in closing unused provided listener: %v", err)
+			if returnableError != nil {
+				returnableError = fmt.Errorf("%v & %v", returnableError, err)
+			} else {
+				returnableError = err
+			}
+		}
+	}
+	providedListeners = []net.Listener{}
+
+	return returnableError
+}
+
+// GetListener obtains a listener for the local network address. The network must be
+// a stream-oriented network: "tcp", "tcp4", "tcp6", "unix" or "unixpacket". It
+// returns an provided net.Listener for the matching network and address, or
+// creates a new one using net.Listen.
+func GetListener(network, address string) (net.Listener, error) {
+	// Add a deferral to say that we've tried to grab a listener
+	defer InformCleanup()
+	switch network {
+	case "tcp", "tcp4", "tcp6":
+		tcpAddr, err := net.ResolveTCPAddr(network, address)
+		if err != nil {
+			return nil, err
+		}
+		return GetListenerTCP(network, tcpAddr)
+	case "unix", "unixpacket":
+		unixAddr, err := net.ResolveUnixAddr(network, address)
+		if err != nil {
+			return nil, err
+		}
+		return GetListenerUnix(network, unixAddr)
+	default:
+		return nil, net.UnknownNetworkError(network)
+	}
+}
+
+// GetListenerTCP announces on the local network address. The network must be:
+// "tcp", "tcp4" or "tcp6". It returns a provided net.Listener for the
+// matching network and address, or creates a new one using net.ListenTCP.
+func GetListenerTCP(network string, address *net.TCPAddr) (*net.TCPListener, error) {
+	if err := getProvidedFDs(); err != nil {
+		return nil, err
+	}
+
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	// look for a provided listener
+	for i, l := range providedListeners {
+		if isSameAddr(l.Addr(), address) {
+			providedListeners = append(providedListeners[:i], providedListeners[i+1:]...)
+
+			activeListeners = append(activeListeners, l)
+			return l.(*net.TCPListener), nil
+		}
+	}
+
+	// no provided listener for this address -> make a fresh listener
+	l, err := net.ListenTCP(network, address)
+	if err != nil {
+		return nil, err
+	}
+	activeListeners = append(activeListeners, l)
+	return l, nil
+}
+
+// GetListenerUnix announces on the local network address. The network must be:
+// "unix" or "unixpacket". It returns a provided net.Listener for the
+// matching network and address, or creates a new one using net.ListenUnix.
+func GetListenerUnix(network string, address *net.UnixAddr) (*net.UnixListener, error) {
+	if err := getProvidedFDs(); err != nil {
+		return nil, err
+	}
+
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	// look for a provided listener
+	for i, l := range providedListeners {
+		if isSameAddr(l.Addr(), address) {
+			providedListeners = append(providedListeners[:i], providedListeners[i+1:]...)
+			activeListeners = append(activeListeners, l)
+			return l.(*net.UnixListener), nil
+		}
+	}
+
+	// make a fresh listener
+	l, err := net.ListenUnix(network, address)
+	if err != nil {
+		return nil, err
+	}
+	activeListeners = append(activeListeners, l)
+	return l, nil
+}
+
+func isSameAddr(a1, a2 net.Addr) bool {
+	// If the addresses are not on the same network fail.
+	if a1.Network() != a2.Network() {
+		return false
+	}
+
+	// If the two addresses have the same string representation they're equal
+	a1s := a1.String()
+	a2s := a2.String()
+	if a1s == a2s {
+		return true
+	}
+
+	// This allows for ipv6 vs ipv4 local addresses to compare as equal. This
+	// scenario is common when listening on localhost.
+	const ipv6prefix = "[::]"
+	a1s = strings.TrimPrefix(a1s, ipv6prefix)
+	a2s = strings.TrimPrefix(a2s, ipv6prefix)
+	const ipv4prefix = "0.0.0.0"
+	a1s = strings.TrimPrefix(a1s, ipv4prefix)
+	a2s = strings.TrimPrefix(a2s, ipv4prefix)
+	return a1s == a2s
+}
+
+func getActiveListeners() []net.Listener {
+	mutex.Lock()
+	defer mutex.Unlock()
+	listeners := make([]net.Listener, len(activeListeners))
+	copy(listeners, activeListeners)
+	return listeners
+}

modules/graceful/restart.go

@@ -0,0 +1,85 @@
+// +build !windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+// This code is heavily inspired by the archived gofacebook/gracenet/net.go handler
+
+package graceful
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"syscall"
+)
+
+var killParent sync.Once
+
+// KillParent sends the kill signal to the parent process if we are a child
+func KillParent() {
+	killParent.Do(func() {
+		if IsChild {
+			ppid := syscall.Getppid()
+			if ppid > 1 {
+				_ = syscall.Kill(ppid, syscall.SIGTERM)
+			}
+		}
+	})
+}
+
+// RestartProcess starts a new process passing it the active listeners. It
+// doesn't fork, but starts a new process using the same environment and
+// arguments as when it was originally started. This allows for a newly
+// deployed binary to be started. It returns the pid of the newly started
+// process when successful.
+func RestartProcess() (int, error) {
+	listeners := getActiveListeners()
+
+	// Extract the fds from the listeners.
+	files := make([]*os.File, len(listeners))
+	for i, l := range listeners {
+		var err error
+		// Now, all our listeners actually have File() functions so instead of
+		// individually casting we just use a hacky interface
+		files[i], err = l.(filer).File()
+		if err != nil {
+			return 0, err
+		}
+		// Remember to close these at the end.
+		defer files[i].Close()
+	}
+
+	// Use the original binary location. This works with symlinks such that if
+	// the file it points to has been changed we will use the updated symlink.
+	argv0, err := exec.LookPath(os.Args[0])
+	if err != nil {
+		return 0, err
+	}
+
+	// Pass on the environment and replace the old count key with the new one.
+	var env []string
+	for _, v := range os.Environ() {
+		if !strings.HasPrefix(v, listenFDs+"=") {
+			env = append(env, v)
+		}
+	}
+	env = append(env, fmt.Sprintf("%s=%d", listenFDs, len(listeners)))
+
+	allFiles := append([]*os.File{os.Stdin, os.Stdout, os.Stderr}, files...)
+	process, err := os.StartProcess(argv0, os.Args, &os.ProcAttr{
+		Dir:   originalWD,
+		Env:   env,
+		Files: allFiles,
+	})
+	if err != nil {
+		return 0, err
+	}
+	return process.Pid, nil
+}
+
+type filer interface {
+	File() (*os.File, error)
+}

modules/graceful/server.go

@@ -0,0 +1,274 @@
+// +build !windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+// This code is highly inspired by endless go
+
+package graceful
+
+import (
+	"crypto/tls"
+	"net"
+	"os"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+type state uint8
+
+const (
+	stateInit state = iota
+	stateRunning
+	stateShuttingDown
+	stateTerminate
+)
+
+var (
+	// RWMutex for when adding servers or shutting down
+	runningServerReg sync.RWMutex
+	runningServerWG  sync.WaitGroup
+	// ensure we only fork once
+	runningServersForked bool
+
+	// DefaultReadTimeOut default read timeout
+	DefaultReadTimeOut time.Duration
+	// DefaultWriteTimeOut default write timeout
+	DefaultWriteTimeOut time.Duration
+	// DefaultMaxHeaderBytes default max header bytes
+	DefaultMaxHeaderBytes int
+
+	// IsChild reports if we are a fork iff LISTEN_FDS is set and our parent PID is not 1
+	IsChild = len(os.Getenv(listenFDs)) > 0 && os.Getppid() > 1
+)
+
+func init() {
+	runningServerReg = sync.RWMutex{}
+	runningServerWG = sync.WaitGroup{}
+
+	DefaultMaxHeaderBytes = 0 // use http.DefaultMaxHeaderBytes - which currently is 1 << 20 (1MB)
+}
+
+// ServeFunction represents a listen.Accept loop
+type ServeFunction = func(net.Listener) error
+
+// Server represents our graceful server
+type Server struct {
+	network         string
+	address         string
+	listener        net.Listener
+	PreSignalHooks  map[os.Signal][]func()
+	PostSignalHooks map[os.Signal][]func()
+	wg              sync.WaitGroup
+	sigChan         chan os.Signal
+	state           state
+	lock            *sync.RWMutex
+	BeforeBegin     func(network, address string)
+	OnShutdown      func()
+}
+
+// WaitForServers waits for all running servers to finish
+func WaitForServers() {
+	runningServerWG.Wait()
+}
+
+// NewServer creates a server on network at provided address
+func NewServer(network, address string) *Server {
+	runningServerReg.Lock()
+	defer runningServerReg.Unlock()
+
+	if IsChild {
+		log.Info("Restarting new server: %s:%s on PID: %d", network, address, os.Getpid())
+	} else {
+		log.Info("Starting new server: %s:%s on PID: %d", network, address, os.Getpid())
+	}
+	srv := &Server{
+		wg:              sync.WaitGroup{},
+		sigChan:         make(chan os.Signal),
+		PreSignalHooks:  map[os.Signal][]func(){},
+		PostSignalHooks: map[os.Signal][]func(){},
+		state:           stateInit,
+		lock:            &sync.RWMutex{},
+		network:         network,
+		address:         address,
+	}
+
+	srv.BeforeBegin = func(network, addr string) {
+		log.Debug("Starting server on %s:%s (PID: %d)", network, addr, syscall.Getpid())
+	}
+
+	return srv
+}
+
+// ListenAndServe listens on the provided network address and then calls Serve
+// to handle requests on incoming connections.
+func (srv *Server) ListenAndServe(serve ServeFunction) error {
+	go srv.handleSignals()
+
+	l, err := GetListener(srv.network, srv.address)
+	if err != nil {
+		log.Error("Unable to GetListener: %v", err)
+		return err
+	}
+
+	srv.listener = newWrappedListener(l, srv)
+
+	KillParent()
+
+	srv.BeforeBegin(srv.network, srv.address)
+
+	return srv.Serve(serve)
+}
+
+// ListenAndServeTLS listens on the provided network address and then calls
+// Serve to handle requests on incoming TLS connections.
+//
+// Filenames containing a certificate and matching private key for the server must
+// be provided. If the certificate is signed by a certificate authority, the
+// certFile should be the concatenation of the server's certificate followed by the
+// CA's certificate.
+func (srv *Server) ListenAndServeTLS(certFile, keyFile string, serve ServeFunction) error {
+	config := &tls.Config{}
+	if config.NextProtos == nil {
+		config.NextProtos = []string{"http/1.1"}
+	}
+
+	config.Certificates = make([]tls.Certificate, 1)
+	var err error
+	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Error("Failed to load https cert file %s for %s:%s: %v", certFile, srv.network, srv.address, err)
+		return err
+	}
+	return srv.ListenAndServeTLSConfig(config, serve)
+}
+
+// ListenAndServeTLSConfig listens on the provided network address and then calls
+// Serve to handle requests on incoming TLS connections.
+func (srv *Server) ListenAndServeTLSConfig(tlsConfig *tls.Config, serve ServeFunction) error {
+	go srv.handleSignals()
+
+	l, err := GetListener(srv.network, srv.address)
+	if err != nil {
+		log.Error("Unable to get Listener: %v", err)
+		return err
+	}
+
+	wl := newWrappedListener(l, srv)
+	srv.listener = tls.NewListener(wl, tlsConfig)
+
+	KillParent()
+	srv.BeforeBegin(srv.network, srv.address)
+
+	return srv.Serve(serve)
+}
+
+// Serve accepts incoming HTTP connections on the wrapped listener l, creating a new
+// service goroutine for each. The service goroutines read requests and then call
+// handler to reply to them. Handler is typically nil, in which case the
+// DefaultServeMux is used.
+//
+// In addition to the standard Serve behaviour each connection is added to a
+// sync.Waitgroup so that all outstanding connections can be served before shutting
+// down the server.
+func (srv *Server) Serve(serve ServeFunction) error {
+	defer log.Debug("Serve() returning... (PID: %d)", syscall.Getpid())
+	srv.setState(stateRunning)
+	runningServerWG.Add(1)
+	err := serve(srv.listener)
+	log.Debug("Waiting for connections to finish... (PID: %d)", syscall.Getpid())
+	srv.wg.Wait()
+	srv.setState(stateTerminate)
+	runningServerWG.Done()
+	// use of closed means that the listeners are closed - i.e. we should be shutting down - return nil
+	if err != nil && strings.Contains(err.Error(), "use of closed") {
+		return nil
+	}
+	return err
+}
+
+func (srv *Server) getState() state {
+	srv.lock.RLock()
+	defer srv.lock.RUnlock()
+
+	return srv.state
+}
+
+func (srv *Server) setState(st state) {
+	srv.lock.Lock()
+	defer srv.lock.Unlock()
+
+	srv.state = st
+}
+
+type wrappedListener struct {
+	net.Listener
+	stopped bool
+	server  *Server
+}
+
+func newWrappedListener(l net.Listener, srv *Server) *wrappedListener {
+	return &wrappedListener{
+		Listener: l,
+		server:   srv,
+	}
+}
+
+func (wl *wrappedListener) Accept() (net.Conn, error) {
+	var c net.Conn
+	// Set keepalive on TCPListeners connections.
+	if tcl, ok := wl.Listener.(*net.TCPListener); ok {
+		tc, err := tcl.AcceptTCP()
+		if err != nil {
+			return nil, err
+		}
+		_ = tc.SetKeepAlive(true)                  // see http.tcpKeepAliveListener
+		_ = tc.SetKeepAlivePeriod(3 * time.Minute) // see http.tcpKeepAliveListener
+		c = tc
+	} else {
+		var err error
+		c, err = wl.Listener.Accept()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	c = wrappedConn{
+		Conn:   c,
+		server: wl.server,
+	}
+
+	wl.server.wg.Add(1)
+	return c, nil
+}
+
+func (wl *wrappedListener) Close() error {
+	if wl.stopped {
+		return syscall.EINVAL
+	}
+
+	wl.stopped = true
+	return wl.Listener.Close()
+}
+
+func (wl *wrappedListener) File() (*os.File, error) {
+	// returns a dup(2) - FD_CLOEXEC flag *not* set so the listening socket can be passed to child processes
+	return wl.Listener.(filer).File()
+}
+
+type wrappedConn struct {
+	net.Conn
+	server *Server
+}
+
+func (w wrappedConn) Close() error {
+	err := w.Conn.Close()
+	if err == nil {
+		w.server.wg.Done()
+	}
+	return err
+}

modules/graceful/server_hooks.go

@@ -0,0 +1,121 @@
+// +build !windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package graceful
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"runtime"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// shutdown closes the listener so that no new connections are accepted
+// and starts a goroutine that will hammer (stop all running requests) the server
+// after setting.GracefulHammerTime.
+func (srv *Server) shutdown() {
+	// only shutdown if we're running.
+	if srv.getState() != stateRunning {
+		return
+	}
+
+	srv.setState(stateShuttingDown)
+	if setting.GracefulHammerTime >= 0 {
+		go srv.hammerTime(setting.GracefulHammerTime)
+	}
+
+	if srv.OnShutdown != nil {
+		srv.OnShutdown()
+	}
+	err := srv.listener.Close()
+	if err != nil {
+		log.Error("PID: %d Listener.Close() error: %v", os.Getpid(), err)
+	} else {
+		log.Info("PID: %d Listener (%s) closed.", os.Getpid(), srv.listener.Addr())
+	}
+}
+
+// hammerTime forces the server to shutdown in a given timeout - whether it
+// finished outstanding requests or not. if Read/WriteTimeout are not set or the
+// max header size is very big a connection could hang...
+//
+// srv.Serve() will not return until all connections are served. this will
+// unblock the srv.wg.Wait() in Serve() thus causing ListenAndServe* functions to
+// return.
+func (srv *Server) hammerTime(d time.Duration) {
+	defer func() {
+		// We call srv.wg.Done() until it panics.
+		// This happens if we call Done() when the WaitGroup counter is already at 0
+		// So if it panics -> we're done, Serve() will return and the
+		// parent will goroutine will exit.
+		if r := recover(); r != nil {
+			log.Error("WaitGroup at 0: Error: %v", r)
+		}
+	}()
+	if srv.getState() != stateShuttingDown {
+		return
+	}
+	time.Sleep(d)
+	log.Warn("Forcefully shutting down parent")
+	for {
+		if srv.getState() == stateTerminate {
+			break
+		}
+		srv.wg.Done()
+
+		// Give other goroutines a chance to finish before we forcibly stop them.
+		runtime.Gosched()
+	}
+}
+
+func (srv *Server) fork() error {
+	runningServerReg.Lock()
+	defer runningServerReg.Unlock()
+
+	// only one server instance should fork!
+	if runningServersForked {
+		return errors.New("another process already forked. Ignoring this one")
+	}
+
+	runningServersForked = true
+
+	// We need to move the file logs to append pids
+	setting.RestartLogsWithPIDSuffix()
+
+	_, err := RestartProcess()
+
+	return err
+}
+
+// RegisterPreSignalHook registers a function to be run before the signal handler for
+// a given signal. These are not mutex locked and should therefore be only called before Serve.
+func (srv *Server) RegisterPreSignalHook(sig os.Signal, f func()) (err error) {
+	for _, s := range hookableSignals {
+		if s == sig {
+			srv.PreSignalHooks[sig] = append(srv.PreSignalHooks[sig], f)
+			return
+		}
+	}
+	err = fmt.Errorf("Signal %v is not supported", sig)
+	return
+}
+
+// RegisterPostSignalHook registers a function to be run after the signal handler for
+// a given signal. These are not mutex locked and should therefore be only called before Serve.
+func (srv *Server) RegisterPostSignalHook(sig os.Signal, f func()) (err error) {
+	for _, s := range hookableSignals {
+		if s == sig {
+			srv.PostSignalHooks[sig] = append(srv.PostSignalHooks[sig], f)
+			return
+		}
+	}
+	err = fmt.Errorf("Signal %v is not supported", sig)
+	return
+}

modules/graceful/server_http.go

@@ -0,0 +1,47 @@
+// +build !windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package graceful
+
+import (
+	"crypto/tls"
+	"net/http"
+)
+
+func newHTTPServer(network, address string, handler http.Handler) (*Server, ServeFunction) {
+	server := NewServer(network, address)
+	httpServer := http.Server{
+		ReadTimeout:    DefaultReadTimeOut,
+		WriteTimeout:   DefaultWriteTimeOut,
+		MaxHeaderBytes: DefaultMaxHeaderBytes,
+		Handler:        handler,
+	}
+	server.OnShutdown = func() {
+		httpServer.SetKeepAlivesEnabled(false)
+	}
+	return server, httpServer.Serve
+}
+
+// HTTPListenAndServe listens on the provided network address and then calls Serve
+// to handle requests on incoming connections.
+func HTTPListenAndServe(network, address string, handler http.Handler) error {
+	server, lHandler := newHTTPServer(network, address, handler)
+	return server.ListenAndServe(lHandler)
+}
+
+// HTTPListenAndServeTLS listens on the provided network address and then calls Serve
+// to handle requests on incoming connections.
+func HTTPListenAndServeTLS(network, address, certFile, keyFile string, handler http.Handler) error {
+	server, lHandler := newHTTPServer(network, address, handler)
+	return server.ListenAndServeTLS(certFile, keyFile, lHandler)
+}
+
+// HTTPListenAndServeTLSConfig listens on the provided network address and then calls Serve
+// to handle requests on incoming connections.
+func HTTPListenAndServeTLSConfig(network, address string, tlsConfig *tls.Config, handler http.Handler) error {
+	server, lHandler := newHTTPServer(network, address, handler)
+	return server.ListenAndServeTLSConfig(tlsConfig, lHandler)
+}

modules/graceful/server_signals.go

@@ -0,0 +1,95 @@
+// +build !windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package graceful
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var hookableSignals []os.Signal
+
+func init() {
+	hookableSignals = []os.Signal{
+		syscall.SIGHUP,
+		syscall.SIGUSR1,
+		syscall.SIGUSR2,
+		syscall.SIGINT,
+		syscall.SIGTERM,
+		syscall.SIGTSTP,
+	}
+}
+
+// handleSignals listens for os Signals and calls any hooked in function that the
+// user had registered with the signal.
+func (srv *Server) handleSignals() {
+	var sig os.Signal
+
+	signal.Notify(
+		srv.sigChan,
+		hookableSignals...,
+	)
+
+	pid := syscall.Getpid()
+	for {
+		sig = <-srv.sigChan
+		srv.preSignalHooks(sig)
+		switch sig {
+		case syscall.SIGHUP:
+			if setting.GracefulRestartable {
+				log.Info("PID: %d. Received SIGHUP. Forking...", pid)
+				err := srv.fork()
+				if err != nil && err.Error() != "another process already forked. Ignoring this one" {
+					log.Error("Error whilst forking from PID: %d : %v", pid, err)
+				}
+			} else {
+				log.Info("PID: %d. Received SIGHUP. Not set restartable. Shutting down...", pid)
+
+				srv.shutdown()
+			}
+		case syscall.SIGUSR1:
+			log.Info("PID %d. Received SIGUSR1.", pid)
+		case syscall.SIGUSR2:
+			log.Warn("PID %d. Received SIGUSR2. Hammering...", pid)
+			srv.hammerTime(0 * time.Second)
+		case syscall.SIGINT:
+			log.Warn("PID %d. Received SIGINT. Shutting down...", pid)
+			srv.shutdown()
+		case syscall.SIGTERM:
+			log.Warn("PID %d. Received SIGTERM. Shutting down...", pid)
+			srv.shutdown()
+		case syscall.SIGTSTP:
+			log.Info("PID %d. Received SIGTSTP.")
+		default:
+			log.Info("PID %d. Received %v.", sig)
+		}
+		srv.postSignalHooks(sig)
+	}
+}
+
+func (srv *Server) preSignalHooks(sig os.Signal) {
+	if _, notSet := srv.PreSignalHooks[sig]; !notSet {
+		return
+	}
+	for _, f := range srv.PreSignalHooks[sig] {
+		f()
+	}
+}
+
+func (srv *Server) postSignalHooks(sig os.Signal) {
+	if _, notSet := srv.PostSignalHooks[sig]; !notSet {
+		return
+	}
+	for _, f := range srv.PostSignalHooks[sig] {
+		f()
+	}
+}

modules/indexer/issues/indexer.go

@@ -5,9 +5,11 @@
 package issues
 
 import (
-	"fmt"
+	"sync"
+	"time"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
@@ -45,78 +47,143 @@ type Indexer interface {
 	Search(kw string, repoID int64, limit, start int) (*SearchResult, error)
 }
 
+type indexerHolder struct {
+	indexer Indexer
+	mutex   sync.RWMutex
+	cond    *sync.Cond
+}
+
+func newIndexerHolder() *indexerHolder {
+	h := &indexerHolder{}
+	h.cond = sync.NewCond(h.mutex.RLocker())
+	return h
+}
+
+func (h *indexerHolder) set(indexer Indexer) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+	h.indexer = indexer
+	h.cond.Broadcast()
+}
+
+func (h *indexerHolder) get() Indexer {
+	h.mutex.RLock()
+	defer h.mutex.RUnlock()
+	if h.indexer == nil {
+		h.cond.Wait()
+	}
+	return h.indexer
+}
+
 var (
+	issueIndexerChannel = make(chan *IndexerData, setting.Indexer.UpdateQueueLength)
 	// issueIndexerQueue queue of issue ids to be updated
 	issueIndexerQueue Queue
-	issueIndexer      Indexer
+	holder            = newIndexerHolder()
 )
 
 // InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until
 // all issue index done.
-func InitIssueIndexer(syncReindex bool) error {
-	var populate bool
-	var dummyQueue bool
-	switch setting.Indexer.IssueType {
-	case "bleve":
-		issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath)
-		exist, err := issueIndexer.Init()
-		if err != nil {
-			return err
-		}
-		populate = !exist
-	case "db":
-		issueIndexer = &DBIndexer{}
-		dummyQueue = true
-	default:
-		return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType)
-	}
-
-	if dummyQueue {
-		issueIndexerQueue = &DummyQueue{}
-		return nil
-	}
-
-	var err error
-	switch setting.Indexer.IssueQueueType {
-	case setting.LevelQueueType:
-		issueIndexerQueue, err = NewLevelQueue(
-			issueIndexer,
-			setting.Indexer.IssueQueueDir,
-			setting.Indexer.IssueQueueBatchNumber)
-		if err != nil {
-			return err
-		}
-	case setting.ChannelQueueType:
-		issueIndexerQueue = NewChannelQueue(issueIndexer, setting.Indexer.IssueQueueBatchNumber)
-	case setting.RedisQueueType:
-		addrs, pass, idx, err := parseConnStr(setting.Indexer.IssueQueueConnStr)
-		if err != nil {
-			return err
-		}
-		issueIndexerQueue, err = NewRedisQueue(addrs, pass, idx, issueIndexer, setting.Indexer.IssueQueueBatchNumber)
-		if err != nil {
-			return err
-		}
-	default:
-		return fmt.Errorf("Unsupported indexer queue type: %v", setting.Indexer.IssueQueueType)
-	}
-
+func InitIssueIndexer(syncReindex bool) {
+	waitChannel := make(chan time.Duration)
 	go func() {
-		err = issueIndexerQueue.Run()
-		if err != nil {
-			log.Error("issueIndexerQueue.Run: %v", err)
+		start := time.Now()
+		log.Info("Initializing Issue Indexer")
+		var populate bool
+		var dummyQueue bool
+		switch setting.Indexer.IssueType {
+		case "bleve":
+			issueIndexer := NewBleveIndexer(setting.Indexer.IssuePath)
+			exist, err := issueIndexer.Init()
+			if err != nil {
+				log.Fatal("Unable to initialize Bleve Issue Indexer: %v", err)
+			}
+			populate = !exist
+			holder.set(issueIndexer)
+		case "db":
+			issueIndexer := &DBIndexer{}
+			holder.set(issueIndexer)
+			dummyQueue = true
+		default:
+			log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
 		}
-	}()
 
-	if populate {
-		if syncReindex {
-			populateIssueIndexer()
+		if dummyQueue {
+			issueIndexerQueue = &DummyQueue{}
 		} else {
-			go populateIssueIndexer()
-		}
-	}
+			var err error
+			switch setting.Indexer.IssueQueueType {
+			case setting.LevelQueueType:
+				issueIndexerQueue, err = NewLevelQueue(
+					holder.get(),
+					setting.Indexer.IssueQueueDir,
+					setting.Indexer.IssueQueueBatchNumber)
+				if err != nil {
+					log.Fatal(
+						"Unable create level queue for issue queue dir: %s batch number: %d : %v",
+						setting.Indexer.IssueQueueDir,
+						setting.Indexer.IssueQueueBatchNumber,
+						err)
+				}
+			case setting.ChannelQueueType:
+				issueIndexerQueue = NewChannelQueue(holder.get(), setting.Indexer.IssueQueueBatchNumber)
+			case setting.RedisQueueType:
+				addrs, pass, idx, err := parseConnStr(setting.Indexer.IssueQueueConnStr)
+				if err != nil {
+					log.Fatal("Unable to parse connection string for RedisQueueType: %s : %v",
+						setting.Indexer.IssueQueueConnStr,
+						err)
+				}
+				issueIndexerQueue, err = NewRedisQueue(addrs, pass, idx, holder.get(), setting.Indexer.IssueQueueBatchNumber)
+				if err != nil {
+					log.Fatal("Unable to create RedisQueue: %s : %v",
+						setting.Indexer.IssueQueueConnStr,
+						err)
+				}
+			default:
+				log.Fatal("Unsupported indexer queue type: %v",
+					setting.Indexer.IssueQueueType)
+			}
 
-	return nil
+			go func() {
+				err = issueIndexerQueue.Run()
+				if err != nil {
+					log.Error("issueIndexerQueue.Run: %v", err)
+				}
+			}()
+		}
+
+		go func() {
+			for data := range issueIndexerChannel {
+				_ = issueIndexerQueue.Push(data)
+			}
+		}()
+
+		if populate {
+			if syncReindex {
+				populateIssueIndexer()
+			} else {
+				go populateIssueIndexer()
+			}
+		}
+		waitChannel <- time.Since(start)
+	}()
+	if syncReindex {
+		<-waitChannel
+	} else if setting.Indexer.StartupTimeout > 0 {
+		go func() {
+			timeout := setting.Indexer.StartupTimeout
+			if graceful.IsChild && setting.GracefulHammerTime > 0 {
+				timeout += setting.GracefulHammerTime
+			}
+			select {
+			case duration := <-waitChannel:
+				log.Info("Issue Indexer Initialization took %v", duration)
+			case <-time.After(timeout):
+				log.Fatal("Issue Indexer Initialization timed-out after: %v", timeout)
+			}
+		}()
+	}
 }
 
 // populateIssueIndexer populate the issue indexer with issue data
@@ -166,13 +233,13 @@ func UpdateIssueIndexer(issue *models.Issue) {
 			comments = append(comments, comment.Content)
 		}
 	}
-	_ = issueIndexerQueue.Push(&IndexerData{
+	issueIndexerChannel <- &IndexerData{
 		ID:       issue.ID,
 		RepoID:   issue.RepoID,
 		Title:    issue.Title,
 		Content:  issue.Content,
 		Comments: comments,
-	})
+	}
 }
 
 // DeleteRepoIssueIndexer deletes repo's all issues indexes
@@ -188,16 +255,16 @@ func DeleteRepoIssueIndexer(repo *models.Repository) {
 		return
 	}
 
-	_ = issueIndexerQueue.Push(&IndexerData{
+	issueIndexerChannel <- &IndexerData{
 		IDs:      ids,
 		IsDelete: true,
-	})
+	}
 }
 
 // SearchIssuesByKeyword search issue ids by keywords and repo id
 func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
 	var issueIDs []int64
-	res, err := issueIndexer.Search(keyword, repoID, 1000, 0)
+	res, err := holder.get().Search(keyword, repoID, 1000, 0)
 	if err != nil {
 		return nil, err
 	}

modules/indexer/issues/indexer_test.go

@@ -5,7 +5,6 @@
 package issues
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -17,11 +16,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func fatalTestError(fmtStr string, args ...interface{}) {
-	fmt.Fprintf(os.Stderr, fmtStr, args...)
-	os.Exit(1)
-}
-
 func TestMain(m *testing.M) {
 	models.MainTest(m, filepath.Join("..", "..", ".."))
 }
@@ -32,9 +26,7 @@ func TestBleveSearchIssues(t *testing.T) {
 	os.RemoveAll(setting.Indexer.IssueQueueDir)
 	os.RemoveAll(setting.Indexer.IssuePath)
 	setting.Indexer.IssueType = "bleve"
-	if err := InitIssueIndexer(true); err != nil {
-		fatalTestError("Error InitIssueIndexer: %v\n", err)
-	}
+	InitIssueIndexer(true)
 
 	time.Sleep(5 * time.Second)
 
@@ -59,9 +51,7 @@ func TestDBSearchIssues(t *testing.T) {
 	assert.NoError(t, models.PrepareTestDatabase())
 
 	setting.Indexer.IssueType = "db"
-	if err := InitIssueIndexer(true); err != nil {
-		fatalTestError("Error InitIssueIndexer: %v\n", err)
-	}
+	InitIssueIndexer(true)
 
 	ids, err := SearchIssuesByKeyword(1, "issue2")
 	assert.NoError(t, err)

modules/indexer/repo.go

@@ -6,6 +6,7 @@ package indexer
 
 import (
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -25,8 +26,36 @@ const (
 	repoIndexerLatestVersion = 4
 )
 
+type bleveIndexerHolder struct {
+	index bleve.Index
+	mutex sync.RWMutex
+	cond  *sync.Cond
+}
+
+func newBleveIndexerHolder() *bleveIndexerHolder {
+	b := &bleveIndexerHolder{}
+	b.cond = sync.NewCond(b.mutex.RLocker())
+	return b
+}
+
+func (r *bleveIndexerHolder) set(index bleve.Index) {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	r.index = index
+	r.cond.Broadcast()
+}
+
+func (r *bleveIndexerHolder) get() bleve.Index {
+	r.mutex.RLock()
+	defer r.mutex.RUnlock()
+	if r.index == nil {
+		r.cond.Wait()
+	}
+	return r.index
+}
+
 // repoIndexer (thread-safe) index for repository contents
-var repoIndexer bleve.Index
+var indexerHolder = newBleveIndexerHolder()
 
 // RepoIndexerOp type of operation to perform on repo indexer
 type RepoIndexerOp int
@@ -73,12 +102,12 @@ func (update RepoIndexerUpdate) AddToFlushingBatch(batch rupture.FlushingBatch)
 
 // InitRepoIndexer initialize repo indexer
 func InitRepoIndexer(populateIndexer func() error) {
-	var err error
-	repoIndexer, err = openIndexer(setting.Indexer.RepoPath, repoIndexerLatestVersion)
+	indexer, err := openIndexer(setting.Indexer.RepoPath, repoIndexerLatestVersion)
 	if err != nil {
 		log.Fatal("InitRepoIndexer: %v", err)
 	}
-	if repoIndexer != nil {
+	if indexer != nil {
+		indexerHolder.set(indexer)
 		return
 	}
 
@@ -92,7 +121,6 @@ func InitRepoIndexer(populateIndexer func() error) {
 
 // createRepoIndexer create a repo indexer if one does not already exist
 func createRepoIndexer(path string, latestVersion int) error {
-	var err error
 	docMapping := bleve.NewDocumentMapping()
 	numericFieldMapping := bleve.NewNumericFieldMapping()
 	numericFieldMapping.IncludeInAll = false
@@ -103,9 +131,9 @@ func createRepoIndexer(path string, latestVersion int) error {
 	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
 
 	mapping := bleve.NewIndexMapping()
-	if err = addUnicodeNormalizeTokenFilter(mapping); err != nil {
+	if err := addUnicodeNormalizeTokenFilter(mapping); err != nil {
 		return err
-	} else if err = mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
+	} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
 		"type":          custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     unicode.Name,
@@ -117,10 +145,12 @@ func createRepoIndexer(path string, latestVersion int) error {
 	mapping.AddDocumentMapping(repoIndexerDocType, docMapping)
 	mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
 
-	repoIndexer, err = bleve.New(path, mapping)
+	indexer, err := bleve.New(path, mapping)
 	if err != nil {
 		return err
 	}
+	indexerHolder.set(indexer)
+
 	return rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{
 		Version: latestVersion,
 	})
@@ -140,14 +170,14 @@ func filenameOfIndexerID(indexerID string) string {
 
 // RepoIndexerBatch batch to add updates to
 func RepoIndexerBatch() rupture.FlushingBatch {
-	return rupture.NewFlushingBatch(repoIndexer, maxBatchSize)
+	return rupture.NewFlushingBatch(indexerHolder.get(), maxBatchSize)
 }
 
 // DeleteRepoFromIndexer delete all of a repo's files from indexer
 func DeleteRepoFromIndexer(repoID int64) error {
 	query := numericEqualityQuery(repoID, "RepoID")
 	searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false)
-	result, err := repoIndexer.Search(searchRequest)
+	result, err := indexerHolder.get().Search(searchRequest)
 	if err != nil {
 		return err
 	}
@@ -196,7 +226,7 @@ func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (i
 	searchRequest.Fields = []string{"Content", "RepoID"}
 	searchRequest.IncludeLocations = true
 
-	result, err := repoIndexer.Search(searchRequest)
+	result, err := indexerHolder.get().Search(searchRequest)
 	if err != nil {
 		return 0, nil, err
 	}

modules/lfs/locks.go

@@ -155,7 +155,9 @@ func PostLockHandler(ctx *context.Context) {
 	}
 
 	var req api.LFSLockRequest
-	dec := json.NewDecoder(ctx.Req.Body().ReadCloser())
+	bodyReader := ctx.Req.Body().ReadCloser()
+	defer bodyReader.Close()
+	dec := json.NewDecoder(bodyReader)
 	if err := dec.Decode(&req); err != nil {
 		writeStatus(ctx, 400)
 		return
@@ -269,7 +271,9 @@ func UnLockHandler(ctx *context.Context) {
 	}
 
 	var req api.LFSLockDeleteRequest
-	dec := json.NewDecoder(ctx.Req.Body().ReadCloser())
+	bodyReader := ctx.Req.Body().ReadCloser()
+	defer bodyReader.Close()
+	dec := json.NewDecoder(bodyReader)
 	if err := dec.Decode(&req); err != nil {
 		writeStatus(ctx, 400)
 		return

modules/lfs/server.go

@@ -327,7 +327,9 @@ func PutHandler(ctx *context.Context) {
 	}
 
 	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
-	if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil {
+	bodyReader := ctx.Req.Body().ReadCloser()
+	defer bodyReader.Close()
+	if err := contentStore.Put(meta, bodyReader); err != nil {
 		ctx.Resp.WriteHeader(500)
 		fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
 		if err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
@@ -434,7 +436,9 @@ func unpack(ctx *context.Context) *RequestVars {
 
 	if r.Method == "POST" { // Maybe also check if +json
 		var p RequestVars
-		dec := json.NewDecoder(r.Body().ReadCloser())
+		bodyReader := r.Body().ReadCloser()
+		defer bodyReader.Close()
+		dec := json.NewDecoder(bodyReader)
 		err := dec.Decode(&p)
 		if err != nil {
 			return rv
@@ -453,7 +457,9 @@ func unpackbatch(ctx *context.Context) *BatchVars {
 	r := ctx.Req
 	var bv BatchVars
 
-	dec := json.NewDecoder(r.Body().ReadCloser())
+	bodyReader := r.Body().ReadCloser()
+	defer bodyReader.Close()
+	dec := json.NewDecoder(bodyReader)
 	err := dec.Decode(&bv)
 	if err != nil {
 		return &bv

modules/markup/html.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/references"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
@@ -36,17 +37,6 @@ var (
 	// While fast, this is also incorrect and lead to false positives.
 	// TODO: fix invalid linking issue
 
-	// mentionPattern matches all mentions in the form of "@user"
-	mentionPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@[0-9a-zA-Z-_\.]+)(?:\s|$|\)|\])`)
-
-	// issueNumericPattern matches string that references to a numeric issue, e.g. #1287
-	issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(#[0-9]+)(?:\s|$|\)|\]|:|\.(\s|$))`)
-	// issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234
-	issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$))`)
-	// crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository
-	// e.g. gogits/gogs#12345
-	crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+#[0-9]+)(?:\s|$|\)|\]|\.(\s|$))`)
-
 	// sha1CurrentPattern matches string that represents a commit SHA, e.g. d8a994ef243349f321568f9e36d5c3f444b99cae
 	// Although SHA1 hashes are 40 chars long, the regex matches the hash from 7 to 40 chars in length
 	// so that abbreviated hash links can be used as well. This matches git and github useability.
@@ -70,6 +60,9 @@ var (
 	linkRegex, _ = xurls.StrictMatchingScheme("https?://")
 )
 
+// CSS class for action keywords (e.g. "closes: #1")
+const keywordClass = "issue-keyword"
+
 // regexp for full links to issues/pulls
 var issueFullPattern *regexp.Regexp
 
@@ -99,15 +92,30 @@ func getIssueFullPattern() *regexp.Regexp {
 	return issueFullPattern
 }
 
-// FindAllMentions matches mention patterns in given content
-// and returns a list of found user names without @ prefix.
-func FindAllMentions(content string) []string {
-	mentions := mentionPattern.FindAllStringSubmatch(content, -1)
-	ret := make([]string, len(mentions))
-	for i, val := range mentions {
-		ret[i] = val[1][1:]
+// CustomLinkURLSchemes allows for additional schemes to be detected when parsing links within text
+func CustomLinkURLSchemes(schemes []string) {
+	schemes = append(schemes, "http", "https")
+	withAuth := make([]string, 0, len(schemes))
+	validScheme := regexp.MustCompile(`^[a-z]+$`)
+	for _, s := range schemes {
+		if !validScheme.MatchString(s) {
+			continue
+		}
+		without := false
+		for _, sna := range xurls.SchemesNoAuthority {
+			if s == sna {
+				without = true
+				break
+			}
+		}
+		if without {
+			s += ":"
+		} else {
+			s += "://"
+		}
+		withAuth = append(withAuth, s)
 	}
-	return ret
+	linkRegex, _ = xurls.StrictMatchingScheme(strings.Join(withAuth, "|"))
 }
 
 // IsSameDomain checks if given url string has the same hostname as current Gitea instance
@@ -142,7 +150,6 @@ var defaultProcessors = []processor{
 	linkProcessor,
 	mentionProcessor,
 	issueIndexPatternProcessor,
-	crossReferenceIssueIndexPatternProcessor,
 	sha1CurrentPatternProcessor,
 	emailAddressProcessor,
 }
@@ -183,7 +190,6 @@ var commitMessageProcessors = []processor{
 	linkProcessor,
 	mentionProcessor,
 	issueIndexPatternProcessor,
-	crossReferenceIssueIndexPatternProcessor,
 	sha1CurrentPatternProcessor,
 	emailAddressProcessor,
 }
@@ -217,7 +223,6 @@ var commitMessageSubjectProcessors = []processor{
 	linkProcessor,
 	mentionProcessor,
 	issueIndexPatternProcessor,
-	crossReferenceIssueIndexPatternProcessor,
 	sha1CurrentPatternProcessor,
 }
 
@@ -330,6 +335,24 @@ func (ctx *postProcessCtx) textNode(node *html.Node) {
 	}
 }
 
+// createKeyword() renders a highlighted version of an action keyword
+func createKeyword(content string) *html.Node {
+	span := &html.Node{
+		Type: html.ElementNode,
+		Data: atom.Span.String(),
+		Attr: []html.Attribute{},
+	}
+	span.Attr = append(span.Attr, html.Attribute{Key: "class", Val: keywordClass})
+
+	text := &html.Node{
+		Type: html.TextNode,
+		Data: content,
+	}
+	span.AppendChild(text)
+
+	return span
+}
+
 func createLink(href, content, class string) *html.Node {
 	a := &html.Node{
 		Type: html.ElementNode,
@@ -377,10 +400,16 @@ func createCodeLink(href, content, class string) *html.Node {
 	return a
 }
 
-// replaceContent takes a text node, and in its content it replaces a section of
-// it with the specified newNode. An example to visualize how this can work can
-// be found here: https://play.golang.org/p/5zP8NnHZ03s
+// replaceContent takes text node, and in its content it replaces a section of
+// it with the specified newNode.
 func replaceContent(node *html.Node, i, j int, newNode *html.Node) {
+	replaceContentList(node, i, j, []*html.Node{newNode})
+}
+
+// replaceContentList takes text node, and in its content it replaces a section of
+// it with the specified newNodes. An example to visualize how this can work can
+// be found here: https://play.golang.org/p/5zP8NnHZ03s
+func replaceContentList(node *html.Node, i, j int, newNodes []*html.Node) {
 	// get the data before and after the match
 	before := node.Data[:i]
 	after := node.Data[j:]
@@ -392,7 +421,9 @@ func replaceContent(node *html.Node, i, j int, newNode *html.Node) {
 	// Get the current next sibling, before which we place the replaced data,
 	// and after that we place the new text node.
 	nextSibling := node.NextSibling
-	node.Parent.InsertBefore(newNode, nextSibling)
+	for _, n := range newNodes {
+		node.Parent.InsertBefore(n, nextSibling)
+	}
 	if after != "" {
 		node.Parent.InsertBefore(&html.Node{
 			Type: html.TextNode,
@@ -402,13 +433,13 @@ func replaceContent(node *html.Node, i, j int, newNode *html.Node) {
 }
 
 func mentionProcessor(_ *postProcessCtx, node *html.Node) {
-	m := mentionPattern.FindStringSubmatchIndex(node.Data)
-	if m == nil {
+	// We replace only the first mention; other mentions will be addressed later
+	found, loc := references.FindFirstMentionBytes([]byte(node.Data))
+	if !found {
 		return
 	}
-	// Replace the mention with a link to the specified user.
-	mention := node.Data[m[2]:m[3]]
-	replaceContent(node, m[2], m[3], createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention"))
+	mention := node.Data[loc.Start:loc.End]
+	replaceContent(node, loc.Start, loc.End, createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention"))
 }
 
 func shortLinkProcessor(ctx *postProcessCtx, node *html.Node) {
@@ -597,45 +628,44 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 	if ctx.metas == nil {
 		return
 	}
-	// default to numeric pattern, unless alphanumeric is requested.
-	pattern := issueNumericPattern
+
+	var (
+		found bool
+		ref   *references.RenderizableReference
+	)
+
 	if ctx.metas["style"] == IssueNameStyleAlphanumeric {
-		pattern = issueAlphanumericPattern
-	}
-
-	match := pattern.FindStringSubmatchIndex(node.Data)
-	if match == nil {
-		return
-	}
-
-	id := node.Data[match[2]:match[3]]
-	var link *html.Node
-	if _, ok := ctx.metas["format"]; ok {
-		// Support for external issue tracker
-		if ctx.metas["style"] == IssueNameStyleAlphanumeric {
-			ctx.metas["index"] = id
-		} else {
-			ctx.metas["index"] = id[1:]
-		}
-		link = createLink(com.Expand(ctx.metas["format"], ctx.metas), id, "issue")
+		found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data)
 	} else {
-		link = createLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "issues", id[1:]), id, "issue")
+		found, ref = references.FindRenderizableReferenceNumeric(node.Data)
 	}
-	replaceContent(node, match[2], match[3], link)
-}
-
-func crossReferenceIssueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
-	m := crossReferenceIssueNumericPattern.FindStringSubmatchIndex(node.Data)
-	if m == nil {
+	if !found {
 		return
 	}
-	ref := node.Data[m[2]:m[3]]
 
-	parts := strings.SplitN(ref, "#", 2)
-	repo, issue := parts[0], parts[1]
+	var link *html.Node
+	reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End]
+	if _, ok := ctx.metas["format"]; ok {
+		ctx.metas["index"] = ref.Issue
+		link = createLink(com.Expand(ctx.metas["format"], ctx.metas), reftext, "issue")
+	} else if ref.Owner == "" {
+		link = createLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "issues", ref.Issue), reftext, "issue")
+	} else {
+		link = createLink(util.URLJoin(setting.AppURL, ref.Owner, ref.Name, "issues", ref.Issue), reftext, "issue")
+	}
 
-	replaceContent(node, m[2], m[3],
-		createLink(util.URLJoin(setting.AppURL, repo, "issues", issue), ref, issue))
+	if ref.Action == references.XRefActionNone {
+		replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link)
+		return
+	}
+
+	// Decorate action keywords
+	keyword := createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End])
+	spaces := &html.Node{
+		Type: html.TextNode,
+		Data: node.Data[ref.ActionLocation.End:ref.RefLocation.Start],
+	}
+	replaceContentList(node, ref.ActionLocation.Start, ref.RefLocation.End, []*html.Node{keyword, spaces, link})
 }
 
 // fullSha1PatternProcessor renders SHA containing URLs

modules/markup/html_internal_test.go

@@ -239,34 +239,6 @@ func TestRender_FullIssueURLs(t *testing.T) {
 		`<a href="http://localhost:3000/gogits/gogs/issues/4" class="issue">#4</a>`)
 }
 
-func TestRegExp_issueNumericPattern(t *testing.T) {
-	trueTestCases := []string{
-		"#1234",
-		"#0",
-		"#1234567890987654321",
-		"  #12",
-		"#12:",
-		"ref: #12: msg",
-	}
-	falseTestCases := []string{
-		"# 1234",
-		"# 0",
-		"# ",
-		"#",
-		"#ABC",
-		"#1A2B",
-		"",
-		"ABC",
-	}
-
-	for _, testCase := range trueTestCases {
-		assert.True(t, issueNumericPattern.MatchString(testCase))
-	}
-	for _, testCase := range falseTestCases {
-		assert.False(t, issueNumericPattern.MatchString(testCase))
-	}
-}
-
 func TestRegExp_sha1CurrentPattern(t *testing.T) {
 	trueTestCases := []string{
 		"d8a994ef243349f321568f9e36d5c3f444b99cae",
@@ -325,70 +297,6 @@ func TestRegExp_anySHA1Pattern(t *testing.T) {
 	}
 }
 
-func TestRegExp_mentionPattern(t *testing.T) {
-	trueTestCases := []string{
-		"@Unknwon",
-		"@ANT_123",
-		"@xxx-DiN0-z-A..uru..s-xxx",
-		"   @lol   ",
-		" @Te-st",
-		"(@gitea)",
-		"[@gitea]",
-	}
-	falseTestCases := []string{
-		"@ 0",
-		"@ ",
-		"@",
-		"",
-		"ABC",
-		"/home/gitea/@gitea",
-		"\"@gitea\"",
-	}
-
-	for _, testCase := range trueTestCases {
-		res := mentionPattern.MatchString(testCase)
-		assert.True(t, res)
-	}
-	for _, testCase := range falseTestCases {
-		res := mentionPattern.MatchString(testCase)
-		assert.False(t, res)
-	}
-}
-
-func TestRegExp_issueAlphanumericPattern(t *testing.T) {
-	trueTestCases := []string{
-		"ABC-1234",
-		"A-1",
-		"RC-80",
-		"ABCDEFGHIJ-1234567890987654321234567890",
-		"ABC-123.",
-		"(ABC-123)",
-		"[ABC-123]",
-		"ABC-123:",
-	}
-	falseTestCases := []string{
-		"RC-08",
-		"PR-0",
-		"ABCDEFGHIJK-1",
-		"PR_1",
-		"",
-		"#ABC",
-		"",
-		"ABC",
-		"GG-",
-		"rm-1",
-		"/home/gitea/ABC-1234",
-		"MY-STRING-ABC-123",
-	}
-
-	for _, testCase := range trueTestCases {
-		assert.True(t, issueAlphanumericPattern.MatchString(testCase))
-	}
-	for _, testCase := range falseTestCases {
-		assert.False(t, issueAlphanumericPattern.MatchString(testCase))
-	}
-}
-
 func TestRegExp_shortLinkPattern(t *testing.T) {
 	trueTestCases := []string{
 		"[[stuff]]",

modules/markup/html_test.go

@@ -89,6 +89,11 @@ func TestRender_links(t *testing.T) {
 	}
 	// Text that should be turned into URL
 
+	defaultCustom := setting.Markdown.CustomURLSchemes
+	setting.Markdown.CustomURLSchemes = []string{"ftp", "magnet"}
+	ReplaceSanitizer()
+	CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes)
+
 	test(
 		"https://www.example.com",
 		`<p><a href="https://www.example.com" rel="nofollow">https://www.example.com</a></p>`)
@@ -131,6 +136,12 @@ func TestRender_links(t *testing.T) {
 	test(
 		"https://username:password@gitea.com",
 		`<p><a href="https://username:password@gitea.com" rel="nofollow">https://username:password@gitea.com</a></p>`)
+	test(
+		"ftp://gitea.com/file.txt",
+		`<p><a href="ftp://gitea.com/file.txt" rel="nofollow">ftp://gitea.com/file.txt</a></p>`)
+	test(
+		"magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&dn=download",
+		`<p><a href="magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&amp;dn=download" rel="nofollow">magnet:?xt=urn:btih:5dee65101db281ac9c46344cd6b175cdcadabcde&amp;dn=download</a></p>`)
 
 	// Test that should *not* be turned into URL
 	test(
@@ -154,6 +165,14 @@ func TestRender_links(t *testing.T) {
 	test(
 		"www",
 		`<p>www</p>`)
+	test(
+		"ftps://gitea.com",
+		`<p>ftps://gitea.com</p>`)
+
+	// Restore previous settings
+	setting.Markdown.CustomURLSchemes = defaultCustom
+	ReplaceSanitizer()
+	CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes)
 }
 
 func TestRender_email(t *testing.T) {

modules/markup/markup.go

@@ -9,12 +9,16 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
 )
 
 // Init initialize regexps for markdown parsing
 func Init() {
 	getIssueFullPattern()
 	NewSanitizer()
+	if len(setting.Markdown.CustomURLSchemes) > 0 {
+		CustomLinkURLSchemes(setting.Markdown.CustomURLSchemes)
+	}
 
 	// since setting maybe changed extensions, this will reload all parser extensions mapping
 	extParsers = make(map[string]Parser)

modules/markup/mdstripper/mdstripper.go

@@ -0,0 +1,260 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package mdstripper
+
+import (
+	"bytes"
+
+	"github.com/russross/blackfriday"
+)
+
+// MarkdownStripper extends blackfriday.Renderer
+type MarkdownStripper struct {
+	blackfriday.Renderer
+	links     []string
+	coallesce bool
+}
+
+const (
+	blackfridayExtensions = 0 |
+		blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
+		blackfriday.EXTENSION_TABLES |
+		blackfriday.EXTENSION_FENCED_CODE |
+		blackfriday.EXTENSION_STRIKETHROUGH |
+		blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK |
+		blackfriday.EXTENSION_DEFINITION_LISTS |
+		blackfriday.EXTENSION_FOOTNOTES |
+		blackfriday.EXTENSION_HEADER_IDS |
+		blackfriday.EXTENSION_AUTO_HEADER_IDS |
+		// Not included in modules/markup/markdown/markdown.go;
+		// required here to process inline links
+		blackfriday.EXTENSION_AUTOLINK
+)
+
+//revive:disable:var-naming Implementing the Rendering interface requires breaking some linting rules
+
+// StripMarkdown parses markdown content by removing all markup and code blocks
+//	in order to extract links and other references
+func StripMarkdown(rawBytes []byte) (string, []string) {
+	stripper := &MarkdownStripper{
+		links: make([]string, 0, 10),
+	}
+	body := blackfriday.Markdown(rawBytes, stripper, blackfridayExtensions)
+	return string(body), stripper.GetLinks()
+}
+
+// StripMarkdownBytes parses markdown content by removing all markup and code blocks
+//	in order to extract links and other references
+func StripMarkdownBytes(rawBytes []byte) ([]byte, []string) {
+	stripper := &MarkdownStripper{
+		links: make([]string, 0, 10),
+	}
+	body := blackfriday.Markdown(rawBytes, stripper, blackfridayExtensions)
+	return body, stripper.GetLinks()
+}
+
+// block-level callbacks
+
+// BlockCode dummy function to proceed with rendering
+func (r *MarkdownStripper) BlockCode(out *bytes.Buffer, text []byte, infoString string) {
+	// Not rendered
+	r.coallesce = false
+}
+
+// BlockQuote dummy function to proceed with rendering
+func (r *MarkdownStripper) BlockQuote(out *bytes.Buffer, text []byte) {
+	// FIXME: perhaps it's better to leave out block quote for this?
+	r.processString(out, text, false)
+}
+
+// BlockHtml dummy function to proceed with rendering
+func (r *MarkdownStripper) BlockHtml(out *bytes.Buffer, text []byte) { //nolint
+	// Not rendered
+	r.coallesce = false
+}
+
+// Header dummy function to proceed with rendering
+func (r *MarkdownStripper) Header(out *bytes.Buffer, text func() bool, level int, id string) {
+	text()
+	r.coallesce = false
+}
+
+// HRule dummy function to proceed with rendering
+func (r *MarkdownStripper) HRule(out *bytes.Buffer) {
+	// Not rendered
+	r.coallesce = false
+}
+
+// List dummy function to proceed with rendering
+func (r *MarkdownStripper) List(out *bytes.Buffer, text func() bool, flags int) {
+	text()
+	r.coallesce = false
+}
+
+// ListItem dummy function to proceed with rendering
+func (r *MarkdownStripper) ListItem(out *bytes.Buffer, text []byte, flags int) {
+	r.processString(out, text, false)
+}
+
+// Paragraph dummy function to proceed with rendering
+func (r *MarkdownStripper) Paragraph(out *bytes.Buffer, text func() bool) {
+	text()
+	r.coallesce = false
+}
+
+// Table dummy function to proceed with rendering
+func (r *MarkdownStripper) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) {
+	r.processString(out, header, false)
+	r.processString(out, body, false)
+}
+
+// TableRow dummy function to proceed with rendering
+func (r *MarkdownStripper) TableRow(out *bytes.Buffer, text []byte) {
+	r.processString(out, text, false)
+}
+
+// TableHeaderCell dummy function to proceed with rendering
+func (r *MarkdownStripper) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {
+	r.processString(out, text, false)
+}
+
+// TableCell dummy function to proceed with rendering
+func (r *MarkdownStripper) TableCell(out *bytes.Buffer, text []byte, flags int) {
+	r.processString(out, text, false)
+}
+
+// Footnotes dummy function to proceed with rendering
+func (r *MarkdownStripper) Footnotes(out *bytes.Buffer, text func() bool) {
+	text()
+}
+
+// FootnoteItem dummy function to proceed with rendering
+func (r *MarkdownStripper) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {
+	r.processString(out, text, false)
+}
+
+// TitleBlock dummy function to proceed with rendering
+func (r *MarkdownStripper) TitleBlock(out *bytes.Buffer, text []byte) {
+	r.processString(out, text, false)
+}
+
+// Span-level callbacks
+
+// AutoLink dummy function to proceed with rendering
+func (r *MarkdownStripper) AutoLink(out *bytes.Buffer, link []byte, kind int) {
+	r.processLink(out, link, []byte{})
+}
+
+// CodeSpan dummy function to proceed with rendering
+func (r *MarkdownStripper) CodeSpan(out *bytes.Buffer, text []byte) {
+	// Not rendered
+	r.coallesce = false
+}
+
+// DoubleEmphasis dummy function to proceed with rendering
+func (r *MarkdownStripper) DoubleEmphasis(out *bytes.Buffer, text []byte) {
+	r.processString(out, text, false)
+}
+
+// Emphasis dummy function to proceed with rendering
+func (r *MarkdownStripper) Emphasis(out *bytes.Buffer, text []byte) {
+	r.processString(out, text, false)
+}
+
+// Image dummy function to proceed with rendering
+func (r *MarkdownStripper) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {
+	// Not rendered
+	r.coallesce = false
+}
+
+// LineBreak dummy function to proceed with rendering
+func (r *MarkdownStripper) LineBreak(out *bytes.Buffer) {
+	// Not rendered
+	r.coallesce = false
+}
+
+// Link dummy function to proceed with rendering
+func (r *MarkdownStripper) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
+	r.processLink(out, link, content)
+}
+
+// RawHtmlTag dummy function to proceed with rendering
+func (r *MarkdownStripper) RawHtmlTag(out *bytes.Buffer, tag []byte) { //nolint
+	// Not rendered
+	r.coallesce = false
+}
+
+// TripleEmphasis dummy function to proceed with rendering
+func (r *MarkdownStripper) TripleEmphasis(out *bytes.Buffer, text []byte) {
+	r.processString(out, text, false)
+}
+
+// StrikeThrough dummy function to proceed with rendering
+func (r *MarkdownStripper) StrikeThrough(out *bytes.Buffer, text []byte) {
+	r.processString(out, text, false)
+}
+
+// FootnoteRef dummy function to proceed with rendering
+func (r *MarkdownStripper) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {
+	// Not rendered
+	r.coallesce = false
+}
+
+// Low-level callbacks
+
+// Entity dummy function to proceed with rendering
+func (r *MarkdownStripper) Entity(out *bytes.Buffer, entity []byte) {
+	// FIXME: literal entities are not parsed; perhaps they should
+	r.coallesce = false
+}
+
+// NormalText dummy function to proceed with rendering
+func (r *MarkdownStripper) NormalText(out *bytes.Buffer, text []byte) {
+	r.processString(out, text, true)
+}
+
+// Header and footer
+
+// DocumentHeader dummy function to proceed with rendering
+func (r *MarkdownStripper) DocumentHeader(out *bytes.Buffer) {
+	r.coallesce = false
+}
+
+// DocumentFooter dummy function to proceed with rendering
+func (r *MarkdownStripper) DocumentFooter(out *bytes.Buffer) {
+	r.coallesce = false
+}
+
+// GetFlags returns rendering flags
+func (r *MarkdownStripper) GetFlags() int {
+	return 0
+}
+
+//revive:enable:var-naming
+
+func doubleSpace(out *bytes.Buffer) {
+	if out.Len() > 0 {
+		out.WriteByte('\n')
+	}
+}
+
+func (r *MarkdownStripper) processString(out *bytes.Buffer, text []byte, coallesce bool) {
+	// Always break-up words
+	if !coallesce || !r.coallesce {
+		doubleSpace(out)
+	}
+	out.Write(text)
+	r.coallesce = coallesce
+}
+func (r *MarkdownStripper) processLink(out *bytes.Buffer, link []byte, content []byte) {
+	// Links are processed out of band
+	r.links = append(r.links, string(link))
+	r.coallesce = false
+}
+
+// GetLinks returns the list of link data collected while parsing
+func (r *MarkdownStripper) GetLinks() []string {
+	return r.links
+}

modules/markup/mdstripper/mdstripper_test.go

@@ -0,0 +1,71 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package mdstripper
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMarkdownStripper(t *testing.T) {
+	type testItem struct {
+		markdown      string
+		expectedText  []string
+		expectedLinks []string
+	}
+
+	list := []testItem{
+		{
+			`
+## This is a title
+
+This is [one](link) to paradise.
+This **is emphasized**.
+This: should coallesce.
+
+` + "```" + `
+This is a code block.
+This should not appear in the output at all.
+` + "```" + `
+
+* Bullet 1
+* Bullet 2
+
+A HIDDEN ` + "`" + `GHOST` + "`" + ` IN THIS LINE.
+		`,
+			[]string{
+				"This is a title",
+				"This is",
+				"to paradise.",
+				"This",
+				"is emphasized",
+				".",
+				"This: should coallesce.",
+				"Bullet 1",
+				"Bullet 2",
+				"A HIDDEN",
+				"IN THIS LINE.",
+			},
+			[]string{
+				"link",
+			}},
+	}
+
+	for _, test := range list {
+		text, links := StripMarkdown([]byte(test.markdown))
+		rawlines := strings.Split(text, "\n")
+		lines := make([]string, 0, len(rawlines))
+		for _, line := range rawlines {
+			line := strings.TrimSpace(line)
+			if line != "" {
+				lines = append(lines, line)
+			}
+		}
+		assert.EqualValues(t, test.expectedText, lines)
+		assert.EqualValues(t, test.expectedLinks, links)
+	}
+}

modules/markup/sanitizer.go

@@ -28,19 +28,28 @@ var sanitizer = &Sanitizer{}
 // entire application lifecycle.
 func NewSanitizer() {
 	sanitizer.init.Do(func() {
-		sanitizer.policy = bluemonday.UGCPolicy()
-		// We only want to allow HighlightJS specific classes for code blocks
-		sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^language-\w+$`)).OnElements("code")
-
-		// Checkboxes
-		sanitizer.policy.AllowAttrs("type").Matching(regexp.MustCompile(`^checkbox$`)).OnElements("input")
-		sanitizer.policy.AllowAttrs("checked", "disabled").OnElements("input")
-
-		// Custom URL-Schemes
-		sanitizer.policy.AllowURLSchemes(setting.Markdown.CustomURLSchemes...)
+		ReplaceSanitizer()
 	})
 }
 
+// ReplaceSanitizer replaces the current sanitizer to account for changes in settings
+func ReplaceSanitizer() {
+	sanitizer = &Sanitizer{}
+	sanitizer.policy = bluemonday.UGCPolicy()
+	// We only want to allow HighlightJS specific classes for code blocks
+	sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^language-\w+$`)).OnElements("code")
+
+	// Checkboxes
+	sanitizer.policy.AllowAttrs("type").Matching(regexp.MustCompile(`^checkbox$`)).OnElements("input")
+	sanitizer.policy.AllowAttrs("checked", "disabled").OnElements("input")
+
+	// Custom URL-Schemes
+	sanitizer.policy.AllowURLSchemes(setting.Markdown.CustomURLSchemes...)
+
+	// Allow keyword markup
+	sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^` + keywordClass + `$`)).OnElements("span")
+}
+
 // Sanitize takes a string that contains a HTML fragment or document and applies policy whitelist.
 func Sanitize(s string) string {
 	NewSanitizer()

modules/migrations/base/downloader.go

@@ -5,6 +5,8 @@
 
 package base
 
+import "code.gitea.io/gitea/modules/structs"
+
 // Downloader downloads the site repo informations
 type Downloader interface {
 	GetRepoInfo() (*Repository, error)
@@ -21,4 +23,5 @@ type Downloader interface {
 type DownloaderFactory interface {
 	Match(opts MigrateOptions) (bool, error)
 	New(opts MigrateOptions) (Downloader, error)
+	GitServiceType() structs.GitServiceType
 }

modules/migrations/base/options.go

@@ -5,22 +5,7 @@
 
 package base
 
-// MigrateOptions defines the way a repository gets migrated
-type MigrateOptions struct {
-	RemoteURL    string
-	AuthUsername string
-	AuthPassword string
-	Name         string
-	Description  string
-	OriginalURL  string
+import "code.gitea.io/gitea/modules/structs"
 
-	Wiki         bool
-	Issues       bool
-	Milestones   bool
-	Labels       bool
-	Releases     bool
-	Comments     bool
-	PullRequests bool
-	Private      bool
-	Mirror       bool
-}
+// MigrateOptions defines the way a repository gets migrated
+type MigrateOptions = structs.MigrateRepoOption

modules/migrations/gitea.go

@@ -22,6 +22,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 
 	gouuid "github.com/satori/go.uuid"
@@ -33,15 +34,17 @@ var (
 
 // GiteaLocalUploader implements an Uploader to gitea sites
 type GiteaLocalUploader struct {
-	doer        *models.User
-	repoOwner   string
-	repoName    string
-	repo        *models.Repository
-	labels      sync.Map
-	milestones  sync.Map
-	issues      sync.Map
-	gitRepo     *git.Repository
-	prHeadCache map[string]struct{}
+	doer           *models.User
+	repoOwner      string
+	repoName       string
+	repo           *models.Repository
+	labels         sync.Map
+	milestones     sync.Map
+	issues         sync.Map
+	gitRepo        *git.Repository
+	prHeadCache    map[string]struct{}
+	userMap        map[int64]int64 // external user id mapping to user id
+	gitServiceType structs.GitServiceType
 }
 
 // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
@@ -51,6 +54,7 @@ func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *Gitea
 		repoOwner:   repoOwner,
 		repoName:    repoName,
 		prHeadCache: make(map[string]struct{}),
+		userMap:     make(map[int64]int64),
 	}
 }
 
@@ -90,16 +94,35 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
 		remoteAddr = u.String()
 	}
 
-	r, err := models.MigrateRepository(g.doer, owner, models.MigrateRepoOptions{
-		Name:                 g.repoName,
-		Description:          repo.Description,
-		OriginalURL:          repo.OriginalURL,
-		IsMirror:             repo.IsMirror,
-		RemoteAddr:           remoteAddr,
-		IsPrivate:            repo.IsPrivate,
-		Wiki:                 opts.Wiki,
-		SyncReleasesWithTags: !opts.Releases, // if didn't get releases, then sync them from tags
+	var r *models.Repository
+	if opts.MigrateToRepoID <= 0 {
+		r, err = models.CreateRepository(g.doer, owner, models.CreateRepoOptions{
+			Name:        g.repoName,
+			Description: repo.Description,
+			OriginalURL: repo.OriginalURL,
+			IsPrivate:   opts.Private,
+			IsMirror:    opts.Mirror,
+			Status:      models.RepositoryBeingMigrated,
+		})
+	} else {
+		r, err = models.GetRepositoryByID(opts.MigrateToRepoID)
+	}
+	if err != nil {
+		return err
+	}
+
+	r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
+		RepoName:       g.repoName,
+		Description:    repo.Description,
+		OriginalURL:    repo.OriginalURL,
+		GitServiceType: opts.GitServiceType,
+		Mirror:         repo.IsMirror,
+		CloneAddr:      remoteAddr,
+		Private:        repo.IsPrivate,
+		Wiki:           opts.Wiki,
+		Releases:       opts.Releases, // if didn't get releases, then sync them from tags
 	})
+
 	g.repo = r
 	if err != nil {
 		return err
@@ -175,20 +198,38 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 	var rels = make([]*models.Release, 0, len(releases))
 	for _, release := range releases {
 		var rel = models.Release{
-			RepoID:           g.repo.ID,
-			PublisherID:      g.doer.ID,
-			TagName:          release.TagName,
-			LowerTagName:     strings.ToLower(release.TagName),
-			Target:           release.TargetCommitish,
-			Title:            release.Name,
-			Sha1:             release.TargetCommitish,
-			Note:             release.Body,
-			IsDraft:          release.Draft,
-			IsPrerelease:     release.Prerelease,
-			IsTag:            false,
-			CreatedUnix:      timeutil.TimeStamp(release.Created.Unix()),
-			OriginalAuthor:   release.PublisherName,
-			OriginalAuthorID: release.PublisherID,
+			RepoID:       g.repo.ID,
+			TagName:      release.TagName,
+			LowerTagName: strings.ToLower(release.TagName),
+			Target:       release.TargetCommitish,
+			Title:        release.Name,
+			Sha1:         release.TargetCommitish,
+			Note:         release.Body,
+			IsDraft:      release.Draft,
+			IsPrerelease: release.Prerelease,
+			IsTag:        false,
+			CreatedUnix:  timeutil.TimeStamp(release.Created.Unix()),
+		}
+
+		userid, ok := g.userMap[release.PublisherID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[release.PublisherID] = userid
+			}
+		}
+
+		if userid > 0 {
+			rel.PublisherID = userid
+		} else {
+			rel.PublisherID = g.doer.ID
+			rel.OriginalAuthor = release.PublisherName
+			rel.OriginalAuthorID = release.PublisherID
 		}
 
 		// calc NumCommits
@@ -266,20 +307,39 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
 		}
 
 		var is = models.Issue{
-			RepoID:           g.repo.ID,
-			Repo:             g.repo,
-			Index:            issue.Number,
-			PosterID:         g.doer.ID,
-			OriginalAuthor:   issue.PosterName,
-			OriginalAuthorID: issue.PosterID,
-			Title:            issue.Title,
-			Content:          issue.Content,
-			IsClosed:         issue.State == "closed",
-			IsLocked:         issue.IsLocked,
-			MilestoneID:      milestoneID,
-			Labels:           labels,
-			CreatedUnix:      timeutil.TimeStamp(issue.Created.Unix()),
+			RepoID:      g.repo.ID,
+			Repo:        g.repo,
+			Index:       issue.Number,
+			Title:       issue.Title,
+			Content:     issue.Content,
+			IsClosed:    issue.State == "closed",
+			IsLocked:    issue.IsLocked,
+			MilestoneID: milestoneID,
+			Labels:      labels,
+			CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
 		}
+
+		userid, ok := g.userMap[issue.PosterID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[issue.PosterID] = userid
+			}
+		}
+
+		if userid > 0 {
+			is.PosterID = userid
+		} else {
+			is.PosterID = g.doer.ID
+			is.OriginalAuthor = issue.PosterName
+			is.OriginalAuthorID = issue.PosterID
+		}
+
 		if issue.Closed != nil {
 			is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
 		}
@@ -313,15 +373,35 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
 			issueID = issueIDStr.(int64)
 		}
 
-		cms = append(cms, &models.Comment{
-			IssueID:          issueID,
-			Type:             models.CommentTypeComment,
-			PosterID:         g.doer.ID,
-			OriginalAuthor:   comment.PosterName,
-			OriginalAuthorID: comment.PosterID,
-			Content:          comment.Content,
-			CreatedUnix:      timeutil.TimeStamp(comment.Created.Unix()),
-		})
+		userid, ok := g.userMap[comment.PosterID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[comment.PosterID] = userid
+			}
+		}
+
+		cm := models.Comment{
+			IssueID:     issueID,
+			Type:        models.CommentTypeComment,
+			Content:     comment.Content,
+			CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
+		}
+
+		if userid > 0 {
+			cm.PosterID = userid
+		} else {
+			cm.PosterID = g.doer.ID
+			cm.OriginalAuthor = comment.PosterName
+			cm.OriginalAuthorID = comment.PosterID
+		}
+
+		cms = append(cms, &cm)
 
 		// TODO: Reactions
 	}
@@ -337,6 +417,28 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
 		if err != nil {
 			return err
 		}
+
+		userid, ok := g.userMap[pr.PosterID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[pr.PosterID] = userid
+			}
+		}
+
+		if userid > 0 {
+			gpr.Issue.PosterID = userid
+		} else {
+			gpr.Issue.PosterID = g.doer.ID
+			gpr.Issue.OriginalAuthor = pr.PosterName
+			gpr.Issue.OriginalAuthorID = pr.PosterID
+		}
+
 		gprs = append(gprs, gpr)
 	}
 	if err := models.InsertPullRequests(gprs...); err != nil {
@@ -442,32 +544,50 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 		head = pr.Head.Ref
 	}
 
-	var pullRequest = models.PullRequest{
-		HeadRepoID:   g.repo.ID,
-		HeadBranch:   head,
-		HeadUserName: g.repoOwner,
-		BaseRepoID:   g.repo.ID,
-		BaseBranch:   pr.Base.Ref,
-		MergeBase:    pr.Base.SHA,
-		Index:        pr.Number,
-		HasMerged:    pr.Merged,
+	var issue = models.Issue{
+		RepoID:      g.repo.ID,
+		Repo:        g.repo,
+		Title:       pr.Title,
+		Index:       pr.Number,
+		Content:     pr.Content,
+		MilestoneID: milestoneID,
+		IsPull:      true,
+		IsClosed:    pr.State == "closed",
+		IsLocked:    pr.IsLocked,
+		Labels:      labels,
+		CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
+	}
 
-		Issue: &models.Issue{
-			RepoID:           g.repo.ID,
-			Repo:             g.repo,
-			Title:            pr.Title,
-			Index:            pr.Number,
-			PosterID:         g.doer.ID,
-			OriginalAuthor:   pr.PosterName,
-			OriginalAuthorID: pr.PosterID,
-			Content:          pr.Content,
-			MilestoneID:      milestoneID,
-			IsPull:           true,
-			IsClosed:         pr.State == "closed",
-			IsLocked:         pr.IsLocked,
-			Labels:           labels,
-			CreatedUnix:      timeutil.TimeStamp(pr.Created.Unix()),
-		},
+	userid, ok := g.userMap[pr.PosterID]
+	if !ok {
+		var err error
+		userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", pr.PosterID))
+		if err != nil {
+			log.Error("GetUserIDByExternalUserID: %v", err)
+		}
+		if userid > 0 {
+			g.userMap[pr.PosterID] = userid
+		}
+	}
+
+	if userid > 0 {
+		issue.PosterID = userid
+	} else {
+		issue.PosterID = g.doer.ID
+		issue.OriginalAuthor = pr.PosterName
+		issue.OriginalAuthorID = pr.PosterID
+	}
+
+	var pullRequest = models.PullRequest{
+		HeadRepoID: g.repo.ID,
+		HeadBranch: head,
+		BaseRepoID: g.repo.ID,
+		BaseBranch: pr.Base.Ref,
+		MergeBase:  pr.Base.SHA,
+		Index:      pr.Number,
+		HasMerged:  pr.Merged,
+
+		Issue: &issue,
 	}
 
 	if pullRequest.Issue.IsClosed && pr.Closed != nil {

modules/migrations/gitea_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
 	"github.com/stretchr/testify/assert"
@@ -29,9 +30,9 @@ func TestGiteaUploadRepo(t *testing.T) {
 		uploader   = NewGiteaLocalUploader(user, user.Name, repoName)
 	)
 
-	err := migrateRepository(downloader, uploader, MigrateOptions{
-		RemoteURL:    "https://github.com/go-xorm/builder",
-		Name:         repoName,
+	err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{
+		CloneAddr:    "https://github.com/go-xorm/builder",
+		RepoName:     repoName,
 		AuthUsername: "",
 
 		Wiki:         true,

modules/migrations/github.go

@@ -14,6 +14,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/structs"
 
 	"github.com/google/go-github/v24/github"
 	"golang.org/x/oauth2"
@@ -34,17 +35,17 @@ type GithubDownloaderV3Factory struct {
 
 // Match returns ture if the migration remote URL matched this downloader factory
 func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) {
-	u, err := url.Parse(opts.RemoteURL)
+	u, err := url.Parse(opts.CloneAddr)
 	if err != nil {
 		return false, err
 	}
 
-	return u.Host == "github.com" && opts.AuthUsername != "", nil
+	return strings.EqualFold(u.Host, "github.com") && opts.AuthUsername != "", nil
 }
 
 // New returns a Downloader related to this factory according MigrateOptions
 func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) {
-	u, err := url.Parse(opts.RemoteURL)
+	u, err := url.Parse(opts.CloneAddr)
 	if err != nil {
 		return nil, err
 	}
@@ -58,6 +59,11 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download
 	return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
 }
 
+// GitServiceType returns the type of git service
+func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
+	return structs.GithubService
+}
+
 // GithubDownloaderV3 implements a Downloader interface to get repository informations
 // from github via APIv3
 type GithubDownloaderV3 struct {

modules/migrations/migrate.go

@@ -6,9 +6,12 @@
 package migrations
 
 import (
+	"fmt"
+
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/structs"
 )
 
 // MigrateOptions is equal to base.MigrateOptions
@@ -27,7 +30,8 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) {
 func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
 	var (
 		downloader base.Downloader
-		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.Name)
+		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.RepoName)
+		theFactory base.DownloaderFactory
 	)
 
 	for _, factory := range factories {
@@ -38,6 +42,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 			if err != nil {
 				return nil, err
 			}
+			theFactory = factory
 			break
 		}
 	}
@@ -50,14 +55,22 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 		opts.Comments = false
 		opts.Issues = false
 		opts.PullRequests = false
-		downloader = NewPlainGitDownloader(ownerName, opts.Name, opts.RemoteURL)
-		log.Trace("Will migrate from git: %s", opts.RemoteURL)
+		opts.GitServiceType = structs.PlainGitService
+		downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
+		log.Trace("Will migrate from git: %s", opts.CloneAddr)
+	} else if opts.GitServiceType == structs.NotMigrated {
+		opts.GitServiceType = theFactory.GitServiceType()
 	}
 
+	uploader.gitServiceType = opts.GitServiceType
 	if err := migrateRepository(downloader, uploader, opts); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}
+
+		if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.CloneAddr, err)); err2 != nil {
+			log.Error("create respotiry notice failed: ", err2)
+		}
 		return nil, err
 	}
 

modules/migrations/update.go

@@ -0,0 +1,53 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/structs"
+)
+
+// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID
+func UpdateMigrationPosterID() {
+	for _, gitService := range structs.SupportedFullGitService {
+		if err := updateMigrationPosterIDByGitService(gitService); err != nil {
+			log.Error("updateMigrationPosterIDByGitService failed: %v", err)
+		}
+	}
+}
+
+func updateMigrationPosterIDByGitService(tp structs.GitServiceType) error {
+	provider := tp.Name()
+	if len(provider) == 0 {
+		return nil
+	}
+
+	const batchSize = 100
+	var start int
+	for {
+		users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{
+			Provider: provider,
+			Start:    start,
+			Limit:    batchSize,
+		})
+		if err != nil {
+			return err
+		}
+
+		for _, user := range users {
+			externalUserID := user.ExternalID
+			if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil {
+				log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err)
+			}
+		}
+
+		if len(users) < batchSize {
+			break
+		}
+		start += len(users)
+	}
+	return nil
+}

modules/notification/base/notifier.go

@@ -21,7 +21,7 @@ type Notifier interface {
 	NotifyNewIssue(*models.Issue)
 	NotifyIssueChangeStatus(*models.User, *models.Issue, bool)
 	NotifyIssueChangeMilestone(doer *models.User, issue *models.Issue)
-	NotifyIssueChangeAssignee(doer *models.User, issue *models.Issue, removed bool)
+	NotifyIssueChangeAssignee(doer *models.User, issue *models.Issue, assignee *models.User, removed bool, comment *models.Comment)
 	NotifyIssueChangeContent(doer *models.User, issue *models.Issue, oldContent string)
 	NotifyIssueClearLabels(doer *models.User, issue *models.Issue)
 	NotifyIssueChangeTitle(doer *models.User, issue *models.Issue, oldTitle string)

modules/notification/base/null.go

@@ -83,7 +83,7 @@ func (*NullNotifier) NotifyIssueChangeContent(doer *models.User, issue *models.I
 }
 
 // NotifyIssueChangeAssignee places a place holder function
-func (*NullNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *models.Issue, removed bool) {
+func (*NullNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *models.Issue, assignee *models.User, removed bool, comment *models.Comment) {
 }
 
 // NotifyIssueClearLabels places a place holder function

modules/notification/mail/mail.go

@@ -5,6 +5,8 @@
 package mail
 
 import (
+	"fmt"
+
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/notification/base"
@@ -88,3 +90,11 @@ func (m *mailNotifier) NotifyPullRequestReview(pr *models.PullRequest, r *models
 		log.Error("MailParticipants: %v", err)
 	}
 }
+
+func (m *mailNotifier) NotifyIssueChangeAssignee(doer *models.User, issue *models.Issue, assignee *models.User, removed bool, comment *models.Comment) {
+	// mail only sent to added assignees and not self-assignee
+	if !removed && doer.ID != assignee.ID && assignee.EmailNotifications() == models.EmailNotificationsEnabled {
+		ct := fmt.Sprintf("Assigned #%d.", issue.Index)
+		mailer.SendIssueAssignedMail(issue, doer, ct, comment, []string{assignee.Email})
+	}
+}

modules/notification/notification.go

@@ -11,6 +11,8 @@ import (
 	"code.gitea.io/gitea/modules/notification/indexer"
 	"code.gitea.io/gitea/modules/notification/mail"
 	"code.gitea.io/gitea/modules/notification/ui"
+	"code.gitea.io/gitea/modules/notification/webhook"
+	"code.gitea.io/gitea/modules/setting"
 )
 
 var (
@@ -23,10 +25,14 @@ func RegisterNotifier(notifier base.Notifier) {
 	notifiers = append(notifiers, notifier)
 }
 
-func init() {
+// NewContext registers notification handlers
+func NewContext() {
 	RegisterNotifier(ui.NewNotifier())
-	RegisterNotifier(mail.NewNotifier())
+	if setting.Service.EnableNotifyMail {
+		RegisterNotifier(mail.NewNotifier())
+	}
 	RegisterNotifier(indexer.NewNotifier())
+	RegisterNotifier(webhook.NewNotifier())
 }
 
 // NotifyCreateIssueComment notifies issue comment related message to notifiers
@@ -136,9 +142,9 @@ func NotifyIssueChangeContent(doer *models.User, issue *models.Issue, oldContent
 }
 
 // NotifyIssueChangeAssignee notifies change content to notifiers
-func NotifyIssueChangeAssignee(doer *models.User, issue *models.Issue, removed bool) {
+func NotifyIssueChangeAssignee(doer *models.User, issue *models.Issue, assignee *models.User, removed bool, comment *models.Comment) {
 	for _, notifier := range notifiers {
-		notifier.NotifyIssueChangeAssignee(doer, issue, removed)
+		notifier.NotifyIssueChangeAssignee(doer, issue, assignee, removed, comment)
 	}
 }
 

modules/notification/webhook/webhook.go

@@ -0,0 +1,67 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package webhook
+
+import (
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/notification/base"
+	api "code.gitea.io/gitea/modules/structs"
+)
+
+type webhookNotifier struct {
+	base.NullNotifier
+}
+
+var (
+	_ base.Notifier = &webhookNotifier{}
+)
+
+// NewNotifier create a new webhookNotifier notifier
+func NewNotifier() base.Notifier {
+	return &webhookNotifier{}
+}
+
+func (m *webhookNotifier) NotifyIssueClearLabels(doer *models.User, issue *models.Issue) {
+	if err := issue.LoadPoster(); err != nil {
+		log.Error("loadPoster: %v", err)
+		return
+	}
+
+	if err := issue.LoadRepo(); err != nil {
+		log.Error("LoadRepo: %v", err)
+		return
+	}
+
+	mode, _ := models.AccessLevel(issue.Poster, issue.Repo)
+	var err error
+	if issue.IsPull {
+		if err = issue.LoadPullRequest(); err != nil {
+			log.Error("LoadPullRequest: %v", err)
+			return
+		}
+
+		err = models.PrepareWebhooks(issue.Repo, models.HookEventPullRequest, &api.PullRequestPayload{
+			Action:      api.HookIssueLabelCleared,
+			Index:       issue.Index,
+			PullRequest: issue.PullRequest.APIFormat(),
+			Repository:  issue.Repo.APIFormat(mode),
+			Sender:      doer.APIFormat(),
+		})
+	} else {
+		err = models.PrepareWebhooks(issue.Repo, models.HookEventIssues, &api.IssuePayload{
+			Action:     api.HookIssueLabelCleared,
+			Index:      issue.Index,
+			Issue:      issue.APIFormat(),
+			Repository: issue.Repo.APIFormat(mode),
+			Sender:     doer.APIFormat(),
+		})
+	}
+	if err != nil {
+		log.Error("PrepareWebhooks [is_pull: %v]: %v", issue.IsPull, err)
+	} else {
+		go models.HookQueue.Add(issue.RepoID)
+	}
+}

modules/password/password.go

@@ -0,0 +1,88 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package password
+
+import (
+	"crypto/rand"
+	"math/big"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var (
+	matchComplexityOnce sync.Once
+	validChars          string
+	requiredChars       []string
+
+	charComplexities = map[string]string{
+		"lower": `abcdefghijklmnopqrstuvwxyz`,
+		"upper": `ABCDEFGHIJKLMNOPQRSTUVWXYZ`,
+		"digit": `0123456789`,
+		"spec":  ` !"#$%&'()*+,-./:;<=>?@[\]^_{|}~` + "`",
+	}
+)
+
+// NewComplexity for preparation
+func NewComplexity() {
+	matchComplexityOnce.Do(func() {
+		setupComplexity(setting.PasswordComplexity)
+	})
+}
+
+func setupComplexity(values []string) {
+	if len(values) != 1 || values[0] != "off" {
+		for _, val := range values {
+			if chars, ok := charComplexities[val]; ok {
+				validChars += chars
+				requiredChars = append(requiredChars, chars)
+			}
+		}
+		if len(requiredChars) == 0 {
+			// No valid character classes found; use all classes as default
+			for _, chars := range charComplexities {
+				validChars += chars
+				requiredChars = append(requiredChars, chars)
+			}
+		}
+	}
+	if validChars == "" {
+		// No complexities to check; provide a sensible default for password generation
+		validChars = charComplexities["lower"] + charComplexities["upper"] + charComplexities["digit"]
+	}
+}
+
+// IsComplexEnough return True if password meets complexity settings
+func IsComplexEnough(pwd string) bool {
+	NewComplexity()
+	if len(validChars) > 0 {
+		for _, req := range requiredChars {
+			if !strings.ContainsAny(req, pwd) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// Generate  a random password
+func Generate(n int) (string, error) {
+	NewComplexity()
+	buffer := make([]byte, n)
+	max := big.NewInt(int64(len(validChars)))
+	for {
+		for j := 0; j < n; j++ {
+			rnd, err := rand.Int(rand.Reader, max)
+			if err != nil {
+				return "", err
+			}
+			buffer[j] = validChars[rnd.Int64()]
+		}
+		if IsComplexEnough(string(buffer)) && string(buffer[0]) != " " && string(buffer[n-1]) != " " {
+			return string(buffer), nil
+		}
+	}
+}

modules/password/password_test.go

@@ -0,0 +1,75 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package password
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestComplexity_IsComplexEnough(t *testing.T) {
+	matchComplexityOnce.Do(func() {})
+
+	testlist := []struct {
+		complexity  []string
+		truevalues  []string
+		falsevalues []string
+	}{
+		{[]string{"lower"}, []string{"abc", "abc!"}, []string{"ABC", "123", "=!$", ""}},
+		{[]string{"upper"}, []string{"ABC"}, []string{"abc", "123", "=!$", "abc!", ""}},
+		{[]string{"digit"}, []string{"123"}, []string{"abc", "ABC", "=!$", "abc!", ""}},
+		{[]string{"spec"}, []string{"=!$", "abc!"}, []string{"abc", "ABC", "123", ""}},
+		{[]string{"off"}, []string{"abc", "ABC", "123", "=!$", "abc!", ""}, nil},
+		{[]string{"lower", "spec"}, []string{"abc!"}, []string{"abc", "ABC", "123", "=!$", "abcABC123", ""}},
+		{[]string{"lower", "upper", "digit"}, []string{"abcABC123"}, []string{"abc", "ABC", "123", "=!$", "abc!", ""}},
+	}
+
+	for _, test := range testlist {
+		testComplextity(test.complexity)
+		for _, val := range test.truevalues {
+			assert.True(t, IsComplexEnough(val))
+		}
+		for _, val := range test.falsevalues {
+			assert.False(t, IsComplexEnough(val))
+		}
+	}
+
+	// Remove settings for other tests
+	testComplextity([]string{"off"})
+}
+
+func TestComplexity_Generate(t *testing.T) {
+	matchComplexityOnce.Do(func() {})
+
+	const maxCount = 50
+	const pwdLen = 50
+
+	test := func(t *testing.T, modes []string) {
+		testComplextity(modes)
+		for i := 0; i < maxCount; i++ {
+			pwd, err := Generate(pwdLen)
+			assert.NoError(t, err)
+			assert.Equal(t, pwdLen, len(pwd))
+			assert.True(t, IsComplexEnough(pwd), "Failed complexities with modes %+v for generated: %s", modes, pwd)
+		}
+	}
+
+	test(t, []string{"lower"})
+	test(t, []string{"upper"})
+	test(t, []string{"lower", "upper", "spec"})
+	test(t, []string{"off"})
+	test(t, []string{""})
+
+	// Remove settings for other tests
+	testComplextity([]string{"off"})
+}
+
+func testComplextity(values []string) {
+	// Cleanup previous values
+	validChars = ""
+	requiredChars = make([]string, 0, len(values))
+	setupComplexity(values)
+}

modules/private/hook.go

@@ -31,11 +31,12 @@ type HookOptions struct {
 	GitAlternativeObjectDirectories string
 	GitQuarantinePath               string
 	ProtectedBranchID               int64
+	IsDeployKey                     bool
 }
 
 // HookPreReceive check whether the provided commits are allowed
 func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string) {
-	reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&gitObjectDirectory=%s&gitAlternativeObjectDirectories=%s&gitQuarantinePath=%s&prID=%d",
+	reqURL := setting.LocalURL + fmt.Sprintf("api/internal/hook/pre-receive/%s/%s?old=%s&new=%s&ref=%s&userID=%d&gitObjectDirectory=%s&gitAlternativeObjectDirectories=%s&gitQuarantinePath=%s&prID=%d&isDeployKey=%t",
 		url.PathEscape(ownerName),
 		url.PathEscape(repoName),
 		url.QueryEscape(opts.OldCommitID),
@@ -46,6 +47,7 @@ func HookPreReceive(ownerName, repoName string, opts HookOptions) (int, string)
 		url.QueryEscape(opts.GitAlternativeObjectDirectories),
 		url.QueryEscape(opts.GitQuarantinePath),
 		opts.ProtectedBranchID,
+		opts.IsDeployKey,
 	)
 
 	resp, err := newInternalRequest(reqURL, "GET").Response()

modules/process/manager.go

@@ -1,4 +1,5 @@
 // Copyright 2014 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
@@ -9,6 +10,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"os/exec"
 	"sync"
 	"time"
@@ -93,6 +95,14 @@ func (pm *Manager) ExecDir(timeout time.Duration, dir, desc, cmdName string, arg
 // Returns its complete stdout and stderr
 // outputs and an error, if any (including timeout)
 func (pm *Manager) ExecDirEnv(timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) {
+	return pm.ExecDirEnvStdIn(timeout, dir, desc, env, nil, cmdName, args...)
+}
+
+// ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its completion
+// up to the given timeout (or DefaultTimeout if -1 is given).
+// Returns its complete stdout and stderr
+// outputs and an error, if any (including timeout)
+func (pm *Manager) ExecDirEnvStdIn(timeout time.Duration, dir, desc string, env []string, stdIn io.Reader, cmdName string, args ...string) (string, string, error) {
 	if timeout == -1 {
 		timeout = 60 * time.Second
 	}
@@ -108,6 +118,10 @@ func (pm *Manager) ExecDirEnv(timeout time.Duration, dir, desc string, env []str
 	cmd.Env = env
 	cmd.Stdout = stdOut
 	cmd.Stderr = stdErr
+	if stdIn != nil {
+		cmd.Stdin = stdIn
+	}
+
 	if err := cmd.Start(); err != nil {
 		return "", "", err
 	}

modules/references/references.go

@@ -0,0 +1,322 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package references
+
+import (
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+
+	"code.gitea.io/gitea/modules/markup/mdstripper"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var (
+	// validNamePattern performs only the most basic validation for user or repository names
+	// Repository name should contain only alphanumeric, dash ('-'), underscore ('_') and dot ('.') characters.
+	validNamePattern = regexp.MustCompile(`^[a-z0-9_.-]+$`)
+
+	// NOTE: All below regex matching do not perform any extra validation.
+	// Thus a link is produced even if the linked entity does not exist.
+	// While fast, this is also incorrect and lead to false positives.
+	// TODO: fix invalid linking issue
+
+	// mentionPattern matches all mentions in the form of "@user"
+	mentionPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@[0-9a-zA-Z-_\.]+)(?:\s|$|\)|\])`)
+	// issueNumericPattern matches string that references to a numeric issue, e.g. #1287
+	issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(#[0-9]+)(?:\s|$|\)|\]|:|\.(\s|$))`)
+	// issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234
+	issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$))`)
+	// crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository
+	// e.g. gogits/gogs#12345
+	crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+#[0-9]+)(?:\s|$|\)|\]|\.(\s|$))`)
+
+	// Same as GitHub. See
+	// https://help.github.com/articles/closing-issues-via-commit-messages
+	issueCloseKeywords  = []string{"close", "closes", "closed", "fix", "fixes", "fixed", "resolve", "resolves", "resolved"}
+	issueReopenKeywords = []string{"reopen", "reopens", "reopened"}
+
+	issueCloseKeywordsPat, issueReopenKeywordsPat *regexp.Regexp
+
+	giteaHostInit sync.Once
+	giteaHost     string
+)
+
+// XRefAction represents the kind of effect a cross reference has once is resolved
+type XRefAction int64
+
+const (
+	// XRefActionNone means the cross-reference is simply a comment
+	XRefActionNone XRefAction = iota // 0
+	// XRefActionCloses means the cross-reference should close an issue if it is resolved
+	XRefActionCloses // 1
+	// XRefActionReopens means the cross-reference should reopen an issue if it is resolved
+	XRefActionReopens // 2
+	// XRefActionNeutered means the cross-reference will no longer affect the source
+	XRefActionNeutered // 3
+)
+
+// IssueReference contains an unverified cross-reference to a local issue or pull request
+type IssueReference struct {
+	Index  int64
+	Owner  string
+	Name   string
+	Action XRefAction
+}
+
+// RenderizableReference contains an unverified cross-reference to with rendering information
+type RenderizableReference struct {
+	Issue          string
+	Owner          string
+	Name           string
+	RefLocation    *RefSpan
+	Action         XRefAction
+	ActionLocation *RefSpan
+}
+
+type rawReference struct {
+	index          int64
+	owner          string
+	name           string
+	action         XRefAction
+	issue          string
+	refLocation    *RefSpan
+	actionLocation *RefSpan
+}
+
+func rawToIssueReferenceList(reflist []*rawReference) []IssueReference {
+	refarr := make([]IssueReference, len(reflist))
+	for i, r := range reflist {
+		refarr[i] = IssueReference{
+			Index:  r.index,
+			Owner:  r.owner,
+			Name:   r.name,
+			Action: r.action,
+		}
+	}
+	return refarr
+}
+
+// RefSpan is the position where the reference was found within the parsed text
+type RefSpan struct {
+	Start int
+	End   int
+}
+
+func makeKeywordsPat(keywords []string) *regexp.Regexp {
+	return regexp.MustCompile(`(?i)(?:\s|^|\(|\[)(` + strings.Join(keywords, `|`) + `):? $`)
+}
+
+func init() {
+	issueCloseKeywordsPat = makeKeywordsPat(issueCloseKeywords)
+	issueReopenKeywordsPat = makeKeywordsPat(issueReopenKeywords)
+}
+
+// getGiteaHostName returns a normalized string with the local host name, with no scheme or port information
+func getGiteaHostName() string {
+	giteaHostInit.Do(func() {
+		if uapp, err := url.Parse(setting.AppURL); err == nil {
+			giteaHost = strings.ToLower(uapp.Host)
+		} else {
+			giteaHost = ""
+		}
+	})
+	return giteaHost
+}
+
+// FindAllMentionsMarkdown matches mention patterns in given content and
+// returns a list of found unvalidated user names **not including** the @ prefix.
+func FindAllMentionsMarkdown(content string) []string {
+	bcontent, _ := mdstripper.StripMarkdownBytes([]byte(content))
+	locations := FindAllMentionsBytes(bcontent)
+	mentions := make([]string, len(locations))
+	for i, val := range locations {
+		mentions[i] = string(bcontent[val.Start+1 : val.End])
+	}
+	return mentions
+}
+
+// FindAllMentionsBytes matches mention patterns in given content
+// and returns a list of locations for the unvalidated user names, including the @ prefix.
+func FindAllMentionsBytes(content []byte) []RefSpan {
+	mentions := mentionPattern.FindAllSubmatchIndex(content, -1)
+	ret := make([]RefSpan, len(mentions))
+	for i, val := range mentions {
+		ret[i] = RefSpan{Start: val[2], End: val[3]}
+	}
+	return ret
+}
+
+// FindFirstMentionBytes matches the first mention in then given content
+// and returns the location of the unvalidated user name, including the @ prefix.
+func FindFirstMentionBytes(content []byte) (bool, RefSpan) {
+	mention := mentionPattern.FindSubmatchIndex(content)
+	if mention == nil {
+		return false, RefSpan{}
+	}
+	return true, RefSpan{Start: mention[2], End: mention[3]}
+}
+
+// FindAllIssueReferencesMarkdown strips content from markdown markup
+// and returns a list of unvalidated references found in it.
+func FindAllIssueReferencesMarkdown(content string) []IssueReference {
+	return rawToIssueReferenceList(findAllIssueReferencesMarkdown(content))
+}
+
+func findAllIssueReferencesMarkdown(content string) []*rawReference {
+	bcontent, links := mdstripper.StripMarkdownBytes([]byte(content))
+	return findAllIssueReferencesBytes(bcontent, links)
+}
+
+// FindAllIssueReferences returns a list of unvalidated references found in a string.
+func FindAllIssueReferences(content string) []IssueReference {
+	return rawToIssueReferenceList(findAllIssueReferencesBytes([]byte(content), []string{}))
+}
+
+// FindRenderizableReferenceNumeric returns the first unvalidated reference found in a string.
+func FindRenderizableReferenceNumeric(content string) (bool, *RenderizableReference) {
+	match := issueNumericPattern.FindStringSubmatchIndex(content)
+	if match == nil {
+		if match = crossReferenceIssueNumericPattern.FindStringSubmatchIndex(content); match == nil {
+			return false, nil
+		}
+	}
+	r := getCrossReference([]byte(content), match[2], match[3], false)
+	if r == nil {
+		return false, nil
+	}
+
+	return true, &RenderizableReference{
+		Issue:          r.issue,
+		Owner:          r.owner,
+		Name:           r.name,
+		RefLocation:    r.refLocation,
+		Action:         r.action,
+		ActionLocation: r.actionLocation,
+	}
+}
+
+// FindRenderizableReferenceAlphanumeric returns the first alphanumeric unvalidated references found in a string.
+func FindRenderizableReferenceAlphanumeric(content string) (bool, *RenderizableReference) {
+	match := issueAlphanumericPattern.FindStringSubmatchIndex(content)
+	if match == nil {
+		return false, nil
+	}
+
+	action, location := findActionKeywords([]byte(content), match[2])
+
+	return true, &RenderizableReference{
+		Issue:          string(content[match[2]:match[3]]),
+		RefLocation:    &RefSpan{Start: match[2], End: match[3]},
+		Action:         action,
+		ActionLocation: location,
+	}
+}
+
+// FindAllIssueReferencesBytes returns a list of unvalidated references found in a byte slice.
+func findAllIssueReferencesBytes(content []byte, links []string) []*rawReference {
+
+	ret := make([]*rawReference, 0, 10)
+
+	matches := issueNumericPattern.FindAllSubmatchIndex(content, -1)
+	for _, match := range matches {
+		if ref := getCrossReference(content, match[2], match[3], false); ref != nil {
+			ret = append(ret, ref)
+		}
+	}
+
+	matches = crossReferenceIssueNumericPattern.FindAllSubmatchIndex(content, -1)
+	for _, match := range matches {
+		if ref := getCrossReference(content, match[2], match[3], false); ref != nil {
+			ret = append(ret, ref)
+		}
+	}
+
+	localhost := getGiteaHostName()
+	for _, link := range links {
+		if u, err := url.Parse(link); err == nil {
+			// Note: we're not attempting to match the URL scheme (http/https)
+			host := strings.ToLower(u.Host)
+			if host != "" && host != localhost {
+				continue
+			}
+			parts := strings.Split(u.EscapedPath(), "/")
+			// /user/repo/issues/3
+			if len(parts) != 5 || parts[0] != "" {
+				continue
+			}
+			if parts[3] != "issues" && parts[3] != "pulls" {
+				continue
+			}
+			// Note: closing/reopening keywords not supported with URLs
+			bytes := []byte(parts[1] + "/" + parts[2] + "#" + parts[4])
+			if ref := getCrossReference(bytes, 0, len(bytes), true); ref != nil {
+				ref.refLocation = nil
+				ret = append(ret, ref)
+			}
+		}
+	}
+
+	return ret
+}
+
+func getCrossReference(content []byte, start, end int, fromLink bool) *rawReference {
+	refid := string(content[start:end])
+	parts := strings.Split(refid, "#")
+	if len(parts) != 2 {
+		return nil
+	}
+	repo, issue := parts[0], parts[1]
+	index, err := strconv.ParseInt(issue, 10, 64)
+	if err != nil {
+		return nil
+	}
+	if repo == "" {
+		if fromLink {
+			// Markdown links must specify owner/repo
+			return nil
+		}
+		action, location := findActionKeywords(content, start)
+		return &rawReference{
+			index:          index,
+			action:         action,
+			issue:          issue,
+			refLocation:    &RefSpan{Start: start, End: end},
+			actionLocation: location,
+		}
+	}
+	parts = strings.Split(strings.ToLower(repo), "/")
+	if len(parts) != 2 {
+		return nil
+	}
+	owner, name := parts[0], parts[1]
+	if !validNamePattern.MatchString(owner) || !validNamePattern.MatchString(name) {
+		return nil
+	}
+	action, location := findActionKeywords(content, start)
+	return &rawReference{
+		index:          index,
+		owner:          owner,
+		name:           name,
+		action:         action,
+		issue:          issue,
+		refLocation:    &RefSpan{Start: start, End: end},
+		actionLocation: location,
+	}
+}
+
+func findActionKeywords(content []byte, start int) (XRefAction, *RefSpan) {
+	m := issueCloseKeywordsPat.FindSubmatchIndex(content[:start])
+	if m != nil {
+		return XRefActionCloses, &RefSpan{Start: m[2], End: m[3]}
+	}
+	m = issueReopenKeywordsPat.FindSubmatchIndex(content[:start])
+	if m != nil {
+		return XRefActionReopens, &RefSpan{Start: m[2], End: m[3]}
+	}
+	return XRefActionNone, nil
+}

modules/references/references_test.go

@@ -0,0 +1,296 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package references
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindAllIssueReferences(t *testing.T) {
+
+	type result struct {
+		Index          int64
+		Owner          string
+		Name           string
+		Issue          string
+		Action         XRefAction
+		RefLocation    *RefSpan
+		ActionLocation *RefSpan
+	}
+
+	type testFixture struct {
+		input    string
+		expected []result
+	}
+
+	fixtures := []testFixture{
+		{
+			"Simply closes: #29 yes",
+			[]result{
+				{29, "", "", "29", XRefActionCloses, &RefSpan{Start: 15, End: 18}, &RefSpan{Start: 7, End: 13}},
+			},
+		},
+		{
+			"#123 no, this is a title.",
+			[]result{},
+		},
+		{
+			" #124 yes, this is a reference.",
+			[]result{
+				{124, "", "", "124", XRefActionNone, &RefSpan{Start: 0, End: 4}, nil},
+			},
+		},
+		{
+			"```\nThis is a code block.\n#723 no, it's a code block.```",
+			[]result{},
+		},
+		{
+			"This `#724` no, it's inline code.",
+			[]result{},
+		},
+		{
+			"This user3/repo4#200 yes.",
+			[]result{
+				{200, "user3", "repo4", "200", XRefActionNone, &RefSpan{Start: 5, End: 20}, nil},
+			},
+		},
+		{
+			"This [one](#919) no, this is a URL fragment.",
+			[]result{},
+		},
+		{
+			"This [two](/user2/repo1/issues/921) yes.",
+			[]result{
+				{921, "user2", "repo1", "921", XRefActionNone, nil, nil},
+			},
+		},
+		{
+			"This [three](/user2/repo1/pulls/922) yes.",
+			[]result{
+				{922, "user2", "repo1", "922", XRefActionNone, nil, nil},
+			},
+		},
+		{
+			"This [four](http://gitea.com:3000/user3/repo4/issues/203) yes.",
+			[]result{
+				{203, "user3", "repo4", "203", XRefActionNone, nil, nil},
+			},
+		},
+		{
+			"This [five](http://github.com/user3/repo4/issues/204) no.",
+			[]result{},
+		},
+		{
+			"This http://gitea.com:3000/user4/repo5/201 no, bad URL.",
+			[]result{},
+		},
+		{
+			"This http://gitea.com:3000/user4/repo5/pulls/202 yes.",
+			[]result{
+				{202, "user4", "repo5", "202", XRefActionNone, nil, nil},
+			},
+		},
+		{
+			"This http://GiTeA.COM:3000/user4/repo6/pulls/205 yes.",
+			[]result{
+				{205, "user4", "repo6", "205", XRefActionNone, nil, nil},
+			},
+		},
+		{
+			"Reopens #15 yes",
+			[]result{
+				{15, "", "", "15", XRefActionReopens, &RefSpan{Start: 8, End: 11}, &RefSpan{Start: 0, End: 7}},
+			},
+		},
+		{
+			"This closes #20 for you yes",
+			[]result{
+				{20, "", "", "20", XRefActionCloses, &RefSpan{Start: 12, End: 15}, &RefSpan{Start: 5, End: 11}},
+			},
+		},
+		{
+			"Do you fix user6/repo6#300 ? yes",
+			[]result{
+				{300, "user6", "repo6", "300", XRefActionCloses, &RefSpan{Start: 11, End: 26}, &RefSpan{Start: 7, End: 10}},
+			},
+		},
+		{
+			"For 999 #1235 no keyword, but yes",
+			[]result{
+				{1235, "", "", "1235", XRefActionNone, &RefSpan{Start: 8, End: 13}, nil},
+			},
+		},
+		{
+			"Which abc. #9434 same as above",
+			[]result{
+				{9434, "", "", "9434", XRefActionNone, &RefSpan{Start: 11, End: 16}, nil},
+			},
+		},
+		{
+			"This closes #600 and reopens #599",
+			[]result{
+				{600, "", "", "600", XRefActionCloses, &RefSpan{Start: 12, End: 16}, &RefSpan{Start: 5, End: 11}},
+				{599, "", "", "599", XRefActionReopens, &RefSpan{Start: 29, End: 33}, &RefSpan{Start: 21, End: 28}},
+			},
+		},
+	}
+
+	// Save original value for other tests that may rely on it
+	prevURL := setting.AppURL
+	setting.AppURL = "https://gitea.com:3000/"
+
+	for _, fixture := range fixtures {
+		expraw := make([]*rawReference, len(fixture.expected))
+		for i, e := range fixture.expected {
+			expraw[i] = &rawReference{
+				index:          e.Index,
+				owner:          e.Owner,
+				name:           e.Name,
+				action:         e.Action,
+				issue:          e.Issue,
+				refLocation:    e.RefLocation,
+				actionLocation: e.ActionLocation,
+			}
+		}
+		expref := rawToIssueReferenceList(expraw)
+		refs := FindAllIssueReferencesMarkdown(fixture.input)
+		assert.EqualValues(t, expref, refs, "Failed to parse: {%s}", fixture.input)
+		rawrefs := findAllIssueReferencesMarkdown(fixture.input)
+		assert.EqualValues(t, expraw, rawrefs, "Failed to parse: {%s}", fixture.input)
+	}
+
+	// Restore for other tests that may rely on the original value
+	setting.AppURL = prevURL
+
+	type alnumFixture struct {
+		input          string
+		issue          string
+		refLocation    *RefSpan
+		action         XRefAction
+		actionLocation *RefSpan
+	}
+
+	alnumFixtures := []alnumFixture{
+		{
+			"This ref ABC-123 is alphanumeric",
+			"ABC-123", &RefSpan{Start: 9, End: 16},
+			XRefActionNone, nil,
+		},
+		{
+			"This closes ABCD-1234 alphanumeric",
+			"ABCD-1234", &RefSpan{Start: 12, End: 21},
+			XRefActionCloses, &RefSpan{Start: 5, End: 11},
+		},
+	}
+
+	for _, fixture := range alnumFixtures {
+		found, ref := FindRenderizableReferenceAlphanumeric(fixture.input)
+		if fixture.issue == "" {
+			assert.False(t, found, "Failed to parse: {%s}", fixture.input)
+		} else {
+			assert.True(t, found, "Failed to parse: {%s}", fixture.input)
+			assert.Equal(t, fixture.issue, ref.Issue, "Failed to parse: {%s}", fixture.input)
+			assert.Equal(t, fixture.refLocation, ref.RefLocation, "Failed to parse: {%s}", fixture.input)
+			assert.Equal(t, fixture.action, ref.Action, "Failed to parse: {%s}", fixture.input)
+			assert.Equal(t, fixture.actionLocation, ref.ActionLocation, "Failed to parse: {%s}", fixture.input)
+		}
+	}
+}
+
+func TestRegExp_mentionPattern(t *testing.T) {
+	trueTestCases := []string{
+		"@Unknwon",
+		"@ANT_123",
+		"@xxx-DiN0-z-A..uru..s-xxx",
+		"   @lol   ",
+		" @Te-st",
+		"(@gitea)",
+		"[@gitea]",
+	}
+	falseTestCases := []string{
+		"@ 0",
+		"@ ",
+		"@",
+		"",
+		"ABC",
+		"/home/gitea/@gitea",
+		"\"@gitea\"",
+	}
+
+	for _, testCase := range trueTestCases {
+		res := mentionPattern.MatchString(testCase)
+		assert.True(t, res)
+	}
+	for _, testCase := range falseTestCases {
+		res := mentionPattern.MatchString(testCase)
+		assert.False(t, res)
+	}
+}
+
+func TestRegExp_issueNumericPattern(t *testing.T) {
+	trueTestCases := []string{
+		"#1234",
+		"#0",
+		"#1234567890987654321",
+		"  #12",
+		"#12:",
+		"ref: #12: msg",
+	}
+	falseTestCases := []string{
+		"# 1234",
+		"# 0",
+		"# ",
+		"#",
+		"#ABC",
+		"#1A2B",
+		"",
+		"ABC",
+	}
+
+	for _, testCase := range trueTestCases {
+		assert.True(t, issueNumericPattern.MatchString(testCase))
+	}
+	for _, testCase := range falseTestCases {
+		assert.False(t, issueNumericPattern.MatchString(testCase))
+	}
+}
+
+func TestRegExp_issueAlphanumericPattern(t *testing.T) {
+	trueTestCases := []string{
+		"ABC-1234",
+		"A-1",
+		"RC-80",
+		"ABCDEFGHIJ-1234567890987654321234567890",
+		"ABC-123.",
+		"(ABC-123)",
+		"[ABC-123]",
+		"ABC-123:",
+	}
+	falseTestCases := []string{
+		"RC-08",
+		"PR-0",
+		"ABCDEFGHIJK-1",
+		"PR_1",
+		"",
+		"#ABC",
+		"",
+		"ABC",
+		"GG-",
+		"rm-1",
+		"/home/gitea/ABC-1234",
+		"MY-STRING-ABC-123",
+	}
+
+	for _, testCase := range trueTestCases {
+		assert.True(t, issueAlphanumericPattern.MatchString(testCase))
+	}
+	for _, testCase := range falseTestCases {
+		assert.False(t, issueAlphanumericPattern.MatchString(testCase))
+	}
+}

modules/repofiles/content.go

@@ -38,6 +38,9 @@ func (ct *ContentType) String() string {
 // GetContentsOrList gets the meta data of a file's contents (*ContentsResponse) if treePath not a tree
 // directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag
 func GetContentsOrList(repo *models.Repository, treePath, ref string) (interface{}, error) {
+	if repo.IsEmpty {
+		return make([]interface{}, 0), nil
+	}
 	if ref == "" {
 		ref = repo.DefaultBranch
 	}

modules/repofiles/content_test.go

@@ -190,3 +190,19 @@ func TestGetContentsOrListErrors(t *testing.T) {
 		assert.Nil(t, fileContentResponse)
 	})
 }
+
+func TestGetContentsOrListOfEmptyRepos(t *testing.T) {
+	models.PrepareTestEnv(t)
+	ctx := test.MockContext(t, "user2/repo15")
+	ctx.SetParams(":id", "15")
+	test.LoadRepo(t, ctx, 15)
+	test.LoadUser(t, ctx, 2)
+	test.LoadGitRepo(t, ctx)
+	repo := ctx.Repo.Repository
+
+	t.Run("empty repo", func(t *testing.T) {
+		contents, err := GetContentsOrList(repo, "", "")
+		assert.NoError(t, err)
+		assert.Empty(t, contents)
+	})
+}

modules/repofiles/file_test.go

@@ -73,7 +73,7 @@ func getExpectedFileResponse() *api.FileResponse {
 		},
 		Verification: &api.PayloadCommitVerification{
 			Verified:  false,
-			Reason:    "",
+			Reason:    "gpg.error.not_signed_commit",
 			Signature: "",
 			Payload:   "",
 		},

modules/repofiles/temp_repo.go

@@ -21,6 +21,8 @@ import (
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/services/gitdiff"
+
+	"github.com/mcuadros/go-version"
 )
 
 // TemporaryUploadRepository is a type to wrap our upload repositories as a shallow clone
@@ -254,7 +256,11 @@ func (t *TemporaryUploadRepository) CommitTree(author, committer *models.User, t
 	authorSig := author.NewGitSig()
 	committerSig := committer.NewGitSig()
 
-	// FIXME: Should we add SSH_ORIGINAL_COMMAND to this
+	binVersion, err := git.BinVersion()
+	if err != nil {
+		return "", fmt.Errorf("Unable to get git version: %v", err)
+	}
+
 	// Because this may call hooks we should pass in the environment
 	env := append(os.Environ(),
 		"GIT_AUTHOR_NAME="+authorSig.Name,
@@ -264,11 +270,29 @@ func (t *TemporaryUploadRepository) CommitTree(author, committer *models.User, t
 		"GIT_COMMITTER_EMAIL="+committerSig.Email,
 		"GIT_COMMITTER_DATE="+commitTimeStr,
 	)
-	commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute,
+
+	messageBytes := new(bytes.Buffer)
+	_, _ = messageBytes.WriteString(message)
+	_, _ = messageBytes.WriteString("\n")
+
+	args := []string{"commit-tree", treeHash, "-p", "HEAD"}
+
+	// Determine if we should sign
+	if version.Compare(binVersion, "1.7.9", ">=") {
+		sign, keyID := t.repo.SignCRUDAction(author, t.basePath, "HEAD")
+		if sign {
+			args = append(args, "-S"+keyID)
+		} else if version.Compare(binVersion, "2.0.0", ">=") {
+			args = append(args, "--no-gpg-sign")
+		}
+	}
+
+	commitHash, stderr, err := process.GetManager().ExecDirEnvStdIn(5*time.Minute,
 		t.basePath,
 		fmt.Sprintf("commitTree (git commit-tree): %s", t.basePath),
 		env,
-		git.GitExecutable, "commit-tree", treeHash, "-p", "HEAD", "-m", message)
+		messageBytes,
+		git.GitExecutable, args...)
 	if err != nil {
 		return "", fmt.Errorf("git commit-tree: %s", stderr)
 	}
@@ -328,6 +352,12 @@ func (t *TemporaryUploadRepository) DiffIndex() (diff *gitdiff.Diff, err error)
 
 // CheckAttribute checks the given attribute of the provided files
 func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...string) (map[string]map[string]string, error) {
+	binVersion, err := git.BinVersion()
+	if err != nil {
+		log.Error("Error retrieving git version: %v", err)
+		return nil, err
+	}
+
 	stdOut := new(bytes.Buffer)
 	stdErr := new(bytes.Buffer)
 
@@ -335,7 +365,14 @@ func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...str
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	cmdArgs := []string{"check-attr", "-z", attribute, "--cached", "--"}
+	cmdArgs := []string{"check-attr", "-z", attribute}
+
+	// git check-attr --cached first appears in git 1.7.8
+	if version.Compare(binVersion, "1.7.8", ">=") {
+		cmdArgs = append(cmdArgs, "--cached")
+	}
+	cmdArgs = append(cmdArgs, "--")
+
 	for _, arg := range args {
 		if arg != "" {
 			cmdArgs = append(cmdArgs, arg)
@@ -353,7 +390,7 @@ func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...str
 	}
 
 	pid := process.GetManager().Add(desc, cmd)
-	err := cmd.Wait()
+	err = cmd.Wait()
 	process.GetManager().Remove(pid)
 
 	if err != nil {

modules/repofiles/update.go

@@ -19,6 +19,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
+	pull_service "code.gitea.io/gitea/services/pull"
 
 	stdcharset "golang.org/x/net/html/charset"
 	"golang.org/x/text/transform"
@@ -313,12 +314,6 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
 		}
 	}
 
-	// Check there is no way this can return multiple infos
-	filename2attribute2info, err := t.CheckAttribute("filter", treePath)
-	if err != nil {
-		return nil, err
-	}
-
 	content := opts.Content
 	if bom {
 		content = string(charset.UTF8BOM) + content
@@ -341,16 +336,23 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
 	opts.Content = content
 	var lfsMetaObject *models.LFSMetaObject
 
-	if setting.LFS.StartServer && filename2attribute2info[treePath] != nil && filename2attribute2info[treePath]["filter"] == "lfs" {
-		// OK so we are supposed to LFS this data!
-		oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content))
+	if setting.LFS.StartServer {
+		// Check there is no way this can return multiple infos
+		filename2attribute2info, err := t.CheckAttribute("filter", treePath)
 		if err != nil {
 			return nil, err
 		}
-		lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID}
-		content = lfsMetaObject.Pointer()
-	}
 
+		if filename2attribute2info[treePath] != nil && filename2attribute2info[treePath]["filter"] == "lfs" {
+			// OK so we are supposed to LFS this data!
+			oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content))
+			if err != nil {
+				return nil, err
+			}
+			lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID}
+			content = lfsMetaObject.Pointer()
+		}
+	}
 	// Add the object to the database
 	objectHash, err := t.HashObject(strings.NewReader(content))
 	if err != nil {
@@ -497,7 +499,7 @@ func PushUpdate(repo *models.Repository, branch string, opts models.PushUpdateOp
 
 	log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name)
 
-	go models.AddTestPullRequestTask(pusher, repo.ID, branch, true)
+	go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true)
 
 	if opts.RefFullName == git.BranchPrefix+repo.DefaultBranch {
 		models.UpdateRepoIndexer(repo)

modules/repofiles/upload.go

@@ -74,9 +74,12 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep
 		infos[i] = uploadInfo{upload: upload}
 	}
 
-	filename2attribute2info, err := t.CheckAttribute("filter", names...)
-	if err != nil {
-		return err
+	var filename2attribute2info map[string]map[string]string
+	if setting.LFS.StartServer {
+		filename2attribute2info, err = t.CheckAttribute("filter", names...)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Copy uploaded files into repository.
@@ -88,7 +91,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep
 		defer file.Close()
 
 		var objectHash string
-		if filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" {
+		if setting.LFS.StartServer && filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" {
 			// Handle LFS
 			// FIXME: Inefficient! this should probably happen in models.Upload
 			oid, err := models.GenerateLFSOid(file)

modules/repofiles/verification.go

@@ -18,10 +18,16 @@ func GetPayloadCommitVerification(commit *git.Commit) *structs.PayloadCommitVeri
 		verification.Signature = commit.Signature.Signature
 		verification.Payload = commit.Signature.Payload
 	}
-	if verification.Reason != "" {
-		verification.Reason = commitVerification.Reason
-	} else if verification.Verified {
-		verification.Reason = "unsigned"
+	if commitVerification.SigningUser != nil {
+		verification.Signer = &structs.PayloadUser{
+			Name:  commitVerification.SigningUser.Name,
+			Email: commitVerification.SigningUser.Email,
+		}
+	}
+	verification.Verified = commitVerification.Verified
+	verification.Reason = commitVerification.Reason
+	if verification.Reason == "" && !verification.Verified {
+		verification.Reason = "gpg.error.not_signed_commit"
 	}
 	return verification
 }

modules/setting/cron.go

@@ -49,6 +49,9 @@ var (
 			Schedule   string
 			OlderThan  time.Duration
 		} `ini:"cron.deleted_branches_cleanup"`
+		UpdateMigrationPosterID struct {
+			Schedule string
+		} `ini:"cron.update_migration_poster_id"`
 	}{
 		UpdateMirror: struct {
 			Enabled    bool
@@ -114,6 +117,11 @@ var (
 			Schedule:   "@every 24h",
 			OlderThan:  24 * time.Hour,
 		},
+		UpdateMigrationPosterID: struct {
+			Schedule string
+		}{
+			Schedule: "@every 24h",
+		},
 	}
 )
 

modules/setting/database.go

@@ -42,12 +42,11 @@ var (
 		DBConnectRetries  int
 		DBConnectBackoff  time.Duration
 		MaxIdleConns      int
+		MaxOpenConns      int
 		ConnMaxLifetime   time.Duration
 		IterateBufferSize int
 	}{
-		Timeout:         500,
-		MaxIdleConns:    0,
-		ConnMaxLifetime: 3 * time.Second,
+		Timeout: 500,
 	}
 )
 
@@ -80,8 +79,13 @@ func InitDBConfig() {
 	Database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
 	Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db"))
 	Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
-	Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(0)
-	Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(3 * time.Second)
+	Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2)
+	if Database.UseMySQL {
+		Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(3 * time.Second)
+	} else {
+		Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(0)
+	}
+	Database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0)
 
 	Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50)
 	Database.LogSQL = sec.Key("LOG_SQL").MustBool(true)

modules/setting/indexer.go

@@ -8,6 +8,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/modules/log"
 
@@ -34,6 +35,7 @@ var (
 		IssueQueueDir         string
 		IssueQueueConnStr     string
 		IssueQueueBatchNumber int
+		StartupTimeout        time.Duration
 		IncludePatterns       []glob.Glob
 		ExcludePatterns       []glob.Glob
 	}{
@@ -67,6 +69,7 @@ func newIndexerService() {
 	Indexer.IssueQueueDir = sec.Key("ISSUE_INDEXER_QUEUE_DIR").MustString(path.Join(AppDataPath, "indexers/issues.queue"))
 	Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString(path.Join(AppDataPath, ""))
 	Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20)
+	Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second)
 }
 
 // IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing

modules/setting/log.go

@@ -6,6 +6,7 @@ package setting
 
 import (
 	"encoding/json"
+	"fmt"
 	golog "log"
 	"os"
 	"path"
@@ -17,6 +18,8 @@ import (
 	ini "gopkg.in/ini.v1"
 )
 
+var filenameSuffix = ""
+
 type defaultLogOptions struct {
 	levelName      string // LogLevel
 	flags          string
@@ -112,7 +115,7 @@ func generateLogConfig(sec *ini.Section, name string, defaults defaultLogOptions
 			panic(err.Error())
 		}
 
-		logConfig["filename"] = logPath
+		logConfig["filename"] = logPath + filenameSuffix
 		logConfig["rotate"] = sec.Key("LOG_ROTATE").MustBool(true)
 		logConfig["maxsize"] = 1 << uint(sec.Key("MAX_SIZE_SHIFT").MustInt(28))
 		logConfig["daily"] = sec.Key("DAILY_ROTATE").MustBool(true)
@@ -277,6 +280,12 @@ func newLogService() {
 	golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT)))
 }
 
+// RestartLogsWithPIDSuffix restarts the logs with a PID suffix on files
+func RestartLogsWithPIDSuffix() {
+	filenameSuffix = fmt.Sprintf(".%d", os.Getpid())
+	NewLogServices(false)
+}
+
 // NewLogServices creates all the log services
 func NewLogServices(disableConsole bool) {
 	newLogService()

modules/setting/repository.go

@@ -65,6 +65,16 @@ var (
 		Issue struct {
 			LockReasons []string
 		} `ini:"repository.issue"`
+
+		Signing struct {
+			SigningKey    string
+			SigningName   string
+			SigningEmail  string
+			InitialCommit []string
+			CRUDActions   []string `ini:"CRUD_ACTIONS"`
+			Merges        []string
+			Wiki          []string
+		} `ini:"repository.signing"`
 	}{
 		AnsiCharset:                             "",
 		ForcePrivate:                            false,
@@ -122,6 +132,25 @@ var (
 		}{
 			LockReasons: strings.Split("Too heated,Off-topic,Spam,Resolved", ","),
 		},
+
+		// Signing settings
+		Signing: struct {
+			SigningKey    string
+			SigningName   string
+			SigningEmail  string
+			InitialCommit []string
+			CRUDActions   []string `ini:"CRUD_ACTIONS"`
+			Merges        []string
+			Wiki          []string
+		}{
+			SigningKey:    "default",
+			SigningName:   "",
+			SigningEmail:  "",
+			InitialCommit: []string{"always"},
+			CRUDActions:   []string{"pubkey", "twofa", "parentsigned"},
+			Merges:        []string{"pubkey", "twofa", "basesigned", "commitssigned"},
+			Wiki:          []string{"never"},
+		},
 	}
 	RepoRootPath string
 	ScriptType   = "bash"

modules/setting/service.go

@@ -23,6 +23,7 @@ var Service struct {
 	ShowRegistrationButton                  bool
 	RequireSignInView                       bool
 	EnableNotifyMail                        bool
+	EnableBasicAuth                         bool
 	EnableReverseProxyAuth                  bool
 	EnableReverseProxyAutoRegister          bool
 	EnableReverseProxyEmail                 bool
@@ -60,6 +61,7 @@ func newService() {
 	Service.EmailDomainWhitelist = sec.Key("EMAIL_DOMAIN_WHITELIST").Strings(",")
 	Service.ShowRegistrationButton = sec.Key("SHOW_REGISTRATION_BUTTON").MustBool(!(Service.DisableRegistration || Service.AllowOnlyExternalRegistration))
 	Service.RequireSignInView = sec.Key("REQUIRE_SIGNIN_VIEW").MustBool()
+	Service.EnableBasicAuth = sec.Key("ENABLE_BASIC_AUTHENTICATION").MustBool(true)
 	Service.EnableReverseProxyAuth = sec.Key("ENABLE_REVERSE_PROXY_AUTHENTICATION").MustBool()
 	Service.EnableReverseProxyAutoRegister = sec.Key("ENABLE_REVERSE_PROXY_AUTO_REGISTRATION").MustBool()
 	Service.EnableReverseProxyEmail = sec.Key("ENABLE_REVERSE_PROXY_EMAIL").MustBool()

modules/setting/setting.go

@@ -87,6 +87,7 @@ var (
 	CertFile             string
 	KeyFile              string
 	StaticRootPath       string
+	StaticCacheTime      time.Duration
 	EnableGzip           bool
 	LandingPageURL       LandingPage
 	UnixSocketPermission uint32
@@ -96,6 +97,9 @@ var (
 	LetsEncryptTOS       bool
 	LetsEncryptDirectory string
 	LetsEncryptEmail     string
+	GracefulRestartable  bool
+	GracefulHammerTime   time.Duration
+	StaticURLPrefix      string
 
 	SSH = struct {
 		Disabled                 bool           `ini:"DISABLE_SSH"`
@@ -146,6 +150,7 @@ var (
 	MinPasswordLength     int
 	ImportLocalPaths      bool
 	DisableGitHooks       bool
+	PasswordComplexity    []string
 	PasswordHashAlgo      string
 
 	// UI settings
@@ -515,7 +520,7 @@ func NewContext() {
 	} else {
 		log.Warn("Custom config '%s' not found, ignore this if you're running first time", CustomConf)
 	}
-	Cfg.NameMapper = ini.AllCapsUnderscore
+	Cfg.NameMapper = ini.SnackCase
 
 	homeDir, err := com.HomeDir()
 	if err != nil {
@@ -561,13 +566,15 @@ func NewContext() {
 	Domain = sec.Key("DOMAIN").MustString("localhost")
 	HTTPAddr = sec.Key("HTTP_ADDR").MustString("0.0.0.0")
 	HTTPPort = sec.Key("HTTP_PORT").MustString("3000")
+	GracefulRestartable = sec.Key("ALLOW_GRACEFUL_RESTARTS").MustBool(true)
+	GracefulHammerTime = sec.Key("GRACEFUL_HAMMER_TIME").MustDuration(60 * time.Second)
 
 	defaultAppURL := string(Protocol) + "://" + Domain
 	if (Protocol == HTTP && HTTPPort != "80") || (Protocol == HTTPS && HTTPPort != "443") {
 		defaultAppURL += ":" + HTTPPort
 	}
 	AppURL = sec.Key("ROOT_URL").MustString(defaultAppURL)
-	AppURL = strings.TrimRight(AppURL, "/") + "/"
+	AppURL = strings.TrimSuffix(AppURL, "/") + "/"
 
 	// Check if has app suburl.
 	appURL, err := url.Parse(AppURL)
@@ -577,6 +584,7 @@ func NewContext() {
 	// Suburl should start with '/' and end without '/', such as '/{subpath}'.
 	// This value is empty if site does not have sub-url.
 	AppSubURL = strings.TrimSuffix(appURL.Path, "/")
+	StaticURLPrefix = strings.TrimSuffix(sec.Key("STATIC_URL_PREFIX").MustString(AppSubURL), "/")
 	AppSubURLDepth = strings.Count(AppSubURL, "/")
 	// Check if Domain differs from AppURL domain than update it to AppURL's domain
 	// TODO: Can be replaced with url.Hostname() when minimal GoLang version is 1.8
@@ -606,6 +614,7 @@ func NewContext() {
 	OfflineMode = sec.Key("OFFLINE_MODE").MustBool()
 	DisableRouterLog = sec.Key("DISABLE_ROUTER_LOG").MustBool()
 	StaticRootPath = sec.Key("STATIC_ROOT_PATH").MustString(AppWorkPath)
+	StaticCacheTime = sec.Key("STATIC_CACHE_TIME").MustDuration(6 * time.Hour)
 	AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data"))
 	EnableGzip = sec.Key("ENABLE_GZIP").MustBool()
 	EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false)
@@ -774,6 +783,15 @@ func NewContext() {
 
 	InternalToken = loadInternalToken(sec)
 
+	cfgdata := sec.Key("PASSWORD_COMPLEXITY").Strings(",")
+	PasswordComplexity = make([]string, 0, len(cfgdata))
+	for _, name := range cfgdata {
+		name := strings.ToLower(strings.Trim(name, `"`))
+		if name != "" {
+			PasswordComplexity = append(PasswordComplexity, name)
+		}
+	}
+
 	sec = Cfg.Section("attachment")
 	AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
 	if !filepath.IsAbs(AttachmentPath) {
@@ -1043,4 +1061,5 @@ func NewServices() {
 	newNotifyMailService()
 	newWebhookService()
 	newIndexerService()
+	newTaskService()
 }

modules/setting/task.go

@@ -0,0 +1,25 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+var (
+	// Task settings
+	Task = struct {
+		QueueType    string
+		QueueLength  int
+		QueueConnStr string
+	}{
+		QueueType:    ChannelQueueType,
+		QueueLength:  1000,
+		QueueConnStr: "addrs=127.0.0.1:6379 db=0",
+	}
+)
+
+func newTaskService() {
+	sec := Cfg.Section("task")
+	Task.QueueType = sec.Key("QUEUE_TYPE").MustString(ChannelQueueType)
+	Task.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000)
+	Task.QueueConnStr = sec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0")
+}

modules/ssh/ssh.go

@@ -183,12 +183,7 @@ func Listen(host string, port int, ciphers []string, keyExchanges []string, macs
 		log.Error("Failed to set Host Key. %s", err)
 	}
 
-	go func() {
-		err := srv.ListenAndServe()
-		if err != nil {
-			log.Error("Failed to serve with builtin SSH server. %s", err)
-		}
-	}()
+	go listen(&srv)
 
 }
 

modules/ssh/ssh_graceful.go

@@ -0,0 +1,30 @@
+// +build !windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package ssh
+
+import (
+	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/gliderlabs/ssh"
+)
+
+func listen(server *ssh.Server) {
+	gracefulServer := graceful.NewServer("tcp", server.Addr)
+
+	err := gracefulServer.ListenAndServe(server.Serve)
+	if err != nil {
+		log.Critical("Failed to start SSH server: %v", err)
+	}
+	log.Info("SSH Listener: %s Closed", server.Addr)
+
+}
+
+// Unused informs our cleanup routine that we will not be using a ssh port
+func Unused() {
+	graceful.InformCleanup()
+}

modules/ssh/ssh_windows.go

@@ -0,0 +1,24 @@
+// +build windows
+
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package ssh
+
+import (
+	"code.gitea.io/gitea/modules/log"
+	"github.com/gliderlabs/ssh"
+)
+
+func listen(server *ssh.Server) {
+	err := server.ListenAndServe()
+	if err != nil {
+		log.Critical("Failed to serve with builtin SSH server. %s", err)
+	}
+}
+
+// Unused does nothing on windows
+func Unused() {
+	// Do nothing
+}

modules/structs/hook.go

@@ -91,10 +91,11 @@ type PayloadCommit struct {
 
 // PayloadCommitVerification represents the GPG verification of a commit
 type PayloadCommitVerification struct {
-	Verified  bool   `json:"verified"`
-	Reason    string `json:"reason"`
-	Signature string `json:"signature"`
-	Payload   string `json:"payload"`
+	Verified  bool         `json:"verified"`
+	Reason    string       `json:"reason"`
+	Signature string       `json:"signature"`
+	Signer    *PayloadUser `json:"signer"`
+	Payload   string       `json:"payload"`
 }
 
 var (
@@ -235,6 +236,7 @@ type IssueCommentPayload struct {
 	Changes    *ChangesPayload        `json:"changes,omitempty"`
 	Repository *Repository            `json:"repository"`
 	Sender     *User                  `json:"sender"`
+	IsPull     bool                   `json:"is_pull"`
 }
 
 // SetSecret modifies the secret of the IssueCommentPayload
@@ -418,6 +420,7 @@ type PullRequestPayload struct {
 	PullRequest *PullRequest    `json:"pull_request"`
 	Repository  *Repository     `json:"repository"`
 	Sender      *User           `json:"sender"`
+	Review      *ReviewPayload  `json:"review"`
 }
 
 // SetSecret modifies the secret of the PullRequestPayload.
@@ -430,6 +433,12 @@ func (p *PullRequestPayload) JSONPayload() ([]byte, error) {
 	return json.MarshalIndent(p, "", "  ")
 }
 
+// ReviewPayload FIXME
+type ReviewPayload struct {
+	Type    string `json:"type"`
+	Content string `json:"content"`
+}
+
 //__________                           .__  __
 //\______   \ ____ ______   ____  _____|__|/  |_  ___________ ___.__.
 // |       _// __ \\____ \ /  _ \/  ___/  \   __\/  _ \_  __ <   |  |

modules/structs/repo.go

@@ -153,6 +153,43 @@ type EditRepoOption struct {
 	Archived *bool `json:"archived,omitempty"`
 }
 
+// GitServiceType represents a git service
+type GitServiceType int
+
+// enumerate all GitServiceType
+const (
+	NotMigrated     GitServiceType = iota // 0 not migrated from external sites
+	PlainGitService                       // 1 plain git service
+	GithubService                         // 2 github.com
+	GiteaService                          // 3 gitea service
+	GitlabService                         // 4 gitlab service
+	GogsService                           // 5 gogs service
+)
+
+// Name represents the service type's name
+// WARNNING: the name have to be equal to that on goth's library
+func (gt GitServiceType) Name() string {
+	switch gt {
+	case GithubService:
+		return "github"
+	case GiteaService:
+		return "gitea"
+	case GitlabService:
+		return "gitlab"
+	case GogsService:
+		return "gogs"
+	}
+	return ""
+}
+
+var (
+	// SupportedFullGitService represents all git services supported to migrate issues/labels/prs and etc.
+	// TODO: add to this list after new git service added
+	SupportedFullGitService = []GitServiceType{
+		GithubService,
+	}
+)
+
 // MigrateRepoOption options for migrating a repository from an external service
 type MigrateRepoOption struct {
 	// required: true
@@ -162,8 +199,18 @@ type MigrateRepoOption struct {
 	// required: true
 	UID int `json:"uid" binding:"Required"`
 	// required: true
-	RepoName    string `json:"repo_name" binding:"Required"`
-	Mirror      bool   `json:"mirror"`
-	Private     bool   `json:"private"`
-	Description string `json:"description"`
+	RepoName        string `json:"repo_name" binding:"Required"`
+	Mirror          bool   `json:"mirror"`
+	Private         bool   `json:"private"`
+	Description     string `json:"description"`
+	OriginalURL     string
+	GitServiceType  GitServiceType
+	Wiki            bool
+	Issues          bool
+	Milestones      bool
+	Labels          bool
+	Releases        bool
+	Comments        bool
+	PullRequests    bool
+	MigrateToRepoID int64
 }

modules/structs/task.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package structs
+
+// TaskType defines task type
+type TaskType int
+
+// all kinds of task types
+const (
+	TaskTypeMigrateRepo TaskType = iota // migrate repository from external or local disk
+)
+
+// Name returns the task type name
+func (taskType TaskType) Name() string {
+	switch taskType {
+	case TaskTypeMigrateRepo:
+		return "Migrate Repository"
+	}
+	return ""
+}
+
+// TaskStatus defines task status
+type TaskStatus int
+
+// enumerate all the kinds of task status
+const (
+	TaskStatusQueue    TaskStatus = iota // 0 task is queue
+	TaskStatusRunning                    // 1 task is running
+	TaskStatusStopped                    // 2 task is stopped
+	TaskStatusFailed                     // 3 task is failed
+	TaskStatusFinished                   // 4 task is finished
+)

modules/task/migrate.go

@@ -0,0 +1,120 @@
+// Copyright 2019 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package task
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations"
+	"code.gitea.io/gitea/modules/notification"
+	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
+)
+
+func handleCreateError(owner *models.User, err error, name string) error {
+	switch {
+	case models.IsErrReachLimitOfRepo(err):
+		return fmt.Errorf("You have already reached your limit of %d repositories", owner.MaxCreationLimit())
+	case models.IsErrRepoAlreadyExist(err):
+		return errors.New("The repository name is already used")
+	case models.IsErrNameReserved(err):
+		return fmt.Errorf("The repository name '%s' is reserved", err.(models.ErrNameReserved).Name)
+	case models.IsErrNamePatternNotAllowed(err):
+		return fmt.Errorf("The pattern '%s' is not allowed in a repository name", err.(models.ErrNamePatternNotAllowed).Pattern)
+	default:
+		return err
+	}
+}
+
+func runMigrateTask(t *models.Task) (err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			var buf bytes.Buffer
+			fmt.Fprintf(&buf, "Handler crashed with error: %v", log.Stack(2))
+
+			err = errors.New(buf.String())
+		}
+
+		if err == nil {
+			err = models.FinishMigrateTask(t)
+			if err == nil {
+				notification.NotifyMigrateRepository(t.Doer, t.Owner, t.Repo)
+				return
+			}
+
+			log.Error("FinishMigrateTask failed: %s", err.Error())
+		}
+
+		t.EndTime = timeutil.TimeStampNow()
+		t.Status = structs.TaskStatusFailed
+		t.Errors = err.Error()
+		if err := t.UpdateCols("status", "errors", "end_time"); err != nil {
+			log.Error("Task UpdateCols failed: %s", err.Error())
+		}
+
+		if t.Repo != nil {
+			if errDelete := models.DeleteRepository(t.Doer, t.OwnerID, t.Repo.ID); errDelete != nil {
+				log.Error("DeleteRepository: %v", errDelete)
+			}
+		}
+	}()
+
+	if err := t.LoadRepo(); err != nil {
+		return err
+	}
+
+	// if repository is ready, then just finsih the task
+	if t.Repo.Status == models.RepositoryReady {
+		return nil
+	}
+
+	if err := t.LoadDoer(); err != nil {
+		return err
+	}
+	if err := t.LoadOwner(); err != nil {
+		return err
+	}
+	t.StartTime = timeutil.TimeStampNow()
+	t.Status = structs.TaskStatusRunning
+	if err := t.UpdateCols("start_time", "status"); err != nil {
+		return err
+	}
+
+	var opts *structs.MigrateRepoOption
+	opts, err = t.MigrateConfig()
+	if err != nil {
+		return err
+	}
+
+	opts.MigrateToRepoID = t.RepoID
+	repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts)
+	if err == nil {
+		notification.NotifyMigrateRepository(t.Doer, t.Owner, repo)
+
+		log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name)
+		return nil
+	}
+
+	if models.IsErrRepoAlreadyExist(err) {
+		return errors.New("The repository name is already used")
+	}
+
+	// remoteAddr may contain credentials, so we sanitize it
+	err = util.URLSanitizedError(err, opts.CloneAddr)
+	if strings.Contains(err.Error(), "Authentication failed") ||
+		strings.Contains(err.Error(), "could not read Username") {
+		return fmt.Errorf("Authentication failed: %v", err.Error())
+	} else if strings.Contains(err.Error(), "fatal:") {
+		return fmt.Errorf("Migration failed: %v", err.Error())
+	}
+
+	return handleCreateError(t.Owner, err, "MigratePost")
+}

modules/task/queue.go

@@ -0,0 +1,14 @@
+// Copyright 2019 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package task
+
+import "code.gitea.io/gitea/models"
+
+// Queue defines an interface to run task queue
+type Queue interface {
+	Run() error
+	Push(*models.Task) error
+	Stop()
+}

modules/task/queue_channel.go

@@ -0,0 +1,48 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package task
+
+import (
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+)
+
+var (
+	_ Queue = &ChannelQueue{}
+)
+
+// ChannelQueue implements
+type ChannelQueue struct {
+	queue chan *models.Task
+}
+
+// NewChannelQueue create a memory channel queue
+func NewChannelQueue(queueLen int) *ChannelQueue {
+	return &ChannelQueue{
+		queue: make(chan *models.Task, queueLen),
+	}
+}
+
+// Run starts to run the queue
+func (c *ChannelQueue) Run() error {
+	for task := range c.queue {
+		err := Run(task)
+		if err != nil {
+			log.Error("Run task failed: %s", err.Error())
+		}
+	}
+	return nil
+}
+
+// Push will push the task ID to queue
+func (c *ChannelQueue) Push(task *models.Task) error {
+	c.queue <- task
+	return nil
+}
+
+// Stop stop the queue
+func (c *ChannelQueue) Stop() {
+	close(c.queue)
+}

modules/task/queue_redis.go

@@ -0,0 +1,130 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package task
+
+import (
+	"encoding/json"
+	"errors"
+	"strconv"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/go-redis/redis"
+)
+
+var (
+	_ Queue = &RedisQueue{}
+)
+
+type redisClient interface {
+	RPush(key string, args ...interface{}) *redis.IntCmd
+	LPop(key string) *redis.StringCmd
+	Ping() *redis.StatusCmd
+}
+
+// RedisQueue redis queue
+type RedisQueue struct {
+	client    redisClient
+	queueName string
+	closeChan chan bool
+}
+
+func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) {
+	fields := strings.Fields(connStr)
+	for _, f := range fields {
+		items := strings.SplitN(f, "=", 2)
+		if len(items) < 2 {
+			continue
+		}
+		switch strings.ToLower(items[0]) {
+		case "addrs":
+			addrs = items[1]
+		case "password":
+			password = items[1]
+		case "db":
+			dbIdx, err = strconv.Atoi(items[1])
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+// NewRedisQueue creates single redis or cluster redis queue
+func NewRedisQueue(addrs string, password string, dbIdx int) (*RedisQueue, error) {
+	dbs := strings.Split(addrs, ",")
+	var queue = RedisQueue{
+		queueName: "task_queue",
+		closeChan: make(chan bool),
+	}
+	if len(dbs) == 0 {
+		return nil, errors.New("no redis host found")
+	} else if len(dbs) == 1 {
+		queue.client = redis.NewClient(&redis.Options{
+			Addr:     strings.TrimSpace(dbs[0]), // use default Addr
+			Password: password,                  // no password set
+			DB:       dbIdx,                     // use default DB
+		})
+	} else {
+		// cluster will ignore db
+		queue.client = redis.NewClusterClient(&redis.ClusterOptions{
+			Addrs:    dbs,
+			Password: password,
+		})
+	}
+	if err := queue.client.Ping().Err(); err != nil {
+		return nil, err
+	}
+	return &queue, nil
+}
+
+// Run starts to run the queue
+func (r *RedisQueue) Run() error {
+	for {
+		select {
+		case <-r.closeChan:
+			return nil
+		case <-time.After(time.Millisecond * 100):
+		}
+
+		bs, err := r.client.LPop(r.queueName).Bytes()
+		if err != nil {
+			if err != redis.Nil {
+				log.Error("LPop failed: %v", err)
+			}
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+
+		var task models.Task
+		err = json.Unmarshal(bs, &task)
+		if err != nil {
+			log.Error("Unmarshal task failed: %s", err.Error())
+		} else {
+			err = Run(&task)
+			if err != nil {
+				log.Error("Run task failed: %s", err.Error())
+			}
+		}
+	}
+}
+
+// Push implements Queue
+func (r *RedisQueue) Push(task *models.Task) error {
+	bs, err := json.Marshal(task)
+	if err != nil {
+		return err
+	}
+	return r.client.RPush(r.queueName, bs).Err()
+}
+
+// Stop stop the queue
+func (r *RedisQueue) Stop() {
+	r.closeChan <- true
+}

modules/task/task.go

@@ -0,0 +1,66 @@
+// Copyright 2019 Gitea. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package task
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
+)
+
+// taskQueue is a global queue of tasks
+var taskQueue Queue
+
+// Run a task
+func Run(t *models.Task) error {
+	switch t.Type {
+	case structs.TaskTypeMigrateRepo:
+		return runMigrateTask(t)
+	default:
+		return fmt.Errorf("Unknow task type: %d", t.Type)
+	}
+}
+
+// Init will start the service to get all unfinished tasks and run them
+func Init() error {
+	switch setting.Task.QueueType {
+	case setting.ChannelQueueType:
+		taskQueue = NewChannelQueue(setting.Task.QueueLength)
+	case setting.RedisQueueType:
+		var err error
+		addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr)
+		if err != nil {
+			return err
+		}
+		taskQueue, err = NewRedisQueue(addrs, pass, idx)
+		if err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType)
+	}
+
+	go func() {
+		if err := taskQueue.Run(); err != nil {
+			log.Error("taskQueue.Run end failed: %v", err)
+		}
+	}()
+
+	return nil
+}
+
+// MigrateRepository add migration repository to task
+func MigrateRepository(doer, u *models.User, opts base.MigrateOptions) error {
+	task, err := models.CreateMigrateTask(doer, u, opts)
+	if err != nil {
+		return err
+	}
+
+	return taskQueue.Push(task)
+}

modules/templates/helper.go

@@ -30,7 +30,7 @@ import (
 	"code.gitea.io/gitea/services/gitdiff"
 	mirror_service "code.gitea.io/gitea/services/mirror"
 
-	"gopkg.in/editorconfig/editorconfig-core-go.v1"
+	"github.com/editorconfig/editorconfig-core-go/v2"
 )
 
 // NewFuncMap returns functions for injecting to templates
@@ -48,6 +48,9 @@ func NewFuncMap() []template.FuncMap {
 		"AppSubUrl": func() string {
 			return setting.AppSubURL
 		},
+		"StaticUrlPrefix": func() string {
+			return setting.StaticURLPrefix
+		},
 		"AppUrl": func() string {
 			return setting.AppURL
 		},
@@ -145,7 +148,11 @@ func NewFuncMap() []template.FuncMap {
 		},
 		"TabSizeClass": func(ec *editorconfig.Editorconfig, filename string) string {
 			if ec != nil {
-				def := ec.GetDefinitionForFilename(filename)
+				def, err := ec.GetDefinitionForFilename(filename)
+				if err != nil {
+					log.Error("tab size class: getting definition for filename: %v", err)
+					return "tab-size-8"
+				}
 				if def.TabWidth > 0 {
 					return fmt.Sprintf("tab-size-%d", def.TabWidth)
 				}
@@ -236,6 +243,8 @@ func NewFuncMap() []template.FuncMap {
 		"CommentMustAsDiff": gitdiff.CommentMustAsDiff,
 		"MirrorAddress":     mirror_service.Address,
 		"MirrorFullAddress": mirror_service.AddressNoCredentials,
+		"MirrorUserName":    mirror_service.Username,
+		"MirrorPassword":    mirror_service.Password,
 	}}
 }
 

modules/util/compare.go

@@ -35,6 +35,16 @@ func ExistsInSlice(target string, slice []string) bool {
 	return i < len(slice)
 }
 
+// IsStringInSlice sequential searches if string exists in slice.
+func IsStringInSlice(target string, slice []string) bool {
+	for i := 0; i < len(slice); i++ {
+		if slice[i] == target {
+			return true
+		}
+	}
+	return false
+}
+
 // IsEqualSlice returns true if slices are equal.
 func IsEqualSlice(target []string, source []string) bool {
 	if len(target) != len(source) {