Merge remote-tracking branch 'upstream/master' into team-grant-all-repos

# Conflicts:
#	models/migrations/migrations.go
#	models/migrations/v90.go
#	models/repo.go

modules/auth/admin.go

@@ -5,9 +5,8 @@
 package auth
 
 import (
-	"gopkg.in/macaron.v1"
-
-	"github.com/go-macaron/binding"
+	"gitea.com/macaron/binding"
+	"gitea.com/macaron/macaron"
 )
 
 // AdminCreateUserForm form for admin to create user

modules/auth/auth.go

@@ -10,18 +10,18 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Unknwon/com"
-	"github.com/go-macaron/binding"
-	"github.com/go-macaron/session"
-	gouuid "github.com/satori/go.uuid"
-	"gopkg.in/macaron.v1"
-
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/validation"
+
+	"gitea.com/macaron/binding"
+	"gitea.com/macaron/macaron"
+	"gitea.com/macaron/session"
+	gouuid "github.com/satori/go.uuid"
+	"github.com/unknwon/com"
 )
 
 // IsAPIPath if URL is an api path
@@ -29,6 +29,11 @@ func IsAPIPath(url string) bool {
 	return strings.HasPrefix(url, "/api/")
 }
 
+// IsAttachmentDownload check if request is a file download (GET) with URL to an attachment
+func IsAttachmentDownload(ctx *macaron.Context) bool {
+	return strings.HasPrefix(ctx.Req.URL.Path, "/attachments/") && ctx.Req.Method == "GET"
+}
+
 // SignedInID returns the id of signed in user.
 func SignedInID(ctx *macaron.Context, sess session.Store) int64 {
 	if !models.HasEngine {
@@ -36,7 +41,7 @@ func SignedInID(ctx *macaron.Context, sess session.Store) int64 {
 	}
 
 	// Check access token.
-	if IsAPIPath(ctx.Req.URL.Path) {
+	if IsAPIPath(ctx.Req.URL.Path) || IsAttachmentDownload(ctx) {
 		tokenSHA := ctx.Query("token")
 		if len(tokenSHA) == 0 {
 			tokenSHA = ctx.Query("access_token")
@@ -68,7 +73,7 @@ func SignedInID(ctx *macaron.Context, sess session.Store) int64 {
 				}
 				return 0
 			}
-			t.UpdatedUnix = util.TimeStampNow()
+			t.UpdatedUnix = timeutil.TimeStampNow()
 			if err = models.UpdateAccessToken(t); err != nil {
 				log.Error("UpdateAccessToken: %v", err)
 			}
@@ -210,7 +215,7 @@ func SignedInUser(ctx *macaron.Context, sess session.Store) (*models.User, bool)
 						return nil, false
 					}
 				}
-				token.UpdatedUnix = util.TimeStampNow()
+				token.UpdatedUnix = timeutil.TimeStampNow()
 				if err = models.UpdateAccessToken(token); err != nil {
 					log.Error("UpdateAccessToken:  %v", err)
 				}
@@ -305,6 +310,10 @@ func validate(errs binding.Errors, data map[string]interface{}, f Form, l macaro
 	}
 
 	data["HasError"] = true
+	// If the field with name errs[0].FieldNames[0] is not found in form
+	// somehow, some code later on will panic on Data["ErrorMsg"].(string).
+	// So initialize it to some default.
+	data["ErrorMsg"] = l.Tr("form.unknown_error")
 	AssignForm(f, data)
 
 	typ := reflect.TypeOf(f)
@@ -315,16 +324,9 @@ func validate(errs binding.Errors, data map[string]interface{}, f Form, l macaro
 		val = val.Elem()
 	}
 
-	for i := 0; i < typ.NumField(); i++ {
-		field := typ.Field(i)
-
+	if field, ok := typ.FieldByName(errs[0].FieldNames[0]); ok {
 		fieldName := field.Tag.Get("form")
-		// Allow ignored fields in the struct
-		if fieldName == "-" {
-			continue
-		}
-
-		if errs[0].FieldNames[0] == field.Name {
+		if fieldName != "-" {
 			data["Err_"+field.Name] = true
 
 			trName := field.Tag.Get("locale")
@@ -355,6 +357,8 @@ func validate(errs binding.Errors, data map[string]interface{}, f Form, l macaro
 				data["ErrorMsg"] = trName + l.Tr("form.url_error")
 			case binding.ERR_INCLUDE:
 				data["ErrorMsg"] = trName + l.Tr("form.include_error", GetInclude(field))
+			case validation.ErrGlobPattern:
+				data["ErrorMsg"] = trName + l.Tr("form.glob_pattern_error", errs[0].Message)
 			default:
 				data["ErrorMsg"] = l.Tr("form.unknown_error") + " " + errs[0].Classification
 			}

modules/auth/auth_form.go

@@ -5,8 +5,8 @@
 package auth
 
 import (
-	"github.com/go-macaron/binding"
-	"gopkg.in/macaron.v1"
+	"gitea.com/macaron/binding"
+	"gitea.com/macaron/macaron"
 )
 
 // AuthenticationForm form for authentication

modules/auth/ldap/ldap.go

@@ -308,12 +308,12 @@ func (ls *Source) UsePagedSearch() bool {
 }
 
 // SearchEntries : search an LDAP source for all users matching userFilter
-func (ls *Source) SearchEntries() []*SearchResult {
+func (ls *Source) SearchEntries() ([]*SearchResult, error) {
 	l, err := dial(ls)
 	if err != nil {
 		log.Error("LDAP Connect error, %s:%v", ls.Host, err)
 		ls.Enabled = false
-		return nil
+		return nil, err
 	}
 	defer l.Close()
 
@@ -321,7 +321,7 @@ func (ls *Source) SearchEntries() []*SearchResult {
 		err := l.Bind(ls.BindDN, ls.BindPassword)
 		if err != nil {
 			log.Debug("Failed to bind as BindDN[%s]: %v", ls.BindDN, err)
-			return nil
+			return nil, err
 		}
 		log.Trace("Bound as BindDN %s", ls.BindDN)
 	} else {
@@ -350,7 +350,7 @@ func (ls *Source) SearchEntries() []*SearchResult {
 	}
 	if err != nil {
 		log.Error("LDAP Search failed unexpectedly! (%v)", err)
-		return nil
+		return nil, err
 	}
 
 	result := make([]*SearchResult, len(sr.Entries))
@@ -368,5 +368,5 @@ func (ls *Source) SearchEntries() []*SearchResult {
 		}
 	}
 
-	return result
+	return result, nil
 }

modules/auth/oauth2/oauth2.go

@@ -19,9 +19,10 @@ import (
 	"github.com/markbates/goth/providers/discord"
 	"github.com/markbates/goth/providers/dropbox"
 	"github.com/markbates/goth/providers/facebook"
+	"github.com/markbates/goth/providers/gitea"
 	"github.com/markbates/goth/providers/github"
 	"github.com/markbates/goth/providers/gitlab"
-	"github.com/markbates/goth/providers/gplus"
+	"github.com/markbates/goth/providers/google"
 	"github.com/markbates/goth/providers/openidConnect"
 	"github.com/markbates/goth/providers/twitter"
 	"github.com/satori/go.uuid"
@@ -165,8 +166,8 @@ func createProvider(providerName, providerType, clientID, clientSecret, openIDCo
 			}
 		}
 		provider = gitlab.NewCustomisedURL(clientID, clientSecret, callbackURL, authURL, tokenURL, profileURL, "read_user")
-	case "gplus":
-		provider = gplus.New(clientID, clientSecret, callbackURL, "email")
+	case "gplus": // named gplus due to legacy gplus -> google migration (Google killed Google+). This ensures old connections still work
+		provider = google.New(clientID, clientSecret, callbackURL)
 	case "openidConnect":
 		if provider, err = openidConnect.New(clientID, clientSecret, callbackURL, openIDConnectAutoDiscoveryURL); err != nil {
 			log.Warn("Failed to create OpenID Connect Provider with name '%s' with url '%s': %v", providerName, openIDConnectAutoDiscoveryURL, err)
@@ -175,6 +176,22 @@ func createProvider(providerName, providerType, clientID, clientSecret, openIDCo
 		provider = twitter.NewAuthenticate(clientID, clientSecret, callbackURL)
 	case "discord":
 		provider = discord.New(clientID, clientSecret, callbackURL, discord.ScopeIdentify, discord.ScopeEmail)
+	case "gitea":
+		authURL := gitea.AuthURL
+		tokenURL := gitea.TokenURL
+		profileURL := gitea.ProfileURL
+		if customURLMapping != nil {
+			if len(customURLMapping.AuthURL) > 0 {
+				authURL = customURLMapping.AuthURL
+			}
+			if len(customURLMapping.TokenURL) > 0 {
+				tokenURL = customURLMapping.TokenURL
+			}
+			if len(customURLMapping.ProfileURL) > 0 {
+				profileURL = customURLMapping.ProfileURL
+			}
+		}
+		provider = gitea.NewCustomisedURL(clientID, clientSecret, callbackURL, authURL, tokenURL, profileURL)
 	}
 
 	// always set the name if provider is created so we can support multiple setups of 1 provider
@@ -192,6 +209,8 @@ func GetDefaultTokenURL(provider string) string {
 		return github.TokenURL
 	case "gitlab":
 		return gitlab.TokenURL
+	case "gitea":
+		return gitea.TokenURL
 	}
 	return ""
 }
@@ -203,6 +222,8 @@ func GetDefaultAuthURL(provider string) string {
 		return github.AuthURL
 	case "gitlab":
 		return gitlab.AuthURL
+	case "gitea":
+		return gitea.AuthURL
 	}
 	return ""
 }
@@ -214,6 +235,8 @@ func GetDefaultProfileURL(provider string) string {
 		return github.ProfileURL
 	case "gitlab":
 		return gitlab.ProfileURL
+	case "gitea":
+		return gitea.ProfileURL
 	}
 	return ""
 }

modules/auth/openid/openid.go

@@ -5,8 +5,9 @@
 package openid
 
 import (
-	"github.com/yohcop/openid-go"
 	"time"
+
+	"github.com/yohcop/openid-go"
 )
 
 // For the demo, we use in-memory infinite storage nonce and discovery

modules/auth/org.go

@@ -9,8 +9,8 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/structs"
 
-	"github.com/go-macaron/binding"
-	"gopkg.in/macaron.v1"
+	"gitea.com/macaron/binding"
+	"gitea.com/macaron/macaron"
 )
 
 // ________                            .__                __  .__
@@ -33,13 +33,14 @@ func (f *CreateOrgForm) Validate(ctx *macaron.Context, errs binding.Errors) bind
 
 // UpdateOrgSettingForm form for updating organization settings
 type UpdateOrgSettingForm struct {
-	Name            string `binding:"Required;AlphaDashDot;MaxSize(40)" locale:"org.org_name_holder"`
-	FullName        string `binding:"MaxSize(100)"`
-	Description     string `binding:"MaxSize(255)"`
-	Website         string `binding:"ValidUrl;MaxSize(255)"`
-	Location        string `binding:"MaxSize(50)"`
-	Visibility      structs.VisibleType
-	MaxRepoCreation int
+	Name                      string `binding:"Required;AlphaDashDot;MaxSize(40)" locale:"org.org_name_holder"`
+	FullName                  string `binding:"MaxSize(100)"`
+	Description               string `binding:"MaxSize(255)"`
+	Website                   string `binding:"ValidUrl;MaxSize(255)"`
+	Location                  string `binding:"MaxSize(50)"`
+	Visibility                structs.VisibleType
+	MaxRepoCreation           int
+	RepoAdminChangeTeamAccess bool
 }
 
 // Validate validates the fields

modules/auth/repo_branch_form.go

@@ -5,8 +5,8 @@
 package auth
 
 import (
-	"github.com/go-macaron/binding"
-	macaron "gopkg.in/macaron.v1"
+	"gitea.com/macaron/binding"
+	"gitea.com/macaron/macaron"
 )
 
 // NewBranchForm form for creating a new branch

modules/auth/repo_form.go

@@ -13,9 +13,9 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/routers/utils"
 
-	"github.com/Unknwon/com"
-	"github.com/go-macaron/binding"
-	macaron "gopkg.in/macaron.v1"
+	"gitea.com/macaron/binding"
+	"gitea.com/macaron/macaron"
+	"github.com/unknwon/com"
 )
 
 // _______________________________________    _________.______________________ _______________.___.
@@ -33,6 +33,7 @@ type CreateRepoForm struct {
 	Description string `binding:"MaxSize(255)"`
 	AutoInit    bool
 	Gitignores  string
+	IssueLabels string
 	License     string
 	Readme      string
 }
@@ -98,13 +99,15 @@ func (f MigrateRepoForm) ParseRemoteAddr(user *models.User) (string, error) {
 
 // RepoSettingForm form for changing repository settings
 type RepoSettingForm struct {
-	RepoName      string `binding:"Required;AlphaDashDot;MaxSize(100)"`
-	Description   string `binding:"MaxSize(255)"`
-	Website       string `binding:"ValidUrl;MaxSize(255)"`
-	Interval      string
-	MirrorAddress string
-	Private       bool
-	EnablePrune   bool
+	RepoName       string `binding:"Required;AlphaDashDot;MaxSize(100)"`
+	Description    string `binding:"MaxSize(255)"`
+	Website        string `binding:"ValidUrl;MaxSize(255)"`
+	Interval       string
+	MirrorAddress  string
+	MirrorUsername string
+	MirrorPassword string
+	Private        bool
+	EnablePrune    bool
 
 	// Advanced settings
 	EnableWiki                       bool
@@ -152,6 +155,8 @@ type ProtectBranchForm struct {
 	EnableMergeWhitelist    bool
 	MergeWhitelistUsers     string
 	MergeWhitelistTeams     string
+	EnableStatusCheck       bool `xorm:"NOT NULL DEFAULT false"`
+	StatusCheckContexts     []string
 	RequiredApprovals       int64
 	ApprovalsWhitelistUsers string
 	ApprovalsWhitelistTeams string
@@ -182,6 +187,7 @@ type WebhookForm struct {
 	PullRequest  bool
 	Repository   bool
 	Active       bool
+	BranchFilter string `binding:"GlobPattern"`
 }
 
 // PushOnly if the hook will be triggered when push

modules/auth/repo_form_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"code.gitea.io/gitea/modules/setting"
+
 	"github.com/stretchr/testify/assert"
 )
 

modules/auth/user_form.go

@@ -11,8 +11,8 @@ import (
 
 	"code.gitea.io/gitea/modules/setting"
 
-	"github.com/go-macaron/binding"
-	macaron "gopkg.in/macaron.v1"
+	"gitea.com/macaron/binding"
+	"gitea.com/macaron/macaron"
 )
 
 // InstallForm form for installation page

modules/auth/user_form_auth_openid.go

@@ -5,8 +5,8 @@
 package auth
 
 import (
-	"github.com/go-macaron/binding"
-	"gopkg.in/macaron.v1"
+	"gitea.com/macaron/binding"
+	"gitea.com/macaron/macaron"
 )
 
 // SignInOpenIDForm form for signing in with OpenID

modules/base/tool.go

@@ -5,7 +5,6 @@
 package base
 
 import (
-	"bytes"
 	"crypto/md5"
 	"crypto/rand"
 	"crypto/sha1"
@@ -13,7 +12,6 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
-	"html/template"
 	"io"
 	"math"
 	"net/http"
@@ -26,21 +24,14 @@ import (
 	"strings"
 	"time"
 	"unicode"
-	"unicode/utf8"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/util"
 
-	"github.com/Unknwon/com"
-	"github.com/Unknwon/i18n"
-	"github.com/gogits/chardet"
+	"github.com/unknwon/com"
 )
 
-// UTF8BOM is the utf-8 byte-order marker
-var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
-
 // EncodeMD5 encodes string to md5 hex value.
 func EncodeMD5(str string) string {
 	m := md5.New()
@@ -68,49 +59,6 @@ func ShortSha(sha1 string) string {
 	return TruncateString(sha1, 10)
 }
 
-// DetectEncoding detect the encoding of content
-func DetectEncoding(content []byte) (string, error) {
-	if utf8.Valid(content) {
-		log.Debug("Detected encoding: utf-8 (fast)")
-		return "UTF-8", nil
-	}
-
-	textDetector := chardet.NewTextDetector()
-	var detectContent []byte
-	if len(content) < 1024 {
-		// Check if original content is valid
-		if _, err := textDetector.DetectBest(content); err != nil {
-			return "", err
-		}
-		times := 1024 / len(content)
-		detectContent = make([]byte, 0, times*len(content))
-		for i := 0; i < times; i++ {
-			detectContent = append(detectContent, content...)
-		}
-	} else {
-		detectContent = content
-	}
-	result, err := textDetector.DetectBest(detectContent)
-	if err != nil {
-		return "", err
-	}
-	if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
-		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
-		return setting.Repository.AnsiCharset, err
-	}
-
-	log.Debug("Detected encoding: %s", result.Charset)
-	return result.Charset, err
-}
-
-// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte
-func RemoveBOMIfPresent(content []byte) []byte {
-	if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
-		return content[3:]
-	}
-	return content
-}
-
 // BasicAuthDecode decode basic auth string
 func BasicAuthDecode(encoded string) (string, string, error) {
 	s, err := base64.StdEncoding.DecodeString(encoded)
@@ -266,154 +214,6 @@ func AvatarLink(email string) string {
 	return SizedAvatarLink(email, DefaultAvatarSize)
 }
 
-// Seconds-based time units
-const (
-	Minute = 60
-	Hour   = 60 * Minute
-	Day    = 24 * Hour
-	Week   = 7 * Day
-	Month  = 30 * Day
-	Year   = 12 * Month
-)
-
-func computeTimeDiff(diff int64, lang string) (int64, string) {
-	diffStr := ""
-	switch {
-	case diff <= 0:
-		diff = 0
-		diffStr = i18n.Tr(lang, "tool.now")
-	case diff < 2:
-		diff = 0
-		diffStr = i18n.Tr(lang, "tool.1s")
-	case diff < 1*Minute:
-		diffStr = i18n.Tr(lang, "tool.seconds", diff)
-		diff = 0
-
-	case diff < 2*Minute:
-		diff -= 1 * Minute
-		diffStr = i18n.Tr(lang, "tool.1m")
-	case diff < 1*Hour:
-		diffStr = i18n.Tr(lang, "tool.minutes", diff/Minute)
-		diff -= diff / Minute * Minute
-
-	case diff < 2*Hour:
-		diff -= 1 * Hour
-		diffStr = i18n.Tr(lang, "tool.1h")
-	case diff < 1*Day:
-		diffStr = i18n.Tr(lang, "tool.hours", diff/Hour)
-		diff -= diff / Hour * Hour
-
-	case diff < 2*Day:
-		diff -= 1 * Day
-		diffStr = i18n.Tr(lang, "tool.1d")
-	case diff < 1*Week:
-		diffStr = i18n.Tr(lang, "tool.days", diff/Day)
-		diff -= diff / Day * Day
-
-	case diff < 2*Week:
-		diff -= 1 * Week
-		diffStr = i18n.Tr(lang, "tool.1w")
-	case diff < 1*Month:
-		diffStr = i18n.Tr(lang, "tool.weeks", diff/Week)
-		diff -= diff / Week * Week
-
-	case diff < 2*Month:
-		diff -= 1 * Month
-		diffStr = i18n.Tr(lang, "tool.1mon")
-	case diff < 1*Year:
-		diffStr = i18n.Tr(lang, "tool.months", diff/Month)
-		diff -= diff / Month * Month
-
-	case diff < 2*Year:
-		diff -= 1 * Year
-		diffStr = i18n.Tr(lang, "tool.1y")
-	default:
-		diffStr = i18n.Tr(lang, "tool.years", diff/Year)
-		diff -= (diff / Year) * Year
-	}
-	return diff, diffStr
-}
-
-// MinutesToFriendly returns a user friendly string with number of minutes
-// converted to hours and minutes.
-func MinutesToFriendly(minutes int, lang string) string {
-	duration := time.Duration(minutes) * time.Minute
-	return TimeSincePro(time.Now().Add(-duration), lang)
-}
-
-// TimeSincePro calculates the time interval and generate full user-friendly string.
-func TimeSincePro(then time.Time, lang string) string {
-	return timeSincePro(then, time.Now(), lang)
-}
-
-func timeSincePro(then, now time.Time, lang string) string {
-	diff := now.Unix() - then.Unix()
-
-	if then.After(now) {
-		return i18n.Tr(lang, "tool.future")
-	}
-	if diff == 0 {
-		return i18n.Tr(lang, "tool.now")
-	}
-
-	var timeStr, diffStr string
-	for {
-		if diff == 0 {
-			break
-		}
-
-		diff, diffStr = computeTimeDiff(diff, lang)
-		timeStr += ", " + diffStr
-	}
-	return strings.TrimPrefix(timeStr, ", ")
-}
-
-func timeSince(then, now time.Time, lang string) string {
-	return timeSinceUnix(then.Unix(), now.Unix(), lang)
-}
-
-func timeSinceUnix(then, now int64, lang string) string {
-	lbl := "tool.ago"
-	diff := now - then
-	if then > now {
-		lbl = "tool.from_now"
-		diff = then - now
-	}
-	if diff <= 0 {
-		return i18n.Tr(lang, "tool.now")
-	}
-
-	_, diffStr := computeTimeDiff(diff, lang)
-	return i18n.Tr(lang, lbl, diffStr)
-}
-
-// RawTimeSince retrieves i18n key of time since t
-func RawTimeSince(t time.Time, lang string) string {
-	return timeSince(t, time.Now(), lang)
-}
-
-// TimeSince calculates the time interval and generate user-friendly string.
-func TimeSince(then time.Time, lang string) template.HTML {
-	return htmlTimeSince(then, time.Now(), lang)
-}
-
-func htmlTimeSince(then, now time.Time, lang string) template.HTML {
-	return template.HTML(fmt.Sprintf(`<span class="time-since" title="%s">%s</span>`,
-		then.Format(setting.TimeFormat),
-		timeSince(then, now, lang)))
-}
-
-// TimeSinceUnix calculates the time interval and generate user-friendly string.
-func TimeSinceUnix(then util.TimeStamp, lang string) template.HTML {
-	return htmlTimeSinceUnix(then, util.TimeStamp(time.Now().Unix()), lang)
-}
-
-func htmlTimeSinceUnix(then, now util.TimeStamp, lang string) template.HTML {
-	return template.HTML(fmt.Sprintf(`<span class="time-since" title="%s">%s</span>`,
-		then.Format(setting.TimeFormat),
-		timeSinceUnix(int64(then), int64(now), lang)))
-}
-
 // Storage space size types
 const (
 	Byte  = 1

modules/base/tool_test.go

@@ -2,43 +2,13 @@ package base
 
 import (
 	"net/url"
-	"os"
 	"testing"
-	"time"
 
 	"code.gitea.io/gitea/modules/setting"
 
-	"github.com/Unknwon/i18n"
-	macaroni18n "github.com/go-macaron/i18n"
 	"github.com/stretchr/testify/assert"
 )
 
-var BaseDate time.Time
-
-// time durations
-const (
-	DayDur   = 24 * time.Hour
-	WeekDur  = 7 * DayDur
-	MonthDur = 30 * DayDur
-	YearDur  = 12 * MonthDur
-)
-
-func TestMain(m *testing.M) {
-	// setup
-	macaroni18n.I18n(macaroni18n.Options{
-		Directory:   "../../options/locale/",
-		DefaultLang: "en-US",
-		Langs:       []string{"en-US"},
-		Names:       []string{"english"},
-	})
-	BaseDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)
-
-	// run the tests
-	retVal := m.Run()
-
-	os.Exit(retVal)
-}
-
 func TestEncodeMD5(t *testing.T) {
 	assert.Equal(t,
 		"3858f62230ac3c915f300c664312c63f",
@@ -64,42 +34,6 @@ func TestShortSha(t *testing.T) {
 	assert.Equal(t, "veryverylo", ShortSha("veryverylong"))
 }
 
-func TestDetectEncoding(t *testing.T) {
-	testSuccess := func(b []byte, expected string) {
-		encoding, err := DetectEncoding(b)
-		assert.NoError(t, err)
-		assert.Equal(t, expected, encoding)
-	}
-	// utf-8
-	b := []byte("just some ascii")
-	testSuccess(b, "UTF-8")
-
-	// utf-8-sig: "hey" (with BOM)
-	b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79}
-	testSuccess(b, "UTF-8")
-
-	// utf-16: "hey<accented G>"
-	b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01}
-	testSuccess(b, "UTF-16LE")
-
-	// iso-8859-1: d<accented e>cor<newline>
-	b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
-	encoding, err := DetectEncoding(b)
-	assert.NoError(t, err)
-	// due to a race condition in `chardet` library, it could either detect
-	// "ISO-8859-1" or "IS0-8859-2" here. Technically either is correct, so
-	// we accept either.
-	assert.Contains(t, encoding, "ISO-8859")
-
-	setting.Repository.AnsiCharset = "placeholder"
-	testSuccess(b, "placeholder")
-
-	// invalid bytes
-	b = []byte{0xfa}
-	_, err = DetectEncoding(b)
-	assert.Error(t, err)
-}
-
 func TestBasicAuthDecode(t *testing.T) {
 	_, _, err := BasicAuthDecode("?")
 	assert.Equal(t, "illegal base64 data at input byte 0", err.Error())
@@ -167,125 +101,6 @@ func TestAvatarLink(t *testing.T) {
 	)
 }
 
-func TestComputeTimeDiff(t *testing.T) {
-	// test that for each offset in offsets,
-	// computeTimeDiff(base + offset) == (offset, str)
-	test := func(base int64, str string, offsets ...int64) {
-		for _, offset := range offsets {
-			diff, diffStr := computeTimeDiff(base+offset, "en")
-			assert.Equal(t, offset, diff)
-			assert.Equal(t, str, diffStr)
-		}
-	}
-	test(0, "now", 0)
-	test(1, "1 second", 0)
-	test(2, "2 seconds", 0)
-	test(Minute, "1 minute", 0, 1, 30, Minute-1)
-	test(2*Minute, "2 minutes", 0, Minute-1)
-	test(Hour, "1 hour", 0, 1, Hour-1)
-	test(5*Hour, "5 hours", 0, Hour-1)
-	test(Day, "1 day", 0, 1, Day-1)
-	test(5*Day, "5 days", 0, Day-1)
-	test(Week, "1 week", 0, 1, Week-1)
-	test(3*Week, "3 weeks", 0, 4*Day+25000)
-	test(Month, "1 month", 0, 1, Month-1)
-	test(10*Month, "10 months", 0, Month-1)
-	test(Year, "1 year", 0, Year-1)
-	test(3*Year, "3 years", 0, Year-1)
-}
-
-func TestMinutesToFriendly(t *testing.T) {
-	// test that a number of minutes yields the expected string
-	test := func(expected string, minutes int) {
-		actual := MinutesToFriendly(minutes, "en")
-		assert.Equal(t, expected, actual)
-	}
-	test("1 minute", 1)
-	test("2 minutes", 2)
-	test("1 hour", 60)
-	test("1 hour, 1 minute", 61)
-	test("1 hour, 2 minutes", 62)
-	test("2 hours", 120)
-}
-
-func TestTimeSince(t *testing.T) {
-	assert.Equal(t, "now", timeSince(BaseDate, BaseDate, "en"))
-
-	// test that each diff in `diffs` yields the expected string
-	test := func(expected string, diffs ...time.Duration) {
-		for _, diff := range diffs {
-			actual := timeSince(BaseDate, BaseDate.Add(diff), "en")
-			assert.Equal(t, i18n.Tr("en", "tool.ago", expected), actual)
-			actual = timeSince(BaseDate.Add(diff), BaseDate, "en")
-			assert.Equal(t, i18n.Tr("en", "tool.from_now", expected), actual)
-		}
-	}
-	test("1 second", time.Second, time.Second+50*time.Millisecond)
-	test("2 seconds", 2*time.Second, 2*time.Second+50*time.Millisecond)
-	test("1 minute", time.Minute, time.Minute+30*time.Second)
-	test("2 minutes", 2*time.Minute, 2*time.Minute+30*time.Second)
-	test("1 hour", time.Hour, time.Hour+30*time.Minute)
-	test("2 hours", 2*time.Hour, 2*time.Hour+30*time.Minute)
-	test("1 day", DayDur, DayDur+12*time.Hour)
-	test("2 days", 2*DayDur, 2*DayDur+12*time.Hour)
-	test("1 week", WeekDur, WeekDur+3*DayDur)
-	test("2 weeks", 2*WeekDur, 2*WeekDur+3*DayDur)
-	test("1 month", MonthDur, MonthDur+15*DayDur)
-	test("2 months", 2*MonthDur, 2*MonthDur+15*DayDur)
-	test("1 year", YearDur, YearDur+6*MonthDur)
-	test("2 years", 2*YearDur, 2*YearDur+6*MonthDur)
-}
-
-func TestTimeSincePro(t *testing.T) {
-	assert.Equal(t, "now", timeSincePro(BaseDate, BaseDate, "en"))
-
-	// test that a difference of `diff` yields the expected string
-	test := func(expected string, diff time.Duration) {
-		actual := timeSincePro(BaseDate, BaseDate.Add(diff), "en")
-		assert.Equal(t, expected, actual)
-		assert.Equal(t, "future", timeSincePro(BaseDate.Add(diff), BaseDate, "en"))
-	}
-	test("1 second", time.Second)
-	test("2 seconds", 2*time.Second)
-	test("1 minute", time.Minute)
-	test("1 minute, 1 second", time.Minute+time.Second)
-	test("1 minute, 59 seconds", time.Minute+59*time.Second)
-	test("2 minutes", 2*time.Minute)
-	test("1 hour", time.Hour)
-	test("1 hour, 1 second", time.Hour+time.Second)
-	test("1 hour, 59 minutes, 59 seconds", time.Hour+59*time.Minute+59*time.Second)
-	test("2 hours", 2*time.Hour)
-	test("1 day", DayDur)
-	test("1 day, 23 hours, 59 minutes, 59 seconds",
-		DayDur+23*time.Hour+59*time.Minute+59*time.Second)
-	test("2 days", 2*DayDur)
-	test("1 week", WeekDur)
-	test("2 weeks", 2*WeekDur)
-	test("1 month", MonthDur)
-	test("3 months", 3*MonthDur)
-	test("1 year", YearDur)
-	test("2 years, 3 months, 1 week, 2 days, 4 hours, 12 minutes, 17 seconds",
-		2*YearDur+3*MonthDur+WeekDur+2*DayDur+4*time.Hour+
-			12*time.Minute+17*time.Second)
-}
-
-func TestHtmlTimeSince(t *testing.T) {
-	setting.TimeFormat = time.UnixDate
-	// test that `diff` yields a result containing `expected`
-	test := func(expected string, diff time.Duration) {
-		actual := htmlTimeSince(BaseDate, BaseDate.Add(diff), "en")
-		assert.Contains(t, actual, `title="Sat Jan  1 00:00:00 UTC 2000"`)
-		assert.Contains(t, actual, expected)
-	}
-	test("1 second", time.Second)
-	test("3 minutes", 3*time.Minute+5*time.Second)
-	test("1 day", DayDur+18*time.Hour)
-	test("1 week", WeekDur+6*DayDur)
-	test("3 months", 3*MonthDur+3*WeekDur)
-	test("2 years", 2*YearDur)
-	test("3 years", 3*YearDur+11*MonthDur+4*WeekDur)
-}
-
 func TestFileSize(t *testing.T) {
 	var size int64 = 512
 	assert.Equal(t, "512B", FileSize(size))

modules/cache/cache.go

@@ -10,7 +10,7 @@ import (
 
 	"code.gitea.io/gitea/modules/setting"
 
-	mc "github.com/go-macaron/cache"
+	mc "gitea.com/macaron/cache"
 )
 
 var conn mc.Cache

modules/charset/charset.go

@@ -0,0 +1,152 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+	"bytes"
+	"fmt"
+	"unicode/utf8"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/gogits/chardet"
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/transform"
+)
+
+// UTF8BOM is the utf-8 byte-order marker
+var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
+
+// ToUTF8WithErr converts content to UTF8 encoding
+func ToUTF8WithErr(content []byte) (string, error) {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil {
+		return "", err
+	} else if charsetLabel == "UTF-8" {
+		return string(RemoveBOMIfPresent(content)), nil
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
+	}
+
+	// If there is an error, we concatenate the nicely decoded part and the
+	// original left over. This way we won't lose much data.
+	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+	if err != nil {
+		result = append(result, content[n:]...)
+	}
+
+	result = RemoveBOMIfPresent(result)
+
+	return string(result), err
+}
+
+// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
+func ToUTF8WithFallback(content []byte) []byte {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil || charsetLabel == "UTF-8" {
+		return RemoveBOMIfPresent(content)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return content
+	}
+
+	// If there is an error, we concatenate the nicely decoded part and the
+	// original left over. This way we won't lose data.
+	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+	if err != nil {
+		return append(result, content[n:]...)
+	}
+
+	return RemoveBOMIfPresent(result)
+}
+
+// ToUTF8 converts content to UTF8 encoding and ignore error
+func ToUTF8(content string) string {
+	res, _ := ToUTF8WithErr([]byte(content))
+	return res
+}
+
+// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
+func ToUTF8DropErrors(content []byte) []byte {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil || charsetLabel == "UTF-8" {
+		return RemoveBOMIfPresent(content)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return content
+	}
+
+	// We ignore any non-decodable parts from the file.
+	// Some parts might be lost
+	var decoded []byte
+	decoder := encoding.NewDecoder()
+	idx := 0
+	for {
+		result, n, err := transform.Bytes(decoder, content[idx:])
+		decoded = append(decoded, result...)
+		if err == nil {
+			break
+		}
+		decoded = append(decoded, ' ')
+		idx = idx + n + 1
+		if idx >= len(content) {
+			break
+		}
+	}
+
+	return RemoveBOMIfPresent(decoded)
+}
+
+// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte
+func RemoveBOMIfPresent(content []byte) []byte {
+	if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
+		return content[3:]
+	}
+	return content
+}
+
+// DetectEncoding detect the encoding of content
+func DetectEncoding(content []byte) (string, error) {
+	if utf8.Valid(content) {
+		log.Debug("Detected encoding: utf-8 (fast)")
+		return "UTF-8", nil
+	}
+
+	textDetector := chardet.NewTextDetector()
+	var detectContent []byte
+	if len(content) < 1024 {
+		// Check if original content is valid
+		if _, err := textDetector.DetectBest(content); err != nil {
+			return "", err
+		}
+		times := 1024 / len(content)
+		detectContent = make([]byte, 0, times*len(content))
+		for i := 0; i < times; i++ {
+			detectContent = append(detectContent, content...)
+		}
+	} else {
+		detectContent = content
+	}
+	result, err := textDetector.DetectBest(detectContent)
+	if err != nil {
+		return "", err
+	}
+	// FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument
+	if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 {
+		log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset)
+		return setting.Repository.AnsiCharset, err
+	}
+
+	log.Debug("Detected encoding: %s", result.Charset)
+	return result.Charset, err
+}

modules/charset/charset_test.go

@@ -0,0 +1,251 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package charset
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRemoveBOMIfPresent(t *testing.T) {
+	res := RemoveBOMIfPresent([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	res = RemoveBOMIfPresent([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+}
+
+func TestToUTF8WithErr(t *testing.T) {
+	var res string
+	var err error
+
+	// Note: golang compiler seems so behave differently depending on the current
+	// locale, so some conversions might behave differently. For that reason, we don't
+	// depend on particular conversions but in expected behaviors.
+
+	res, err = ToUTF8WithErr([]byte{0x41, 0x42, 0x43})
+	assert.NoError(t, err)
+	assert.Equal(t, "ABC", res)
+
+	// "áéíóú"
+	res, err = ToUTF8WithErr([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.NoError(t, err)
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
+
+	// "áéíóú"
+	res, err = ToUTF8WithErr([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
+		0xc3, 0xba})
+	assert.NoError(t, err)
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
+
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
+	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
+
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
+	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
+
+	res, err = ToUTF8WithErr([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e})
+	assert.NoError(t, err)
+	stringMustStartWith(t, "Hola,", res)
+	stringMustEndWith(t, "AAA.", res)
+
+	// Japanese (Shift-JIS)
+	// 日属秘ぞしちゅ。
+	res, err = ToUTF8WithErr([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
+		0xBF, 0x82, 0xE3, 0x81, 0x42})
+	assert.NoError(t, err)
+	assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82},
+		[]byte(res))
+
+	res, err = ToUTF8WithErr([]byte{0x00, 0x00, 0x00, 0x00})
+	assert.NoError(t, err)
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
+}
+
+func TestToUTF8WithFallback(t *testing.T) {
+	// "ABC"
+	res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43})
+	assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
+
+	// "áéíóú"
+	res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	// UTF8 BOM + "áéíóú"
+	res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	// "Hola, así cómo ños"
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+	assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63,
+		0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res)
+
+	// "Hola, así cómo "
+	minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
+
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	// Japanese (Shift-JIS)
+	// "日属秘ぞしちゅ。"
+	res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
+	assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82}, res)
+
+	res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00})
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+}
+
+func TestToUTF8(t *testing.T) {
+
+	// Note: golang compiler seems so behave differently depending on the current
+	// locale, so some conversions might behave differently. For that reason, we don't
+	// depend on particular conversions but in expected behaviors.
+
+	res := ToUTF8(string([]byte{0x41, 0x42, 0x43}))
+	assert.Equal(t, "ABC", res)
+
+	// "áéíóú"
+	res = ToUTF8(string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}))
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
+
+	// BOM + "áéíóú"
+	res = ToUTF8(string([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
+		0xc3, 0xba}))
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
+
+	// Latin1
+	// Hola, así cómo ños
+	res = ToUTF8(string([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}))
+	assert.Equal(t, []byte{0x48, 0x6f, 0x6c, 0x61, 0x2c, 0x20, 0x61, 0x73, 0xc3, 0xad, 0x20, 0x63,
+		0xc3, 0xb3, 0x6d, 0x6f, 0x20, 0xc3, 0xb1, 0x6f, 0x73}, []byte(res))
+
+	// Latin1
+	// Hola, así cómo \x07ños
+	res = ToUTF8(string([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
+		0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}))
+	// Hola,
+	bytesMustStartWith(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C}, []byte(res))
+
+	// This test FAILS
+	// res = ToUTF8("Hola, así cómo \x81ños")
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	// assert.Regexp(t, "^Hola, así cómo", res)
+
+	// Japanese (Shift-JIS)
+	// 日属秘ぞしちゅ。
+	res = ToUTF8(string([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
+		0xBF, 0x82, 0xE3, 0x81, 0x42}))
+	assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82},
+		[]byte(res))
+
+	res = ToUTF8("\x00\x00\x00\x00")
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
+}
+
+func TestToUTF8DropErrors(t *testing.T) {
+	// "ABC"
+	res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43})
+	assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
+
+	// "áéíóú"
+	res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	// UTF8 BOM + "áéíóú"
+	res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
+	assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
+
+	// "Hola, así cómo ños"
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
+	assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73}, res)
+
+	// "Hola, así cómo "
+	minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
+
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
+	// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
+	assert.Equal(t, minmatch, res[0:len(minmatch)])
+
+	// Japanese (Shift-JIS)
+	// "日属秘ぞしちゅ。"
+	res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
+	assert.Equal(t, []byte{0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
+		0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82}, res)
+
+	res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
+	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+}
+
+func TestDetectEncoding(t *testing.T) {
+	testSuccess := func(b []byte, expected string) {
+		encoding, err := DetectEncoding(b)
+		assert.NoError(t, err)
+		assert.Equal(t, expected, encoding)
+	}
+	// utf-8
+	b := []byte("just some ascii")
+	testSuccess(b, "UTF-8")
+
+	// utf-8-sig: "hey" (with BOM)
+	b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79}
+	testSuccess(b, "UTF-8")
+
+	// utf-16: "hey<accented G>"
+	b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01}
+	testSuccess(b, "UTF-16LE")
+
+	// iso-8859-1: d<accented e>cor<newline>
+	b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a}
+	encoding, err := DetectEncoding(b)
+	assert.NoError(t, err)
+	// due to a race condition in `chardet` library, it could either detect
+	// "ISO-8859-1" or "IS0-8859-2" here. Technically either is correct, so
+	// we accept either.
+	assert.Contains(t, encoding, "ISO-8859")
+
+	setting.Repository.AnsiCharset = "placeholder"
+	testSuccess(b, "placeholder")
+
+	// invalid bytes
+	b = []byte{0xfa}
+	_, err = DetectEncoding(b)
+	assert.Error(t, err)
+}
+
+func stringMustStartWith(t *testing.T, expected string, value string) {
+	assert.Equal(t, expected, string(value[:len(expected)]))
+}
+
+func stringMustEndWith(t *testing.T, expected string, value string) {
+	assert.Equal(t, expected, string(value[len(value)-len(expected):]))
+}
+
+func bytesMustStartWith(t *testing.T, expected []byte, value []byte) {
+	assert.Equal(t, expected, value[:len(expected)])
+}

modules/context/api.go

@@ -10,14 +10,13 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/go-macaron/csrf"
-
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
-	"gopkg.in/macaron.v1"
+	"gitea.com/macaron/csrf"
+	"gitea.com/macaron/macaron"
 )
 
 // APIContext is a specific macaron context for API service

modules/context/auth.go

@@ -10,8 +10,9 @@ import (
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"github.com/go-macaron/csrf"
-	macaron "gopkg.in/macaron.v1"
+
+	"gitea.com/macaron/csrf"
+	"gitea.com/macaron/macaron"
 )
 
 // ToggleOptions contains required or check options

modules/context/context.go

@@ -20,12 +20,13 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
-	"github.com/Unknwon/com"
-	"github.com/go-macaron/cache"
-	"github.com/go-macaron/csrf"
-	"github.com/go-macaron/i18n"
-	"github.com/go-macaron/session"
-	"gopkg.in/macaron.v1"
+
+	"gitea.com/macaron/cache"
+	"gitea.com/macaron/csrf"
+	"gitea.com/macaron/i18n"
+	"gitea.com/macaron/macaron"
+	"gitea.com/macaron/session"
+	"github.com/unknwon/com"
 )
 
 // Context represents context of a request.
@@ -249,6 +250,19 @@ func Contexter() macaron.Handler {
 		if ctx.Query("go-get") == "1" {
 			ownerName := c.Params(":username")
 			repoName := c.Params(":reponame")
+			trimmedRepoName := strings.TrimSuffix(repoName, ".git")
+
+			if ownerName == "" || trimmedRepoName == "" {
+				_, _ = c.Write([]byte(`<!doctype html>
+<html>
+	<body>
+		invalid import path
+	</body>
+</html>
+`))
+				c.WriteHeader(400)
+				return
+			}
 			branchName := "master"
 
 			repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName)
@@ -276,7 +290,7 @@ func Contexter() macaron.Handler {
 	</body>
 </html>
 `, map[string]string{
-				"GoGetImport":    ComposeGoGetImport(ownerName, strings.TrimSuffix(repoName, ".git")),
+				"GoGetImport":    ComposeGoGetImport(ownerName, trimmedRepoName),
 				"CloneLink":      models.ComposeHTTPSCloneURL(ownerName, repoName),
 				"GoDocDirectory": prefix + "{/dir}",
 				"GoDocFile":      prefix + "{/dir}/{file}#L{line}",

modules/context/org.go

@@ -9,7 +9,8 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/setting"
-	macaron "gopkg.in/macaron.v1"
+
+	"gitea.com/macaron/macaron"
 )
 
 // Organization contains organization context

modules/context/pagination.go

@@ -10,7 +10,7 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/Unknwon/paginater"
+	"github.com/unknwon/paginater"
 )
 
 // Pagination provides a pagination via Paginater and additional configurations for the link params used in rendering

modules/context/panic.go

@@ -20,7 +20,8 @@ import (
 	"fmt"
 
 	"code.gitea.io/gitea/modules/log"
-	macaron "gopkg.in/macaron.v1"
+
+	"gitea.com/macaron/macaron"
 )
 
 // Recovery returns a middleware that recovers from any panics and writes a 500 and a log if so.
@@ -30,7 +31,7 @@ func Recovery() macaron.Handler {
 	return func(ctx *Context) {
 		defer func() {
 			if err := recover(); err != nil {
-				combinedErr := fmt.Errorf("%s\n%s", err, string(log.Stack(2)))
+				combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2))
 				ctx.ServerError("PANIC:", combinedErr)
 			}
 		}()

modules/context/permission.go

@@ -8,7 +8,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 
-	macaron "gopkg.in/macaron.v1"
+	"gitea.com/macaron/macaron"
 )
 
 // RequireRepoAdmin returns a macaron middleware for requiring repository admin permission

modules/context/repo.go

@@ -18,9 +18,9 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
-	"github.com/Unknwon/com"
+	"gitea.com/macaron/macaron"
+	"github.com/unknwon/com"
 	"gopkg.in/editorconfig/editorconfig-core-go.v1"
-	"gopkg.in/macaron.v1"
 )
 
 // PullRequest contains informations to make a pull request
@@ -201,10 +201,14 @@ func ComposeGoGetImport(owner, repo string) string {
 // .netrc file.
 func EarlyResponseForGoGetMeta(ctx *Context) {
 	username := ctx.Params(":username")
-	reponame := ctx.Params(":reponame")
+	reponame := strings.TrimSuffix(ctx.Params(":reponame"), ".git")
+	if username == "" || reponame == "" {
+		ctx.PlainText(400, []byte("invalid repository path"))
+		return
+	}
 	ctx.PlainText(200, []byte(com.Expand(`<meta name="go-import" content="{GoGetImport} git {CloneLink}">`,
 		map[string]string{
-			"GoGetImport": ComposeGoGetImport(username, strings.TrimSuffix(reponame, ".git")),
+			"GoGetImport": ComposeGoGetImport(username, reponame),
 			"CloneLink":   models.ComposeHTTPSCloneURL(username, reponame),
 		})))
 }
@@ -391,6 +395,7 @@ func RepoAssignment() macaron.Handler {
 		ctx.Data["Owner"] = ctx.Repo.Repository.Owner
 		ctx.Data["IsRepositoryOwner"] = ctx.Repo.IsOwner()
 		ctx.Data["IsRepositoryAdmin"] = ctx.Repo.IsAdmin()
+		ctx.Data["RepoOwnerIsOrganization"] = repo.Owner.IsOrganization()
 		ctx.Data["CanWriteCode"] = ctx.Repo.CanWrite(models.UnitTypeCode)
 		ctx.Data["CanWriteIssues"] = ctx.Repo.CanWrite(models.UnitTypeIssues)
 		ctx.Data["CanWritePulls"] = ctx.Repo.CanWrite(models.UnitTypePullRequests)

modules/cron/cron.go

@@ -1,4 +1,5 @@
 // Copyright 2014 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
@@ -7,15 +8,43 @@ package cron
 import (
 	"time"
 
-	"github.com/gogits/cron"
-
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/sync"
+	mirror_service "code.gitea.io/gitea/services/mirror"
+
+	"github.com/gogs/cron"
+)
+
+const (
+	mirrorUpdate           = "mirror_update"
+	gitFsck                = "git_fsck"
+	checkRepos             = "check_repos"
+	archiveCleanup         = "archive_cleanup"
+	syncExternalUsers      = "sync_external_users"
+	deletedBranchesCleanup = "deleted_branches_cleanup"
 )
 
 var c = cron.New()
 
+// Prevent duplicate running tasks.
+var taskStatusTable = sync.NewStatusTable()
+
+// Func defines a cron function body
+type Func func()
+
+// WithUnique wrap a cron func with an unique running check
+func WithUnique(name string, body Func) Func {
+	return func() {
+		if !taskStatusTable.StartIfNotRunning(name) {
+			return
+		}
+		defer taskStatusTable.Stop(name)
+		body()
+	}
+}
+
 // NewContext begins cron tasks
 func NewContext() {
 	var (
@@ -23,69 +52,69 @@ func NewContext() {
 		err   error
 	)
 	if setting.Cron.UpdateMirror.Enabled {
-		entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, models.MirrorUpdate)
+		entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, WithUnique(mirrorUpdate, mirror_service.Update))
 		if err != nil {
 			log.Fatal("Cron[Update mirrors]: %v", err)
 		}
 		if setting.Cron.UpdateMirror.RunAtStart {
 			entry.Prev = time.Now()
 			entry.ExecTimes++
-			go models.MirrorUpdate()
+			go WithUnique(mirrorUpdate, mirror_service.Update)()
 		}
 	}
 	if setting.Cron.RepoHealthCheck.Enabled {
-		entry, err = c.AddFunc("Repository health check", setting.Cron.RepoHealthCheck.Schedule, models.GitFsck)
+		entry, err = c.AddFunc("Repository health check", setting.Cron.RepoHealthCheck.Schedule, WithUnique(gitFsck, models.GitFsck))
 		if err != nil {
 			log.Fatal("Cron[Repository health check]: %v", err)
 		}
 		if setting.Cron.RepoHealthCheck.RunAtStart {
 			entry.Prev = time.Now()
 			entry.ExecTimes++
-			go models.GitFsck()
+			go WithUnique(gitFsck, models.GitFsck)()
 		}
 	}
 	if setting.Cron.CheckRepoStats.Enabled {
-		entry, err = c.AddFunc("Check repository statistics", setting.Cron.CheckRepoStats.Schedule, models.CheckRepoStats)
+		entry, err = c.AddFunc("Check repository statistics", setting.Cron.CheckRepoStats.Schedule, WithUnique(checkRepos, models.CheckRepoStats))
 		if err != nil {
 			log.Fatal("Cron[Check repository statistics]: %v", err)
 		}
 		if setting.Cron.CheckRepoStats.RunAtStart {
 			entry.Prev = time.Now()
 			entry.ExecTimes++
-			go models.CheckRepoStats()
+			go WithUnique(checkRepos, models.CheckRepoStats)()
 		}
 	}
 	if setting.Cron.ArchiveCleanup.Enabled {
-		entry, err = c.AddFunc("Clean up old repository archives", setting.Cron.ArchiveCleanup.Schedule, models.DeleteOldRepositoryArchives)
+		entry, err = c.AddFunc("Clean up old repository archives", setting.Cron.ArchiveCleanup.Schedule, WithUnique(archiveCleanup, models.DeleteOldRepositoryArchives))
 		if err != nil {
 			log.Fatal("Cron[Clean up old repository archives]: %v", err)
 		}
 		if setting.Cron.ArchiveCleanup.RunAtStart {
 			entry.Prev = time.Now()
 			entry.ExecTimes++
-			go models.DeleteOldRepositoryArchives()
+			go WithUnique(archiveCleanup, models.DeleteOldRepositoryArchives)()
 		}
 	}
 	if setting.Cron.SyncExternalUsers.Enabled {
-		entry, err = c.AddFunc("Synchronize external users", setting.Cron.SyncExternalUsers.Schedule, models.SyncExternalUsers)
+		entry, err = c.AddFunc("Synchronize external users", setting.Cron.SyncExternalUsers.Schedule, WithUnique(syncExternalUsers, models.SyncExternalUsers))
 		if err != nil {
 			log.Fatal("Cron[Synchronize external users]: %v", err)
 		}
 		if setting.Cron.SyncExternalUsers.RunAtStart {
 			entry.Prev = time.Now()
 			entry.ExecTimes++
-			go models.SyncExternalUsers()
+			go WithUnique(syncExternalUsers, models.SyncExternalUsers)()
 		}
 	}
 	if setting.Cron.DeletedBranchesCleanup.Enabled {
-		entry, err = c.AddFunc("Remove old deleted branches", setting.Cron.DeletedBranchesCleanup.Schedule, models.RemoveOldDeletedBranches)
+		entry, err = c.AddFunc("Remove old deleted branches", setting.Cron.DeletedBranchesCleanup.Schedule, WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches))
 		if err != nil {
 			log.Fatal("Cron[Remove old deleted branches]: %v", err)
 		}
 		if setting.Cron.DeletedBranchesCleanup.RunAtStart {
 			entry.Prev = time.Now()
 			entry.ExecTimes++
-			go models.RemoveOldDeletedBranches()
+			go WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches)()
 		}
 	}
 	c.Start()

modules/git/commit.go

@@ -10,6 +10,11 @@ import (
 	"bytes"
 	"container/list"
 	"fmt"
+	"image"
+	"image/color"
+	_ "image/gif"  // for processing gif images
+	_ "image/jpeg" // for processing jpeg images
+	_ "image/png"  // for processing png images
 	"io"
 	"net/http"
 	"strconv"
@@ -158,6 +163,43 @@ func (c *Commit) IsImageFile(name string) bool {
 	return isImage
 }
 
+// ImageMetaData represents metadata of an image file
+type ImageMetaData struct {
+	ColorModel color.Model
+	Width      int
+	Height     int
+	ByteSize   int64
+}
+
+// ImageInfo returns information about the dimensions of an image
+func (c *Commit) ImageInfo(name string) (*ImageMetaData, error) {
+	if !c.IsImageFile(name) {
+		return nil, nil
+	}
+
+	blob, err := c.GetBlobByPath(name)
+	if err != nil {
+		return nil, err
+	}
+	reader, err := blob.DataAsync()
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+	config, _, err := image.DecodeConfig(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	metadata := ImageMetaData{
+		ColorModel: config.ColorModel,
+		Width:      config.Width,
+		Height:     config.Height,
+		ByteSize:   blob.Size(),
+	}
+	return &metadata, nil
+}
+
 // GetCommitByPath return the commit of relative path object.
 func (c *Commit) GetCommitByPath(relpath string) (*Commit, error) {
 	return c.repo.getCommitByPathWithID(c.ID, relpath)
@@ -169,6 +211,7 @@ func AddChanges(repoPath string, all bool, files ...string) error {
 	if all {
 		cmd.AddArguments("--all")
 	}
+	cmd.AddArguments("--")
 	_, err := cmd.AddArguments(files...).RunInDir(repoPath)
 	return err
 }
@@ -304,10 +347,21 @@ func (c *Commit) GetFilesChangedSinceCommit(pastCommit string) ([]string, error)
 }
 
 // FileChangedSinceCommit Returns true if the file given has changed since the the past commit
+// YOU MUST ENSURE THAT pastCommit is a valid commit ID.
 func (c *Commit) FileChangedSinceCommit(filename, pastCommit string) (bool, error) {
 	return c.repo.FileChangedBetweenCommits(filename, pastCommit, c.ID.String())
 }
 
+// HasFile returns true if the file given exists on this commit
+// This does only mean it's there - it does not mean the file was changed during the commit.
+func (c *Commit) HasFile(filename string) (bool, error) {
+	_, err := c.GetBlobByPath(filename)
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
 // GetSubModules get all the sub modules of current revision git tree
 func (c *Commit) GetSubModules() (*ObjectCache, error) {
 	if c.submoduleCache != nil {

modules/git/commit_info.go

@@ -25,7 +25,7 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom
 		defer commitGraphFile.Close()
 	}
 
-	c, err := commitNodeIndex.Get(plumbing.Hash(commit.ID))
+	c, err := commitNodeIndex.Get(commit.ID)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -68,7 +68,9 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom
 	// get it for free during the tree traversal and it's used for listing
 	// pages to display information about newest commit for a given path.
 	var treeCommit *Commit
-	if rev, ok := revs[""]; ok {
+	if treePath == "" {
+		treeCommit = commit
+	} else if rev, ok := revs[""]; ok {
 		treeCommit = convertCommit(rev)
 	}
 	return commitsInfo, treeCommit, nil

modules/git/commit_info_test.go

@@ -28,21 +28,27 @@ func cloneRepo(url, dir, name string) (string, error) {
 func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
 	// these test case are specific to the repo1 test repo
 	testCases := []struct {
-		CommitID    string
-		Path        string
-		ExpectedIDs map[string]string
+		CommitID           string
+		Path               string
+		ExpectedIDs        map[string]string
+		ExpectedTreeCommit string
 	}{
 		{"8d92fc957a4d7cfd98bc375f0b7bb189a0d6c9f2", "", map[string]string{
 			"file1.txt": "95bb4d39648ee7e325106df01a621c530863a653",
 			"file2.txt": "8d92fc957a4d7cfd98bc375f0b7bb189a0d6c9f2",
-		}},
+		}, "8d92fc957a4d7cfd98bc375f0b7bb189a0d6c9f2"},
 		{"2839944139e0de9737a044f78b0e4b40d989a9e3", "", map[string]string{
 			"file1.txt":   "2839944139e0de9737a044f78b0e4b40d989a9e3",
 			"branch1.txt": "9c9aef8dd84e02bc7ec12641deb4c930a7c30185",
-		}},
+		}, "2839944139e0de9737a044f78b0e4b40d989a9e3"},
 		{"5c80b0245c1c6f8343fa418ec374b13b5d4ee658", "branch2", map[string]string{
 			"branch2.txt": "5c80b0245c1c6f8343fa418ec374b13b5d4ee658",
-		}},
+		}, "5c80b0245c1c6f8343fa418ec374b13b5d4ee658"},
+		{"feaf4ba6bc635fec442f46ddd4512416ec43c2c2", "", map[string]string{
+			"file1.txt": "95bb4d39648ee7e325106df01a621c530863a653",
+			"file2.txt": "8d92fc957a4d7cfd98bc375f0b7bb189a0d6c9f2",
+			"foo":       "37991dec2c8e592043f47155ce4808d4580f9123",
+		}, "feaf4ba6bc635fec442f46ddd4512416ec43c2c2"},
 	}
 	for _, testCase := range testCases {
 		commit, err := repo1.GetCommit(testCase.CommitID)
@@ -51,7 +57,8 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
 		assert.NoError(t, err)
 		entries, err := tree.ListEntries()
 		assert.NoError(t, err)
-		commitsInfo, _, err := entries.GetCommitsInfo(commit, testCase.Path, nil)
+		commitsInfo, treeCommit, err := entries.GetCommitsInfo(commit, testCase.Path, nil)
+		assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
 		assert.NoError(t, err)
 		assert.Len(t, commitsInfo, len(testCase.ExpectedIDs))
 		for _, commitInfo := range commitsInfo {

modules/git/hook.go

@@ -12,7 +12,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/Unknwon/com"
+	"github.com/unknwon/com"
 )
 
 // hookNames is a list of Git server hooks' name that are supported.

modules/git/notes.go

@@ -7,7 +7,7 @@ package git
 import (
 	"io/ioutil"
 
-	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/object"
 )
 
 // NotesRef is the git ref where Gitea will look for git-notes data.
@@ -27,13 +27,28 @@ func GetNote(repo *Repository, commitID string, note *Note) error {
 		return err
 	}
 
-	entry, err := notes.GetTreeEntryByPath(commitID)
-	if err != nil {
-		return err
+	remainingCommitID := commitID
+	path := ""
+	currentTree := notes.Tree.gogitTree
+	var file *object.File
+	for len(remainingCommitID) > 2 {
+		file, err = currentTree.File(remainingCommitID)
+		if err == nil {
+			path += remainingCommitID
+			break
+		}
+		if err == object.ErrFileNotFound {
+			currentTree, err = currentTree.Tree(remainingCommitID[0:2])
+			path += remainingCommitID[0:2] + "/"
+			remainingCommitID = remainingCommitID[2:]
+		}
+		if err != nil {
+			return err
+		}
 	}
 
-	blob := entry.Blob()
-	dataRc, err := blob.DataAsync()
+	blob := file.Blob
+	dataRc, err := blob.Reader()
 	if err != nil {
 		return err
 	}
@@ -45,26 +60,21 @@ func GetNote(repo *Repository, commitID string, note *Note) error {
 	}
 	note.Message = d
 
-	commit, err := repo.gogitRepo.CommitObject(plumbing.Hash(notes.ID))
-	if err != nil {
-		return err
-	}
-
 	commitNodeIndex, commitGraphFile := repo.CommitNodeIndex()
 	if commitGraphFile != nil {
 		defer commitGraphFile.Close()
 	}
 
-	commitNode, err := commitNodeIndex.Get(commit.Hash)
-	if err != nil {
-		return nil
-	}
-
-	lastCommits, err := getLastCommitForPaths(commitNode, "", []string{commitID})
+	commitNode, err := commitNodeIndex.Get(notes.ID)
 	if err != nil {
 		return err
 	}
-	note.Commit = convertCommit(lastCommits[commitID])
+
+	lastCommits, err := getLastCommitForPaths(commitNode, "", []string{path})
+	if err != nil {
+		return err
+	}
+	note.Commit = convertCommit(lastCommits[path])
 
 	return nil
 }

modules/git/notes_test.go

@@ -22,3 +22,17 @@ func TestGetNotes(t *testing.T) {
 	assert.Equal(t, []byte("Note contents\n"), note.Message)
 	assert.Equal(t, "Vladimir Panteleev", note.Commit.Author.Name)
 }
+
+func TestGetNestedNotes(t *testing.T) {
+	repoPath := filepath.Join(testReposDir, "repo3_notes")
+	repo, err := OpenRepository(repoPath)
+	assert.NoError(t, err)
+
+	note := Note{}
+	err = GetNote(repo, "3e668dbfac39cbc80a9ff9c61eb565d944453ba4", &note)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("Note 2"), note.Message)
+	err = GetNote(repo, "ba0a96fa63532d6c5087ecef070b0250ed72fa47", &note)
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("Note 1"), note.Message)
+}

modules/git/parse.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"strconv"
 
-	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/filemode"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 )
@@ -57,7 +56,7 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
 			return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
 		}
 		entry.ID = id
-		entry.gogitTreeEntry.Hash = plumbing.Hash(id)
+		entry.gogitTreeEntry.Hash = id
 		pos += 41 // skip over sha and trailing space
 
 		end := pos + bytes.IndexByte(data[pos:], '\n')

modules/git/repo.go

@@ -17,7 +17,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Unknwon/com"
+	"github.com/unknwon/com"
 	"gopkg.in/src-d/go-billy.v4/osfs"
 	gogit "gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing/cache"
@@ -187,8 +187,7 @@ func Pull(repoPath string, opts PullRemoteOptions) error {
 	if opts.All {
 		cmd.AddArguments("--all")
 	} else {
-		cmd.AddArguments(opts.Remote)
-		cmd.AddArguments(opts.Branch)
+		cmd.AddArguments("--", opts.Remote, opts.Branch)
 	}
 
 	if opts.Timeout <= 0 {
@@ -213,7 +212,7 @@ func Push(repoPath string, opts PushOptions) error {
 	if opts.Force {
 		cmd.AddArguments("-f")
 	}
-	cmd.AddArguments(opts.Remote, opts.Branch)
+	cmd.AddArguments("--", opts.Remote, opts.Branch)
 	_, err := cmd.RunInDirWithEnv(repoPath, opts.Env)
 	return err
 }

modules/git/repo_blame.go

@@ -20,5 +20,5 @@ func (repo *Repository) LineBlame(revision, path, file string, line uint) (*Comm
 	if len(res) < 40 {
 		return nil, fmt.Errorf("invalid result of blame: %s", res)
 	}
-	return repo.GetCommit(string(res[:40]))
+	return repo.GetCommit(res[:40])
 }

modules/git/repo_blob.go

@@ -9,7 +9,7 @@ import (
 )
 
 func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
-	encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, plumbing.Hash(id))
+	encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id)
 	if err != nil {
 		return nil, ErrNotExist{id.String(), ""}
 	}

modules/git/repo_branch.go

@@ -135,7 +135,7 @@ func (repo *Repository) DeleteBranch(name string, opts DeleteBranchOptions) erro
 		cmd.AddArguments("-d")
 	}
 
-	cmd.AddArguments(name)
+	cmd.AddArguments("--", name)
 	_, err := cmd.RunInDir(repo.Path)
 
 	return err

modules/git/repo_commit.go

@@ -86,9 +86,9 @@ func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
 func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
 	var tagObject *object.Tag
 
-	gogitCommit, err := repo.gogitRepo.CommitObject(plumbing.Hash(id))
+	gogitCommit, err := repo.gogitRepo.CommitObject(id)
 	if err == plumbing.ErrObjectNotFound {
-		tagObject, err = repo.gogitRepo.TagObject(plumbing.Hash(id))
+		tagObject, err = repo.gogitRepo.TagObject(id)
 		if err == nil {
 			gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target)
 		}
@@ -117,20 +117,26 @@ func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
 	return commit, nil
 }
 
-// GetCommit returns commit object of by ID string.
-func (repo *Repository) GetCommit(commitID string) (*Commit, error) {
+// ConvertToSHA1 returns a Hash object from a potential ID string
+func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) {
 	if len(commitID) != 40 {
 		var err error
-		actualCommitID, err := NewCommand("rev-parse", commitID).RunInDir(repo.Path)
+		actualCommitID, err := NewCommand("rev-parse", "--verify", commitID).RunInDir(repo.Path)
 		if err != nil {
-			if strings.Contains(err.Error(), "unknown revision or path") {
-				return nil, ErrNotExist{commitID, ""}
+			if strings.Contains(err.Error(), "unknown revision or path") ||
+				strings.Contains(err.Error(), "fatal: Needed a single revision") {
+				return SHA1{}, ErrNotExist{commitID, ""}
 			}
-			return nil, err
+			return SHA1{}, err
 		}
 		commitID = actualCommitID
 	}
-	id, err := NewIDFromString(commitID)
+	return NewIDFromString(commitID)
+}
+
+// GetCommit returns commit object of by ID string.
+func (repo *Repository) GetCommit(commitID string) (*Commit, error) {
+	id, err := repo.ConvertToSHA1(commitID)
 	if err != nil {
 		return nil, err
 	}
@@ -202,36 +208,57 @@ func (repo *Repository) commitsByRange(id SHA1, page int) (*list.List, error) {
 }
 
 func (repo *Repository) searchCommits(id SHA1, opts SearchCommitsOptions) (*list.List, error) {
-	cmd := NewCommand("log", id.String(), "-100", "-i", prettyLogFormat)
+	cmd := NewCommand("log", id.String(), "-100", prettyLogFormat)
+	args := []string{"-i"}
+	if len(opts.Authors) > 0 {
+		for _, v := range opts.Authors {
+			args = append(args, "--author="+v)
+		}
+	}
+	if len(opts.Committers) > 0 {
+		for _, v := range opts.Committers {
+			args = append(args, "--committer="+v)
+		}
+	}
+	if len(opts.After) > 0 {
+		args = append(args, "--after="+opts.After)
+	}
+	if len(opts.Before) > 0 {
+		args = append(args, "--before="+opts.Before)
+	}
+	if opts.All {
+		args = append(args, "--all")
+	}
 	if len(opts.Keywords) > 0 {
 		for _, v := range opts.Keywords {
 			cmd.AddArguments("--grep=" + v)
 		}
 	}
-	if len(opts.Authors) > 0 {
-		for _, v := range opts.Authors {
-			cmd.AddArguments("--author=" + v)
-		}
-	}
-	if len(opts.Committers) > 0 {
-		for _, v := range opts.Committers {
-			cmd.AddArguments("--committer=" + v)
-		}
-	}
-	if len(opts.After) > 0 {
-		cmd.AddArguments("--after=" + opts.After)
-	}
-	if len(opts.Before) > 0 {
-		cmd.AddArguments("--before=" + opts.Before)
-	}
-	if opts.All {
-		cmd.AddArguments("--all")
-	}
+	cmd.AddArguments(args...)
 	stdout, err := cmd.RunInDirBytes(repo.Path)
 	if err != nil {
 		return nil, err
 	}
-	return repo.parsePrettyFormatLogToList(stdout)
+	if len(stdout) != 0 {
+		stdout = append(stdout, '\n')
+	}
+	if len(opts.Keywords) > 0 {
+		for _, v := range opts.Keywords {
+			if len(v) >= 4 {
+				hashCmd := NewCommand("log", "-1", prettyLogFormat)
+				hashCmd.AddArguments(args...)
+				hashCmd.AddArguments(v)
+				hashMatching, err := hashCmd.RunInDirBytes(repo.Path)
+				if err != nil || bytes.Contains(stdout, hashMatching) {
+					continue
+				}
+				stdout = append(stdout, hashMatching...)
+				stdout = append(stdout, '\n')
+			}
+		}
+	}
+
+	return repo.parsePrettyFormatLogToList(bytes.TrimSuffix(stdout, []byte{'\n'}))
 }
 
 func (repo *Repository) getFilesChanged(id1, id2 string) ([]string, error) {
@@ -243,6 +270,7 @@ func (repo *Repository) getFilesChanged(id1, id2 string) ([]string, error) {
 }
 
 // FileChangedBetweenCommits Returns true if the file changed between commit IDs id1 and id2
+// You must ensure that id1 and id2 are valid commit ids.
 func (repo *Repository) FileChangedBetweenCommits(filename, id1, id2 string) (bool, error) {
 	stdout, err := NewCommand("diff", "--name-only", "-z", id1, id2, "--", filename).RunInDirBytes(repo.Path)
 	if err != nil {
@@ -266,6 +294,16 @@ func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) (
 	return repo.parsePrettyFormatLogToList(stdout)
 }
 
+// CommitsByFileAndRangeNoFollow return the commits according revison file and the page
+func (repo *Repository) CommitsByFileAndRangeNoFollow(revision, file string, page int) (*list.List, error) {
+	stdout, err := NewCommand("log", revision, "--skip="+strconv.Itoa((page-1)*50),
+		"--max-count="+strconv.Itoa(CommitsRangeSize), prettyLogFormat, "--", file).RunInDirBytes(repo.Path)
+	if err != nil {
+		return nil, err
+	}
+	return repo.parsePrettyFormatLogToList(stdout)
+}
+
 // FilesCountBetween return the number of files changed between two commits
 func (repo *Repository) FilesCountBetween(startCommitID, endCommitID string) (int, error) {
 	stdout, err := NewCommand("diff", "--name-only", startCommitID+"..."+endCommitID).RunInDir(repo.Path)

modules/git/repo_commit_test.go

@@ -40,8 +40,9 @@ func TestRepository_GetCommitBranches(t *testing.T) {
 func TestGetTagCommitWithSignature(t *testing.T) {
 	bareRepo1Path := filepath.Join(testReposDir, "repo1_bare")
 	bareRepo1, err := OpenRepository(bareRepo1Path)
-	commit, err := bareRepo1.GetCommit("3ad28a9149a2864384548f3d17ed7f38014c9e8a")
+	assert.NoError(t, err)
 
+	commit, err := bareRepo1.GetCommit("3ad28a9149a2864384548f3d17ed7f38014c9e8a")
 	assert.NoError(t, err)
 	assert.NotNil(t, commit)
 	assert.NotNil(t, commit.Signature)
@@ -52,8 +53,10 @@ func TestGetTagCommitWithSignature(t *testing.T) {
 func TestGetCommitWithBadCommitID(t *testing.T) {
 	bareRepo1Path := filepath.Join(testReposDir, "repo1_bare")
 	bareRepo1, err := OpenRepository(bareRepo1Path)
+	assert.NoError(t, err)
+
 	commit, err := bareRepo1.GetCommit("bad_branch")
 	assert.Nil(t, commit)
 	assert.Error(t, err)
-	assert.EqualError(t, err, "object does not exist [id: bad_branch, rel_path: ]")
+	assert.True(t, IsErrNotExist(err))
 }

modules/git/repo_commitgraph.go

@@ -10,6 +10,7 @@ import (
 	"path"
 
 	gitealog "code.gitea.io/gitea/modules/log"
+
 	"gopkg.in/src-d/go-git.v4/plumbing/format/commitgraph"
 	cgobject "gopkg.in/src-d/go-git.v4/plumbing/object/commitgraph"
 )

modules/git/repo_compare.go

@@ -39,7 +39,7 @@ func (repo *Repository) GetMergeBase(tmpRemote string, base, head string) (strin
 		}
 	}
 
-	stdout, err := NewCommand("merge-base", base, head).RunInDir(repo.Path)
+	stdout, err := NewCommand("merge-base", "--", base, head).RunInDir(repo.Path)
 	return strings.TrimSpace(stdout), base, err
 }
 
@@ -54,7 +54,7 @@ func (repo *Repository) GetCompareInfo(basePath, baseBranch, headBranch string)
 	if repo.Path != basePath {
 		// Add a temporary remote
 		tmpRemote = strconv.FormatInt(time.Now().UnixNano(), 10)
-		if err = repo.AddRemote(tmpRemote, basePath, true); err != nil {
+		if err = repo.AddRemote(tmpRemote, basePath, false); err != nil {
 			return nil, fmt.Errorf("AddRemote: %v", err)
 		}
 		defer func() {

modules/git/repo_index.go

@@ -12,7 +12,7 @@ import (
 // ReadTreeToIndex reads a treeish to the index
 func (repo *Repository) ReadTreeToIndex(treeish string) error {
 	if len(treeish) != 40 {
-		res, err := NewCommand("rev-parse", treeish).RunInDir(repo.Path)
+		res, err := NewCommand("rev-parse", "--verify", treeish).RunInDir(repo.Path)
 		if err != nil {
 			return err
 		}

modules/git/repo_ref.go

@@ -34,13 +34,13 @@ func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
 			refType := string(ObjectCommit)
 			if ref.Name().IsTag() {
 				// tags can be of type `commit` (lightweight) or `tag` (annotated)
-				if tagType, _ := repo.GetTagType(SHA1(ref.Hash())); err == nil {
+				if tagType, _ := repo.GetTagType(ref.Hash()); err == nil {
 					refType = tagType
 				}
 			}
 			r := &Reference{
 				Name:   ref.Name().String(),
-				Object: SHA1(ref.Hash()),
+				Object: ref.Hash(),
 				Type:   refType,
 				repo:   repo,
 			}

modules/git/repo_stats_test.go

@@ -18,17 +18,18 @@ func TestRepository_GetCodeActivityStats(t *testing.T) {
 	assert.NoError(t, err)
 
 	timeFrom, err := time.Parse(time.RFC3339, "2016-01-01T00:00:00+00:00")
+	assert.NoError(t, err)
 
 	code, err := bareRepo1.GetCodeActivityStats(timeFrom, "")
 	assert.NoError(t, err)
 	assert.NotNil(t, code)
 
-	assert.EqualValues(t, 8, code.CommitCount)
-	assert.EqualValues(t, 2, code.AuthorCount)
-	assert.EqualValues(t, 8, code.CommitCountInAllBranches)
+	assert.EqualValues(t, 9, code.CommitCount)
+	assert.EqualValues(t, 3, code.AuthorCount)
+	assert.EqualValues(t, 9, code.CommitCountInAllBranches)
 	assert.EqualValues(t, 10, code.Additions)
 	assert.EqualValues(t, 1, code.Deletions)
-	assert.Len(t, code.Authors, 2)
+	assert.Len(t, code.Authors, 3)
 	assert.Contains(t, code.Authors, "tris.git@shoddynet.org")
 	assert.EqualValues(t, 3, code.Authors["tris.git@shoddynet.org"])
 	assert.EqualValues(t, 5, code.Authors[""])

modules/git/repo_tag.go

@@ -29,13 +29,13 @@ func (repo *Repository) IsTagExist(name string) bool {
 
 // CreateTag create one tag in the repository
 func (repo *Repository) CreateTag(name, revision string) error {
-	_, err := NewCommand("tag", name, revision).RunInDir(repo.Path)
+	_, err := NewCommand("tag", "--", name, revision).RunInDir(repo.Path)
 	return err
 }
 
 // CreateAnnotatedTag create one annotated tag in the repository
 func (repo *Repository) CreateAnnotatedTag(name, message, revision string) error {
-	_, err := NewCommand("tag", "-a", "-m", message, name, revision).RunInDir(repo.Path)
+	_, err := NewCommand("tag", "-a", "-m", message, "--", name, revision).RunInDir(repo.Path)
 	return err
 }
 
@@ -153,15 +153,18 @@ func (repo *Repository) GetTagNameBySHA(sha string) (string, error) {
 
 // GetTagID returns the object ID for a tag (annotated tags have both an object SHA AND a commit SHA)
 func (repo *Repository) GetTagID(name string) (string, error) {
-	stdout, err := NewCommand("show-ref", name).RunInDir(repo.Path)
+	stdout, err := NewCommand("show-ref", "--tags", "--", name).RunInDir(repo.Path)
 	if err != nil {
 		return "", err
 	}
-	fields := strings.Fields(stdout)
-	if len(fields) != 2 {
-		return "", ErrNotExist{ID: name}
+	// Make sure exact match is used: "v1" != "release/v1"
+	for _, line := range strings.Split(stdout, "\n") {
+		fields := strings.Fields(line)
+		if len(fields) == 2 && fields[1] == "refs/tags/"+name {
+			return fields[0], nil
+		}
 	}
-	return fields[0], nil
+	return "", ErrNotExist{ID: name}
 }
 
 // GetTag returns a Git tag by given name.

modules/git/repo_tag_test.go

@@ -62,6 +62,16 @@ func TestRepository_GetTag(t *testing.T) {
 	assert.NotEqual(t, aTagID, aTag.Object.String())
 	assert.EqualValues(t, aTagCommitID, aTag.Object.String())
 	assert.EqualValues(t, "tag", aTag.Type)
+
+	rTagCommitID := "8006ff9adbf0cb94da7dad9e537e53817f9fa5c0"
+	rTagName := "release/" + lTagName
+	bareRepo1.CreateTag(rTagName, rTagCommitID)
+	rTagID, err := bareRepo1.GetTagID(rTagName)
+	assert.NoError(t, err)
+	assert.EqualValues(t, rTagCommitID, rTagID)
+	oTagID, err := bareRepo1.GetTagID(lTagName)
+	assert.NoError(t, err)
+	assert.EqualValues(t, lTagCommitID, oTagID)
 }
 
 func TestRepository_GetAnnotatedTag(t *testing.T) {

modules/git/repo_tree.go

@@ -10,12 +10,10 @@ import (
 	"os"
 	"strings"
 	"time"
-
-	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
 func (repo *Repository) getTree(id SHA1) (*Tree, error) {
-	gogitTree, err := repo.gogitRepo.TreeObject(plumbing.Hash(id))
+	gogitTree, err := repo.gogitRepo.TreeObject(id)
 	if err != nil {
 		return nil, err
 	}
@@ -28,7 +26,7 @@ func (repo *Repository) getTree(id SHA1) (*Tree, error) {
 // GetTree find the tree object in the repository.
 func (repo *Repository) GetTree(idStr string) (*Tree, error) {
 	if len(idStr) != 40 {
-		res, err := NewCommand("rev-parse", idStr).RunInDir(repo.Path)
+		res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
 		if err != nil {
 			return nil, err
 		}
@@ -41,7 +39,7 @@ func (repo *Repository) GetTree(idStr string) (*Tree, error) {
 		return nil, err
 	}
 	resolvedID := id
-	commitObject, err := repo.gogitRepo.CommitObject(plumbing.Hash(id))
+	commitObject, err := repo.gogitRepo.CommitObject(id)
 	if err == nil {
 		id = SHA1(commitObject.TreeHash)
 	}
@@ -63,7 +61,7 @@ type CommitTreeOpts struct {
 
 // CommitTree creates a commit from a given tree id for the user with provided message
 func (repo *Repository) CommitTree(sig *Signature, tree *Tree, opts CommitTreeOpts) (SHA1, error) {
-	commitTimeStr := time.Now().Format(time.UnixDate)
+	commitTimeStr := time.Now().Format(time.RFC3339)
 
 	// Because this may call hooks we should pass in the environment
 	env := append(os.Environ(),

modules/git/tests/repos/repo1_bare/logs/HEAD

@@ -0,0 +1 @@
+37991dec2c8e592043f47155ce4808d4580f9123 feaf4ba6bc635fec442f46ddd4512416ec43c2c2 silverwind <me@silverwind.io> 1563741799 +0200	push

modules/git/tests/repos/repo1_bare/logs/refs/heads/master

@@ -0,0 +1 @@
+37991dec2c8e592043f47155ce4808d4580f9123 feaf4ba6bc635fec442f46ddd4512416ec43c2c2 silverwind <me@silverwind.io> 1563741799 +0200	push

modules/git/tests/repos/repo1_bare/refs/heads/master

@@ -1 +1 @@
-37991dec2c8e592043f47155ce4808d4580f9123
+feaf4ba6bc635fec442f46ddd4512416ec43c2c2

modules/git/tests/repos/repo3_notes/COMMIT_EDITMSG

@@ -0,0 +1 @@
+2

modules/git/tests/repos/repo3_notes/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

modules/git/tests/repos/repo3_notes/config

@@ -0,0 +1,7 @@
+[core]
+	repositoryformatversion = 0
+	filemode = false
+	bare = false
+	logallrefupdates = true
+	symlinks = false
+	ignorecase = true

modules/git/tests/repos/repo3_notes/description

@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.

modules/git/tests/repos/repo3_notes/logs/HEAD

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 ba0a96fa63532d6c5087ecef070b0250ed72fa47 Filip Navara <filip.navara@gmail.com> 1567767895 +0200	commit (initial): 1
+ba0a96fa63532d6c5087ecef070b0250ed72fa47 3e668dbfac39cbc80a9ff9c61eb565d944453ba4 Filip Navara <filip.navara@gmail.com> 1567767909 +0200	commit: 2

modules/git/tests/repos/repo3_notes/logs/refs/heads/master

@@ -0,0 +1,2 @@
+0000000000000000000000000000000000000000 ba0a96fa63532d6c5087ecef070b0250ed72fa47 Filip Navara <filip.navara@gmail.com> 1567767895 +0200	commit (initial): 1
+ba0a96fa63532d6c5087ecef070b0250ed72fa47 3e668dbfac39cbc80a9ff9c61eb565d944453ba4 Filip Navara <filip.navara@gmail.com> 1567767909 +0200	commit: 2

modules/git/tests/repos/repo3_notes/objects/3e/668dbfac39cbc80a9ff9c61eb565d944453ba4

@@ -0,0 +1,3 @@
+x
ťŽ;Â0@™s
+ďH•ă&v*!ÄÄČś4�JôŁ(p~
+G`|oxziťç©
‘;´š3Đ –ÂČŢ÷6 ś$`¦"NRäŃşXlałiÍK�¨¨ĺŢ÷4rň$§\P0"yĚŁPQ'F_í±V¸NĎi�›ľµ*śĘ—şĺG—ű¬ÓłKë|ëY„eŔ
ŽH�f·űfË™ÜăEm

modules/git/tests/repos/repo3_notes/objects/65/4c8b6b63c08bf37f638d3f521626b7fbbd4d37

@@ -0,0 +1 @@
+x
ќЌ;В0©}Љн‘"Зїu$„RQr‡•ЩK1F–Йщ1ЃnfЉчR-%w�Сzcбґ{д%7Аўh#Сx¶€ЙQ¤айfXС»?jѓKЮт®ґS#8ЙЧ¦зПЦ{ЎјM©–3М> †Ёќ…Ј6Z«QЗmзїФя8В

modules/git/tests/repos/repo3_notes/refs/heads/master

@@ -0,0 +1 @@
+3e668dbfac39cbc80a9ff9c61eb565d944453ba4

modules/git/tests/repos/repo3_notes/refs/notes/commits

@@ -0,0 +1 @@
+654c8b6b63c08bf37f638d3f521626b7fbbd4d37

modules/git/tree.go

@@ -63,7 +63,7 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) {
 }
 
 func (t *Tree) loadTreeObject() error {
-	gogitTree, err := t.repo.gogitRepo.TreeObject(plumbing.Hash(t.ID))
+	gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID)
 	if err != nil {
 		return err
 	}

modules/git/tree_blob.go

@@ -9,7 +9,6 @@ import (
 	"path"
 	"strings"
 
-	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/filemode"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 )
@@ -23,7 +22,7 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
 			gogitTreeEntry: &object.TreeEntry{
 				Name: "",
 				Mode: filemode.Dir,
-				Hash: plumbing.Hash(t.ID),
+				Hash: t.ID,
 			},
 		}, nil
 	}

modules/git/tree_entry.go

@@ -108,6 +108,11 @@ func (te *TreeEntry) IsRegular() bool {
 	return te.gogitTreeEntry.Mode == filemode.Regular
 }
 
+// IsExecutable if the entry is an executable file (not necessarily binary)
+func (te *TreeEntry) IsExecutable() bool {
+	return te.gogitTreeEntry.Mode == filemode.Executable
+}
+
 // Blob returns the blob object the entry
 func (te *TreeEntry) Blob() *Blob {
 	encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash)

modules/gzip/gzip.go

@@ -15,8 +15,8 @@ import (
 	"strings"
 	"sync"
 
+	"gitea.com/macaron/macaron"
 	"github.com/klauspost/compress/gzip"
-	"gopkg.in/macaron.v1"
 )
 
 const (

modules/gzip/gzip_test.go

@@ -12,9 +12,9 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"gitea.com/macaron/macaron"
 	gzipp "github.com/klauspost/compress/gzip"
 	"github.com/stretchr/testify/assert"
-	macaron "gopkg.in/macaron.v1"
 )
 
 func setup(sampleResponse []byte) (*macaron.Macaron, *[]byte) {

modules/highlight/highlight.go

@@ -19,55 +19,101 @@ var (
 	}
 
 	// File names that are representing highlight classes.
-	highlightFileNames = map[string]bool{
-		"dockerfile": true,
-		"makefile":   true,
+	highlightFileNames = map[string]string{
+		"dockerfile":     "dockerfile",
+		"makefile":       "makefile",
+		"gnumakefile":    "makefile",
+		"cmakelists.txt": "cmake",
 	}
 
 	// Extensions that are same as highlight classes.
+	// See hljs.listLanguages() for list of language names.
 	highlightExts = map[string]struct{}{
-		".arm":   {},
-		".as":    {},
-		".sh":    {},
-		".cs":    {},
-		".cpp":   {},
-		".c":     {},
-		".css":   {},
-		".cmake": {},
-		".bat":   {},
-		".dart":  {},
-		".patch": {},
-		".erl":   {},
-		".go":    {},
-		".html":  {},
-		".xml":   {},
-		".hs":    {},
-		".ini":   {},
-		".json":  {},
-		".java":  {},
-		".js":    {},
-		".less":  {},
-		".lua":   {},
-		".php":   {},
-		".py":    {},
-		".rb":    {},
-		".rs":    {},
-		".scss":  {},
-		".sql":   {},
-		".scala": {},
-		".swift": {},
-		".ts":    {},
-		".vb":    {},
-		".yml":   {},
-		".yaml":  {},
+		".applescript": {},
+		".arm":         {},
+		".as":          {},
+		".bash":        {},
+		".bat":         {},
+		".c":           {},
+		".cmake":       {},
+		".cpp":         {},
+		".cs":          {},
+		".css":         {},
+		".dart":        {},
+		".diff":        {},
+		".django":      {},
+		".go":          {},
+		".gradle":      {},
+		".groovy":      {},
+		".haml":        {},
+		".handlebars":  {},
+		".html":        {},
+		".ini":         {},
+		".java":        {},
+		".json":        {},
+		".less":        {},
+		".lua":         {},
+		".php":         {},
+		".scala":       {},
+		".scss":        {},
+		".sql":         {},
+		".swift":       {},
+		".ts":          {},
+		".xml":         {},
+		".yaml":        {},
 	}
 
 	// Extensions that are not same as highlight classes.
 	highlightMapping = map[string]string{
-		".txt":     "nohighlight",
+		".ahk":     "autohotkey",
+		".crmsh":   "crmsh",
+		".dash":    "shell",
+		".erl":     "erlang",
 		".escript": "erlang",
 		".ex":      "elixir",
 		".exs":     "elixir",
+		".f":       "fortran",
+		".f77":     "fortran",
+		".f90":     "fortran",
+		".f95":     "fortran",
+		".feature": "gherkin",
+		".fish":    "shell",
+		".for":     "fortran",
+		".hbs":     "handlebars",
+		".hs":      "haskell",
+		".hx":      "haxe",
+		".js":      "javascript",
+		".jsx":     "javascript",
+		".ksh":     "shell",
+		".kt":      "kotlin",
+		".l":       "ocaml",
+		".ls":      "livescript",
+		".md":      "markdown",
+		".mjs":     "javascript",
+		".mli":     "ocaml",
+		".mll":     "ocaml",
+		".mly":     "ocaml",
+		".patch":   "diff",
+		".pl":      "perl",
+		".pm":      "perl",
+		".ps1":     "powershell",
+		".psd1":    "powershell",
+		".psm1":    "powershell",
+		".py":      "python",
+		".pyw":     "python",
+		".rb":      "ruby",
+		".rs":      "rust",
+		".scpt":    "applescript",
+		".scptd":   "applescript",
+		".sh":      "bash",
+		".tcsh":    "shell",
+		".ts":      "typescript",
+		".tsx":     "typescript",
+		".txt":     "plaintext",
+		".vb":      "vbnet",
+		".vbs":     "vbscript",
+		".yml":     "yaml",
+		".zsh":     "shell",
 	}
 )
 
@@ -87,8 +133,8 @@ func FileNameToHighlightClass(fname string) string {
 		return "nohighlight"
 	}
 
-	if highlightFileNames[fname] {
-		return fname
+	if name, ok := highlightFileNames[fname]; ok {
+		return name
 	}
 
 	ext := path.Ext(fname)

modules/indexer/issues/queue_disk.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
+
 	"github.com/lunny/levelqueue"
 )
 

modules/indexer/issues/queue_redis.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
+
 	"github.com/go-redis/redis"
 )
 

modules/indexer/repo.go

@@ -12,9 +12,7 @@ import (
 
 	"github.com/blevesearch/bleve"
 	"github.com/blevesearch/bleve/analysis/analyzer/custom"
-	"github.com/blevesearch/bleve/analysis/token/camelcase"
 	"github.com/blevesearch/bleve/analysis/token/lowercase"
-	"github.com/blevesearch/bleve/analysis/token/unique"
 	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
 	"github.com/blevesearch/bleve/search/query"
 	"github.com/ethantkoenig/rupture"
@@ -24,7 +22,7 @@ const (
 	repoIndexerAnalyzer = "repoIndexerAnalyzer"
 	repoIndexerDocType  = "repoIndexerDocType"
 
-	repoIndexerLatestVersion = 1
+	repoIndexerLatestVersion = 4
 )
 
 // repoIndexer (thread-safe) index for repository contents
@@ -111,7 +109,7 @@ func createRepoIndexer(path string, latestVersion int) error {
 		"type":          custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     unicode.Name,
-		"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name, unique.Name},
+		"token_filters": []string{unicodeNormalizeName, lowercase.Name},
 	}); err != nil {
 		return err
 	}

modules/lfs/locks.go

@@ -97,7 +97,7 @@ func GetListLockHandler(ctx *context.Context) {
 			})
 			return
 		}
-		lock, err := models.GetLFSLockByID(int64(v))
+		lock, err := models.GetLFSLockByID(v)
 		handleLockListOut(ctx, repository, lock, err)
 		return
 	}

modules/lfs/server.go

@@ -17,8 +17,8 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
+	"gitea.com/macaron/macaron"
 	"github.com/dgrijalva/jwt-go"
-	"gopkg.in/macaron.v1"
 )
 
 const (

modules/log/colors.go

@@ -378,9 +378,9 @@ func (cv *ColoredValue) Format(s fmt.State, c rune) {
 			return
 		}
 	}
-	s.Write([]byte(*cv.colorBytes))
+	s.Write(*cv.colorBytes)
 	fmt.Fprintf(&protectedANSIWriter{w: s}, fmtString(s, c), *(cv.Value))
-	s.Write([]byte(*cv.resetBytes))
+	s.Write(*cv.resetBytes)
 }
 
 // SetColorBytes will allow a user to set the colorBytes of a colored value

modules/log/level.go

@@ -101,7 +101,7 @@ func (l *Level) UnmarshalJSON(b []byte) error {
 
 	switch v := tmp.(type) {
 	case string:
-		*l = FromString(string(v))
+		*l = FromString(v)
 	case int:
 		*l = FromString(Level(v).String())
 	default:

modules/log/log.go

@@ -8,13 +8,35 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"sync"
 )
 
+type loggerMap struct {
+	sync.Map
+}
+
+func (m *loggerMap) Load(k string) (*Logger, bool) {
+	v, ok := m.Map.Load(k)
+	if !ok {
+		return nil, false
+	}
+	l, ok := v.(*Logger)
+	return l, ok
+}
+
+func (m *loggerMap) Store(k string, v *Logger) {
+	m.Map.Store(k, v)
+}
+
+func (m *loggerMap) Delete(k string) {
+	m.Map.Delete(k)
+}
+
 var (
 	// DEFAULT is the name of the default logger
 	DEFAULT = "default"
 	// NamedLoggers map of named loggers
-	NamedLoggers = make(map[string]*Logger)
+	NamedLoggers loggerMap
 	prefix       string
 )
 
@@ -25,16 +47,16 @@ func NewLogger(bufLen int64, name, provider, config string) *Logger {
 		CriticalWithSkip(1, "Unable to create default logger: %v", err)
 		panic(err)
 	}
-	return NamedLoggers[DEFAULT]
+	l, _ := NamedLoggers.Load(DEFAULT)
+	return l
 }
 
 // NewNamedLogger creates a new named logger for a given configuration
 func NewNamedLogger(name string, bufLen int64, subname, provider, config string) error {
-	logger, ok := NamedLoggers[name]
+	logger, ok := NamedLoggers.Load(name)
 	if !ok {
 		logger = newLogger(name, bufLen)
-
-		NamedLoggers[name] = logger
+		NamedLoggers.Store(name, logger)
 	}
 
 	return logger.SetLogger(subname, provider, config)
@@ -42,16 +64,16 @@ func NewNamedLogger(name string, bufLen int64, subname, provider, config string)
 
 // DelNamedLogger closes and deletes the named logger
 func DelNamedLogger(name string) {
-	l, ok := NamedLoggers[name]
+	l, ok := NamedLoggers.Load(name)
 	if ok {
-		delete(NamedLoggers, name)
+		NamedLoggers.Delete(name)
 		l.Close()
 	}
 }
 
 // DelLogger removes the named sublogger from the default logger
 func DelLogger(name string) error {
-	logger := NamedLoggers[DEFAULT]
+	logger, _ := NamedLoggers.Load(DEFAULT)
 	found, err := logger.DelLogger(name)
 	if !found {
 		Trace("Log %s not found, no need to delete", name)
@@ -61,21 +83,24 @@ func DelLogger(name string) error {
 
 // GetLogger returns either a named logger or the default logger
 func GetLogger(name string) *Logger {
-	logger, ok := NamedLoggers[name]
+	logger, ok := NamedLoggers.Load(name)
 	if ok {
 		return logger
 	}
-	return NamedLoggers[DEFAULT]
+	logger, _ = NamedLoggers.Load(DEFAULT)
+	return logger
 }
 
 // GetLevel returns the minimum logger level
 func GetLevel() Level {
-	return NamedLoggers[DEFAULT].GetLevel()
+	l, _ := NamedLoggers.Load(DEFAULT)
+	return l.GetLevel()
 }
 
 // GetStacktraceLevel returns the minimum logger level
 func GetStacktraceLevel() Level {
-	return NamedLoggers[DEFAULT].GetStacktraceLevel()
+	l, _ := NamedLoggers.Load(DEFAULT)
+	return l.GetStacktraceLevel()
 }
 
 // Trace records trace log
@@ -169,18 +194,18 @@ func IsFatal() bool {
 
 // Close closes all the loggers
 func Close() {
-	l, ok := NamedLoggers[DEFAULT]
+	l, ok := NamedLoggers.Load(DEFAULT)
 	if !ok {
 		return
 	}
-	delete(NamedLoggers, DEFAULT)
+	NamedLoggers.Delete(DEFAULT)
 	l.Close()
 }
 
 // Log a message with defined skip and at logging level
 // A skip of 0 refers to the caller of this command
 func Log(skip int, level Level, format string, v ...interface{}) {
-	l, ok := NamedLoggers[DEFAULT]
+	l, ok := NamedLoggers.Load(DEFAULT)
 	if ok {
 		l.Log(skip+1, level, format, v...)
 	}
@@ -195,7 +220,8 @@ type LoggerAsWriter struct {
 // NewLoggerAsWriter creates a Writer representation of the logger with setable log level
 func NewLoggerAsWriter(level string, ourLoggers ...*Logger) *LoggerAsWriter {
 	if len(ourLoggers) == 0 {
-		ourLoggers = []*Logger{NamedLoggers[DEFAULT]}
+		l, _ := NamedLoggers.Load(DEFAULT)
+		ourLoggers = []*Logger{l}
 	}
 	l := &LoggerAsWriter{
 		ourLoggers: ourLoggers,

modules/log/log_test.go

@@ -143,7 +143,7 @@ func TestNewNamedLogger(t *testing.T) {
 	level := INFO
 	err := NewNamedLogger("test", 0, "console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String()))
 	assert.NoError(t, err)
-	logger := NamedLoggers["test"]
+	logger, _ := NamedLoggers.Load("test")
 	assert.Equal(t, level, logger.GetLevel())
 
 	written, closed := baseConsoleTest(t, logger)

modules/log/writer.go

@@ -203,7 +203,7 @@ func (logger *WriterLogger) createMsg(buf *[]byte, event *Event) {
 	(&protectedANSIWriter{
 		w:    &baw,
 		mode: pawMode,
-	}).Write([]byte(msg))
+	}).Write(msg)
 	*buf = baw
 
 	if event.stacktrace != "" && logger.StacktraceLevel <= event.level {

modules/mailer/mailer.go

@@ -1,303 +0,0 @@
-// Copyright 2014 The Gogs Authors. All rights reserved.
-// Copyright 2017 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package mailer
-
-import (
-	"bytes"
-	"crypto/tls"
-	"fmt"
-	"io"
-	"net"
-	"net/smtp"
-	"os"
-	"os/exec"
-	"strings"
-	"time"
-
-	"code.gitea.io/gitea/modules/base"
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
-
-	"github.com/jaytaylor/html2text"
-	"gopkg.in/gomail.v2"
-)
-
-// Message mail body and log info
-type Message struct {
-	Info string // Message information for log purpose.
-	*gomail.Message
-}
-
-// NewMessageFrom creates new mail message object with custom From header.
-func NewMessageFrom(to []string, fromDisplayName, fromAddress, subject, body string) *Message {
-	log.Trace("NewMessageFrom (body):\n%s", body)
-
-	msg := gomail.NewMessage()
-	msg.SetAddressHeader("From", fromAddress, fromDisplayName)
-	msg.SetHeader("To", to...)
-	if len(setting.MailService.SubjectPrefix) > 0 {
-		msg.SetHeader("Subject", setting.MailService.SubjectPrefix+" "+subject)
-	} else {
-		msg.SetHeader("Subject", subject)
-	}
-	msg.SetDateHeader("Date", time.Now())
-	msg.SetHeader("X-Auto-Response-Suppress", "All")
-
-	plainBody, err := html2text.FromString(body)
-	if err != nil || setting.MailService.SendAsPlainText {
-		if strings.Contains(base.TruncateString(body, 100), "<html>") {
-			log.Warn("Mail contains HTML but configured to send as plain text.")
-		}
-		msg.SetBody("text/plain", plainBody)
-	} else {
-		msg.SetBody("text/plain", plainBody)
-		msg.AddAlternative("text/html", body)
-	}
-
-	return &Message{
-		Message: msg,
-	}
-}
-
-// NewMessage creates new mail message object with default From header.
-func NewMessage(to []string, subject, body string) *Message {
-	return NewMessageFrom(to, setting.MailService.FromName, setting.MailService.FromEmail, subject, body)
-}
-
-type loginAuth struct {
-	username, password string
-}
-
-// LoginAuth SMTP AUTH LOGIN Auth Handler
-func LoginAuth(username, password string) smtp.Auth {
-	return &loginAuth{username, password}
-}
-
-// Start start SMTP login auth
-func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
-	return "LOGIN", []byte{}, nil
-}
-
-// Next next step of SMTP login auth
-func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
-	if more {
-		switch string(fromServer) {
-		case "Username:":
-			return []byte(a.username), nil
-		case "Password:":
-			return []byte(a.password), nil
-		default:
-			return nil, fmt.Errorf("unknown fromServer: %s", string(fromServer))
-		}
-	}
-	return nil, nil
-}
-
-// Sender SMTP mail sender
-type smtpSender struct {
-}
-
-// Send send email
-func (s *smtpSender) Send(from string, to []string, msg io.WriterTo) error {
-	opts := setting.MailService
-
-	host, port, err := net.SplitHostPort(opts.Host)
-	if err != nil {
-		return err
-	}
-
-	tlsconfig := &tls.Config{
-		InsecureSkipVerify: opts.SkipVerify,
-		ServerName:         host,
-	}
-
-	if opts.UseCertificate {
-		cert, err := tls.LoadX509KeyPair(opts.CertFile, opts.KeyFile)
-		if err != nil {
-			return err
-		}
-		tlsconfig.Certificates = []tls.Certificate{cert}
-	}
-
-	conn, err := net.Dial("tcp", net.JoinHostPort(host, port))
-	if err != nil {
-		return err
-	}
-	defer conn.Close()
-
-	isSecureConn := opts.IsTLSEnabled || (strings.HasSuffix(port, "465"))
-	// Start TLS directly if the port ends with 465 (SMTPS protocol)
-	if isSecureConn {
-		conn = tls.Client(conn, tlsconfig)
-	}
-
-	client, err := smtp.NewClient(conn, host)
-	if err != nil {
-		return fmt.Errorf("NewClient: %v", err)
-	}
-
-	if !opts.DisableHelo {
-		hostname := opts.HeloHostname
-		if len(hostname) == 0 {
-			hostname, err = os.Hostname()
-			if err != nil {
-				return err
-			}
-		}
-
-		if err = client.Hello(hostname); err != nil {
-			return fmt.Errorf("Hello: %v", err)
-		}
-	}
-
-	// If not using SMTPS, always use STARTTLS if available
-	hasStartTLS, _ := client.Extension("STARTTLS")
-	if !isSecureConn && hasStartTLS {
-		if err = client.StartTLS(tlsconfig); err != nil {
-			return fmt.Errorf("StartTLS: %v", err)
-		}
-	}
-
-	canAuth, options := client.Extension("AUTH")
-	if canAuth && len(opts.User) > 0 {
-		var auth smtp.Auth
-
-		if strings.Contains(options, "CRAM-MD5") {
-			auth = smtp.CRAMMD5Auth(opts.User, opts.Passwd)
-		} else if strings.Contains(options, "PLAIN") {
-			auth = smtp.PlainAuth("", opts.User, opts.Passwd, host)
-		} else if strings.Contains(options, "LOGIN") {
-			// Patch for AUTH LOGIN
-			auth = LoginAuth(opts.User, opts.Passwd)
-		}
-
-		if auth != nil {
-			if err = client.Auth(auth); err != nil {
-				return fmt.Errorf("Auth: %v", err)
-			}
-		}
-	}
-
-	if err = client.Mail(from); err != nil {
-		return fmt.Errorf("Mail: %v", err)
-	}
-
-	for _, rec := range to {
-		if err = client.Rcpt(rec); err != nil {
-			return fmt.Errorf("Rcpt: %v", err)
-		}
-	}
-
-	w, err := client.Data()
-	if err != nil {
-		return fmt.Errorf("Data: %v", err)
-	} else if _, err = msg.WriteTo(w); err != nil {
-		return fmt.Errorf("WriteTo: %v", err)
-	} else if err = w.Close(); err != nil {
-		return fmt.Errorf("Close: %v", err)
-	}
-
-	return client.Quit()
-}
-
-// Sender sendmail mail sender
-type sendmailSender struct {
-}
-
-// Send send email
-func (s *sendmailSender) Send(from string, to []string, msg io.WriterTo) error {
-	var err error
-	var closeError error
-	var waitError error
-
-	args := []string{"-F", from, "-i"}
-	args = append(args, setting.MailService.SendmailArgs...)
-	args = append(args, to...)
-	log.Trace("Sending with: %s %v", setting.MailService.SendmailPath, args)
-	cmd := exec.Command(setting.MailService.SendmailPath, args...)
-	pipe, err := cmd.StdinPipe()
-
-	if err != nil {
-		return err
-	}
-
-	if err = cmd.Start(); err != nil {
-		return err
-	}
-
-	_, err = msg.WriteTo(pipe)
-
-	// we MUST close the pipe or sendmail will hang waiting for more of the message
-	// Also we should wait on our sendmail command even if something fails
-	closeError = pipe.Close()
-	waitError = cmd.Wait()
-	if err != nil {
-		return err
-	} else if closeError != nil {
-		return closeError
-	} else {
-		return waitError
-	}
-}
-
-// Sender sendmail mail sender
-type dummySender struct {
-}
-
-// Send send email
-func (s *dummySender) Send(from string, to []string, msg io.WriterTo) error {
-	buf := bytes.Buffer{}
-	if _, err := msg.WriteTo(&buf); err != nil {
-		return err
-	}
-	log.Info("Mail From: %s To: %v Body: %s", from, to, buf.String())
-	return nil
-}
-
-func processMailQueue() {
-	for msg := range mailQueue {
-		log.Trace("New e-mail sending request %s: %s", msg.GetHeader("To"), msg.Info)
-		if err := gomail.Send(Sender, msg.Message); err != nil {
-			log.Error("Failed to send emails %s: %s - %v", msg.GetHeader("To"), msg.Info, err)
-		} else {
-			log.Trace("E-mails sent %s: %s", msg.GetHeader("To"), msg.Info)
-		}
-	}
-}
-
-var mailQueue chan *Message
-
-// Sender sender for sending mail synchronously
-var Sender gomail.Sender
-
-// NewContext start mail queue service
-func NewContext() {
-	// Need to check if mailQueue is nil because in during reinstall (user had installed
-	// before but swithed install lock off), this function will be called again
-	// while mail queue is already processing tasks, and produces a race condition.
-	if setting.MailService == nil || mailQueue != nil {
-		return
-	}
-
-	switch setting.MailService.MailerType {
-	case "smtp":
-		Sender = &smtpSender{}
-	case "sendmail":
-		Sender = &sendmailSender{}
-	case "dummy":
-		Sender = &dummySender{}
-	}
-
-	mailQueue = make(chan *Message, setting.MailService.QueueLength)
-	go processMailQueue()
-}
-
-// SendAsync send mail asynchronous
-func SendAsync(msg *Message) {
-	go func() {
-		mailQueue <- msg
-	}()
-}

modules/markup/csv/csv.go

@@ -9,12 +9,18 @@ import (
 	"encoding/csv"
 	"html"
 	"io"
+	"regexp"
+	"strings"
 
 	"code.gitea.io/gitea/modules/markup"
+	"code.gitea.io/gitea/modules/util"
 )
 
+var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
+
 func init() {
 	markup.RegisterParser(Parser{})
+
 }
 
 // Parser implements markup.Parser for orgmode
@@ -28,12 +34,13 @@ func (Parser) Name() string {
 
 // Extensions implements markup.Parser
 func (Parser) Extensions() []string {
-	return []string{".csv"}
+	return []string{".csv", ".tsv"}
 }
 
 // Render implements markup.Parser
-func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+func (p Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
 	rd := csv.NewReader(bytes.NewReader(rawBytes))
+	rd.Comma = p.bestDelimiter(rawBytes)
 	var tmpBlock bytes.Buffer
 	tmpBlock.WriteString(`<table class="table">`)
 	for {
@@ -50,9 +57,57 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string,
 			tmpBlock.WriteString(html.EscapeString(field))
 			tmpBlock.WriteString("</td>")
 		}
-		tmpBlock.WriteString("<tr>")
+		tmpBlock.WriteString("</tr>")
 	}
 	tmpBlock.WriteString("</table>")
 
 	return tmpBlock.Bytes()
 }
+
+// bestDelimiter scores the input CSV data against delimiters, and returns the best match.
+// Reads at most 10k bytes & 10 lines.
+func (p Parser) bestDelimiter(data []byte) rune {
+	maxLines := 10
+	maxBytes := util.Min(len(data), 1e4)
+	text := string(data[:maxBytes])
+	text = quoteRegexp.ReplaceAllLiteralString(text, "")
+	lines := strings.SplitN(text, "\n", maxLines+1)
+	lines = lines[:util.Min(maxLines, len(lines))]
+
+	delimiters := []rune{',', ';', '\t', '|'}
+	bestDelim := delimiters[0]
+	bestScore := 0.0
+	for _, delim := range delimiters {
+		score := p.scoreDelimiter(lines, delim)
+		if score > bestScore {
+			bestScore = score
+			bestDelim = delim
+		}
+	}
+
+	return bestDelim
+}
+
+// scoreDelimiter uses a count & regularity metric to evaluate a delimiter against lines of CSV
+func (Parser) scoreDelimiter(lines []string, delim rune) (score float64) {
+	countTotal := 0
+	countLineMax := 0
+	linesNotEqual := 0
+
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+
+		countLine := strings.Count(line, string(delim))
+		countTotal += countLine
+		if countLine != countLineMax {
+			if countLineMax != 0 {
+				linesNotEqual++
+			}
+			countLineMax = util.Max(countLine, countLineMax)
+		}
+	}
+
+	return float64(countTotal) * (1 - float64(linesNotEqual)/float64(len(lines)))
+}

modules/markup/csv/csv_test.go

@@ -13,9 +13,14 @@ import (
 func TestRenderCSV(t *testing.T) {
 	var parser Parser
 	var kases = map[string]string{
-		"a":     "<table class=\"table\"><tr><td>a</td><tr></table>",
-		"1,2":   "<table class=\"table\"><tr><td>1</td><td>2</td><tr></table>",
-		"<br/>": "<table class=\"table\"><tr><td>&lt;br/&gt;</td><tr></table>",
+		"a":                         "<table class=\"table\"><tr><td>a</td></tr></table>",
+		"1,2":                       "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
+		"1;2":                       "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
+		"1\t2":                      "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
+		"1|2":                       "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
+		"1,2,3;4,5,6;7,8,9\na;b;c":  "<table class=\"table\"><tr><td>1,2,3</td><td>4,5,6</td><td>7,8,9</td></tr><tr><td>a</td><td>b</td><td>c</td></tr></table>",
+		"\"1,2,3,4\";\"a\nb\"\nc;d": "<table class=\"table\"><tr><td>1,2,3,4</td><td>a\nb</td></tr><tr><td>c</td><td>d</td></tr></table>",
+		"<br/>":                     "<table class=\"table\"><tr><td>&lt;br/&gt;</td></tr></table>",
 	}
 
 	for k, v := range kases {

modules/markup/html.go

@@ -13,10 +13,12 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 
-	"github.com/Unknwon/com"
+	"github.com/unknwon/com"
 	"golang.org/x/net/html"
 	"golang.org/x/net/html/atom"
 	"mvdan.cc/xurls/v2"
@@ -38,9 +40,9 @@ var (
 	mentionPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@[0-9a-zA-Z-_\.]+)(?:\s|$|\)|\])`)
 
 	// issueNumericPattern matches string that references to a numeric issue, e.g. #1287
-	issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(#[0-9]+)(?:\s|$|\)|\]|\.(\s|$))`)
+	issueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(#[0-9]+)(?:\s|$|\)|\]|:|\.(\s|$))`)
 	// issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234
-	issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|\.(\s|$))`)
+	issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$))`)
 	// crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository
 	// e.g. gogits/gogs#12345
 	crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+#[0-9]+)(?:\s|$|\)|\]|\.(\s|$))`)
@@ -209,6 +211,40 @@ func RenderCommitMessage(
 	return ctx.postProcess(rawHTML)
 }
 
+var commitMessageSubjectProcessors = []processor{
+	fullIssuePatternProcessor,
+	fullSha1PatternProcessor,
+	linkProcessor,
+	mentionProcessor,
+	issueIndexPatternProcessor,
+	crossReferenceIssueIndexPatternProcessor,
+	sha1CurrentPatternProcessor,
+}
+
+// RenderCommitMessageSubject will use the same logic as PostProcess and
+// RenderCommitMessage, but will disable the shortLinkProcessor and
+// emailAddressProcessor, will add a defaultLinkProcessor if defaultLink is set,
+// which changes every text node into a link to the passed default link.
+func RenderCommitMessageSubject(
+	rawHTML []byte,
+	urlPrefix, defaultLink string,
+	metas map[string]string,
+) ([]byte, error) {
+	ctx := &postProcessCtx{
+		metas:     metas,
+		urlPrefix: urlPrefix,
+		procs:     commitMessageSubjectProcessors,
+	}
+	if defaultLink != "" {
+		// we don't have to fear data races, because being
+		// commitMessageSubjectProcessors of fixed len and cap, every time we
+		// append something to it the slice is realloc+copied, so append always
+		// generates the slice ex-novo.
+		ctx.procs = append(ctx.procs, genDefaultLinkProcessor(defaultLink))
+	}
+	return ctx.postProcess(rawHTML)
+}
+
 // RenderDescriptionHTML will use similar logic as PostProcess, but will
 // use a single special linkProcessor.
 func RenderDescriptionHTML(
@@ -294,12 +330,17 @@ func (ctx *postProcessCtx) textNode(node *html.Node) {
 	}
 }
 
-func createLink(href, content string) *html.Node {
+func createLink(href, content, class string) *html.Node {
 	a := &html.Node{
 		Type: html.ElementNode,
 		Data: atom.A.String(),
 		Attr: []html.Attribute{{Key: "href", Val: href}},
 	}
+
+	if class != "" {
+		a.Attr = append(a.Attr, html.Attribute{Key: "class", Val: class})
+	}
+
 	text := &html.Node{
 		Type: html.TextNode,
 		Data: content,
@@ -309,12 +350,17 @@ func createLink(href, content string) *html.Node {
 	return a
 }
 
-func createCodeLink(href, content string) *html.Node {
+func createCodeLink(href, content, class string) *html.Node {
 	a := &html.Node{
 		Type: html.ElementNode,
 		Data: atom.A.String(),
 		Attr: []html.Attribute{{Key: "href", Val: href}},
 	}
+
+	if class != "" {
+		a.Attr = append(a.Attr, html.Attribute{Key: "class", Val: class})
+	}
+
 	text := &html.Node{
 		Type: html.TextNode,
 		Data: content,
@@ -362,7 +408,7 @@ func mentionProcessor(_ *postProcessCtx, node *html.Node) {
 	}
 	// Replace the mention with a link to the specified user.
 	mention := node.Data[m[2]:m[3]]
-	replaceContent(node, m[2], m[3], createLink(util.URLJoin(setting.AppURL, mention[1:]), mention))
+	replaceContent(node, m[2], m[3], createLink(util.URLJoin(setting.AppURL, mention[1:]), mention, "mention"))
 }
 
 func shortLinkProcessor(ctx *postProcessCtx, node *html.Node) {
@@ -438,7 +484,7 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
 
 	name += tail
 	image := false
-	switch ext := filepath.Ext(string(link)); ext {
+	switch ext := filepath.Ext(link); ext {
 	// fast path: empty string, ignore
 	case "":
 		break
@@ -482,7 +528,7 @@ func shortLinkProcessorFull(ctx *postProcessCtx, node *html.Node, noLink bool) {
 			title = props["alt"]
 		}
 		if title == "" {
-			title = path.Base(string(name))
+			title = path.Base(name)
 		}
 		alt := props["alt"]
 		if alt == "" {
@@ -539,11 +585,11 @@ func fullIssuePatternProcessor(ctx *postProcessCtx, node *html.Node) {
 	if matchOrg == ctx.metas["user"] && matchRepo == ctx.metas["repo"] {
 		// TODO if m[4]:m[5] is not nil, then link is to a comment,
 		// and we should indicate that in the text somehow
-		replaceContent(node, m[0], m[1], createLink(link, id))
+		replaceContent(node, m[0], m[1], createLink(link, id, "issue"))
 
 	} else {
 		orgRepoID := matchOrg + "/" + matchRepo + id
-		replaceContent(node, m[0], m[1], createLink(link, orgRepoID))
+		replaceContent(node, m[0], m[1], createLink(link, orgRepoID, "issue"))
 	}
 }
 
@@ -571,9 +617,9 @@ func issueIndexPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 		} else {
 			ctx.metas["index"] = id[1:]
 		}
-		link = createLink(com.Expand(ctx.metas["format"], ctx.metas), id)
+		link = createLink(com.Expand(ctx.metas["format"], ctx.metas), id, "issue")
 	} else {
-		link = createLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "issues", id[1:]), id)
+		link = createLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "issues", id[1:]), id, "issue")
 	}
 	replaceContent(node, match[2], match[3], link)
 }
@@ -589,7 +635,7 @@ func crossReferenceIssueIndexPatternProcessor(ctx *postProcessCtx, node *html.No
 	repo, issue := parts[0], parts[1]
 
 	replaceContent(node, m[2], m[3],
-		createLink(util.URLJoin(setting.AppURL, repo, "issues", issue), ref))
+		createLink(util.URLJoin(setting.AppURL, repo, "issues", issue), ref, issue))
 }
 
 // fullSha1PatternProcessor renders SHA containing URLs
@@ -640,12 +686,15 @@ func fullSha1PatternProcessor(ctx *postProcessCtx, node *html.Node) {
 		text += " (" + hash + ")"
 	}
 
-	replaceContent(node, start, end, createCodeLink(urlFull, text))
+	replaceContent(node, start, end, createCodeLink(urlFull, text, "commit"))
 }
 
 // sha1CurrentPatternProcessor renders SHA1 strings to corresponding links that
 // are assumed to be in the same repository.
 func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) {
+	if ctx.metas == nil || ctx.metas["user"] == "" || ctx.metas["repo"] == "" || ctx.metas["repoPath"] == "" {
+		return
+	}
 	m := sha1CurrentPattern.FindStringSubmatchIndex(node.Data)
 	if m == nil {
 		return
@@ -657,8 +706,17 @@ func sha1CurrentPatternProcessor(ctx *postProcessCtx, node *html.Node) {
 	// but that is not always the case.
 	// Although unlikely, deadbeef and 1234567 are valid short forms of SHA1 hash
 	// as used by git and github for linking and thus we have to do similar.
+	// Because of this, we check to make sure that a matched hash is actually
+	// a commit in the repository before making it a link.
+	if _, err := git.NewCommand("rev-parse", "--verify", hash).RunInDirBytes(ctx.metas["repoPath"]); err != nil {
+		if !strings.Contains(err.Error(), "fatal: Needed a single revision") {
+			log.Debug("sha1CurrentPatternProcessor git rev-parse: %v", err)
+		}
+		return
+	}
+
 	replaceContent(node, m[2], m[3],
-		createCodeLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "commit", hash), base.ShortSha(hash)))
+		createCodeLink(util.URLJoin(setting.AppURL, ctx.metas["user"], ctx.metas["repo"], "commit", hash), base.ShortSha(hash), "commit"))
 }
 
 // emailAddressProcessor replaces raw email addresses with a mailto: link.
@@ -668,7 +726,7 @@ func emailAddressProcessor(ctx *postProcessCtx, node *html.Node) {
 		return
 	}
 	mail := node.Data[m[2]:m[3]]
-	replaceContent(node, m[2], m[3], createLink("mailto:"+mail, mail))
+	replaceContent(node, m[2], m[3], createLink("mailto:"+mail, mail, "mailto"))
 }
 
 // linkProcessor creates links for any HTTP or HTTPS URL not captured by
@@ -679,7 +737,7 @@ func linkProcessor(ctx *postProcessCtx, node *html.Node) {
 		return
 	}
 	uri := node.Data[m[0]:m[1]]
-	replaceContent(node, m[0], m[1], createLink(uri, uri))
+	replaceContent(node, m[0], m[1], createLink(uri, uri, "link"))
 }
 
 func genDefaultLinkProcessor(defaultLink string) processor {
@@ -693,7 +751,10 @@ func genDefaultLinkProcessor(defaultLink string) processor {
 		node.Type = html.ElementNode
 		node.Data = "a"
 		node.DataAtom = atom.A
-		node.Attr = []html.Attribute{{Key: "href", Val: defaultLink}}
+		node.Attr = []html.Attribute{
+			{Key: "href", Val: defaultLink},
+			{Key: "class", Val: "default-link"},
+		}
 		node.FirstChild, node.LastChild = ch, ch
 	}
 }

modules/markup/html_internal_test.go

@@ -20,18 +20,22 @@ const Repo = "gogits/gogs"
 const AppSubURL = AppURL + Repo + "/"
 
 // alphanumLink an HTML link to an alphanumeric-style issue
-func alphanumIssueLink(baseURL string, name string) string {
-	return link(util.URLJoin(baseURL, name), name)
+func alphanumIssueLink(baseURL, class, name string) string {
+	return link(util.URLJoin(baseURL, name), class, name)
 }
 
 // numericLink an HTML to a numeric-style issue
-func numericIssueLink(baseURL string, index int) string {
-	return link(util.URLJoin(baseURL, strconv.Itoa(index)), fmt.Sprintf("#%d", index))
+func numericIssueLink(baseURL, class string, index int) string {
+	return link(util.URLJoin(baseURL, strconv.Itoa(index)), class, fmt.Sprintf("#%d", index))
 }
 
 // link an HTML link
-func link(href, contents string) string {
-	return fmt.Sprintf("<a href=\"%s\">%s</a>", href, contents)
+func link(href, class, contents string) string {
+	if class != "" {
+		class = " class=\"" + class + "\""
+	}
+
+	return fmt.Sprintf("<a href=\"%s\"%s>%s</a>", href, class, contents)
 }
 
 var numericMetas = map[string]string{
@@ -89,13 +93,13 @@ func TestRender_IssueIndexPattern2(t *testing.T) {
 	test := func(s, expectedFmt string, indices ...int) {
 		links := make([]interface{}, len(indices))
 		for i, index := range indices {
-			links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, "issues"), index)
+			links[i] = numericIssueLink(util.URLJoin(setting.AppSubURL, "issues"), "issue", index)
 		}
 		expectedNil := fmt.Sprintf(expectedFmt, links...)
 		testRenderIssueIndexPattern(t, s, expectedNil, &postProcessCtx{metas: localMetas})
 
 		for i, index := range indices {
-			links[i] = numericIssueLink("https://someurl.com/someUser/someRepo/", index)
+			links[i] = numericIssueLink("https://someurl.com/someUser/someRepo/", "issue", index)
 		}
 		expectedNum := fmt.Sprintf(expectedFmt, links...)
 		testRenderIssueIndexPattern(t, s, expectedNum, &postProcessCtx{metas: numericMetas})
@@ -118,6 +122,10 @@ func TestRender_IssueIndexPattern2(t *testing.T) {
 	test("wow (#54321 #1243)", "wow (%s %s)", 54321, 1243)
 	test("(#4)(#5)", "(%s)(%s)", 4, 5)
 	test("#1 (#4321) test", "%s (%s) test", 1, 4321)
+
+	// should render with :
+	test("#1234: test", "%s: test", 1234)
+	test("wow (#54321: test)", "wow (%s: test)", 54321)
 }
 
 func TestRender_IssueIndexPattern3(t *testing.T) {
@@ -154,7 +162,7 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
 	test := func(s, expectedFmt string, names ...string) {
 		links := make([]interface{}, len(names))
 		for i, name := range names {
-			links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", name)
+			links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "issue", name)
 		}
 		expected := fmt.Sprintf(expectedFmt, links...)
 		testRenderIssueIndexPattern(t, s, expected, &postProcessCtx{metas: alphanumericMetas})
@@ -193,17 +201,17 @@ func TestRender_AutoLink(t *testing.T) {
 
 	// render valid issue URLs
 	test(util.URLJoin(setting.AppSubURL, "issues", "3333"),
-		numericIssueLink(util.URLJoin(setting.AppSubURL, "issues"), 3333))
+		numericIssueLink(util.URLJoin(setting.AppSubURL, "issues"), "issue", 3333))
 
 	// render valid commit URLs
 	tmp := util.URLJoin(AppSubURL, "commit", "d8a994ef243349f321568f9e36d5c3f444b99cae")
-	test(tmp, "<a href=\""+tmp+"\"><code class=\"nohighlight\">d8a994ef24</code></a>")
+	test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24</code></a>")
 	tmp += "#diff-2"
-	test(tmp, "<a href=\""+tmp+"\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>")
+	test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>")
 
 	// render other commit URLs
 	tmp = "https://external-link.gitea.io/go-gitea/gitea/commit/d8a994ef243349f321568f9e36d5c3f444b99cae#diff-2"
-	test(tmp, "<a href=\""+tmp+"\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>")
+	test(tmp, "<a href=\""+tmp+"\" class=\"commit\"><code class=\"nohighlight\">d8a994ef24 (diff-2)</code></a>")
 }
 
 func TestRender_FullIssueURLs(t *testing.T) {
@@ -224,11 +232,11 @@ func TestRender_FullIssueURLs(t *testing.T) {
 	test("Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6",
 		"Here is a link https://git.osgeo.org/gogs/postgis/postgis/pulls/6")
 	test("Look here http://localhost:3000/person/repo/issues/4",
-		`Look here <a href="http://localhost:3000/person/repo/issues/4">person/repo#4</a>`)
+		`Look here <a href="http://localhost:3000/person/repo/issues/4" class="issue">person/repo#4</a>`)
 	test("http://localhost:3000/person/repo/issues/4#issuecomment-1234",
-		`<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234">person/repo#4</a>`)
+		`<a href="http://localhost:3000/person/repo/issues/4#issuecomment-1234" class="issue">person/repo#4</a>`)
 	test("http://localhost:3000/gogits/gogs/issues/4",
-		`<a href="http://localhost:3000/gogits/gogs/issues/4">#4</a>`)
+		`<a href="http://localhost:3000/gogits/gogs/issues/4" class="issue">#4</a>`)
 }
 
 func TestRegExp_issueNumericPattern(t *testing.T) {
@@ -237,6 +245,8 @@ func TestRegExp_issueNumericPattern(t *testing.T) {
 		"#0",
 		"#1234567890987654321",
 		"  #12",
+		"#12:",
+		"ref: #12: msg",
 	}
 	falseTestCases := []string{
 		"# 1234",
@@ -354,6 +364,7 @@ func TestRegExp_issueAlphanumericPattern(t *testing.T) {
 		"ABC-123.",
 		"(ABC-123)",
 		"[ABC-123]",
+		"ABC-123:",
 	}
 	falseTestCases := []string{
 		"RC-08",

modules/markup/html_test.go

@@ -17,8 +17,9 @@ import (
 )
 
 var localMetas = map[string]string{
-	"user": "gogits",
-	"repo": "gogs",
+	"user":     "gogits",
+	"repo":     "gogs",
+	"repoPath": "../../integrations/gitea-repositories-meta/user13/repo11.git/",
 }
 
 func TestRender_Commits(t *testing.T) {
@@ -27,22 +28,23 @@ func TestRender_Commits(t *testing.T) {
 
 	test := func(input, expected string) {
 		buffer := RenderString(".md", input, setting.AppSubURL, localMetas)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
-	var sha = "b6dd6210eaebc915fd5be5579c58cce4da2e2579"
+	var sha = "65f1bf27bc3bf70f64657658635e66094edbcb4d"
 	var commit = util.URLJoin(AppSubURL, "commit", sha)
 	var subtree = util.URLJoin(commit, "src")
 	var tree = strings.Replace(subtree, "/commit/", "/tree/", -1)
 
-	test(sha, `<p><a href="`+commit+`" rel="nofollow"><code>b6dd6210ea</code></a></p>`)
-	test(sha[:7], `<p><a href="`+commit[:len(commit)-(40-7)]+`" rel="nofollow"><code>b6dd621</code></a></p>`)
-	test(sha[:39], `<p><a href="`+commit[:len(commit)-(40-39)]+`" rel="nofollow"><code>b6dd6210ea</code></a></p>`)
-	test(commit, `<p><a href="`+commit+`" rel="nofollow"><code>b6dd6210ea</code></a></p>`)
-	test(tree, `<p><a href="`+tree+`" rel="nofollow"><code>b6dd6210ea/src</code></a></p>`)
-	test("commit "+sha, `<p>commit <a href="`+commit+`" rel="nofollow"><code>b6dd6210ea</code></a></p>`)
+	test(sha, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`)
+	test(sha[:7], `<p><a href="`+commit[:len(commit)-(40-7)]+`" rel="nofollow"><code>65f1bf2</code></a></p>`)
+	test(sha[:39], `<p><a href="`+commit[:len(commit)-(40-39)]+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`)
+	test(commit, `<p><a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`)
+	test(tree, `<p><a href="`+tree+`" rel="nofollow"><code>65f1bf27bc/src</code></a></p>`)
+	test("commit "+sha, `<p>commit <a href="`+commit+`" rel="nofollow"><code>65f1bf27bc</code></a></p>`)
 	test("/home/gitea/"+sha, "<p>/home/gitea/"+sha+"</p>")
-
+	test("deadbeef", `<p>deadbeef</p>`)
+	test("d27ace93", `<p>d27ace93</p>`)
 }
 
 func TestRender_CrossReferences(t *testing.T) {
@@ -51,7 +53,7 @@ func TestRender_CrossReferences(t *testing.T) {
 
 	test := func(input, expected string) {
 		buffer := RenderString("a.md", input, setting.AppSubURL, localMetas)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
 	test(
@@ -83,7 +85,7 @@ func TestRender_links(t *testing.T) {
 
 	test := func(input, expected string) {
 		buffer := RenderString("a.md", input, setting.AppSubURL, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 	// Text that should be turned into URL
 
@@ -160,7 +162,7 @@ func TestRender_email(t *testing.T) {
 
 	test := func(input, expected string) {
 		buffer := RenderString("a.md", input, setting.AppSubURL, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 	// Text that should be turned into email link
 
@@ -214,9 +216,9 @@ func TestRender_ShortLinks(t *testing.T) {
 
 	test := func(input, expected, expectedWiki string) {
 		buffer := markdown.RenderString(input, tree, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 		buffer = markdown.RenderWiki([]byte(input), setting.AppSubURL, localMetas)
-		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(buffer))
 	}
 
 	rawtree := util.URLJoin(AppSubURL, "raw", "master")

modules/markup/markdown/markdown.go

@@ -153,7 +153,7 @@ func RenderRaw(body []byte, urlPrefix string, wikiMarkdown bool) []byte {
 	}
 
 	body = blackfriday.Markdown(body, renderer, exts)
-	return body
+	return markup.SanitizeBytes(body)
 }
 
 var (

modules/markup/markdown/markdown_test.go

@@ -21,8 +21,9 @@ const AppSubURL = AppURL + Repo + "/"
 
 // these values should match the Repo const above
 var localMetas = map[string]string{
-	"user": "gogits",
-	"repo": "gogs",
+	"user":     "gogits",
+	"repo":     "gogs",
+	"repoPath": "../../../integrations/gitea-repositories-meta/user13/repo11.git/",
 }
 
 func TestRender_StandardLinks(t *testing.T) {
@@ -31,7 +32,7 @@ func TestRender_StandardLinks(t *testing.T) {
 
 	test := func(input, expected, expectedWiki string) {
 		buffer := RenderString(input, setting.AppSubURL, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 		bufferWiki := RenderWiki([]byte(input), setting.AppSubURL, nil)
 		assert.Equal(t, strings.TrimSpace(expectedWiki), strings.TrimSpace(bufferWiki))
 	}
@@ -74,7 +75,7 @@ func TestRender_Images(t *testing.T) {
 
 	test := func(input, expected string) {
 		buffer := RenderString(input, setting.AppSubURL, nil)
-		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(string(buffer)))
+		assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
 	}
 
 	url := "../../.images/src/02/train.jpg"
@@ -103,7 +104,7 @@ func testAnswers(baseURLContent, baseURLImages string) []string {
 <li><a href="` + baseURLContent + `/Tips" rel="nofollow">Tips</a></li>
 </ul>
 
-<p>See commit <a href="http://localhost:3000/gogits/gogs/commit/fc7f44dadf" rel="nofollow"><code>fc7f44dadf</code></a></p>
+<p>See commit <a href="http://localhost:3000/gogits/gogs/commit/65f1bf27bc" rel="nofollow"><code>65f1bf27bc</code></a></p>
 
 <p>Ideas and codes</p>
 
@@ -194,7 +195,7 @@ var sameCases = []string{
 - [[Links, Language bindings, Engine bindings|Links]]
 - [[Tips]]
 
-See commit fc7f44dadf
+See commit 65f1bf27bc
 
 Ideas and codes
 

modules/migrations/base/downloader.go

@@ -8,6 +8,7 @@ package base
 // Downloader downloads the site repo informations
 type Downloader interface {
 	GetRepoInfo() (*Repository, error)
+	GetTopics() ([]string, error)
 	GetMilestones() ([]*Milestone, error)
 	GetReleases() ([]*Release, error)
 	GetLabels() ([]*Label, error)

modules/migrations/base/release.go

@@ -25,6 +25,9 @@ type Release struct {
 	Body            string
 	Draft           bool
 	Prerelease      bool
+	PublisherID     int64
+	PublisherName   string
+	PublisherEmail  string
 	Assets          []ReleaseAsset
 	Created         time.Time
 	Published       time.Time

modules/migrations/base/uploader.go

@@ -9,6 +9,7 @@ package base
 type Uploader interface {
 	MaxBatchInsertSize(tp string) int
 	CreateRepo(repo *Repository, opts MigrateOptions) error
+	CreateTopics(topic ...string) error
 	CreateMilestones(milestones ...*Milestone) error
 	CreateReleases(releases ...*Release) error
 	CreateLabels(labels ...*Label) error

modules/migrations/git.go

@@ -38,6 +38,11 @@ func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
 	}, nil
 }
 
+// GetTopics returns empty list for plain git repo
+func (g *PlainGitDownloader) GetTopics() ([]string, error) {
+	return []string{}, nil
+}
+
 // GetMilestones returns milestones
 func (g *PlainGitDownloader) GetMilestones() ([]*base.Milestone, error) {
 	return nil, ErrNotSupported