Git LFS support v2 (#122)

* Import github.com/git-lfs/lfs-test-server as lfs module base

Imported commit is 3968aac269a77b73924649b9412ae03f7ccd3198

Removed:

Dockerfile CONTRIBUTING.md mgmt* script/ vendor/ kvlogger.go
.dockerignore .gitignore README.md

* Remove config, add JWT support from github.com/mgit-at/lfs-test-server

Imported commit f0cdcc5a01599c5a955dc1bbf683bb4acecdba83

* Add LFS settings

* Add LFS meta object model

* Add LFS routes and initialization

* Import github.com/dgrijalva/jwt-go into vendor/

* Adapt LFS module: handlers, routing, meta store

* Move LFS routes to /user/repo/info/lfs/*

* Add request header checks to LFS BatchHandler / PostHandler

* Implement LFS basic authentication

* Rework JWT secret generation / load

* Implement LFS SSH token authentication with JWT

Specification: https://github.com/github/git-lfs/tree/master/docs/api

* Integrate LFS settings into install process

* Remove LFS objects when repository is deleted

Only removes objects from content store when deleted repo is the only
referencing repository

* Make LFS module stateless

Fixes bug where LFS would not work after installation without
restarting Gitea

* Change 500 'Internal Server Error' to 400 'Bad Request'

* Change sql query to xorm call

* Remove unneeded type from LFS module

* Change internal imports to code.gitea.io/gitea/

* Add Gitea authors copyright

* Change basic auth realm to "gitea-lfs"

* Add unique indexes to LFS model

* Use xorm count function in LFS check on repository delete

* Return io.ReadCloser from content store and close after usage

* Add LFS info to runWeb()

* Export LFS content store base path

* LFS file download from UI

* Work around git-lfs client issue with unauthenticated requests

Returning a dummy Authorization header for unauthenticated requests
lets git-lfs client skip asking for auth credentials
See: https://github.com/github/git-lfs/issues/1088

* Fix unauthenticated UI downloads from public repositories

* Authentication check order, Finish LFS file view logic

* Ignore LFS hooks if installed for current OS user

Fixes Gitea UI actions for repositories tracking LFS files.
Checks for minimum needed git version by parsing the semantic version
string.

* Hide LFS metafile diff from commit view, marking as binary

* Show LFS notice if file in commit view is tracked

* Add notbefore/nbf JWT claim

* Correct lint suggestions - comments for structs and functions

- Add comments to LFS model
- Function comment for GetRandomBytesAsBase64
- LFS server function comments and lint variable suggestion

* Move secret generation code out of conditional

Ensures no LFS code may run with an empty secret

* Do not hand out JWT tokens if LFS server support is disabled

models/git_diff.go

@@ -200,6 +200,7 @@ type DiffFile struct {
 	IsCreated          bool
 	IsDeleted          bool
 	IsBin              bool
+	IsLFSFile          bool
 	IsRenamed          bool
 	IsSubmodule        bool
 	Sections           []*DiffSection
@@ -245,6 +246,7 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (*
 		leftLine, rightLine int
 		lineCount           int
 		curFileLinesCount   int
+		curFileLFSPrefix    bool
 	)
 
 	input := bufio.NewReader(reader)
@@ -268,6 +270,28 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (*
 			continue
 		}
 
+		trimLine := strings.Trim(line, "+- ")
+
+		if trimLine == LFSMetaFileIdentifier {
+			curFileLFSPrefix = true
+		}
+
+		if curFileLFSPrefix && strings.HasPrefix(trimLine, LFSMetaFileOidPrefix) {
+			oid := strings.TrimPrefix(trimLine, LFSMetaFileOidPrefix)
+
+			if len(oid) == 64 {
+				m := &LFSMetaObject{Oid: oid}
+				count, err := x.Count(m)
+
+				if err == nil && count > 0 {
+					curFile.IsBin = true
+					curFile.IsLFSFile = true
+					curSection.Lines = nil
+					break
+				}
+			}
+		}
+
 		curFileLinesCount++
 		lineCount++
 
@@ -354,6 +378,7 @@ func ParsePatch(maxLines, maxLineCharacteres, maxFiles int, reader io.Reader) (*
 				break
 			}
 			curFileLinesCount = 0
+			curFileLFSPrefix = false
 
 			// Check file diff type and is submodule.
 			for {

models/lfs.go

@@ -0,0 +1,122 @@
+package models
+
+import (
+	"errors"
+	"github.com/go-xorm/xorm"
+	"time"
+)
+
+// LFSMetaObject stores metadata for LFS tracked files.
+type LFSMetaObject struct {
+	ID           int64     `xorm:"pk autoincr"`
+	Oid          string    `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	Size         int64     `xorm:"NOT NULL"`
+	RepositoryID int64     `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	Existing     bool      `xorm:"-"`
+	Created      time.Time `xorm:"-"`
+	CreatedUnix  int64
+}
+
+// LFSTokenResponse defines the JSON structure in which the JWT token is stored.
+// This structure is fetched via SSH and passed by the Git LFS client to the server
+// endpoint for authorization.
+type LFSTokenResponse struct {
+	Header map[string]string `json:"header"`
+	Href   string            `json:"href"`
+}
+
+var (
+	// ErrLFSObjectNotExist is returned from lfs models functions in order
+	// to differentiate between database and missing object errors.
+	ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist")
+)
+
+const (
+	// LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files.
+	// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
+	LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+
+	// LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
+	LFSMetaFileOidPrefix = "oid sha256:"
+)
+
+// NewLFSMetaObject stores a given populated LFSMetaObject structure in the database
+// if it is not already present.
+func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
+	var err error
+
+	has, err := x.Get(m)
+	if err != nil {
+		return nil, err
+	}
+
+	if has {
+		m.Existing = true
+		return m, nil
+	}
+
+	sess := x.NewSession()
+	defer sessionRelease(sess)
+	if err = sess.Begin(); err != nil {
+		return nil, err
+	}
+
+	if _, err = sess.Insert(m); err != nil {
+		return nil, err
+	}
+
+	return m, sess.Commit()
+}
+
+// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
+// It may return ErrLFSObjectNotExist or a database error. If the error is nil,
+// the returned pointer is a valid LFSMetaObject.
+func GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error) {
+	if len(oid) == 0 {
+		return nil, ErrLFSObjectNotExist
+	}
+
+	m := &LFSMetaObject{Oid: oid}
+	has, err := x.Get(m)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, ErrLFSObjectNotExist
+	}
+	return m, nil
+}
+
+// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
+// It may return ErrLFSObjectNotExist or a database error.
+func RemoveLFSMetaObjectByOid(oid string) error {
+	if len(oid) == 0 {
+		return ErrLFSObjectNotExist
+	}
+
+	sess := x.NewSession()
+	defer sessionRelease(sess)
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	m := &LFSMetaObject{Oid: oid}
+
+	if _, err := sess.Delete(m); err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}
+
+// BeforeInsert sets the time at which the LFSMetaObject was created.
+func (m *LFSMetaObject) BeforeInsert() {
+	m.CreatedUnix = time.Now().Unix()
+}
+
+// AfterSet stores the LFSMetaObject creation time in the database as local time.
+func (m *LFSMetaObject) AfterSet(colName string, _ xorm.Cell) {
+	switch colName {
+	case "created_unix":
+		m.Created = time.Unix(m.CreatedUnix, 0).Local()
+	}
+}

models/models.go

@@ -79,7 +79,7 @@ func init() {
 		new(Mirror), new(Release), new(LoginSource), new(Webhook),
 		new(UpdateTask), new(HookTask),
 		new(Team), new(OrgUser), new(TeamUser), new(TeamRepo),
-		new(Notice), new(EmailAddress))
+		new(Notice), new(EmailAddress), new(LFSMetaObject))
 
 	gonicNames := []string{"SSL", "UID"}
 	for _, name := range gonicNames {

models/repo.go

@@ -1480,6 +1480,35 @@ func DeleteRepository(uid, repoID int64) error {
 		RemoveAllWithNotice("Delete attachment", attachmentPaths[i])
 	}
 
+	// Remove LFS objects
+	var lfsObjects []*LFSMetaObject
+
+	if err = sess.Where("repository_id=?", repoID).Find(&lfsObjects); err != nil {
+		return err
+	}
+
+	for _, v := range lfsObjects {
+		count, err := sess.Count(&LFSMetaObject{Oid: v.Oid})
+
+		if err != nil {
+			return err
+		}
+
+		if count > 1 {
+			continue
+		}
+
+		oidPath := filepath.Join(v.Oid[0:2], v.Oid[2:4], v.Oid[4:len(v.Oid)])
+		err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath))
+		if err != nil {
+			return err
+		}
+	}
+
+	if _, err := sess.Delete(&LFSMetaObject{RepositoryID: repoID}); err != nil {
+		return err
+	}
+
 	if err = sess.Commit(); err != nil {
 		return fmt.Errorf("Commit: %v", err)
 	}
@@ -2240,6 +2269,34 @@ func ForkRepository(u *User, oldRepo *Repository, name, desc string) (_ *Reposit
 		return nil, fmt.Errorf("createUpdateHook: %v", err)
 	}
 
+	//Commit repo to get Fork ID
+	err = sess.Commit()
+	if err != nil {
+		return nil, err
+	}
+	sessionRelease(sess)
+
+	// Copy LFS meta objects in new session
+	sess = x.NewSession()
+	defer sessionRelease(sess)
+	if err = sess.Begin(); err != nil {
+		return nil, err
+	}
+
+	var lfsObjects []*LFSMetaObject
+
+	if err = sess.Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil {
+		return nil, err
+	}
+
+	for _, v := range lfsObjects {
+		v.ID = 0
+		v.RepositoryID = repo.ID
+		if _, err = sess.Insert(v); err != nil {
+			return nil, err
+		}
+	}
+
 	return repo, sess.Commit()
 }