tdsds.com/gitea
1
1
Fork 0
mirror of https://github.com/go-gitea/gitea synced 2025-07-22 18:28:37 +00:00
Code Releases Activity

Add LFS Migration and Mirror (#14726)

Browse Source 
* Implemented LFS client.

* Implemented scanning for pointer files.

* Implemented downloading of lfs files.

* Moved model-dependent code into services.

* Removed models dependency. Added TryReadPointerFromBuffer.

* Migrated code from service to module.

* Centralised storage creation.

* Removed dependency from models.

* Moved ContentStore into modules.

* Share structs between server and client.

* Moved method to services.

* Implemented lfs download on clone.

* Implemented LFS sync on clone and mirror update.

* Added form fields.

* Updated templates.

* Fixed condition.

* Use alternate endpoint.

* Added missing methods.

* Fixed typo and make linter happy.

* Detached pointer parser from gogit dependency.

* Fixed TestGetLFSRange test.

* Added context to support cancellation.

* Use ReadFull to probably read more data.

* Removed duplicated code from models.

* Moved scan implementation into pointer_scanner_nogogit.

* Changed method name.

* Added comments.

* Added more/specific log/error messages.

* Embedded lfs.Pointer into models.LFSMetaObject.

* Moved code from models to module.

* Moved code from models to module.

* Moved code from models to module.

* Reduced pointer usage.

* Embedded type.

* Use promoted fields.

* Fixed unexpected eof.

* Added unit tests.

* Implemented migration of local file paths.

* Show an error on invalid LFS endpoints.

* Hide settings if not used.

* Added LFS info to mirror struct.

* Fixed comment.

* Check LFS endpoint.

* Manage LFS settings from mirror page.

* Fixed selector.

* Adjusted selector.

* Added more tests.

* Added local filesystem migration test.

* Fixed typo.

* Reset settings.

* Added special windows path handling.

* Added unit test for HTTPClient.

* Added unit test for BasicTransferAdapter.

* Moved into util package.

* Test if LFS endpoint is allowed.

* Added support for git://

* Just use a static placeholder as the displayed url may be invalid.

* Reverted to original code.

* Added "Advanced Settings".

* Updated wording.

* Added discovery info link.

* Implemented suggestion.

* Fixed missing format parameter.

* Added Pointer.IsValid().

* Always remove model on error.

* Added suggestions.

* Use channel instead of array.

* Update routers/repo/migrate.go

* fmt

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
KN4CK3R
2021-04-09 00:25:57 +02:00
committed by GitHub
parent f544414a23
commit c03e488e14
75 changed files with 2159 additions and 711 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
routers/repo/lfs.go
View File

@@ -5,7 +5,6 @@
package repo
import (
"bufio"
"bytes"
"fmt"
gotemplate "html/template"
@@ -15,7 +14,6 @@ import (
"path"
"strconv"
"strings"
"sync"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
@@ -266,7 +264,7 @@ func LFSFileGet(ctx *context.Context) {
return
}
ctx.Data["LFSFile"] = meta
dataRc, err := lfs.ReadMetaObject(meta)
dataRc, err := lfs.ReadMetaObject(meta.Pointer)
if err != nil {
ctx.ServerError("LFSFileGet", err)
return
@@ -385,9 +383,8 @@ func LFSFileFind(ctx *context.Context) {
ctx.Data["PageIsSettingsLFS"] = true
var hash git.SHA1
if len(sha) == 0 {
meta := models.LFSMetaObject{Oid: oid, Size: size}
pointer := meta.Pointer()
hash = git.ComputeBlobHash([]byte(pointer))
pointer := lfs.Pointer{Oid: oid, Size: size}
hash = git.ComputeBlobHash([]byte(pointer.StringContent()))
sha = hash.String()
} else {
hash = git.MustIDFromString(sha)
@@ -421,158 +418,99 @@ func LFSPointerFiles(ctx *context.Context) {
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
basePath := ctx.Repo.Repository.RepoPath()
err = func() error {
pointerChan := make(chan lfs.PointerBlob)
errChan := make(chan error, 1)
go lfs.SearchPointerBlobs(ctx.Req.Context(), ctx.Repo.GitRepo, pointerChan, errChan)
pointerChan := make(chan pointerResult)
numPointers := 0
var numAssociated, numNoExist, numAssociatable int
catFileCheckReader, catFileCheckWriter := io.Pipe()
shasToBatchReader, shasToBatchWriter := io.Pipe()
catFileBatchReader, catFileBatchWriter := io.Pipe()
errChan := make(chan error, 1)
wg := sync.WaitGroup{}
wg.Add(5)
type pointerResult struct {
SHA string
Oid string
Size int64
InRepo bool
Exists bool
Accessible bool
}
var numPointers, numAssociated, numNoExist, numAssociatable int
results := []pointerResult{}
go func() {
defer wg.Done()
pointers := make([]pointerResult, 0, 50)
for pointer := range pointerChan {
pointers = append(pointers, pointer)
if pointer.InRepo {
contentStore := lfs.NewContentStore()
repo := ctx.Repo.Repository
for pointerBlob := range pointerChan {
numPointers++
result := pointerResult{
SHA: pointerBlob.Hash,
Oid: pointerBlob.Oid,
Size: pointerBlob.Size,
}
if _, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid); err != nil {
if err != models.ErrLFSObjectNotExist {
return err
}
} else {
result.InRepo = true
}
result.Exists, err = contentStore.Exists(pointerBlob.Pointer)
if err != nil {
return err
}
if result.Exists {
if !result.InRepo {
// Can we fix?
// OK well that's "simple"
// - we need to check whether current user has access to a repo that has access to the file
result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid)
if err != nil {
return err
}
} else {
result.Accessible = true
}
}
if result.InRepo {
numAssociated++
}
if !pointer.Exists {
if !result.Exists {
numNoExist++
}
if !pointer.InRepo && pointer.Accessible {
if !result.InRepo && result.Accessible {
numAssociatable++
}
results = append(results, result)
}
numPointers = len(pointers)
ctx.Data["Pointers"] = pointers
err, has := <-errChan
if has {
return err
}
ctx.Data["Pointers"] = results
ctx.Data["NumPointers"] = numPointers
ctx.Data["NumAssociated"] = numAssociated
ctx.Data["NumAssociatable"] = numAssociatable
ctx.Data["NumNoExist"] = numNoExist
ctx.Data["NumNotAssociated"] = numPointers - numAssociated
return nil
}()
go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User)
go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
if git.CheckGitVersionAtLeast("2.6.0") != nil {
revListReader, revListWriter := io.Pipe()
shasToCheckReader, shasToCheckWriter := io.Pipe()
wg.Add(2)
go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
} else {
go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
if err != nil {
ctx.ServerError("LFSPointerFiles", err)
return
}
wg.Wait()
select {
case err, has := <-errChan:
if has {
ctx.ServerError("LFSPointerFiles", err)
}
default:
}
ctx.HTML(http.StatusOK, tplSettingsLFSPointers)
}
type pointerResult struct {
SHA string
Oid string
Size int64
InRepo bool
Exists bool
Accessible bool
}
func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
defer wg.Done()
defer catFileBatchReader.Close()
contentStore := lfs.ContentStore{ObjectStorage: storage.LFS}
bufferedReader := bufio.NewReader(catFileBatchReader)
buf := make([]byte, 1025)
for {
// File descriptor line: sha
sha, err := bufferedReader.ReadString(' ')
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
// Throw away the blob
if _, err := bufferedReader.ReadString(' '); err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
sizeStr, err := bufferedReader.ReadString('\n')
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
pointerBuf := buf[:size+1]
if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
pointerBuf = pointerBuf[:size]
// Now we need to check if the pointerBuf is an LFS pointer
pointer := lfs.IsPointerFile(&pointerBuf)
if pointer == nil {
continue
}
result := pointerResult{
SHA: strings.TrimSpace(sha),
Oid: pointer.Oid,
Size: pointer.Size,
}
// Then we need to check that this pointer is in the db
if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
if err != models.ErrLFSObjectNotExist {
_ = catFileBatchReader.CloseWithError(err)
break
}
} else {
result.InRepo = true
}
result.Exists, err = contentStore.Exists(pointer)
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
if result.Exists {
if !result.InRepo {
// Can we fix?
// OK well that's "simple"
// - we need to check whether current user has access to a repo that has access to the file
result.Accessible, err = models.LFSObjectAccessible(user, result.Oid)
if err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
} else {
result.Accessible = true
}
}
pointerChan <- result
}
close(pointerChan)
}
// LFSAutoAssociate auto associates accessible lfs files
func LFSAutoAssociate(ctx *context.Context) {
if !setting.LFS.StartServer {