Add LFS Migration and Mirror (#14726)

* Implemented LFS client.

* Implemented scanning for pointer files.

* Implemented downloading of lfs files.

* Moved model-dependent code into services.

* Removed models dependency. Added TryReadPointerFromBuffer.

* Migrated code from service to module.

* Centralised storage creation.

* Removed dependency from models.

* Moved ContentStore into modules.

* Share structs between server and client.

* Moved method to services.

* Implemented lfs download on clone.

* Implemented LFS sync on clone and mirror update.

* Added form fields.

* Updated templates.

* Fixed condition.

* Use alternate endpoint.

* Added missing methods.

* Fixed typo and make linter happy.

* Detached pointer parser from gogit dependency.

* Fixed TestGetLFSRange test.

* Added context to support cancellation.

* Use ReadFull to probably read more data.

* Removed duplicated code from models.

* Moved scan implementation into pointer_scanner_nogogit.

* Changed method name.

* Added comments.

* Added more/specific log/error messages.

* Embedded lfs.Pointer into models.LFSMetaObject.

* Moved code from models to module.

* Moved code from models to module.

* Moved code from models to module.

* Reduced pointer usage.

* Embedded type.

* Use promoted fields.

* Fixed unexpected eof.

* Added unit tests.

* Implemented migration of local file paths.

* Show an error on invalid LFS endpoints.

* Hide settings if not used.

* Added LFS info to mirror struct.

* Fixed comment.

* Check LFS endpoint.

* Manage LFS settings from mirror page.

* Fixed selector.

* Adjusted selector.

* Added more tests.

* Added local filesystem migration test.

* Fixed typo.

* Reset settings.

* Added special windows path handling.

* Added unit test for HTTPClient.

* Added unit test for BasicTransferAdapter.

* Moved into util package.

* Test if LFS endpoint is allowed.

* Added support for git://

* Just use a static placeholder as the displayed url may be invalid.

* Reverted to original code.

* Added "Advanced Settings".

* Updated wording.

* Added discovery info link.

* Implemented suggestion.

* Fixed missing format parameter.

* Added Pointer.IsValid().

* Always remove model on error.

* Added suggestions.

* Use channel instead of array.

* Update routers/repo/migrate.go

* fmt

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>

routers/api/v1/repo/migrate.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/convert"
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/migrations/base"
@@ -101,27 +102,7 @@ func Migrate(ctx *context.APIContext) {
 		err = migrations.IsMigrateURLAllowed(remoteAddr, ctx.User)
 	}
 	if err != nil {
-		if models.IsErrInvalidCloneAddr(err) {
-			addrErr := err.(*models.ErrInvalidCloneAddr)
-			switch {
-			case addrErr.IsURLError:
-				ctx.Error(http.StatusUnprocessableEntity, "", err)
-			case addrErr.IsPermissionDenied:
-				if addrErr.LocalPath {
-					ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.")
-				} else if len(addrErr.PrivateNet) == 0 {
-					ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from blocked hosts.")
-				} else {
-					ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from private IPs.")
-				}
-			case addrErr.IsInvalidPath:
-				ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.")
-			default:
-				ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", "Unknown error type (ErrInvalidCloneAddr): "+err.Error())
-			}
-		} else {
-			ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", err)
-		}
+		handleRemoteAddrError(ctx, err)
 		return
 	}
 
@@ -137,12 +118,29 @@ func Migrate(ctx *context.APIContext) {
 		return
 	}
 
+	form.LFS = form.LFS && setting.LFS.StartServer
+
+	if form.LFS && len(form.LFSEndpoint) > 0 {
+		ep := lfs.DetermineEndpoint("", form.LFSEndpoint)
+		if ep == nil {
+			ctx.Error(http.StatusInternalServerError, "", ctx.Tr("repo.migrate.invalid_lfs_endpoint"))
+			return
+		}
+		err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User)
+		if err != nil {
+			handleRemoteAddrError(ctx, err)
+			return
+		}
+	}
+
 	var opts = migrations.MigrateOptions{
 		CloneAddr:      remoteAddr,
 		RepoName:       form.RepoName,
 		Description:    form.Description,
 		Private:        form.Private || setting.Repository.ForcePrivate,
 		Mirror:         form.Mirror,
+		LFS:            form.LFS,
+		LFSEndpoint:    form.LFSEndpoint,
 		AuthUsername:   form.AuthUsername,
 		AuthPassword:   form.AuthPassword,
 		AuthToken:      form.AuthToken,
@@ -245,3 +243,27 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA
 		}
 	}
 }
+
+func handleRemoteAddrError(ctx *context.APIContext, err error) {
+	if models.IsErrInvalidCloneAddr(err) {
+		addrErr := err.(*models.ErrInvalidCloneAddr)
+		switch {
+		case addrErr.IsURLError:
+			ctx.Error(http.StatusUnprocessableEntity, "", err)
+		case addrErr.IsPermissionDenied:
+			if addrErr.LocalPath {
+				ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.")
+			} else if len(addrErr.PrivateNet) == 0 {
+				ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from blocked hosts.")
+			} else {
+				ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from private IPs.")
+			}
+		case addrErr.IsInvalidPath:
+			ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.")
+		default:
+			ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", "Unknown error type (ErrInvalidCloneAddr): "+err.Error())
+		}
+	} else {
+		ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", err)
+	}
+}

routers/repo/download.go

@@ -96,12 +96,13 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error {
 		}
 	}()
 
-	if meta, _ := lfs.ReadPointerFile(dataRc); meta != nil {
-		meta, _ = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid)
+	pointer, _ := lfs.ReadPointer(dataRc)
+	if pointer.IsValid() {
+		meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid)
 		if meta == nil {
 			return ServeBlob(ctx, blob)
 		}
-		lfsDataRc, err := lfs.ReadMetaObject(meta)
+		lfsDataRc, err := lfs.ReadMetaObject(meta.Pointer)
 		if err != nil {
 			return err
 		}

routers/repo/lfs.go

@@ -5,7 +5,6 @@
 package repo
 
 import (
-	"bufio"
 	"bytes"
 	"fmt"
 	gotemplate "html/template"
@@ -15,7 +14,6 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
@@ -266,7 +264,7 @@ func LFSFileGet(ctx *context.Context) {
 		return
 	}
 	ctx.Data["LFSFile"] = meta
-	dataRc, err := lfs.ReadMetaObject(meta)
+	dataRc, err := lfs.ReadMetaObject(meta.Pointer)
 	if err != nil {
 		ctx.ServerError("LFSFileGet", err)
 		return
@@ -385,9 +383,8 @@ func LFSFileFind(ctx *context.Context) {
 	ctx.Data["PageIsSettingsLFS"] = true
 	var hash git.SHA1
 	if len(sha) == 0 {
-		meta := models.LFSMetaObject{Oid: oid, Size: size}
-		pointer := meta.Pointer()
-		hash = git.ComputeBlobHash([]byte(pointer))
+		pointer := lfs.Pointer{Oid: oid, Size: size}
+		hash = git.ComputeBlobHash([]byte(pointer.StringContent()))
 		sha = hash.String()
 	} else {
 		hash = git.MustIDFromString(sha)
@@ -421,158 +418,99 @@ func LFSPointerFiles(ctx *context.Context) {
 	}
 	ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
 
-	basePath := ctx.Repo.Repository.RepoPath()
+	err = func() error {
+		pointerChan := make(chan lfs.PointerBlob)
+		errChan := make(chan error, 1)
+		go lfs.SearchPointerBlobs(ctx.Req.Context(), ctx.Repo.GitRepo, pointerChan, errChan)
 
-	pointerChan := make(chan pointerResult)
+		numPointers := 0
+		var numAssociated, numNoExist, numAssociatable int
 
-	catFileCheckReader, catFileCheckWriter := io.Pipe()
-	shasToBatchReader, shasToBatchWriter := io.Pipe()
-	catFileBatchReader, catFileBatchWriter := io.Pipe()
-	errChan := make(chan error, 1)
-	wg := sync.WaitGroup{}
-	wg.Add(5)
+		type pointerResult struct {
+			SHA        string
+			Oid        string
+			Size       int64
+			InRepo     bool
+			Exists     bool
+			Accessible bool
+		}
 
-	var numPointers, numAssociated, numNoExist, numAssociatable int
+		results := []pointerResult{}
 
-	go func() {
-		defer wg.Done()
-		pointers := make([]pointerResult, 0, 50)
-		for pointer := range pointerChan {
-			pointers = append(pointers, pointer)
-			if pointer.InRepo {
+		contentStore := lfs.NewContentStore()
+		repo := ctx.Repo.Repository
+
+		for pointerBlob := range pointerChan {
+			numPointers++
+
+			result := pointerResult{
+				SHA:  pointerBlob.Hash,
+				Oid:  pointerBlob.Oid,
+				Size: pointerBlob.Size,
+			}
+
+			if _, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid); err != nil {
+				if err != models.ErrLFSObjectNotExist {
+					return err
+				}
+			} else {
+				result.InRepo = true
+			}
+
+			result.Exists, err = contentStore.Exists(pointerBlob.Pointer)
+			if err != nil {
+				return err
+			}
+
+			if result.Exists {
+				if !result.InRepo {
+					// Can we fix?
+					// OK well that's "simple"
+					// - we need to check whether current user has access to a repo that has access to the file
+					result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid)
+					if err != nil {
+						return err
+					}
+				} else {
+					result.Accessible = true
+				}
+			}
+
+			if result.InRepo {
 				numAssociated++
 			}
-			if !pointer.Exists {
+			if !result.Exists {
 				numNoExist++
 			}
-			if !pointer.InRepo && pointer.Accessible {
+			if !result.InRepo && result.Accessible {
 				numAssociatable++
 			}
+
+			results = append(results, result)
 		}
-		numPointers = len(pointers)
-		ctx.Data["Pointers"] = pointers
+
+		err, has := <-errChan
+		if has {
+			return err
+		}
+
+		ctx.Data["Pointers"] = results
 		ctx.Data["NumPointers"] = numPointers
 		ctx.Data["NumAssociated"] = numAssociated
 		ctx.Data["NumAssociatable"] = numAssociatable
 		ctx.Data["NumNoExist"] = numNoExist
 		ctx.Data["NumNotAssociated"] = numPointers - numAssociated
+
+		return nil
 	}()
-	go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User)
-	go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
-	go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
-	if git.CheckGitVersionAtLeast("2.6.0") != nil {
-		revListReader, revListWriter := io.Pipe()
-		shasToCheckReader, shasToCheckWriter := io.Pipe()
-		wg.Add(2)
-		go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
-		go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
-		go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
-	} else {
-		go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
+	if err != nil {
+		ctx.ServerError("LFSPointerFiles", err)
+		return
 	}
-	wg.Wait()
 
-	select {
-	case err, has := <-errChan:
-		if has {
-			ctx.ServerError("LFSPointerFiles", err)
-		}
-	default:
-	}
 	ctx.HTML(http.StatusOK, tplSettingsLFSPointers)
 }
 
-type pointerResult struct {
-	SHA        string
-	Oid        string
-	Size       int64
-	InRepo     bool
-	Exists     bool
-	Accessible bool
-}
-
-func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
-	defer wg.Done()
-	defer catFileBatchReader.Close()
-	contentStore := lfs.ContentStore{ObjectStorage: storage.LFS}
-
-	bufferedReader := bufio.NewReader(catFileBatchReader)
-	buf := make([]byte, 1025)
-	for {
-		// File descriptor line: sha
-		sha, err := bufferedReader.ReadString(' ')
-		if err != nil {
-			_ = catFileBatchReader.CloseWithError(err)
-			break
-		}
-		// Throw away the blob
-		if _, err := bufferedReader.ReadString(' '); err != nil {
-			_ = catFileBatchReader.CloseWithError(err)
-			break
-		}
-		sizeStr, err := bufferedReader.ReadString('\n')
-		if err != nil {
-			_ = catFileBatchReader.CloseWithError(err)
-			break
-		}
-		size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
-		if err != nil {
-			_ = catFileBatchReader.CloseWithError(err)
-			break
-		}
-		pointerBuf := buf[:size+1]
-		if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
-			_ = catFileBatchReader.CloseWithError(err)
-			break
-		}
-		pointerBuf = pointerBuf[:size]
-		// Now we need to check if the pointerBuf is an LFS pointer
-		pointer := lfs.IsPointerFile(&pointerBuf)
-		if pointer == nil {
-			continue
-		}
-
-		result := pointerResult{
-			SHA:  strings.TrimSpace(sha),
-			Oid:  pointer.Oid,
-			Size: pointer.Size,
-		}
-
-		// Then we need to check that this pointer is in the db
-		if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
-			if err != models.ErrLFSObjectNotExist {
-				_ = catFileBatchReader.CloseWithError(err)
-				break
-			}
-		} else {
-			result.InRepo = true
-		}
-
-		result.Exists, err = contentStore.Exists(pointer)
-		if err != nil {
-			_ = catFileBatchReader.CloseWithError(err)
-			break
-		}
-
-		if result.Exists {
-			if !result.InRepo {
-				// Can we fix?
-				// OK well that's "simple"
-				// - we need to check whether current user has access to a repo that has access to the file
-				result.Accessible, err = models.LFSObjectAccessible(user, result.Oid)
-				if err != nil {
-					_ = catFileBatchReader.CloseWithError(err)
-					break
-				}
-			} else {
-				result.Accessible = true
-			}
-		}
-		pointerChan <- result
-	}
-	close(pointerChan)
-}
-
 // LFSAutoAssociate auto associates accessible lfs files
 func LFSAutoAssociate(ctx *context.Context) {
 	if !setting.LFS.StartServer {

routers/repo/migrate.go

@@ -12,6 +12,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/setting"
@@ -47,6 +48,7 @@ func Migrate(ctx *context.Context) {
 
 	ctx.Data["private"] = getRepoPrivate(ctx)
 	ctx.Data["mirror"] = ctx.Query("mirror") == "1"
+	ctx.Data["lfs"] = ctx.Query("lfs") == "1"
 	ctx.Data["wiki"] = ctx.Query("wiki") == "1"
 	ctx.Data["milestones"] = ctx.Query("milestones") == "1"
 	ctx.Data["labels"] = ctx.Query("labels") == "1"
@@ -114,6 +116,34 @@ func handleMigrateError(ctx *context.Context, owner *models.User, err error, nam
 	}
 }
 
+func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplName, form *forms.MigrateRepoForm) {
+	if models.IsErrInvalidCloneAddr(err) {
+		addrErr := err.(*models.ErrInvalidCloneAddr)
+		switch {
+		case addrErr.IsProtocolInvalid:
+			ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tpl, form)
+		case addrErr.IsURLError:
+			ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
+		case addrErr.IsPermissionDenied:
+			if addrErr.LocalPath {
+				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, form)
+			} else if len(addrErr.PrivateNet) == 0 {
+				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form)
+			} else {
+				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, form)
+			}
+		case addrErr.IsInvalidPath:
+			ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, form)
+		default:
+			log.Error("Error whilst updating url: %v", err)
+			ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
+		}
+	} else {
+		log.Error("Error whilst updating url: %v", err)
+		ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
+	}
+}
+
 // MigratePost response for migrating from external git repository
 func MigratePost(ctx *context.Context) {
 	form := web.GetForm(ctx).(*forms.MigrateRepoForm)
@@ -144,35 +174,28 @@ func MigratePost(ctx *context.Context) {
 		err = migrations.IsMigrateURLAllowed(remoteAddr, ctx.User)
 	}
 	if err != nil {
-		if models.IsErrInvalidCloneAddr(err) {
-			ctx.Data["Err_CloneAddr"] = true
-			addrErr := err.(*models.ErrInvalidCloneAddr)
-			switch {
-			case addrErr.IsProtocolInvalid:
-				ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tpl, &form)
-			case addrErr.IsURLError:
-				ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form)
-			case addrErr.IsPermissionDenied:
-				if addrErr.LocalPath {
-					ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, &form)
-				} else if len(addrErr.PrivateNet) == 0 {
-					ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, &form)
-				} else {
-					ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, &form)
-				}
-			case addrErr.IsInvalidPath:
-				ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, &form)
-			default:
-				log.Error("Error whilst updating url: %v", err)
-				ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form)
-			}
-		} else {
-			log.Error("Error whilst updating url: %v", err)
-			ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form)
-		}
+		ctx.Data["Err_CloneAddr"] = true
+		handleMigrateRemoteAddrError(ctx, err, tpl, form)
 		return
 	}
 
+	form.LFS = form.LFS && setting.LFS.StartServer
+
+	if form.LFS && len(form.LFSEndpoint) > 0 {
+		ep := lfs.DetermineEndpoint("", form.LFSEndpoint)
+		if ep == nil {
+			ctx.Data["Err_LFSEndpoint"] = true
+			ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tpl, &form)
+			return
+		}
+		err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User)
+		if err != nil {
+			ctx.Data["Err_LFSEndpoint"] = true
+			handleMigrateRemoteAddrError(ctx, err, tpl, form)
+			return
+		}
+	}
+
 	var opts = migrations.MigrateOptions{
 		OriginalURL:    form.CloneAddr,
 		GitServiceType: serviceType,
@@ -181,6 +204,8 @@ func MigratePost(ctx *context.Context) {
 		Description:    form.Description,
 		Private:        form.Private || setting.Repository.ForcePrivate,
 		Mirror:         form.Mirror && !setting.Repository.DisableMirrors,
+		LFS:            form.LFS,
+		LFSEndpoint:    form.LFSEndpoint,
 		AuthUsername:   form.AuthUsername,
 		AuthPassword:   form.AuthPassword,
 		AuthToken:      form.AuthToken,

routers/repo/setting.go

@@ -17,6 +17,7 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/repository"
@@ -170,30 +171,8 @@ func SettingsPost(ctx *context.Context) {
 			err = migrations.IsMigrateURLAllowed(address, ctx.User)
 		}
 		if err != nil {
-			if models.IsErrInvalidCloneAddr(err) {
-				ctx.Data["Err_MirrorAddress"] = true
-				addrErr := err.(*models.ErrInvalidCloneAddr)
-				switch {
-				case addrErr.IsProtocolInvalid:
-					ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tplSettingsOptions, &form)
-				case addrErr.IsURLError:
-					ctx.RenderWithErr(ctx.Tr("form.url_error"), tplSettingsOptions, &form)
-				case addrErr.IsPermissionDenied:
-					if addrErr.LocalPath {
-						ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, &form)
-					} else if len(addrErr.PrivateNet) == 0 {
-						ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, &form)
-					} else {
-						ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, &form)
-					}
-				case addrErr.IsInvalidPath:
-					ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, &form)
-				default:
-					ctx.ServerError("Unknown error", err)
-				}
-			}
 			ctx.Data["Err_MirrorAddress"] = true
-			ctx.RenderWithErr(ctx.Tr("repo.mirror_address_url_invalid"), tplSettingsOptions, &form)
+			handleSettingRemoteAddrError(ctx, err, form)
 			return
 		}
 
@@ -202,6 +181,30 @@ func SettingsPost(ctx *context.Context) {
 			return
 		}
 
+		form.LFS = form.LFS && setting.LFS.StartServer
+
+		if len(form.LFSEndpoint) > 0 {
+			ep := lfs.DetermineEndpoint("", form.LFSEndpoint)
+			if ep == nil {
+				ctx.Data["Err_LFSEndpoint"] = true
+				ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tplSettingsOptions, &form)
+				return
+			}
+			err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User)
+			if err != nil {
+				ctx.Data["Err_LFSEndpoint"] = true
+				handleSettingRemoteAddrError(ctx, err, form)
+				return
+			}
+		}
+
+		ctx.Repo.Mirror.LFS = form.LFS
+		ctx.Repo.Mirror.LFSEndpoint = form.LFSEndpoint
+		if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil {
+			ctx.ServerError("UpdateMirror", err)
+			return
+		}
+
 		ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success"))
 		ctx.Redirect(repo.Link() + "/settings")
 
@@ -615,6 +618,31 @@ func SettingsPost(ctx *context.Context) {
 	}
 }
 
+func handleSettingRemoteAddrError(ctx *context.Context, err error, form *forms.RepoSettingForm) {
+	if models.IsErrInvalidCloneAddr(err) {
+		addrErr := err.(*models.ErrInvalidCloneAddr)
+		switch {
+		case addrErr.IsProtocolInvalid:
+			ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tplSettingsOptions, form)
+		case addrErr.IsURLError:
+			ctx.RenderWithErr(ctx.Tr("form.url_error"), tplSettingsOptions, form)
+		case addrErr.IsPermissionDenied:
+			if addrErr.LocalPath {
+				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, form)
+			} else if len(addrErr.PrivateNet) == 0 {
+				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, form)
+			} else {
+				ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, form)
+			}
+		case addrErr.IsInvalidPath:
+			ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, form)
+		default:
+			ctx.ServerError("Unknown error", err)
+		}
+	}
+	ctx.RenderWithErr(ctx.Tr("repo.mirror_address_url_invalid"), tplSettingsOptions, form)
+}
+
 // Collaboration render a repository's collaboration page
 func Collaboration(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("repo.settings")

routers/repo/view.go

@@ -274,43 +274,42 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 
 		// FIXME: what happens when README file is an image?
 		if isTextFile && setting.LFS.StartServer {
-			meta := lfs.IsPointerFile(&buf)
-			if meta != nil {
-				meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid)
+			pointer, _ := lfs.ReadPointerFromBuffer(buf)
+			if pointer.IsValid() {
+				meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid)
 				if err != nil && err != models.ErrLFSObjectNotExist {
 					ctx.ServerError("GetLFSMetaObject", err)
 					return
 				}
-			}
+				if meta != nil {
+					ctx.Data["IsLFSFile"] = true
+					isLFSFile = true
 
-			if meta != nil {
-				ctx.Data["IsLFSFile"] = true
-				isLFSFile = true
+					// OK read the lfs object
+					var err error
+					dataRc, err = lfs.ReadMetaObject(pointer)
+					if err != nil {
+						ctx.ServerError("ReadMetaObject", err)
+						return
+					}
+					defer dataRc.Close()
 
-				// OK read the lfs object
-				var err error
-				dataRc, err = lfs.ReadMetaObject(meta)
-				if err != nil {
-					ctx.ServerError("ReadMetaObject", err)
-					return
+					buf = make([]byte, 1024)
+					n, err = dataRc.Read(buf)
+					if err != nil {
+						ctx.ServerError("Data", err)
+						return
+					}
+					buf = buf[:n]
+
+					isTextFile = base.IsTextFile(buf)
+					ctx.Data["IsTextFile"] = isTextFile
+
+					fileSize = meta.Size
+					ctx.Data["FileSize"] = meta.Size
+					filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name))
+					ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64)
 				}
-				defer dataRc.Close()
-
-				buf = make([]byte, 1024)
-				n, err = dataRc.Read(buf)
-				if err != nil {
-					ctx.ServerError("Data", err)
-					return
-				}
-				buf = buf[:n]
-
-				isTextFile = base.IsTextFile(buf)
-				ctx.Data["IsTextFile"] = isTextFile
-
-				fileSize = meta.Size
-				ctx.Data["FileSize"] = meta.Size
-				filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name))
-				ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64)
 			}
 		}
 
@@ -400,39 +399,39 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 
 	//Check for LFS meta file
 	if isTextFile && setting.LFS.StartServer {
-		meta := lfs.IsPointerFile(&buf)
-		if meta != nil {
-			meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid)
+		pointer, _ := lfs.ReadPointerFromBuffer(buf)
+		if pointer.IsValid() {
+			meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid)
 			if err != nil && err != models.ErrLFSObjectNotExist {
 				ctx.ServerError("GetLFSMetaObject", err)
 				return
 			}
-		}
-		if meta != nil {
-			isLFSFile = true
+			if meta != nil {
+				isLFSFile = true
 
-			// OK read the lfs object
-			var err error
-			dataRc, err = lfs.ReadMetaObject(meta)
-			if err != nil {
-				ctx.ServerError("ReadMetaObject", err)
-				return
+				// OK read the lfs object
+				var err error
+				dataRc, err = lfs.ReadMetaObject(pointer)
+				if err != nil {
+					ctx.ServerError("ReadMetaObject", err)
+					return
+				}
+				defer dataRc.Close()
+
+				buf = make([]byte, 1024)
+				n, err = dataRc.Read(buf)
+				// Error EOF don't mean there is an error, it just means we read to
+				// the end
+				if err != nil && err != io.EOF {
+					ctx.ServerError("Data", err)
+					return
+				}
+				buf = buf[:n]
+
+				isTextFile = base.IsTextFile(buf)
+				fileSize = meta.Size
+				ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath)
 			}
-			defer dataRc.Close()
-
-			buf = make([]byte, 1024)
-			n, err = dataRc.Read(buf)
-			// Error EOF don't mean there is an error, it just means we read to
-			// the end
-			if err != nil && err != io.EOF {
-				ctx.ServerError("Data", err)
-				return
-			}
-			buf = buf[:n]
-
-			isTextFile = base.IsTextFile(buf)
-			fileSize = meta.Size
-			ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath)
 		}
 	}
 

routers/routes/web.go

@@ -16,7 +16,6 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/httpcache"
-	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/metrics"
 	"code.gitea.io/gitea/modules/public"
@@ -38,6 +37,7 @@ import (
 	"code.gitea.io/gitea/routers/user"
 	userSetting "code.gitea.io/gitea/routers/user/setting"
 	"code.gitea.io/gitea/services/forms"
+	"code.gitea.io/gitea/services/lfs"
 	"code.gitea.io/gitea/services/mailer"
 
 	// to registers all internal adapters