Refactor LFS SSH and internal routers (#32473)

Gitea instance keeps reporting a lot of errors like "LFS SSH transfer connection denied, pure SSH protocol is disabled". When starting debugging the problem, there are more problems found. Try to address most of them:

* avoid unnecessary server side error logs (change `fail()` to not log them)
* figure out the broken tests/user2/lfs.git (added comments)
* avoid `migratePushMirrors` failure when a repository doesn't exist (ignore them)
* avoid "Authorization" (internal&lfs) header conflicts, remove the tricky "swapAuth" and use "X-Gitea-Internal-Auth"
* make internal token comparing constant time (it wasn't a serous problem because in a real world it's nearly impossible to timing-attack the token, but good to fix and backport)
* avoid duplicate routers (introduce AddOwnerRepoGitLFSRoutes)
* avoid "internal (private)" routes using session/web context (they should use private context)
* fix incorrect "path" usages (use "filepath")
* fix incorrect mocked route point handling (need to check func nil correctly)
* split some tests from "git general tests" to "git misc tests" (to keep "git_general_test.go" simple)

Still no correct result for Git LFS SSH tests. So the code is kept there
(`tests/integration/git_lfs_ssh_test.go`) and a FIXME explains the details.

modules/git/batch_reader.go

@@ -146,9 +146,8 @@ func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufi
 }
 
 // ReadBatchLine reads the header line from cat-file --batch
-// We expect:
-// <sha> SP <type> SP <size> LF
-// sha is a hex encoded here
+// We expect: <oid> SP <type> SP <size> LF
+// then leaving the rest of the stream "<contents> LF" to be read
 func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) {
 	typ, err = rd.ReadString('\n')
 	if err != nil {

modules/lfstransfer/backend/backend.go

@@ -33,12 +33,12 @@ var _ transfer.Backend = &GiteaBackend{}
 
 // GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API
 type GiteaBackend struct {
-	ctx    context.Context
-	server *url.URL
-	op     string
-	token  string
-	itoken string
-	logger transfer.Logger
+	ctx          context.Context
+	server       *url.URL
+	op           string
+	authToken    string
+	internalAuth string
+	logger       transfer.Logger
 }
 
 func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (transfer.Backend, error) {
@@ -48,7 +48,7 @@ func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (t
 		return nil, err
 	}
 	server = server.JoinPath("api/internal/repo", repo, "info/lfs")
-	return &GiteaBackend{ctx: ctx, server: server, op: op, token: token, itoken: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil
+	return &GiteaBackend{ctx: ctx, server: server, op: op, authToken: token, internalAuth: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil
 }
 
 // Batch implements transfer.Backend
@@ -73,10 +73,10 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans
 	}
 	url := g.server.JoinPath("objects/batch").String()
 	headers := map[string]string{
-		headerAuthorisation: g.itoken,
-		headerAuthX:         g.token,
-		headerAccept:        mimeGitLFS,
-		headerContentType:   mimeGitLFS,
+		headerAuthorization:     g.authToken,
+		headerGiteaInternalAuth: g.internalAuth,
+		headerAccept:            mimeGitLFS,
+		headerContentType:       mimeGitLFS,
 	}
 	req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
 	resp, err := req.Response()
@@ -119,7 +119,7 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans
 				}
 				idMapStr := base64.StdEncoding.EncodeToString(idMapBytes)
 				item.Args[argID] = idMapStr
-				if authHeader, ok := action.Header[headerAuthorisation]; ok {
+				if authHeader, ok := action.Header[headerAuthorization]; ok {
 					authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader))
 					item.Args[argToken] = authHeaderB64
 				}
@@ -142,7 +142,7 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans
 				}
 				idMapStr := base64.StdEncoding.EncodeToString(idMapBytes)
 				item.Args[argID] = idMapStr
-				if authHeader, ok := action.Header[headerAuthorisation]; ok {
+				if authHeader, ok := action.Header[headerAuthorization]; ok {
 					authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader))
 					item.Args[argToken] = authHeaderB64
 				}
@@ -183,9 +183,9 @@ func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser,
 	}
 	url := action.Href
 	headers := map[string]string{
-		headerAuthorisation: g.itoken,
-		headerAuthX:         g.token,
-		headerAccept:        mimeOctetStream,
+		headerAuthorization:     g.authToken,
+		headerGiteaInternalAuth: g.internalAuth,
+		headerAccept:            mimeOctetStream,
 	}
 	req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil)
 	resp, err := req.Response()
@@ -229,10 +229,10 @@ func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer
 	}
 	url := action.Href
 	headers := map[string]string{
-		headerAuthorisation: g.itoken,
-		headerAuthX:         g.token,
-		headerContentType:   mimeOctetStream,
-		headerContentLength: strconv.FormatInt(size, 10),
+		headerAuthorization:     g.authToken,
+		headerGiteaInternalAuth: g.internalAuth,
+		headerContentType:       mimeOctetStream,
+		headerContentLength:     strconv.FormatInt(size, 10),
 	}
 	reqBytes, err := io.ReadAll(r)
 	if err != nil {
@@ -279,10 +279,10 @@ func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (trans
 	}
 	url := action.Href
 	headers := map[string]string{
-		headerAuthorisation: g.itoken,
-		headerAuthX:         g.token,
-		headerAccept:        mimeGitLFS,
-		headerContentType:   mimeGitLFS,
+		headerAuthorization:     g.authToken,
+		headerGiteaInternalAuth: g.internalAuth,
+		headerAccept:            mimeGitLFS,
+		headerContentType:       mimeGitLFS,
 	}
 	req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
 	resp, err := req.Response()

modules/lfstransfer/backend/lock.go

@@ -21,17 +21,17 @@ import (
 var _ transfer.LockBackend = &giteaLockBackend{}
 
 type giteaLockBackend struct {
-	ctx    context.Context
-	g      *GiteaBackend
-	server *url.URL
-	token  string
-	itoken string
-	logger transfer.Logger
+	ctx          context.Context
+	g            *GiteaBackend
+	server       *url.URL
+	authToken    string
+	internalAuth string
+	logger       transfer.Logger
 }
 
 func newGiteaLockBackend(g *GiteaBackend) transfer.LockBackend {
 	server := g.server.JoinPath("locks")
-	return &giteaLockBackend{ctx: g.ctx, g: g, server: server, token: g.token, itoken: g.itoken, logger: g.logger}
+	return &giteaLockBackend{ctx: g.ctx, g: g, server: server, authToken: g.authToken, internalAuth: g.internalAuth, logger: g.logger}
 }
 
 // Create implements transfer.LockBackend
@@ -45,10 +45,10 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) {
 	}
 	url := g.server.String()
 	headers := map[string]string{
-		headerAuthorisation: g.itoken,
-		headerAuthX:         g.token,
-		headerAccept:        mimeGitLFS,
-		headerContentType:   mimeGitLFS,
+		headerAuthorization:     g.authToken,
+		headerGiteaInternalAuth: g.internalAuth,
+		headerAccept:            mimeGitLFS,
+		headerContentType:       mimeGitLFS,
 	}
 	req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
 	resp, err := req.Response()
@@ -97,10 +97,10 @@ func (g *giteaLockBackend) Unlock(lock transfer.Lock) error {
 	}
 	url := g.server.JoinPath(lock.ID(), "unlock").String()
 	headers := map[string]string{
-		headerAuthorisation: g.itoken,
-		headerAuthX:         g.token,
-		headerAccept:        mimeGitLFS,
-		headerContentType:   mimeGitLFS,
+		headerAuthorization:     g.authToken,
+		headerGiteaInternalAuth: g.internalAuth,
+		headerAccept:            mimeGitLFS,
+		headerContentType:       mimeGitLFS,
 	}
 	req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
 	resp, err := req.Response()
@@ -180,10 +180,10 @@ func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, er
 	urlq.RawQuery = v.Encode()
 	url := urlq.String()
 	headers := map[string]string{
-		headerAuthorisation: g.itoken,
-		headerAuthX:         g.token,
-		headerAccept:        mimeGitLFS,
-		headerContentType:   mimeGitLFS,
+		headerAuthorization:     g.authToken,
+		headerGiteaInternalAuth: g.internalAuth,
+		headerAccept:            mimeGitLFS,
+		headerContentType:       mimeGitLFS,
 	}
 	req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil)
 	resp, err := req.Response()

modules/lfstransfer/backend/util.go

@@ -20,11 +20,11 @@ import (
 
 // HTTP headers
 const (
-	headerAccept        = "Accept"
-	headerAuthorisation = "Authorization"
-	headerAuthX         = "X-Auth"
-	headerContentType   = "Content-Type"
-	headerContentLength = "Content-Length"
+	headerAccept            = "Accept"
+	headerAuthorization     = "Authorization"
+	headerGiteaInternalAuth = "X-Gitea-Internal-Auth"
+	headerContentType       = "Content-Type"
+	headerContentLength     = "Content-Length"
 )
 
 // MIME types

modules/private/internal.go

@@ -43,7 +43,7 @@ Ensure you are running in the correct environment or set the correct configurati
 	req := httplib.NewRequest(url, method).
 		SetContext(ctx).
 		Header("X-Real-IP", getClientIP()).
-		Header("Authorization", fmt.Sprintf("Bearer %s", setting.InternalToken)).
+		Header("X-Gitea-Internal-Auth", fmt.Sprintf("Bearer %s", setting.InternalToken)).
 		SetTLSClientConfig(&tls.Config{
 			InsecureSkipVerify: true,
 			ServerName:         setting.Domain,

modules/web/route.go

@@ -6,6 +6,7 @@ package web
 import (
 	"net/http"
 	"net/url"
+	"reflect"
 	"strings"
 
 	"code.gitea.io/gitea/modules/setting"
@@ -82,15 +83,23 @@ func (r *Router) getPattern(pattern string) string {
 	return strings.TrimSuffix(newPattern, "/")
 }
 
+func isNilOrFuncNil(v any) bool {
+	if v == nil {
+		return true
+	}
+	r := reflect.ValueOf(v)
+	return r.Kind() == reflect.Func && r.IsNil()
+}
+
 func (r *Router) wrapMiddlewareAndHandler(h []any) ([]func(http.Handler) http.Handler, http.HandlerFunc) {
 	handlerProviders := make([]func(http.Handler) http.Handler, 0, len(r.curMiddlewares)+len(h)+1)
 	for _, m := range r.curMiddlewares {
-		if m != nil {
+		if !isNilOrFuncNil(m) {
 			handlerProviders = append(handlerProviders, toHandlerProvider(m))
 		}
 	}
 	for _, m := range h {
-		if h != nil {
+		if !isNilOrFuncNil(m) {
 			handlerProviders = append(handlerProviders, toHandlerProvider(m))
 		}
 	}