Refactor LFS SSH and internal routers (#32473)

Gitea instance keeps reporting a lot of errors like "LFS SSH transfer connection denied, pure SSH protocol is disabled". When starting debugging the problem, there are more problems found. Try to address most of them:

* avoid unnecessary server side error logs (change `fail()` to not log them)
* figure out the broken tests/user2/lfs.git (added comments)
* avoid `migratePushMirrors` failure when a repository doesn't exist (ignore them)
* avoid "Authorization" (internal&lfs) header conflicts, remove the tricky "swapAuth" and use "X-Gitea-Internal-Auth"
* make internal token comparing constant time (it wasn't a serous problem because in a real world it's nearly impossible to timing-attack the token, but good to fix and backport)
* avoid duplicate routers (introduce AddOwnerRepoGitLFSRoutes)
* avoid "internal (private)" routes using session/web context (they should use private context)
* fix incorrect "path" usages (use "filepath")
* fix incorrect mocked route point handling (need to check func nil correctly)
* split some tests from "git general tests" to "git misc tests" (to keep "git_general_test.go" simple)

Still no correct result for Git LFS SSH tests. So the code is kept there
(`tests/integration/git_lfs_ssh_test.go`) and a FIXME explains the details.

routers/common/lfs.go

@@ -0,0 +1,29 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package common
+
+import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/web"
+	"code.gitea.io/gitea/services/lfs"
+)
+
+func AddOwnerRepoGitLFSRoutes(m *web.Router, middlewares ...any) {
+	// shared by web and internal routers
+	m.Group("/{username}/{reponame}/info/lfs", func() {
+		m.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler)
+		m.Put("/objects/{oid}/{size}", lfs.UploadHandler)
+		m.Get("/objects/{oid}/{filename}", lfs.DownloadHandler)
+		m.Get("/objects/{oid}", lfs.DownloadHandler)
+		m.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler)
+		m.Group("/locks", func() {
+			m.Get("/", lfs.GetListLockHandler)
+			m.Post("/", lfs.PostLockHandler)
+			m.Post("/verify", lfs.VerifyLockHandler)
+			m.Post("/{lid}/unlock", lfs.UnLockHandler)
+		}, lfs.CheckAcceptMediaType)
+		m.Any("/*", http.NotFound)
+	}, middlewares...)
+}

routers/private/internal.go

@@ -5,6 +5,7 @@
 package private
 
 import (
+	"crypto/subtle"
 	"net/http"
 	"strings"
 
@@ -14,28 +15,30 @@ import (
 	"code.gitea.io/gitea/modules/web"
 	"code.gitea.io/gitea/routers/common"
 	"code.gitea.io/gitea/services/context"
-	"code.gitea.io/gitea/services/lfs"
 
 	"gitea.com/go-chi/binding"
 	chi_middleware "github.com/go-chi/chi/v5/middleware"
 )
 
-// CheckInternalToken check internal token is set
-func CheckInternalToken(next http.Handler) http.Handler {
+const RouterMockPointInternalLFS = "internal-lfs"
+
+func authInternal(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		tokens := req.Header.Get("Authorization")
-		fields := strings.SplitN(tokens, " ", 2)
 		if setting.InternalToken == "" {
 			log.Warn(`The INTERNAL_TOKEN setting is missing from the configuration file: %q, internal API can't work.`, setting.CustomConf)
 			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
 			return
 		}
-		if len(fields) != 2 || fields[0] != "Bearer" || fields[1] != setting.InternalToken {
+
+		tokens := req.Header.Get("X-Gitea-Internal-Auth") // TODO: use something like JWT or HMAC to avoid passing the token in the clear
+		after, found := strings.CutPrefix(tokens, "Bearer ")
+		authSucceeded := found && subtle.ConstantTimeCompare([]byte(after), []byte(setting.InternalToken)) == 1
+		if !authSucceeded {
 			log.Debug("Forbidden attempt to access internal url: Authorization header: %s", tokens)
 			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
-		} else {
-			next.ServeHTTP(w, req)
+			return
 		}
+		next.ServeHTTP(w, req)
 	})
 }
 
@@ -48,20 +51,12 @@ func bind[T any](_ T) any {
 	}
 }
 
-// SwapAuthToken swaps Authorization header with X-Auth header
-func swapAuthToken(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		req.Header.Set("Authorization", req.Header.Get("X-Auth"))
-		next.ServeHTTP(w, req)
-	})
-}
-
 // Routes registers all internal APIs routes to web application.
 // These APIs will be invoked by internal commands for example `gitea serv` and etc.
 func Routes() *web.Router {
 	r := web.NewRouter()
 	r.Use(context.PrivateContexter())
-	r.Use(CheckInternalToken)
+	r.Use(authInternal)
 	// Log the real ip address of the request from SSH is really helpful for diagnosing sometimes.
 	// Since internal API will be sent only from Gitea sub commands and it's under control (checked by InternalToken), we can trust the headers.
 	r.Use(chi_middleware.RealIP)
@@ -90,25 +85,13 @@ func Routes() *web.Router {
 	r.Post("/restore_repo", RestoreRepo)
 	r.Post("/actions/generate_actions_runner_token", GenerateActionsRunnerToken)
 
-	r.Group("/repo/{username}/{reponame}", func() {
-		r.Group("/info/lfs", func() {
-			r.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler)
-			r.Put("/objects/{oid}/{size}", lfs.UploadHandler)
-			r.Get("/objects/{oid}/{filename}", lfs.DownloadHandler)
-			r.Get("/objects/{oid}", lfs.DownloadHandler)
-			r.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler)
-			r.Group("/locks", func() {
-				r.Get("/", lfs.GetListLockHandler)
-				r.Post("/", lfs.PostLockHandler)
-				r.Post("/verify", lfs.VerifyLockHandler)
-				r.Post("/{lid}/unlock", lfs.UnLockHandler)
-			}, lfs.CheckAcceptMediaType)
-			r.Any("/*", func(ctx *context.Context) {
-				ctx.NotFound("", nil)
-			})
-		}, swapAuthToken)
-	}, common.Sessioner(), context.Contexter())
-	// end "/repo/{username}/{reponame}": git (LFS) API mirror
+	r.Group("/repo", func() {
+		// FIXME: it is not right to use context.Contexter here because all routes here should use PrivateContext
+		common.AddOwnerRepoGitLFSRoutes(r, func(ctx *context.PrivateContext) {
+			webContext := &context.Context{Base: ctx.Base}
+			ctx.AppendContextValue(context.WebContextKey, webContext)
+		}, web.RouterMockPoint(RouterMockPointInternalLFS))
+	})
 
 	return r
 }

routers/web/web.go

@@ -44,7 +44,6 @@ import (
 	auth_service "code.gitea.io/gitea/services/auth"
 	"code.gitea.io/gitea/services/context"
 	"code.gitea.io/gitea/services/forms"
-	"code.gitea.io/gitea/services/lfs"
 
 	_ "code.gitea.io/gitea/modules/session" // to registers all internal adapters
 
@@ -1598,23 +1597,8 @@ func registerRoutes(m *web.Router) {
 		m.Post("/action/{action}", reqSignIn, repo.Action)
 	}, ignSignIn, context.RepoAssignment, context.RepoRef())
 
+	common.AddOwnerRepoGitLFSRoutes(m, ignSignInAndCsrf, lfsServerEnabled)
 	m.Group("/{username}/{reponame}", func() {
-		m.Group("/info/lfs", func() {
-			m.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler)
-			m.Put("/objects/{oid}/{size}", lfs.UploadHandler)
-			m.Get("/objects/{oid}/{filename}", lfs.DownloadHandler)
-			m.Get("/objects/{oid}", lfs.DownloadHandler)
-			m.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler)
-			m.Group("/locks", func() {
-				m.Get("/", lfs.GetListLockHandler)
-				m.Post("/", lfs.PostLockHandler)
-				m.Post("/verify", lfs.VerifyLockHandler)
-				m.Post("/{lid}/unlock", lfs.UnLockHandler)
-			}, lfs.CheckAcceptMediaType)
-			m.Any("/*", func(ctx *context.Context) {
-				ctx.NotFound("", nil)
-			})
-		}, ignSignInAndCsrf, lfsServerEnabled)
 		gitHTTPRouters(m)
 	})
 	// end "/{username}/{reponame}.git": git support