Refactor request context (#32956)

Introduce RequestContext: is a short-lived context that is used to store
request-specific data.

RequestContext could be used to clean form tmp files, close context git
repo, and do some tracing in the future.

Then a lot of legacy code could be removed or improved. For example:
most `ctx.Repo.GitRepo.Close()` could be removed because the git repo
could be closed when the request is done.

routers/api/actions/artifacts.go

@@ -126,11 +126,10 @@ func ArtifactsRoutes(prefix string) *web.Router {
 func ArtifactContexter() func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-			base, baseCleanUp := context.NewBaseContext(resp, req)
-			defer baseCleanUp()
+			base := context.NewBaseContext(resp, req)
 
 			ctx := &ArtifactContext{Base: base}
-			ctx.AppendContextValue(artifactContextKey, ctx)
+			ctx.SetContextValue(artifactContextKey, ctx)
 
 			// action task call server api with Bearer ACTIONS_RUNTIME_TOKEN
 			// we should verify the ACTIONS_RUNTIME_TOKEN

routers/api/actions/artifactsv4.go

@@ -126,12 +126,9 @@ type artifactV4Routes struct {
 func ArtifactV4Contexter() func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-			base, baseCleanUp := context.NewBaseContext(resp, req)
-			defer baseCleanUp()
-
+			base := context.NewBaseContext(resp, req)
 			ctx := &ArtifactContext{Base: base}
-			ctx.AppendContextValue(artifactContextKey, ctx)
-
+			ctx.SetContextValue(artifactContextKey, ctx)
 			next.ServeHTTP(ctx.Resp, ctx.Req)
 		})
 	}

routers/api/v1/repo/branch.go

@@ -729,15 +729,11 @@ func CreateBranchProtection(ctx *context.APIContext) {
 	} else {
 		if !isPlainRule {
 			if ctx.Repo.GitRepo == nil {
-				ctx.Repo.GitRepo, err = gitrepo.OpenRepository(ctx, ctx.Repo.Repository)
+				ctx.Repo.GitRepo, err = gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx, ctx.Repo.Repository)
 				if err != nil {
 					ctx.Error(http.StatusInternalServerError, "OpenRepository", err)
 					return
 				}
-				defer func() {
-					ctx.Repo.GitRepo.Close()
-					ctx.Repo.GitRepo = nil
-				}()
 			}
 			// FIXME: since we only need to recheck files protected rules, we could improve this
 			matchedBranches, err := git_model.FindAllMatchedBranches(ctx, ctx.Repo.Repository.ID, ruleName)
@@ -1061,15 +1057,11 @@ func EditBranchProtection(ctx *context.APIContext) {
 	} else {
 		if !isPlainRule {
 			if ctx.Repo.GitRepo == nil {
-				ctx.Repo.GitRepo, err = gitrepo.OpenRepository(ctx, ctx.Repo.Repository)
+				ctx.Repo.GitRepo, err = gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx, ctx.Repo.Repository)
 				if err != nil {
 					ctx.Error(http.StatusInternalServerError, "OpenRepository", err)
 					return
 				}
-				defer func() {
-					ctx.Repo.GitRepo.Close()
-					ctx.Repo.GitRepo = nil
-				}()
 			}
 
 			// FIXME: since we only need to recheck files protected rules, we could improve this

routers/api/v1/repo/compare.go

@@ -44,13 +44,12 @@ func CompareDiff(ctx *context.APIContext) {
 	//     "$ref": "#/responses/notFound"
 
 	if ctx.Repo.GitRepo == nil {
-		gitRepo, err := gitrepo.OpenRepository(ctx, ctx.Repo.Repository)
+		var err error
+		ctx.Repo.GitRepo, err = gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx, ctx.Repo.Repository)
 		if err != nil {
 			ctx.Error(http.StatusInternalServerError, "OpenRepository", err)
 			return
 		}
-		ctx.Repo.GitRepo = gitRepo
-		defer gitRepo.Close()
 	}
 
 	infoPath := ctx.PathParam("*")

routers/api/v1/repo/download.go

@@ -28,13 +28,12 @@ func DownloadArchive(ctx *context.APIContext) {
 	}
 
 	if ctx.Repo.GitRepo == nil {
-		gitRepo, err := gitrepo.OpenRepository(ctx, ctx.Repo.Repository)
+		var err error
+		ctx.Repo.GitRepo, err = gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx, ctx.Repo.Repository)
 		if err != nil {
 			ctx.Error(http.StatusInternalServerError, "OpenRepository", err)
 			return
 		}
-		ctx.Repo.GitRepo = gitRepo
-		defer gitRepo.Close()
 	}
 
 	r, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, ctx.PathParam("*"), tp)

routers/api/v1/repo/file.go

@@ -287,13 +287,12 @@ func GetArchive(ctx *context.APIContext) {
 	//     "$ref": "#/responses/notFound"
 
 	if ctx.Repo.GitRepo == nil {
-		gitRepo, err := gitrepo.OpenRepository(ctx, ctx.Repo.Repository)
+		var err error
+		ctx.Repo.GitRepo, err = gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx, ctx.Repo.Repository)
 		if err != nil {
 			ctx.Error(http.StatusInternalServerError, "OpenRepository", err)
 			return
 		}
-		ctx.Repo.GitRepo = gitRepo
-		defer gitRepo.Close()
 	}
 
 	archiveDownload(ctx)

routers/api/v1/repo/repo.go

@@ -726,12 +726,11 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
 
 	if ctx.Repo.GitRepo == nil && !repo.IsEmpty {
 		var err error
-		ctx.Repo.GitRepo, err = gitrepo.OpenRepository(ctx, repo)
+		ctx.Repo.GitRepo, err = gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx, repo)
 		if err != nil {
 			ctx.Error(http.StatusInternalServerError, "Unable to OpenRepository", err)
 			return err
 		}
-		defer ctx.Repo.GitRepo.Close()
 	}
 
 	// Default branch only updated if changed and exist or the repository is empty

routers/api/v1/repo/transfer.go

@@ -100,7 +100,7 @@ func Transfer(ctx *context.APIContext) {
 	}
 
 	if ctx.Repo.GitRepo != nil {
-		ctx.Repo.GitRepo.Close()
+		_ = ctx.Repo.GitRepo.Close()
 		ctx.Repo.GitRepo = nil
 	}
 

routers/common/errpage_test.go

@@ -12,8 +12,8 @@ import (
 	"testing"
 
 	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/reqctx"
 	"code.gitea.io/gitea/modules/test"
-	"code.gitea.io/gitea/modules/web/middleware"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -21,7 +21,7 @@ import (
 func TestRenderPanicErrorPage(t *testing.T) {
 	w := httptest.NewRecorder()
 	req := &http.Request{URL: &url.URL{}}
-	req = req.WithContext(middleware.WithContextData(context.Background()))
+	req = req.WithContext(reqctx.NewRequestContextForTest(context.Background()))
 	RenderPanicErrorPage(w, req, errors.New("fake panic error (for test only)"))
 	respContent := w.Body.String()
 	assert.Contains(t, respContent, `class="page-content status-page-500"`)

routers/common/middleware.go

@@ -4,16 +4,14 @@
 package common
 
 import (
-	go_context "context"
 	"fmt"
 	"net/http"
 	"strings"
 
 	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/httplib"
-	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/reqctx"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/web/middleware"
 	"code.gitea.io/gitea/modules/web/routing"
 	"code.gitea.io/gitea/services/context"
 
@@ -24,54 +22,12 @@ import (
 
 // ProtocolMiddlewares returns HTTP protocol related middlewares, and it provides a global panic recovery
 func ProtocolMiddlewares() (handlers []any) {
-	// make sure chi uses EscapedPath(RawPath) as RoutePath, then "%2f" could be handled correctly
-	handlers = append(handlers, func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-			ctx := chi.RouteContext(req.Context())
-			if req.URL.RawPath == "" {
-				ctx.RoutePath = req.URL.EscapedPath()
-			} else {
-				ctx.RoutePath = req.URL.RawPath
-			}
-			next.ServeHTTP(resp, req)
-		})
-	})
+	// the order is important
+	handlers = append(handlers, ChiRoutePathHandler())   // make sure chi has correct paths
+	handlers = append(handlers, RequestContextHandler()) //	// prepare the context and panic recovery
 
-	// prepare the ContextData and panic recovery
-	handlers = append(handlers, func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-			defer func() {
-				if err := recover(); err != nil {
-					RenderPanicErrorPage(resp, req, err) // it should never panic
-				}
-			}()
-			req = req.WithContext(middleware.WithContextData(req.Context()))
-			req = req.WithContext(go_context.WithValue(req.Context(), httplib.RequestContextKey, req))
-			next.ServeHTTP(resp, req)
-		})
-	})
-
-	// wrap the request and response, use the process context and add it to the process manager
-	handlers = append(handlers, func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-			ctx, _, finished := process.GetManager().AddTypedContext(req.Context(), fmt.Sprintf("%s: %s", req.Method, req.RequestURI), process.RequestProcessType, true)
-			defer finished()
-			next.ServeHTTP(context.WrapResponseWriter(resp), req.WithContext(cache.WithCacheContext(ctx)))
-		})
-	})
-
-	if setting.ReverseProxyLimit > 0 {
-		opt := proxy.NewForwardedHeadersOptions().
-			WithForwardLimit(setting.ReverseProxyLimit).
-			ClearTrustedProxies()
-		for _, n := range setting.ReverseProxyTrustedProxies {
-			if !strings.Contains(n, "/") {
-				opt.AddTrustedProxy(n)
-			} else {
-				opt.AddTrustedNetwork(n)
-			}
-		}
-		handlers = append(handlers, proxy.ForwardedHeaders(opt))
+	if setting.ReverseProxyLimit > 0 && len(setting.ReverseProxyTrustedProxies) > 0 {
+		handlers = append(handlers, ForwardedHeadersHandler(setting.ReverseProxyLimit, setting.ReverseProxyTrustedProxies))
 	}
 
 	if setting.IsRouteLogEnabled() {
@@ -85,6 +41,59 @@ func ProtocolMiddlewares() (handlers []any) {
 	return handlers
 }
 
+func RequestContextHandler() func(h http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+			profDesc := fmt.Sprintf("%s: %s", req.Method, req.RequestURI)
+			ctx, finished := reqctx.NewRequestContext(req.Context(), profDesc)
+			defer finished()
+
+			defer func() {
+				if err := recover(); err != nil {
+					RenderPanicErrorPage(resp, req, err) // it should never panic
+				}
+			}()
+
+			ds := reqctx.GetRequestDataStore(ctx)
+			req = req.WithContext(cache.WithCacheContext(ctx))
+			ds.SetContextValue(httplib.RequestContextKey, req)
+			ds.AddCleanUp(func() {
+				if req.MultipartForm != nil {
+					_ = req.MultipartForm.RemoveAll() // remove the temp files buffered to tmp directory
+				}
+			})
+			next.ServeHTTP(context.WrapResponseWriter(resp), req)
+		})
+	}
+}
+
+func ChiRoutePathHandler() func(h http.Handler) http.Handler {
+	// make sure chi uses EscapedPath(RawPath) as RoutePath, then "%2f" could be handled correctly
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+			ctx := chi.RouteContext(req.Context())
+			if req.URL.RawPath == "" {
+				ctx.RoutePath = req.URL.EscapedPath()
+			} else {
+				ctx.RoutePath = req.URL.RawPath
+			}
+			next.ServeHTTP(resp, req)
+		})
+	}
+}
+
+func ForwardedHeadersHandler(limit int, trustedProxies []string) func(h http.Handler) http.Handler {
+	opt := proxy.NewForwardedHeadersOptions().WithForwardLimit(limit).ClearTrustedProxies()
+	for _, n := range trustedProxies {
+		if !strings.Contains(n, "/") {
+			opt.AddTrustedProxy(n)
+		} else {
+			opt.AddTrustedNetwork(n)
+		}
+	}
+	return proxy.ForwardedHeaders(opt)
+}
+
 func Sessioner() func(next http.Handler) http.Handler {
 	return session.Sessioner(session.Options{
 		Provider:       setting.SessionConfig.Provider,

routers/install/install.go

@@ -25,6 +25,7 @@ import (
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/optional"
+	"code.gitea.io/gitea/modules/reqctx"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/templates"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -61,15 +62,11 @@ func Contexter() func(next http.Handler) http.Handler {
 	envConfigKeys := setting.CollectEnvConfigKeys()
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-			base, baseCleanUp := context.NewBaseContext(resp, req)
-			defer baseCleanUp()
-
+			base := context.NewBaseContext(resp, req)
 			ctx := context.NewWebContext(base, rnd, session.GetSession(req))
-			ctx.AppendContextValue(context.WebContextKey, ctx)
+			ctx.SetContextValue(context.WebContextKey, ctx)
 			ctx.Data.MergeFrom(middleware.CommonTemplateContextData())
-			ctx.Data.MergeFrom(middleware.ContextData{
-				"Context":        ctx, // TODO: use "ctx" in template and remove this
-				"locale":         ctx.Locale,
+			ctx.Data.MergeFrom(reqctx.ContextData{
 				"Title":          ctx.Locale.Tr("install.install"),
 				"PageIsInstall":  true,
 				"DbTypeNames":    dbTypeNames,

routers/private/internal.go

@@ -63,8 +63,8 @@ func Routes() *web.Router {
 	r.Post("/ssh/{id}/update/{repoid}", UpdatePublicKeyInRepo)
 	r.Post("/ssh/log", bind(private.SSHLogOption{}), SSHLog)
 	r.Post("/hook/pre-receive/{owner}/{repo}", RepoAssignment, bind(private.HookOptions{}), HookPreReceive)
-	r.Post("/hook/post-receive/{owner}/{repo}", context.OverrideContext, bind(private.HookOptions{}), HookPostReceive)
-	r.Post("/hook/proc-receive/{owner}/{repo}", context.OverrideContext, RepoAssignment, bind(private.HookOptions{}), HookProcReceive)
+	r.Post("/hook/post-receive/{owner}/{repo}", context.OverrideContext(), bind(private.HookOptions{}), HookPostReceive)
+	r.Post("/hook/proc-receive/{owner}/{repo}", context.OverrideContext(), RepoAssignment, bind(private.HookOptions{}), HookProcReceive)
 	r.Post("/hook/set-default-branch/{owner}/{repo}/{branch}", RepoAssignment, SetDefaultBranch)
 	r.Get("/serv/none/{keyid}", ServNoCommand)
 	r.Get("/serv/command/{keyid}/{owner}/{repo}", ServCommand)
@@ -88,7 +88,7 @@ func Routes() *web.Router {
 		// Fortunately, the LFS handlers are able to handle requests without a complete web context
 		common.AddOwnerRepoGitLFSRoutes(r, func(ctx *context.PrivateContext) {
 			webContext := &context.Context{Base: ctx.Base}
-			ctx.AppendContextValue(context.WebContextKey, webContext)
+			ctx.SetContextValue(context.WebContextKey, webContext)
 		})
 	})
 

routers/private/internal_repo.go

@@ -4,7 +4,6 @@
 package private
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 
@@ -17,40 +16,29 @@ import (
 
 // This file contains common functions relating to setting the Repository for the internal routes
 
-// RepoAssignment assigns the repository and gitrepository to the private context
-func RepoAssignment(ctx *gitea_context.PrivateContext) context.CancelFunc {
+// RepoAssignment assigns the repository and git repository to the private context
+func RepoAssignment(ctx *gitea_context.PrivateContext) {
 	ownerName := ctx.PathParam(":owner")
 	repoName := ctx.PathParam(":repo")
 
 	repo := loadRepository(ctx, ownerName, repoName)
 	if ctx.Written() {
 		// Error handled in loadRepository
-		return nil
+		return
 	}
 
-	gitRepo, err := gitrepo.OpenRepository(ctx, repo)
+	gitRepo, err := gitrepo.RepositoryFromRequestContextOrOpen(ctx, ctx, repo)
 	if err != nil {
 		log.Error("Failed to open repository: %s/%s Error: %v", ownerName, repoName, err)
 		ctx.JSON(http.StatusInternalServerError, private.Response{
 			Err: fmt.Sprintf("Failed to open repository: %s/%s Error: %v", ownerName, repoName, err),
 		})
-		return nil
+		return
 	}
-
 	ctx.Repo = &gitea_context.Repository{
 		Repository: repo,
 		GitRepo:    gitRepo,
 	}
-
-	// We opened it, we should close it
-	cancel := func() {
-		// If it's been set to nil then assume someone else has closed it.
-		if ctx.Repo.GitRepo != nil {
-			ctx.Repo.GitRepo.Close()
-		}
-	}
-
-	return cancel
 }
 
 func loadRepository(ctx *gitea_context.PrivateContext, ownerName, repoName string) *repo_model.Repository {

routers/web/web.go

@@ -4,7 +4,6 @@
 package web
 
 import (
-	gocontext "context"
 	"net/http"
 	"strings"
 
@@ -1521,24 +1520,23 @@ func registerRoutes(m *web.Router) {
 
 		m.Group("/blob_excerpt", func() {
 			m.Get("/{sha}", repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.ExcerptBlob)
-		}, func(ctx *context.Context) gocontext.CancelFunc {
+		}, func(ctx *context.Context) {
 			// FIXME: refactor this function, use separate routes for wiki/code
 			if ctx.FormBool("wiki") {
 				ctx.Data["PageIsWiki"] = true
 				repo.MustEnableWiki(ctx)
-				return nil
+				return
 			}
 
 			if ctx.Written() {
-				return nil
+				return
 			}
-			cancel := context.RepoRef()(ctx)
+			context.RepoRef()(ctx)
 			if ctx.Written() {
-				return cancel
+				return
 			}
 
 			repo.MustBeNotEmpty(ctx)
-			return cancel
 		})
 
 		m.Group("/media", func() {