Add API endpoint to request contents of multiple files simultaniously (#34139)

Adds an API POST endpoint under `/repos/{owner}/{repo}/file-contents`
which receives a list of paths and returns a list of the contents of
these files.

This API endpoint will be helpful for applications like headless CMS
(reference: https://github.com/sveltia/sveltia-cms/issues/198) which
need to retrieve a large number of files by reducing the amount of
needed API calls.

Close #33495

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

routers/api/v1/api.go

@@ -1389,14 +1389,17 @@ func Routes() *web.Router {
 				m.Post("/diffpatch", reqRepoWriter(unit.TypeCode), reqToken(), bind(api.ApplyDiffPatchFileOptions{}), mustNotBeArchived, repo.ApplyDiffPatch)
 				m.Group("/contents", func() {
 					m.Get("", repo.GetContentsList)
-					m.Post("", reqToken(), bind(api.ChangeFilesOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.ChangeFiles)
 					m.Get("/*", repo.GetContents)
+					m.Post("", reqToken(), bind(api.ChangeFilesOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.ChangeFiles)
 					m.Group("/*", func() {
 						m.Post("", bind(api.CreateFileOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.CreateFile)
 						m.Put("", bind(api.UpdateFileOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.UpdateFile)
 						m.Delete("", bind(api.DeleteFileOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.DeleteFile)
 					}, reqToken())
-				}, reqRepoReader(unit.TypeCode))
+				}, reqRepoReader(unit.TypeCode), context.ReferencesGitRepo())
+				m.Combo("/file-contents", reqRepoReader(unit.TypeCode), context.ReferencesGitRepo()).
+					Get(repo.GetFileContentsGet).
+					Post(bind(api.GetFilesOptions{}), repo.GetFileContentsPost) // POST method requires "write" permission, so we also support "GET" method above
 				m.Get("/signing-key.gpg", misc.SigningKey)
 				m.Group("/topics", func() {
 					m.Combo("").Get(repo.ListTopics).

routers/api/v1/repo/file.go

@@ -16,16 +16,18 @@ import (
 
 	git_model "code.gitea.io/gitea/models/git"
 	repo_model "code.gitea.io/gitea/models/repo"
-	"code.gitea.io/gitea/models/unit"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/httpcache"
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 	api "code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web"
+	"code.gitea.io/gitea/routers/api/v1/utils"
 	"code.gitea.io/gitea/routers/common"
 	"code.gitea.io/gitea/services/context"
 	pull_service "code.gitea.io/gitea/services/pull"
@@ -375,7 +377,7 @@ func GetEditorconfig(ctx *context.APIContext) {
 	//   required: true
 	// - name: ref
 	//   in: query
-	//   description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)"
+	//   description: "The name of the commit/branch/tag. Default to the repository’s default branch."
 	//   type: string
 	//   required: false
 	// responses:
@@ -410,11 +412,6 @@ func canWriteFiles(ctx *context.APIContext, branch string) bool {
 		!ctx.Repo.Repository.IsArchived
 }
 
-// canReadFiles returns true if repository is readable and user has proper access level.
-func canReadFiles(r *context.Repository) bool {
-	return r.Permission.CanRead(unit.TypeCode)
-}
-
 func base64Reader(s string) (io.ReadSeeker, error) {
 	b, err := base64.StdEncoding.DecodeString(s)
 	if err != nil {
@@ -894,6 +891,17 @@ func DeleteFile(ctx *context.APIContext) {
 	}
 }
 
+func resolveRefCommit(ctx *context.APIContext, ref string, minCommitIDLen ...int) *utils.RefCommit {
+	ref = util.IfZero(ref, ctx.Repo.Repository.DefaultBranch)
+	refCommit, err := utils.ResolveRefCommit(ctx, ctx.Repo.Repository, ref, minCommitIDLen...)
+	if errors.Is(err, util.ErrNotExist) {
+		ctx.APIErrorNotFound(err)
+	} else if err != nil {
+		ctx.APIErrorInternal(err)
+	}
+	return refCommit
+}
+
 // GetContents Get the metadata and contents (if a file) of an entry in a repository, or a list of entries if a dir
 func GetContents(ctx *context.APIContext) {
 	// swagger:operation GET /repos/{owner}/{repo}/contents/{filepath} repository repoGetContents
@@ -919,7 +927,7 @@ func GetContents(ctx *context.APIContext) {
 	//   required: true
 	// - name: ref
 	//   in: query
-	//   description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)"
+	//   description: "The name of the commit/branch/tag. Default to the repository’s default branch."
 	//   type: string
 	//   required: false
 	// responses:
@@ -928,18 +936,13 @@ func GetContents(ctx *context.APIContext) {
 	//   "404":
 	//     "$ref": "#/responses/notFound"
 
-	if !canReadFiles(ctx.Repo) {
-		ctx.APIErrorInternal(repo_model.ErrUserDoesNotHaveAccessToRepo{
-			UserID:   ctx.Doer.ID,
-			RepoName: ctx.Repo.Repository.LowerName,
-		})
+	treePath := ctx.PathParam("*")
+	refCommit := resolveRefCommit(ctx, ctx.FormTrim("ref"))
+	if ctx.Written() {
 		return
 	}
 
-	treePath := ctx.PathParam("*")
-	ref := ctx.FormTrim("ref")
-
-	if fileList, err := files_service.GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref); err != nil {
+	if fileList, err := files_service.GetContentsOrList(ctx, ctx.Repo.Repository, refCommit, treePath); err != nil {
 		if git.IsErrNotExist(err) {
 			ctx.APIErrorNotFound("GetContentsOrList", err)
 			return
@@ -970,7 +973,7 @@ func GetContentsList(ctx *context.APIContext) {
 	//   required: true
 	// - name: ref
 	//   in: query
-	//   description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)"
+	//   description: "The name of the commit/branch/tag. Default to the repository’s default branch."
 	//   type: string
 	//   required: false
 	// responses:
@@ -982,3 +985,102 @@ func GetContentsList(ctx *context.APIContext) {
 	// same as GetContents(), this function is here because swagger fails if path is empty in GetContents() interface
 	GetContents(ctx)
 }
+
+func GetFileContentsGet(ctx *context.APIContext) {
+	// swagger:operation GET /repos/{owner}/{repo}/file-contents repository repoGetFileContents
+	// ---
+	// summary: Get the metadata and contents of requested files
+	// description: See the POST method. This GET method supports to use JSON encoded request body in query parameter.
+	// produces:
+	// - application/json
+	// parameters:
+	// - name: owner
+	//   in: path
+	//   description: owner of the repo
+	//   type: string
+	//   required: true
+	// - name: repo
+	//   in: path
+	//   description: name of the repo
+	//   type: string
+	//   required: true
+	// - name: ref
+	//   in: query
+	//   description: "The name of the commit/branch/tag. Default to the repository’s default branch."
+	//   type: string
+	//   required: false
+	// - name: body
+	//   in: query
+	//   description: "The JSON encoded body (see the POST request): {\"files\": [\"filename1\", \"filename2\"]}"
+	//   type: string
+	//   required: true
+	// responses:
+	//   "200":
+	//     "$ref": "#/responses/ContentsListResponse"
+	//   "404":
+	//     "$ref": "#/responses/notFound"
+
+	// POST method requires "write" permission, so we also support this "GET" method
+	handleGetFileContents(ctx)
+}
+
+func GetFileContentsPost(ctx *context.APIContext) {
+	// swagger:operation POST /repos/{owner}/{repo}/file-contents repository repoGetFileContentsPost
+	// ---
+	// summary: Get the metadata and contents of requested files
+	// description: Uses automatic pagination based on default page size and
+	// 							max response size and returns the maximum allowed number of files.
+	//							Files which could not be retrieved are null. Files which are too large
+	//							are being returned with `encoding == null`, `content == null` and `size > 0`,
+	//							they can be requested separately by using the `download_url`.
+	// produces:
+	// - application/json
+	// parameters:
+	// - name: owner
+	//   in: path
+	//   description: owner of the repo
+	//   type: string
+	//   required: true
+	// - name: repo
+	//   in: path
+	//   description: name of the repo
+	//   type: string
+	//   required: true
+	// - name: ref
+	//   in: query
+	//   description: "The name of the commit/branch/tag. Default to the repository’s default branch."
+	//   type: string
+	//   required: false
+	// - name: body
+	//   in: body
+	//   required: true
+	//   schema:
+	//     "$ref": "#/definitions/GetFilesOptions"
+	// responses:
+	//   "200":
+	//     "$ref": "#/responses/ContentsListResponse"
+	//   "404":
+	//     "$ref": "#/responses/notFound"
+
+	// This is actually a "read" request, but we need to accept a "files" list, then POST method seems easy to use.
+	// But the permission system requires that the caller must have "write" permission to use POST method.
+	// At the moment there is no other way to get around the permission check, so there is a "GET" workaround method above.
+	handleGetFileContents(ctx)
+}
+
+func handleGetFileContents(ctx *context.APIContext) {
+	opts, ok := web.GetForm(ctx).(*api.GetFilesOptions)
+	if !ok {
+		err := json.Unmarshal(util.UnsafeStringToBytes(ctx.FormString("body")), &opts)
+		if err != nil {
+			ctx.APIError(http.StatusBadRequest, "invalid body parameter")
+			return
+		}
+	}
+	refCommit := resolveRefCommit(ctx, ctx.FormTrim("ref"))
+	if ctx.Written() {
+		return
+	}
+	filesResponse := files_service.GetContentsListFromTreePaths(ctx, ctx.Repo.Repository, refCommit, opts.Files)
+	ctx.JSON(http.StatusOK, util.SliceNilAsEmpty(filesResponse))
+}

routers/api/v1/repo/status.go

@@ -177,20 +177,14 @@ func GetCommitStatusesByRef(ctx *context.APIContext) {
 	//   "404":
 	//     "$ref": "#/responses/notFound"
 
-	filter := utils.ResolveRefOrSha(ctx, ctx.PathParam("ref"))
+	refCommit := resolveRefCommit(ctx, ctx.PathParam("ref"), 7)
 	if ctx.Written() {
 		return
 	}
-
-	getCommitStatuses(ctx, filter) // By default filter is maybe the raw SHA
+	getCommitStatuses(ctx, refCommit.CommitID)
 }
 
-func getCommitStatuses(ctx *context.APIContext, sha string) {
-	if len(sha) == 0 {
-		ctx.APIError(http.StatusBadRequest, nil)
-		return
-	}
-	sha = utils.MustConvertToSHA1(ctx.Base, ctx.Repo, sha)
+func getCommitStatuses(ctx *context.APIContext, commitID string) {
 	repo := ctx.Repo.Repository
 
 	listOptions := utils.GetListOptions(ctx)
@@ -198,12 +192,12 @@ func getCommitStatuses(ctx *context.APIContext, sha string) {
 	statuses, maxResults, err := db.FindAndCount[git_model.CommitStatus](ctx, &git_model.CommitStatusOptions{
 		ListOptions: listOptions,
 		RepoID:      repo.ID,
-		SHA:         sha,
+		SHA:         commitID,
 		SortType:    ctx.FormTrim("sort"),
 		State:       ctx.FormTrim("state"),
 	})
 	if err != nil {
-		ctx.APIErrorInternal(fmt.Errorf("GetCommitStatuses[%s, %s, %d]: %w", repo.FullName(), sha, ctx.FormInt("page"), err))
+		ctx.APIErrorInternal(fmt.Errorf("GetCommitStatuses[%s, %s, %d]: %w", repo.FullName(), commitID, ctx.FormInt("page"), err))
 		return
 	}
 
@@ -257,16 +251,16 @@ func GetCombinedCommitStatusByRef(ctx *context.APIContext) {
 	//   "404":
 	//     "$ref": "#/responses/notFound"
 
-	sha := utils.ResolveRefOrSha(ctx, ctx.PathParam("ref"))
+	refCommit := resolveRefCommit(ctx, ctx.PathParam("ref"), 7)
 	if ctx.Written() {
 		return
 	}
 
 	repo := ctx.Repo.Repository
 
-	statuses, count, err := git_model.GetLatestCommitStatus(ctx, repo.ID, sha, utils.GetListOptions(ctx))
+	statuses, count, err := git_model.GetLatestCommitStatus(ctx, repo.ID, refCommit.Commit.ID.String(), utils.GetListOptions(ctx))
 	if err != nil {
-		ctx.APIErrorInternal(fmt.Errorf("GetLatestCommitStatus[%s, %s]: %w", repo.FullName(), sha, err))
+		ctx.APIErrorInternal(fmt.Errorf("GetLatestCommitStatus[%s, %s]: %w", repo.FullName(), refCommit.CommitID, err))
 		return
 	}
 

routers/api/v1/settings/settings.go

@@ -43,6 +43,7 @@ func GetGeneralAPISettings(ctx *context.APIContext) {
 		DefaultPagingNum:       setting.API.DefaultPagingNum,
 		DefaultGitTreesPerPage: setting.API.DefaultGitTreesPerPage,
 		DefaultMaxBlobSize:     setting.API.DefaultMaxBlobSize,
+		DefaultMaxResponseSize: setting.API.DefaultMaxResponseSize,
 	})
 }
 

routers/api/v1/swagger/options.go

@@ -118,6 +118,9 @@ type swaggerParameterBodies struct {
 	// in:body
 	EditAttachmentOptions api.EditAttachmentOptions
 
+	// in:body
+	GetFilesOptions api.GetFilesOptions
+
 	// in:body
 	ChangeFilesOptions api.ChangeFilesOptions
 

routers/api/v1/utils/git.go

@@ -4,48 +4,48 @@
 package utils
 
 import (
-	gocontext "context"
 	"errors"
-	"fmt"
-	"net/http"
 
+	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/gitrepo"
-	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/reqctx"
 	"code.gitea.io/gitea/services/context"
 )
 
-// ResolveRefOrSha resolve ref to sha if exist
-func ResolveRefOrSha(ctx *context.APIContext, ref string) string {
-	if len(ref) == 0 {
-		ctx.APIError(http.StatusBadRequest, nil)
-		return ""
+type RefCommit struct {
+	InputRef string
+	RefName  git.RefName
+	Commit   *git.Commit
+	CommitID string
+}
+
+// ResolveRefCommit resolve ref to a commit if exist
+func ResolveRefCommit(ctx reqctx.RequestContext, repo *repo_model.Repository, inputRef string, minCommitIDLen ...int) (_ *RefCommit, err error) {
+	gitRepo, err := gitrepo.RepositoryFromRequestContextOrOpen(ctx, repo)
+	if err != nil {
+		return nil, err
 	}
-
-	sha := ref
-	// Search branches and tags
-	for _, refType := range []string{"heads", "tags"} {
-		refSHA, lastMethodName, err := searchRefCommitByType(ctx, refType, ref)
-		if err != nil {
-			ctx.APIErrorInternal(fmt.Errorf("%s: %w", lastMethodName, err))
-			return ""
-		}
-		if refSHA != "" {
-			sha = refSHA
-			break
-		}
+	refCommit := RefCommit{InputRef: inputRef}
+	if gitrepo.IsBranchExist(ctx, repo, inputRef) {
+		refCommit.RefName = git.RefNameFromBranch(inputRef)
+	} else if gitrepo.IsTagExist(ctx, repo, inputRef) {
+		refCommit.RefName = git.RefNameFromTag(inputRef)
+	} else if git.IsStringLikelyCommitID(git.ObjectFormatFromName(repo.ObjectFormatName), inputRef, minCommitIDLen...) {
+		refCommit.RefName = git.RefNameFromCommit(inputRef)
 	}
-
-	sha = MustConvertToSHA1(ctx, ctx.Repo, sha)
-
-	if ctx.Repo.GitRepo != nil {
-		err := ctx.Repo.GitRepo.AddLastCommitCache(ctx.Repo.Repository.GetCommitsCountCacheKey(ref, ref != sha), ctx.Repo.Repository.FullName(), sha)
-		if err != nil {
-			log.Error("Unable to get commits count for %s in %s. Error: %v", sha, ctx.Repo.Repository.FullName(), err)
-		}
+	if refCommit.RefName == "" {
+		return nil, git.ErrNotExist{ID: inputRef}
 	}
+	if refCommit.Commit, err = gitRepo.GetCommit(refCommit.RefName.String()); err != nil {
+		return nil, err
+	}
+	refCommit.CommitID = refCommit.Commit.ID.String()
+	return &refCommit, nil
+}
 
-	return sha
+func NewRefCommit(refName git.RefName, commit *git.Commit) *RefCommit {
+	return &RefCommit{InputRef: refName.ShortName(), RefName: refName, Commit: commit, CommitID: commit.ID.String()}
 }
 
 // GetGitRefs return git references based on filter
@@ -59,42 +59,3 @@ func GetGitRefs(ctx *context.APIContext, filter string) ([]*git.Reference, strin
 	refs, err := ctx.Repo.GitRepo.GetRefsFiltered(filter)
 	return refs, "GetRefsFiltered", err
 }
-
-func searchRefCommitByType(ctx *context.APIContext, refType, filter string) (string, string, error) {
-	refs, lastMethodName, err := GetGitRefs(ctx, refType+"/"+filter) // Search by type
-	if err != nil {
-		return "", lastMethodName, err
-	}
-	if len(refs) > 0 {
-		return refs[0].Object.String(), "", nil // Return found SHA
-	}
-	return "", "", nil
-}
-
-// ConvertToObjectID returns a full-length SHA1 from a potential ID string
-func ConvertToObjectID(ctx gocontext.Context, repo *context.Repository, commitID string) (git.ObjectID, error) {
-	objectFormat := repo.GetObjectFormat()
-	if len(commitID) == objectFormat.FullLength() && objectFormat.IsValid(commitID) {
-		sha, err := git.NewIDFromString(commitID)
-		if err == nil {
-			return sha, nil
-		}
-	}
-
-	gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo.Repository)
-	if err != nil {
-		return objectFormat.EmptyObjectID(), fmt.Errorf("RepositoryFromContextOrOpen: %w", err)
-	}
-	defer closer.Close()
-
-	return gitRepo.ConvertToGitID(commitID)
-}
-
-// MustConvertToSHA1 returns a full-length SHA1 string from a potential ID string, or returns origin input if it can't convert to SHA1
-func MustConvertToSHA1(ctx gocontext.Context, repo *context.Repository, commitID string) string {
-	sha, err := ConvertToObjectID(ctx, repo, commitID)
-	if err != nil {
-		return commitID
-	}
-	return sha.String()
-}