Refactor repo contents API and add "contents-ext" API (#34822)

See the updated swagger document for details.

services/repository/files/content.go

@@ -5,13 +5,14 @@ package files
 
 import (
 	"context"
-	"fmt"
+	"io"
 	"net/url"
 	"path"
+	"strings"
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
-	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
@@ -34,54 +35,50 @@ func (ct *ContentType) String() string {
 	return string(*ct)
 }
 
+type GetContentsOrListOptions struct {
+	TreePath                 string
+	IncludeSingleFileContent bool // include the file's content when the tree path is a file
+	IncludeLfsMetadata       bool
+}
+
 // GetContentsOrList gets the metadata of a file's contents (*ContentsResponse) if treePath not a tree
 // directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag
-func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, refCommit *utils.RefCommit, treePath string) (any, error) {
-	if repo.IsEmpty {
-		return make([]any, 0), nil
+func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, opts GetContentsOrListOptions) (ret api.ContentsExtResponse, _ error) {
+	entry, err := prepareGetContentsEntry(refCommit, &opts.TreePath)
+	if repo.IsEmpty && opts.TreePath == "" {
+		return api.ContentsExtResponse{DirContents: make([]*api.ContentsResponse, 0)}, nil
 	}
-
-	// Check that the path given in opts.treePath is valid (not a git path)
-	cleanTreePath := CleanGitTreePath(treePath)
-	if cleanTreePath == "" && treePath != "" {
-		return nil, ErrFilenameInvalid{
-			Path: treePath,
-		}
-	}
-	treePath = cleanTreePath
-
-	// Get the commit object for the ref
-	commit := refCommit.Commit
-
-	entry, err := commit.GetTreeEntryByPath(treePath)
 	if err != nil {
-		return nil, err
+		return ret, err
 	}
 
+	// get file contents
 	if entry.Type() != "tree" {
-		return GetContents(ctx, repo, refCommit, treePath, false)
+		ret.FileContents, err = getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts)
+		return ret, err
 	}
 
-	// We are in a directory, so we return a list of FileContentResponse objects
-	var fileList []*api.ContentsResponse
-
-	gitTree, err := commit.SubTree(treePath)
+	// list directory contents
+	gitTree, err := refCommit.Commit.SubTree(opts.TreePath)
 	if err != nil {
-		return nil, err
+		return ret, err
 	}
 	entries, err := gitTree.ListEntries()
 	if err != nil {
-		return nil, err
+		return ret, err
 	}
+	ret.DirContents = make([]*api.ContentsResponse, 0, len(entries))
 	for _, e := range entries {
-		subTreePath := path.Join(treePath, e.Name())
-		fileContentResponse, err := GetContents(ctx, repo, refCommit, subTreePath, true)
+		subOpts := opts
+		subOpts.TreePath = path.Join(opts.TreePath, e.Name())
+		subOpts.IncludeSingleFileContent = false // never include file content when listing a directory
+		fileContentResponse, err := GetFileContents(ctx, repo, gitRepo, refCommit, subOpts)
 		if err != nil {
-			return nil, err
+			return ret, err
 		}
-		fileList = append(fileList, fileContentResponse)
+		ret.DirContents = append(ret.DirContents, fileContentResponse)
 	}
-	return fileList, nil
+	return ret, nil
 }
 
 // GetObjectTypeFromTreeEntry check what content is behind it
@@ -100,35 +97,36 @@ func GetObjectTypeFromTreeEntry(entry *git.TreeEntry) ContentType {
 	}
 }
 
-// GetContents gets the metadata on a file's contents. Ref can be a branch, commit or tag
-func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *utils.RefCommit, treePath string, forList bool) (*api.ContentsResponse, error) {
+func prepareGetContentsEntry(refCommit *utils.RefCommit, treePath *string) (*git.TreeEntry, error) {
 	// Check that the path given in opts.treePath is valid (not a git path)
-	cleanTreePath := CleanGitTreePath(treePath)
-	if cleanTreePath == "" && treePath != "" {
-		return nil, ErrFilenameInvalid{
-			Path: treePath,
-		}
-	}
-	treePath = cleanTreePath
-
-	gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
-	if err != nil {
-		return nil, err
-	}
-	defer closer.Close()
-
-	commit := refCommit.Commit
-	entry, err := commit.GetTreeEntryByPath(treePath)
-	if err != nil {
-		return nil, err
+	cleanTreePath := CleanGitTreePath(*treePath)
+	if cleanTreePath == "" && *treePath != "" {
+		return nil, ErrFilenameInvalid{Path: *treePath}
 	}
+	*treePath = cleanTreePath
 
+	// Only allow safe ref types
 	refType := refCommit.RefName.RefType()
 	if refType != git.RefTypeBranch && refType != git.RefTypeTag && refType != git.RefTypeCommit {
-		return nil, fmt.Errorf("no commit found for the ref [ref: %s]", refCommit.RefName)
+		return nil, util.NewNotExistErrorf("no commit found for the ref [ref: %s]", refCommit.RefName)
 	}
 
-	selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(treePath) + "?ref=" + url.QueryEscape(refCommit.InputRef))
+	return refCommit.Commit.GetTreeEntryByPath(*treePath)
+}
+
+// GetFileContents gets the metadata on a file's contents. Ref can be a branch, commit or tag
+func GetFileContents(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, opts GetContentsOrListOptions) (*api.ContentsResponse, error) {
+	entry, err := prepareGetContentsEntry(refCommit, &opts.TreePath)
+	if err != nil {
+		return nil, err
+	}
+	return getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts)
+}
+
+func getFileContentsByEntryInternal(_ context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, entry *git.TreeEntry, opts GetContentsOrListOptions) (*api.ContentsResponse, error) {
+	refType := refCommit.RefName.RefType()
+	commit := refCommit.Commit
+	selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(opts.TreePath) + "?ref=" + url.QueryEscape(refCommit.InputRef))
 	if err != nil {
 		return nil, err
 	}
@@ -139,7 +137,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *ut
 		return nil, err
 	}
 
-	lastCommit, err := commit.GetCommitByPath(treePath)
+	lastCommit, err := refCommit.Commit.GetCommitByPath(opts.TreePath)
 	if err != nil {
 		return nil, err
 	}
@@ -147,7 +145,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *ut
 	// All content types have these fields in populated
 	contentsResponse := &api.ContentsResponse{
 		Name:          entry.Name(),
-		Path:          treePath,
+		Path:          opts.TreePath,
 		SHA:           entry.ID.String(),
 		LastCommitSHA: lastCommit.ID.String(),
 		Size:          entry.Size(),
@@ -170,13 +168,18 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *ut
 	if entry.IsRegular() || entry.IsExecutable() {
 		contentsResponse.Type = string(ContentTypeRegular)
 		// if it is listing the repo root dir, don't waste system resources on reading content
-		if !forList {
-			blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String())
+		if opts.IncludeSingleFileContent {
+			blobResponse, err := GetBlobBySHA(repo, gitRepo, entry.ID.String())
+			if err != nil {
+				return nil, err
+			}
+			contentsResponse.Encoding, contentsResponse.Content = blobResponse.Encoding, blobResponse.Content
+			contentsResponse.LfsOid, contentsResponse.LfsSize = blobResponse.LfsOid, blobResponse.LfsSize
+		} else if opts.IncludeLfsMetadata {
+			contentsResponse.LfsOid, contentsResponse.LfsSize, err = parsePossibleLfsPointerBlob(gitRepo, entry.ID.String())
 			if err != nil {
 				return nil, err
 			}
-			contentsResponse.Encoding = blobResponse.Encoding
-			contentsResponse.Content = blobResponse.Content
 		}
 	} else if entry.IsDir() {
 		contentsResponse.Type = string(ContentTypeDir)
@@ -190,7 +193,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *ut
 		contentsResponse.Target = &targetFromContent
 	} else if entry.IsSubModule() {
 		contentsResponse.Type = string(ContentTypeSubmodule)
-		submodule, err := commit.GetSubModule(treePath)
+		submodule, err := commit.GetSubModule(opts.TreePath)
 		if err != nil {
 			return nil, err
 		}
@@ -200,7 +203,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *ut
 	}
 	// Handle links
 	if entry.IsRegular() || entry.IsLink() || entry.IsExecutable() {
-		downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(treePath))
+		downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(opts.TreePath))
 		if err != nil {
 			return nil, err
 		}
@@ -208,7 +211,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *ut
 		contentsResponse.DownloadURL = &downloadURLString
 	}
 	if !entry.IsSubModule() {
-		htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(treePath))
+		htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(opts.TreePath))
 		if err != nil {
 			return nil, err
 		}
@@ -228,8 +231,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *ut
 	return contentsResponse, nil
 }
 
-// GetBlobBySHA get the GitBlobResponse of a repository using a sha hash.
-func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) {
+func GetBlobBySHA(repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) {
 	gitBlob, err := gitRepo.GetBlob(sha)
 	if err != nil {
 		return nil, err
@@ -239,12 +241,49 @@ func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git
 		URL:  repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()),
 		Size: gitBlob.Size(),
 	}
-	if gitBlob.Size() <= setting.API.DefaultMaxBlobSize {
-		content, err := gitBlob.GetBlobContentBase64()
-		if err != nil {
-			return nil, err
-		}
-		ret.Encoding, ret.Content = util.ToPointer("base64"), &content
+
+	blobSize := gitBlob.Size()
+	if blobSize > setting.API.DefaultMaxBlobSize {
+		return ret, nil
+	}
+
+	var originContent *strings.Builder
+	if 0 < blobSize && blobSize < lfs.MetaFileMaxSize {
+		originContent = &strings.Builder{}
+	}
+
+	content, err := gitBlob.GetBlobContentBase64(originContent)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.Encoding, ret.Content = util.ToPointer("base64"), &content
+	if originContent != nil {
+		ret.LfsOid, ret.LfsSize = parsePossibleLfsPointerBuffer(strings.NewReader(originContent.String()))
 	}
 	return ret, nil
 }
+
+func parsePossibleLfsPointerBuffer(r io.Reader) (*string, *int64) {
+	p, _ := lfs.ReadPointer(r)
+	if p.IsValid() {
+		return &p.Oid, &p.Size
+	}
+	return nil, nil
+}
+
+func parsePossibleLfsPointerBlob(gitRepo *git.Repository, sha string) (*string, *int64, error) {
+	gitBlob, err := gitRepo.GetBlob(sha)
+	if err != nil {
+		return nil, nil, err
+	}
+	if gitBlob.Size() > lfs.MetaFileMaxSize {
+		return nil, nil, nil // not a LFS pointer
+	}
+	buf, err := gitBlob.GetBlobContent(lfs.MetaFileMaxSize)
+	if err != nil {
+		return nil, nil, err
+	}
+	oid, size := parsePossibleLfsPointerBuffer(strings.NewReader(buf))
+	return oid, size, nil
+}