Refactor repo contents API and add "contents-ext" API (#34822)

See the updated swagger document for details.

modules/git/blob.go

@@ -9,6 +9,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"io"
+	"strings"
 
 	"code.gitea.io/gitea/modules/typesniffer"
 	"code.gitea.io/gitea/modules/util"
@@ -63,33 +64,37 @@ func (b *Blob) GetBlobLineCount(w io.Writer) (int, error) {
 	}
 }
 
-// GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string
-func (b *Blob) GetBlobContentBase64() (string, error) {
+// GetBlobContentBase64 Reads the content of the blob with a base64 encoding and returns the encoded string
+func (b *Blob) GetBlobContentBase64(originContent *strings.Builder) (string, error) {
 	dataRc, err := b.DataAsync()
 	if err != nil {
 		return "", err
 	}
 	defer dataRc.Close()
 
-	pr, pw := io.Pipe()
-	encoder := base64.NewEncoder(base64.StdEncoding, pw)
-
-	go func() {
-		_, err := io.Copy(encoder, dataRc)
-		_ = encoder.Close()
-
-		if err != nil {
-			_ = pw.CloseWithError(err)
-		} else {
-			_ = pw.Close()
+	base64buf := &strings.Builder{}
+	encoder := base64.NewEncoder(base64.StdEncoding, base64buf)
+	buf := make([]byte, 32*1024)
+loop:
+	for {
+		n, err := dataRc.Read(buf)
+		if n > 0 {
+			if originContent != nil {
+				_, _ = originContent.Write(buf[:n])
+			}
+			if _, err := encoder.Write(buf[:n]); err != nil {
+				return "", err
+			}
+		}
+		switch {
+		case errors.Is(err, io.EOF):
+			break loop
+		case err != nil:
+			return "", err
 		}
-	}()
-
-	out, err := io.ReadAll(pr)
-	if err != nil {
-		return "", err
 	}
-	return string(out), nil
+	_ = encoder.Close()
+	return base64buf.String(), nil
 }
 
 // GuessContentType guesses the content type of the blob.

modules/git/tree_entry_nogogit.go

@@ -18,7 +18,7 @@ type TreeEntry struct {
 	sized     bool
 }
 
-// Name returns the name of the entry
+// Name returns the name of the entry (base name)
 func (te *TreeEntry) Name() string {
 	return te.name
 }

modules/lfs/pointer.go

@@ -15,15 +15,13 @@ import (
 	"strings"
 )
 
+// spec: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
 const (
-	blobSizeCutoff = 1024
+	MetaFileMaxSize = 1024 // spec says the maximum size of a pointer file must be smaller than 1024
 
-	// MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
-	// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
-	MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+	MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" // the first line of a pointer file
 
-	// MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
-	MetaFileOidPrefix = "oid sha256:"
+	MetaFileOidPrefix = "oid sha256:" // spec says the only supported hash is sha256 at the moment
 )
 
 var (
@@ -39,7 +37,7 @@ var (
 
 // ReadPointer tries to read LFS pointer data from the reader
 func ReadPointer(reader io.Reader) (Pointer, error) {
-	buf := make([]byte, blobSizeCutoff)
+	buf := make([]byte, MetaFileMaxSize)
 	n, err := io.ReadFull(reader, buf)
 	if err != nil && err != io.ErrUnexpectedEOF {
 		return Pointer{}, err
@@ -65,6 +63,7 @@ func ReadPointerFromBuffer(buf []byte) (Pointer, error) {
 		return p, ErrInvalidStructure
 	}
 
+	// spec says "key/value pairs MUST be sorted alphabetically in ascending order (version is exception and must be the first)"
 	oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
 	if len(oid) != 64 || !oidPattern.MatchString(oid) {
 		return p, ErrInvalidOIDFormat

modules/lfs/pointer_scanner_gogit.go

@@ -31,7 +31,7 @@ func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan c
 			default:
 			}
 
-			if blob.Size > blobSizeCutoff {
+			if blob.Size > MetaFileMaxSize {
 				return nil
 			}
 

modules/structs/git_blob.go

@@ -10,4 +10,7 @@ type GitBlobResponse struct {
 	URL      string  `json:"url"`
 	SHA      string  `json:"sha"`
 	Size     int64   `json:"size"`
+
+	LfsOid  *string `json:"lfs_oid,omitempty"`
+	LfsSize *int64  `json:"lfs_size,omitempty"`
 }

modules/structs/repo_file.go

@@ -119,6 +119,11 @@ type FileLinksResponse struct {
 	HTMLURL *string `json:"html"`
 }
 
+type ContentsExtResponse struct {
+	FileContents *ContentsResponse   `json:"file_contents,omitempty"`
+	DirContents  []*ContentsResponse `json:"dir_contents,omitempty"`
+}
+
 // ContentsResponse contains information about a repo's entry's (dir, file, symlink, submodule) metadata and content
 type ContentsResponse struct {
 	Name          string `json:"name"`
@@ -145,6 +150,9 @@ type ContentsResponse struct {
 	// `submodule_git_url` is populated when `type` is `submodule`, otherwise null
 	SubmoduleGitURL *string            `json:"submodule_git_url"`
 	Links           *FileLinksResponse `json:"_links"`
+
+	LfsOid  *string `json:"lfs_oid"`
+	LfsSize *int64  `json:"lfs_size"`
 }
 
 // FileCommitResponse contains information generated from a Git commit for a repo's file.