Improve listing performance by using go-git (#6478)

* Use go-git for tree reading and commit info lookup.

Signed-off-by: Filip Navara <navara@emclient.com>

* Use TreeEntry.IsRegular() instead of ObjectType that was removed.

Signed-off-by: Filip Navara <navara@emclient.com>

* Use the treePath to optimize commit info search.

Signed-off-by: Filip Navara <navara@emclient.com>

* Extract the latest commit at treePath along with the other commits.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix listing commit info for a directory that was created in one commit and never modified after.

Signed-off-by: Filip Navara <navara@emclient.com>

* Avoid nearly all external 'git' invocations when doing directory listing (.editorconfig code path is still hit).

Signed-off-by: Filip Navara <navara@emclient.com>

* Use go-git for reading blobs.

Signed-off-by: Filip Navara <navara@emclient.com>

* Make SHA1 type alias for plumbing.Hash in go-git.

Signed-off-by: Filip Navara <navara@emclient.com>

* Make Signature type alias for object.Signature in go-git.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix GetCommitsInfo for repository with only one commit.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix PGP signature verification.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix issues with walking commit graph across merges.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix typo in condition.

Signed-off-by: Filip Navara <navara@emclient.com>

* Speed up loading branch list by keeping the repository reference (and thus all the loaded packfile indexes).

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix lising submodules.

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix build

Signed-off-by: Filip Navara <navara@emclient.com>

* Add back commit cache because of name-rev

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix tests

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix code style

* Fix spelling

* Address PR feedback

Signed-off-by: Filip Navara <navara@emclient.com>

* Update vendor module list

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix getting trees by commit id

Signed-off-by: Filip Navara <navara@emclient.com>

* Fix remaining unit test failures

* Fix GetTreeBySHA

* Avoid running `git name-rev` if not necessary

Signed-off-by: Filip Navara <navara@emclient.com>

* Move Branch code to git module

* Clean up GPG signature verification and fix it for tagged commits

* Address PR feedback (import formatting, copyright headers)

* Make blob lookup by SHA working

* Update tests to use public API

* Allow getting content from any type of object through the blob interface

* Change test to actually expect the object content that is in the GIT repository

* Change one more test to actually expect the object content that is in the GIT repository

* Add comments

modules/repofiles/blob_test.go

@@ -27,7 +27,7 @@ func TestGetBlobBySHA(t *testing.T) {
 
 	gbr, err := GetBlobBySHA(ctx.Repo.Repository, ctx.Params(":sha"))
 	expectedGBR := &api.GitBlobResponse{
-		Content:  "Y29tbWl0IDY1ZjFiZjI3YmMzYmY3MGY2NDY1NzY1ODYzNWU2NjA5NGVkYmNiNGQKQXV0aG9yOiB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+CkRhdGU6ICAgU3VuIE1hciAxOSAxNjo0Nzo1OSAyMDE3IC0wNDAwCgogICAgSW5pdGlhbCBjb21taXQKCmRpZmYgLS1naXQgYS9SRUFETUUubWQgYi9SRUFETUUubWQKbmV3IGZpbGUgbW9kZSAxMDA2NDQKaW5kZXggMDAwMDAwMC4uNGI0ODUxYQotLS0gL2Rldi9udWxsCisrKyBiL1JFQURNRS5tZApAQCAtMCwwICsxLDMgQEAKKyMgcmVwbzEKKworRGVzY3JpcHRpb24gZm9yIHJlcG8xClwgTm8gbmV3bGluZSBhdCBlbmQgb2YgZmlsZQo=",
+		Content:  "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK",
 		Encoding: "base64",
 		URL:      "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d",
 		SHA:      "65f1bf27bc3bf70f64657658635e66094edbcb4d",

modules/repofiles/content.go

@@ -61,7 +61,7 @@ func GetFileContents(repo *models.Repository, treePath, ref string) (*api.FileCo
 		HTMLURL:     htmlURL.String(),
 		GitURL:      gitURL.String(),
 		DownloadURL: downloadURL.String(),
-		Type:        string(entry.Type),
+		Type:        entry.Type(),
 		Links: &api.FileLinksResponse{
 			Self:    selfURL.String(),
 			GitURL:  gitURL.String(),

modules/repofiles/temp_repo.go

@@ -58,7 +58,7 @@ func (t *TemporaryUploadRepository) Clone(branch string) error {
 		fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath),
 		"git", "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil {
 		if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched {
-			return models.ErrBranchNotExist{
+			return git.ErrBranchNotExist{
 				Name: branch,
 			}
 		} else if matched, _ := regexp.MatchString(".* repository .* does not exist.*", stderr); matched {

modules/repofiles/tree.go

@@ -23,7 +23,7 @@ func GetTreeBySHA(repo *models.Repository, sha string, page, perPage int, recurs
 		}
 	}
 	tree := new(api.GitTreeResponse)
-	tree.SHA = gitTree.ID.String()
+	tree.SHA = gitTree.CommitID.String()
 	tree.URL = repo.APIURL() + "/git/trees/" + tree.SHA
 	var entries git.Entries
 	if recursive {
@@ -74,11 +74,12 @@ func GetTreeBySHA(repo *models.Repository, sha string, page, perPage int, recurs
 	tree.Entries = make([]api.GitEntry, rangeEnd-rangeStart)
 	for e := rangeStart; e < rangeEnd; e++ {
 		i := e - rangeStart
-		tree.Entries[i].Path = entries[e].Name()
-		tree.Entries[i].Mode = fmt.Sprintf("%06x", entries[e].Mode())
-		tree.Entries[i].Type = string(entries[e].Type)
-		tree.Entries[i].Size = entries[e].Size()
-		tree.Entries[i].SHA = entries[e].ID.String()
+
+		tree.Entries[e].Path = entries[e].Name()
+		tree.Entries[e].Mode = fmt.Sprintf("%06o", entries[e].Mode())
+		tree.Entries[e].Type = entries[e].Type()
+		tree.Entries[e].Size = entries[e].Size()
+		tree.Entries[e].SHA = entries[e].ID.String()
 
 		if entries[e].IsDir() {
 			copy(treeURL[copyPos:], entries[e].ID.String())

modules/repofiles/tree_test.go

@@ -46,5 +46,6 @@ func TestGetTreeBySHA(t *testing.T) {
 		Page:       1,
 		TotalCount: 1,
 	}
-	assert.EqualValues(t, tree, expectedTree)
+
+	assert.EqualValues(t, expectedTree, tree)
 }

modules/repofiles/update.go

@@ -62,7 +62,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
 				BranchName: opts.NewBranch,
 			}
 		}
-		if err != nil && !models.IsErrBranchNotExist(err) {
+		if err != nil && !git.IsErrBranchNotExist(err) {
 			return nil, err
 		}
 	} else {