Sync branches into databases (#22743)

Related #14180
Related #25233 
Related #22639
Close #19786
Related #12763 

This PR will change all the branches retrieve method from reading git
data to read database to reduce git read operations.

- [x] Sync git branches information into database when push git data
- [x] Create a new table `Branch`, merge some columns of `DeletedBranch`
into `Branch` table and drop the table `DeletedBranch`.
- [x] Read `Branch` table when visit `code` -> `branch` page
- [x] Read `Branch` table when list branch names in `code` page dropdown
- [x] Read `Branch` table when list git ref compare page
- [x] Provide a button in admin page to manually sync all branches.
- [x] Sync branches if repository is not empty but database branches are
empty when visiting pages with branches list
- [x] Use `commit_time desc` as the default FindBranch order by to keep
consistent as before and deleted branches will be always at the end.

---------

Co-authored-by: Jason Song <i@wolfogre.com>

services/convert/convert.go

@@ -50,7 +50,7 @@ func ToEmailSearch(email *user_model.SearchEmailResult) *api.Email {
 }
 
 // ToBranch convert a git.Commit and git.Branch to an api.Branch
-func ToBranch(ctx context.Context, repo *repo_model.Repository, b *git.Branch, c *git.Commit, bp *git_model.ProtectedBranch, user *user_model.User, isRepoAdmin bool) (*api.Branch, error) {
+func ToBranch(ctx context.Context, repo *repo_model.Repository, branchName string, c *git.Commit, bp *git_model.ProtectedBranch, user *user_model.User, isRepoAdmin bool) (*api.Branch, error) {
 	if bp == nil {
 		var hasPerm bool
 		var canPush bool
@@ -65,11 +65,11 @@ func ToBranch(ctx context.Context, repo *repo_model.Repository, b *git.Branch, c
 			if err != nil {
 				return nil, err
 			}
-			canPush = issues_model.CanMaintainerWriteToBranch(perms, b.Name, user)
+			canPush = issues_model.CanMaintainerWriteToBranch(perms, branchName, user)
 		}
 
 		return &api.Branch{
-			Name:                b.Name,
+			Name:                branchName,
 			Commit:              ToPayloadCommit(ctx, repo, c),
 			Protected:           false,
 			RequiredApprovals:   0,
@@ -81,7 +81,7 @@ func ToBranch(ctx context.Context, repo *repo_model.Repository, b *git.Branch, c
 	}
 
 	branch := &api.Branch{
-		Name:                b.Name,
+		Name:                branchName,
 		Commit:              ToPayloadCommit(ctx, repo, c),
 		Protected:           true,
 		RequiredApprovals:   bp.RequiredApprovals,

services/migrations/dump.go

@@ -642,7 +642,7 @@ func (g *RepositoryDumper) Finish() error {
 
 // DumpRepository dump repository according MigrateOptions to a local directory
 func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error {
-	doer, err := user_model.GetAdminUser()
+	doer, err := user_model.GetAdminUser(ctx)
 	if err != nil {
 		return err
 	}
@@ -705,7 +705,7 @@ func updateOptionsUnits(opts *base.MigrateOptions, units []string) error {
 
 // RestoreRepository restore a repository from the disk directory
 func RestoreRepository(ctx context.Context, baseDir, ownerName, repoName string, units []string, validation bool) error {
-	doer, err := user_model.GetAdminUser()
+	doer, err := user_model.GetAdminUser(ctx)
 	if err != nil {
 		return err
 	}

services/pull/pull.go

@@ -170,7 +170,7 @@ func ChangeTargetBranch(ctx context.Context, pr *issues_model.PullRequest, doer
 		return err
 	}
 	if branchesEqual {
-		return models.ErrBranchesEqual{
+		return git_model.ErrBranchesEqual{
 			HeadBranchName: pr.HeadBranch,
 			BaseBranchName: targetBranch,
 		}
@@ -338,7 +338,7 @@ func AddTestPullRequestTask(doer *user_model.User, repoID int64, branch string,
 		for _, pr := range prs {
 			divergence, err := GetDiverging(ctx, pr)
 			if err != nil {
-				if models.IsErrBranchDoesNotExist(err) && !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) {
+				if git_model.IsErrBranchNotExist(err) && !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) {
 					log.Warn("Cannot test PR %s/%d: head_branch %s no longer exists", pr.BaseRepo.Name, pr.IssueID, pr.HeadBranch)
 				} else {
 					log.Error("GetDiverging: %v", err)

services/pull/temp_repo.go

@@ -11,7 +11,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"code.gitea.io/gitea/models"
+	git_model "code.gitea.io/gitea/models/git"
 	issues_model "code.gitea.io/gitea/models/issues"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
@@ -181,7 +181,7 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest)
 		Run(prCtx.RunOpts()); err != nil {
 		cancel()
 		if !git.IsBranchExist(ctx, pr.HeadRepo.RepoPath(), pr.HeadBranch) {
-			return nil, nil, models.ErrBranchDoesNotExist{
+			return nil, nil, git_model.ErrBranchNotExist{
 				BranchName: pr.HeadBranch,
 			}
 		}

services/pull/update.go

@@ -7,7 +7,6 @@ import (
 	"context"
 	"fmt"
 
-	"code.gitea.io/gitea/models"
 	git_model "code.gitea.io/gitea/models/git"
 	issues_model "code.gitea.io/gitea/models/issues"
 	access_model "code.gitea.io/gitea/models/perm/access"
@@ -168,7 +167,7 @@ func GetDiverging(ctx context.Context, pr *issues_model.PullRequest) (*git.Diver
 	log.Trace("GetDiverging[%-v]: compare commits", pr)
 	prCtx, cancel, err := createTemporaryRepoForPR(ctx, pr)
 	if err != nil {
-		if !models.IsErrBranchDoesNotExist(err) {
+		if !git_model.IsErrBranchNotExist(err) {
 			log.Error("CreateTemporaryRepoForPR %-v: %v", pr, err)
 		}
 		return nil, err

services/repository/adopt.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/models/db"
+	git_model "code.gitea.io/gitea/models/git"
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/container"
@@ -146,7 +147,15 @@ func adoptRepository(ctx context.Context, repoPath string, u *user_model.User, r
 			}
 		}
 	}
-	branches, _, _ := gitRepo.GetBranchNames(0, 0)
+
+	branches, _ := git_model.FindBranchNames(ctx, git_model.FindBranchOptions{
+		RepoID: repo.ID,
+		ListOptions: db.ListOptions{
+			ListAll: true,
+		},
+		IsDeletedBranch: util.OptionalBoolFalse,
+	})
+
 	found := false
 	hasDefault := false
 	hasMaster := false

services/repository/branch.go

@@ -10,13 +10,21 @@ import (
 	"strings"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
+	issues_model "code.gitea.io/gitea/models/issues"
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/notification"
+	"code.gitea.io/gitea/modules/queue"
 	repo_module "code.gitea.io/gitea/modules/repository"
+	"code.gitea.io/gitea/modules/util"
+	files_service "code.gitea.io/gitea/services/repository/files"
+
+	"xorm.io/builder"
 )
 
 // CreateNewBranch creates a new repository branch
@@ -27,7 +35,7 @@ func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_mode
 	}
 
 	if !git.IsBranchExist(ctx, repo.RepoPath(), oldBranchName) {
-		return models.ErrBranchDoesNotExist{
+		return git_model.ErrBranchNotExist{
 			BranchName: oldBranchName,
 		}
 	}
@@ -40,16 +48,165 @@ func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_mode
 		if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) {
 			return err
 		}
-		return fmt.Errorf("Push: %w", err)
+		return fmt.Errorf("push: %w", err)
 	}
 
 	return nil
 }
 
-// GetBranches returns branches from the repository, skipping skip initial branches and
-// returning at most limit branches, or all branches if limit is 0.
-func GetBranches(ctx context.Context, repo *repo_model.Repository, skip, limit int) ([]*git.Branch, int, error) {
-	return git.GetBranchesByPath(ctx, repo.RepoPath(), skip, limit)
+// Branch contains the branch information
+type Branch struct {
+	DBBranch          *git_model.Branch
+	IsProtected       bool
+	IsIncluded        bool
+	CommitsAhead      int
+	CommitsBehind     int
+	LatestPullRequest *issues_model.PullRequest
+	MergeMovedOn      bool
+}
+
+// LoadBranches loads branches from the repository limited by page & pageSize.
+func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, isDeletedBranch util.OptionalBool, page, pageSize int) (*Branch, []*Branch, int64, error) {
+	defaultDBBranch, err := git_model.GetBranch(ctx, repo.ID, repo.DefaultBranch)
+	if err != nil {
+		return nil, nil, 0, err
+	}
+
+	branchOpts := git_model.FindBranchOptions{
+		RepoID:          repo.ID,
+		IsDeletedBranch: isDeletedBranch,
+		ListOptions: db.ListOptions{
+			Page:     page,
+			PageSize: pageSize,
+		},
+	}
+
+	totalNumOfBranches, err := git_model.CountBranches(ctx, branchOpts)
+	if err != nil {
+		return nil, nil, 0, err
+	}
+
+	branchOpts.ExcludeBranchNames = []string{repo.DefaultBranch}
+
+	dbBranches, err := git_model.FindBranches(ctx, branchOpts)
+	if err != nil {
+		return nil, nil, 0, err
+	}
+
+	if err := dbBranches.LoadDeletedBy(ctx); err != nil {
+		return nil, nil, 0, err
+	}
+	if err := dbBranches.LoadPusher(ctx); err != nil {
+		return nil, nil, 0, err
+	}
+
+	rules, err := git_model.FindRepoProtectedBranchRules(ctx, repo.ID)
+	if err != nil {
+		return nil, nil, 0, err
+	}
+
+	repoIDToRepo := map[int64]*repo_model.Repository{}
+	repoIDToRepo[repo.ID] = repo
+
+	repoIDToGitRepo := map[int64]*git.Repository{}
+	repoIDToGitRepo[repo.ID] = gitRepo
+
+	branches := make([]*Branch, 0, len(dbBranches))
+	for i := range dbBranches {
+		branch, err := loadOneBranch(ctx, repo, dbBranches[i], &rules, repoIDToRepo, repoIDToGitRepo)
+		if err != nil {
+			return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err)
+		}
+
+		branches = append(branches, branch)
+	}
+
+	// Always add the default branch
+	log.Debug("loadOneBranch: load default: '%s'", defaultDBBranch.Name)
+	defaultBranch, err := loadOneBranch(ctx, repo, defaultDBBranch, &rules, repoIDToRepo, repoIDToGitRepo)
+	if err != nil {
+		return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err)
+	}
+
+	return defaultBranch, branches, totalNumOfBranches, nil
+}
+
+func loadOneBranch(ctx context.Context, repo *repo_model.Repository, dbBranch *git_model.Branch, protectedBranches *git_model.ProtectedBranchRules,
+	repoIDToRepo map[int64]*repo_model.Repository,
+	repoIDToGitRepo map[int64]*git.Repository,
+) (*Branch, error) {
+	log.Trace("loadOneBranch: '%s'", dbBranch.Name)
+
+	branchName := dbBranch.Name
+	p := protectedBranches.GetFirstMatched(branchName)
+	isProtected := p != nil
+
+	divergence := &git.DivergeObject{
+		Ahead:  -1,
+		Behind: -1,
+	}
+
+	// it's not default branch
+	if repo.DefaultBranch != dbBranch.Name && !dbBranch.IsDeleted {
+		var err error
+		divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName)
+		if err != nil {
+			log.Error("CountDivergingCommits: %v", err)
+		}
+	}
+
+	pr, err := issues_model.GetLatestPullRequestByHeadInfo(repo.ID, branchName)
+	if err != nil {
+		return nil, fmt.Errorf("GetLatestPullRequestByHeadInfo: %v", err)
+	}
+	headCommit := dbBranch.CommitID
+
+	mergeMovedOn := false
+	if pr != nil {
+		pr.HeadRepo = repo
+		if err := pr.LoadIssue(ctx); err != nil {
+			return nil, fmt.Errorf("LoadIssue: %v", err)
+		}
+		if repo, ok := repoIDToRepo[pr.BaseRepoID]; ok {
+			pr.BaseRepo = repo
+		} else if err := pr.LoadBaseRepo(ctx); err != nil {
+			return nil, fmt.Errorf("LoadBaseRepo: %v", err)
+		} else {
+			repoIDToRepo[pr.BaseRepoID] = pr.BaseRepo
+		}
+		pr.Issue.Repo = pr.BaseRepo
+
+		if pr.HasMerged {
+			baseGitRepo, ok := repoIDToGitRepo[pr.BaseRepoID]
+			if !ok {
+				baseGitRepo, err = git.OpenRepository(ctx, pr.BaseRepo.RepoPath())
+				if err != nil {
+					return nil, fmt.Errorf("OpenRepository: %v", err)
+				}
+				defer baseGitRepo.Close()
+				repoIDToGitRepo[pr.BaseRepoID] = baseGitRepo
+			}
+			pullCommit, err := baseGitRepo.GetRefCommitID(pr.GetGitRefName())
+			if err != nil && !git.IsErrNotExist(err) {
+				return nil, fmt.Errorf("GetBranchCommitID: %v", err)
+			}
+			if err == nil && headCommit != pullCommit {
+				// the head has moved on from the merge - we shouldn't delete
+				mergeMovedOn = true
+			}
+		}
+	}
+
+	isIncluded := divergence.Ahead == 0 && repo.DefaultBranch != branchName
+	return &Branch{
+		DBBranch:          dbBranch,
+		IsProtected:       isProtected,
+		IsIncluded:        isIncluded,
+		CommitsAhead:      divergence.Ahead,
+		CommitsBehind:     divergence.Behind,
+		LatestPullRequest: pr,
+		MergeMovedOn:      mergeMovedOn,
+	}, nil
 }
 
 func GetBranchCommitID(ctx context.Context, repo *repo_model.Repository, branch string) (string, error) {
@@ -62,17 +219,17 @@ func checkBranchName(ctx context.Context, repo *repo_model.Repository, name stri
 		branchRefName := strings.TrimPrefix(refName, git.BranchPrefix)
 		switch {
 		case branchRefName == name:
-			return models.ErrBranchAlreadyExists{
+			return git_model.ErrBranchAlreadyExists{
 				BranchName: name,
 			}
 		// If branchRefName like a/b but we want to create a branch named a then we have a conflict
 		case strings.HasPrefix(branchRefName, name+"/"):
-			return models.ErrBranchNameConflict{
+			return git_model.ErrBranchNameConflict{
 				BranchName: branchRefName,
 			}
 			// Conversely if branchRefName like a but we want to create a branch named a/b then we also have a conflict
 		case strings.HasPrefix(name, branchRefName+"/"):
-			return models.ErrBranchNameConflict{
+			return git_model.ErrBranchNameConflict{
 				BranchName: branchRefName,
 			}
 		case refName == git.TagPrefix+name:
@@ -101,7 +258,7 @@ func CreateNewBranchFromCommit(ctx context.Context, doer *user_model.User, repo
 		if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) {
 			return err
 		}
-		return fmt.Errorf("Push: %w", err)
+		return fmt.Errorf("push: %w", err)
 	}
 
 	return nil
@@ -169,13 +326,28 @@ func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.R
 		return git_model.ErrBranchIsProtected
 	}
 
+	rawBranch, err := git_model.GetBranch(ctx, repo.ID, branchName)
+	if err != nil {
+		return fmt.Errorf("GetBranch: %vc", err)
+	}
+
+	if rawBranch.IsDeleted {
+		return nil
+	}
+
 	commit, err := gitRepo.GetBranchCommit(branchName)
 	if err != nil {
 		return err
 	}
 
-	if err := gitRepo.DeleteBranch(branchName, git.DeleteBranchOptions{
-		Force: true,
+	if err := db.WithTx(ctx, func(ctx context.Context) error {
+		if err := git_model.AddDeletedBranch(ctx, repo.ID, branchName, doer.ID); err != nil {
+			return err
+		}
+
+		return gitRepo.DeleteBranch(branchName, git.DeleteBranchOptions{
+			Force: true,
+		})
 	}); err != nil {
 		return err
 	}
@@ -196,3 +368,45 @@ func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.R
 
 	return nil
 }
+
+type BranchSyncOptions struct {
+	RepoID int64
+}
+
+// branchSyncQueue represents a queue to handle branch sync jobs.
+var branchSyncQueue *queue.WorkerPoolQueue[*BranchSyncOptions]
+
+func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions {
+	for _, opts := range items {
+		_, err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, 0)
+		if err != nil {
+			log.Error("syncRepoBranches [%d] failed: %v", opts.RepoID, err)
+		}
+	}
+	return nil
+}
+
+func addRepoToBranchSyncQueue(repoID, doerID int64) error {
+	return branchSyncQueue.Push(&BranchSyncOptions{
+		RepoID: repoID,
+	})
+}
+
+func initBranchSyncQueue(ctx context.Context) error {
+	branchSyncQueue = queue.CreateUniqueQueue(ctx, "branch_sync", handlerBranchSync)
+	if branchSyncQueue == nil {
+		return errors.New("unable to create branch_sync queue")
+	}
+	go graceful.GetManager().RunWithCancel(branchSyncQueue)
+
+	return nil
+}
+
+func AddAllRepoBranchesToSyncQueue(ctx context.Context, doerID int64) error {
+	if err := db.Iterate(ctx, builder.Eq{"is_empty": false}, func(ctx context.Context, repo *repo_model.Repository) error {
+		return addRepoToBranchSyncQueue(repo.ID, doerID)
+	}); err != nil {
+		return fmt.Errorf("run sync all branches failed: %v", err)
+	}
+	return nil
+}

services/repository/files/patch.go

@@ -58,7 +58,7 @@ func (opts *ApplyDiffPatchOptions) Validate(ctx context.Context, repo *repo_mode
 	if opts.NewBranch != opts.OldBranch {
 		existingBranch, err := gitRepo.GetBranch(opts.NewBranch)
 		if existingBranch != nil {
-			return models.ErrBranchAlreadyExists{
+			return git_model.ErrBranchAlreadyExists{
 				BranchName: opts.NewBranch,
 			}
 		}

services/repository/files/update.go

@@ -197,7 +197,7 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use
 	if opts.NewBranch != opts.OldBranch {
 		existingBranch, err := gitRepo.GetBranch(opts.NewBranch)
 		if existingBranch != nil {
-			return nil, models.ErrBranchAlreadyExists{
+			return nil, git_model.ErrBranchAlreadyExists{
 				BranchName: opts.NewBranch,
 			}
 		}

services/repository/fork.go

@@ -157,7 +157,15 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork
 		if err = repo_module.CreateDelegateHooks(repoPath); err != nil {
 			return fmt.Errorf("createDelegateHooks: %w", err)
 		}
-		return nil
+
+		gitRepo, err := git.OpenRepository(txCtx, repo.RepoPath())
+		if err != nil {
+			return fmt.Errorf("OpenRepository: %w", err)
+		}
+		defer gitRepo.Close()
+
+		_, err = repo_module.SyncRepoBranchesWithRepo(txCtx, repo, gitRepo, doer.ID)
+		return err
 	})
 	needsRollbackInPanic = false
 	if err != nil {

services/repository/push.go

@@ -93,7 +93,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 	defer gitRepo.Close()
 
 	if err = repo_module.UpdateRepoSize(ctx, repo); err != nil {
-		log.Error("Failed to update size for repository: %v", err)
+		return fmt.Errorf("Failed to update size for repository: %v", err)
 	}
 
 	addTags := make([]string, 0, len(optsList))
@@ -259,8 +259,8 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 
 				notification.NotifyPushCommits(ctx, pusher, repo, opts, commits)
 
-				if err = git_model.RemoveDeletedBranchByName(ctx, repo.ID, branch); err != nil {
-					log.Error("models.RemoveDeletedBranch %s/%s failed: %v", repo.ID, branch, err)
+				if err = git_model.UpdateBranch(ctx, repo.ID, branch, newCommit.ID.String(), newCommit.CommitMessage, opts.PusherID, newCommit.Committer.When); err != nil {
+					return fmt.Errorf("git_model.UpdateBranch %s:%s failed: %v", repo.FullName(), branch, err)
 				}
 
 				// Cache for big repository
@@ -273,8 +273,9 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 					// close all related pulls
 					log.Error("close related pull request failed: %v", err)
 				}
-				if err := git_model.AddDeletedBranch(db.DefaultContext, repo.ID, branch, opts.OldCommitID, pusher.ID); err != nil {
-					log.Warn("AddDeletedBranch: %v", err)
+
+				if err := git_model.AddDeletedBranch(db.DefaultContext, repo.ID, branch, pusher.ID); err != nil {
+					return fmt.Errorf("AddDeletedBranch %s:%s failed: %v", repo.FullName(), branch, err)
 				}
 			}
 

services/repository/repository.go

@@ -17,6 +17,7 @@ import (
 	system_model "code.gitea.io/gitea/models/system"
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/notification"
 	repo_module "code.gitea.io/gitea/modules/repository"
@@ -100,7 +101,10 @@ func Init() error {
 	}
 	system_model.RemoveAllWithNotice(db.DefaultContext, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath)
 	system_model.RemoveAllWithNotice(db.DefaultContext, "Clean up temporary repositories", repo_module.LocalCopyPath())
-	return initPushQueue()
+	if err := initPushQueue(); err != nil {
+		return err
+	}
+	return initBranchSyncQueue(graceful.GetManager().ShutdownContext())
 }
 
 // UpdateRepository updates a repository