1
1
mirror of https://github.com/go-gitea/gitea synced 2025-07-22 18:28:37 +00:00

Option to delay conflict checking of old pull requests until page view (#27779)

`[repository.pull-request] DELAY_CHECK_FOR_INACTIVE_DAYS` is a new
setting to delay the mergeable check for pull requests that have been
inactive for the specified number of days.

This avoids potentially long delays for big repositories with many pull
requests. and reduces system load overall when there are many
repositories or pull requests.

When viewing the PR, checking will start immediately and the PR merge
box will automatically reload when complete. Accessing the PR through
the API will also start checking immediately.

The default value of `7` provides a balance between system load, and
keeping behavior similar to what it was before both for users and API
access. With `0` all conflict checking will be delayed, while `-1`
always checks immediately to restore the previous behavior.

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Brecht Van Lommel
2025-04-24 21:26:57 +02:00
committed by GitHub
parent d1ad8e1e80
commit a9343896f4
32 changed files with 437 additions and 244 deletions

View File

@@ -204,7 +204,7 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git.
return nil, fmt.Errorf("failed to update pull ref. Error: %w", err)
}
pull_service.AddToTaskQueue(ctx, pr)
pull_service.StartPullRequestCheckImmediately(ctx, pr)
err = pr.LoadIssue(ctx)
if err != nil {
return nil, fmt.Errorf("failed to load pull issue. Error: %w", err)

View File

@@ -289,7 +289,7 @@ func handlePullRequestAutoMerge(pullID int64, sha string) {
}
if err := pull_service.CheckPullMergeable(ctx, doer, &perm, pr, pull_service.MergeCheckTypeGeneral, false); err != nil {
if errors.Is(err, pull_service.ErrUserNotAllowedToMerge) {
if errors.Is(err, pull_service.ErrNotReadyToMerge) {
log.Info("%-v was scheduled to automerge by an unauthorized user", pr)
return
}

View File

@@ -556,7 +556,7 @@ func (g *GiteaLocalUploader) CreatePullRequests(ctx context.Context, prs ...*bas
}
for _, pr := range gprs {
g.issues[pr.Issue.Index] = pr.Issue
pull.AddToTaskQueue(ctx, pr)
pull.StartPullRequestCheckImmediately(ctx, pr)
}
return nil
}

View File

@@ -10,6 +10,7 @@ import (
"fmt"
"strconv"
"strings"
"time"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
@@ -25,6 +26,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
asymkey_service "code.gitea.io/gitea/services/asymkey"
notify_service "code.gitea.io/gitea/services/notify"
@@ -34,27 +36,88 @@ import (
var prPatchCheckerQueue *queue.WorkerPoolQueue[string]
var (
ErrIsClosed = errors.New("pull is closed")
ErrUserNotAllowedToMerge = ErrDisallowedToMerge{}
ErrHasMerged = errors.New("has already been merged")
ErrIsWorkInProgress = errors.New("work in progress PRs cannot be merged")
ErrIsChecking = errors.New("cannot merge while conflict checking is in progress")
ErrNotMergeableState = errors.New("not in mergeable state")
ErrDependenciesLeft = errors.New("is blocked by an open dependency")
ErrIsClosed = errors.New("pull is closed")
ErrNoPermissionToMerge = errors.New("no permission to merge")
ErrNotReadyToMerge = errors.New("not ready to merge")
ErrHasMerged = errors.New("has already been merged")
ErrIsWorkInProgress = errors.New("work in progress PRs cannot be merged")
ErrIsChecking = errors.New("cannot merge while conflict checking is in progress")
ErrNotMergeableState = errors.New("not in mergeable state")
ErrDependenciesLeft = errors.New("is blocked by an open dependency")
)
// AddToTaskQueue adds itself to pull request test task queue.
func AddToTaskQueue(ctx context.Context, pr *issues_model.PullRequest) {
func markPullRequestStatusAsChecking(ctx context.Context, pr *issues_model.PullRequest) bool {
pr.Status = issues_model.PullRequestStatusChecking
err := pr.UpdateColsIfNotMerged(ctx, "status")
if err != nil {
log.Error("AddToTaskQueue(%-v).UpdateCols.(add to queue): %v", pr, err)
log.Error("UpdateColsIfNotMerged failed, pr: %-v, err: %v", pr, err)
return false
}
pr, err = issues_model.GetPullRequestByID(ctx, pr.ID)
if err != nil {
log.Error("GetPullRequestByID failed, pr: %-v, err: %v", pr, err)
return false
}
return pr.Status == issues_model.PullRequestStatusChecking
}
var AddPullRequestToCheckQueue = realAddPullRequestToCheckQueue
func realAddPullRequestToCheckQueue(prID int64) {
err := prPatchCheckerQueue.Push(strconv.FormatInt(prID, 10))
if err != nil && !errors.Is(err, queue.ErrAlreadyInQueue) {
log.Error("Error adding %v to the pull requests check queue: %v", prID, err)
}
}
func StartPullRequestCheckImmediately(ctx context.Context, pr *issues_model.PullRequest) {
if !markPullRequestStatusAsChecking(ctx, pr) {
return
}
log.Trace("Adding %-v to the test pull requests queue", pr)
err = prPatchCheckerQueue.Push(strconv.FormatInt(pr.ID, 10))
if err != nil && err != queue.ErrAlreadyInQueue {
log.Error("Error adding %-v to the test pull requests queue: %v", pr, err)
AddPullRequestToCheckQueue(pr.ID)
}
// StartPullRequestCheckDelayable will delay the check if the pull request was not updated recently.
// When the "base" branch gets updated, all PRs targeting that "base" branch need to re-check whether
// they are mergeable.
// When there are too many stale PRs, each "base" branch update will consume a lot of system resources.
// So we can delay the checks for PRs that were not updated recently, only mark their status as
// "checking", and then next time when these PRs are updated or viewed, the real checks will run.
func StartPullRequestCheckDelayable(ctx context.Context, pr *issues_model.PullRequest) {
if !markPullRequestStatusAsChecking(ctx, pr) {
return
}
if setting.Repository.PullRequest.DelayCheckForInactiveDays >= 0 {
if err := pr.LoadIssue(ctx); err != nil {
return
}
duration := 24 * time.Hour * time.Duration(setting.Repository.PullRequest.DelayCheckForInactiveDays)
if pr.Issue.UpdatedUnix.AddDuration(duration) <= timeutil.TimeStampNow() {
return
}
}
AddPullRequestToCheckQueue(pr.ID)
}
func StartPullRequestCheckOnView(ctx context.Context, pr *issues_model.PullRequest) {
// TODO: its correctness totally depends on the "unique queue" feature and the global lock.
// So duplicate "start" requests will be ignored if there is already a task in the queue or one is running.
// Ideally in the future we should decouple the "unique queue" feature from the "start" request.
if pr.Status == issues_model.PullRequestStatusChecking {
if setting.IsInTesting {
// In testing mode, there might be an "immediate" queue, which is not a real queue, everything is executed in the same goroutine
// So we can't use the global lock here, otherwise it will cause a deadlock.
AddPullRequestToCheckQueue(pr.ID)
} else {
// When a PR check starts, the task is popped from the queue and the task handler acquires the global lock
// So we need to acquire the global lock here to prevent from duplicate tasks
_, _ = globallock.TryLockAndDo(ctx, getPullWorkingLockKey(pr.ID), func(ctx context.Context) error {
AddPullRequestToCheckQueue(pr.ID) // the queue is a unique queue and won't add the same task again
return nil
})
}
}
}
@@ -84,7 +147,7 @@ func CheckPullMergeable(stdCtx context.Context, doer *user_model.User, perm *acc
log.Error("Error whilst checking if %-v is allowed to merge %-v: %v", doer, pr, err)
return err
} else if !allowedMerge {
return ErrUserNotAllowedToMerge
return ErrNoPermissionToMerge
}
if mergeCheckType == MergeCheckTypeManually {
@@ -105,7 +168,7 @@ func CheckPullMergeable(stdCtx context.Context, doer *user_model.User, perm *acc
}
if err := CheckPullBranchProtections(ctx, pr, false); err != nil {
if !IsErrDisallowedToMerge(err) {
if !errors.Is(err, ErrNotReadyToMerge) {
log.Error("Error whilst checking pull branch protection for %-v: %v", pr, err)
return err
}
@@ -172,10 +235,10 @@ func isSignedIfRequired(ctx context.Context, pr *issues_model.PullRequest, doer
return sign, err
}
// checkAndUpdateStatus checks if pull request is possible to leaving checking status,
// markPullRequestAsMergeable checks if pull request is possible to leaving checking status,
// and set to be either conflict or mergeable.
func checkAndUpdateStatus(ctx context.Context, pr *issues_model.PullRequest) {
// If status has not been changed to conflict by testPatch then we are mergeable
func markPullRequestAsMergeable(ctx context.Context, pr *issues_model.PullRequest) {
// If status has not been changed to conflict by testPullRequestTmpRepoBranchMergeable then we are mergeable
if pr.Status == issues_model.PullRequestStatusChecking {
pr.Status = issues_model.PullRequestStatusMergeable
}
@@ -310,6 +373,10 @@ func manuallyMerged(ctx context.Context, pr *issues_model.PullRequest) bool {
// InitializePullRequests checks and tests untested patches of pull requests.
func InitializePullRequests(ctx context.Context) {
// If we prefer to delay the checks, then no need to do any check during startup, there should be not much difference
if setting.Repository.PullRequest.DelayCheckForInactiveDays >= 0 {
return
}
prs, err := issues_model.GetPullRequestIDsByCheckStatus(ctx, issues_model.PullRequestStatusChecking)
if err != nil {
log.Error("Find Checking PRs: %v", err)
@@ -320,24 +387,12 @@ func InitializePullRequests(ctx context.Context) {
case <-ctx.Done():
return
default:
log.Trace("Adding PR[%d] to the pull requests patch checking queue", prID)
if err := prPatchCheckerQueue.Push(strconv.FormatInt(prID, 10)); err != nil {
log.Error("Error adding PR[%d] to the pull requests patch checking queue %v", prID, err)
}
AddPullRequestToCheckQueue(prID)
}
}
}
// handle passed PR IDs and test the PRs
func handler(items ...string) []string {
for _, s := range items {
id, _ := strconv.ParseInt(s, 10, 64)
testPR(id)
}
return nil
}
func testPR(id int64) {
func checkPullRequestMergeable(id int64) {
ctx := graceful.GetManager().HammerContext()
releaser, err := globallock.Lock(ctx, getPullWorkingLockKey(id))
if err != nil {
@@ -351,7 +406,7 @@ func testPR(id int64) {
pr, err := issues_model.GetPullRequestByID(ctx, id)
if err != nil {
log.Error("Unable to GetPullRequestByID[%d] for testPR: %v", id, err)
log.Error("Unable to GetPullRequestByID[%d] for checkPullRequestMergeable: %v", id, err)
return
}
@@ -370,15 +425,15 @@ func testPR(id int64) {
return
}
if err := TestPatch(pr); err != nil {
log.Error("testPatch[%-v]: %v", pr, err)
if err := testPullRequestBranchMergeable(pr); err != nil {
log.Error("testPullRequestTmpRepoBranchMergeable[%-v]: %v", pr, err)
pr.Status = issues_model.PullRequestStatusError
if err := pr.UpdateCols(ctx, "status"); err != nil {
log.Error("update pr [%-v] status to PullRequestStatusError failed: %v", pr, err)
}
return
}
checkAndUpdateStatus(ctx, pr)
markPullRequestAsMergeable(ctx, pr)
}
// CheckPRsForBaseBranch check all pulls with baseBrannch
@@ -387,17 +442,21 @@ func CheckPRsForBaseBranch(ctx context.Context, baseRepo *repo_model.Repository,
if err != nil {
return err
}
for _, pr := range prs {
AddToTaskQueue(ctx, pr)
StartPullRequestCheckImmediately(ctx, pr)
}
return nil
}
// Init runs the task queue to test all the checking status pull requests
func Init() error {
prPatchCheckerQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "pr_patch_checker", handler)
prPatchCheckerQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "pr_patch_checker", func(items ...string) []string {
for _, s := range items {
id, _ := strconv.ParseInt(s, 10, 64)
checkPullRequestMergeable(id)
}
return nil
})
if prPatchCheckerQueue == nil {
return errors.New("unable to create pr_patch_checker queue")

View File

@@ -36,7 +36,7 @@ func TestPullRequest_AddToTaskQueue(t *testing.T) {
assert.NoError(t, err)
pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: 2})
AddToTaskQueue(db.DefaultContext, pr)
StartPullRequestCheckImmediately(db.DefaultContext, pr)
assert.Eventually(t, func() bool {
pr = unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: 2})

View File

@@ -518,25 +518,6 @@ func IsUserAllowedToMerge(ctx context.Context, pr *issues_model.PullRequest, p a
return false, nil
}
// ErrDisallowedToMerge represents an error that a branch is protected and the current user is not allowed to modify it.
type ErrDisallowedToMerge struct {
Reason string
}
// IsErrDisallowedToMerge checks if an error is an ErrDisallowedToMerge.
func IsErrDisallowedToMerge(err error) bool {
_, ok := err.(ErrDisallowedToMerge)
return ok
}
func (err ErrDisallowedToMerge) Error() string {
return fmt.Sprintf("not allowed to merge [reason: %s]", err.Reason)
}
func (err ErrDisallowedToMerge) Unwrap() error {
return util.ErrPermissionDenied
}
// CheckPullBranchProtections checks whether the PR is ready to be merged (reviews and status checks)
func CheckPullBranchProtections(ctx context.Context, pr *issues_model.PullRequest, skipProtectedFilesCheck bool) (err error) {
if err = pr.LoadBaseRepo(ctx); err != nil {
@@ -556,31 +537,21 @@ func CheckPullBranchProtections(ctx context.Context, pr *issues_model.PullReques
return err
}
if !isPass {
return ErrDisallowedToMerge{
Reason: "Not all required status checks successful",
}
return util.ErrorWrap(ErrNotReadyToMerge, "Not all required status checks successful")
}
if !issues_model.HasEnoughApprovals(ctx, pb, pr) {
return ErrDisallowedToMerge{
Reason: "Does not have enough approvals",
}
return util.ErrorWrap(ErrNotReadyToMerge, "Does not have enough approvals")
}
if issues_model.MergeBlockedByRejectedReview(ctx, pb, pr) {
return ErrDisallowedToMerge{
Reason: "There are requested changes",
}
return util.ErrorWrap(ErrNotReadyToMerge, "There are requested changes")
}
if issues_model.MergeBlockedByOfficialReviewRequests(ctx, pb, pr) {
return ErrDisallowedToMerge{
Reason: "There are official review requests",
}
return util.ErrorWrap(ErrNotReadyToMerge, "There are official review requests")
}
if issues_model.MergeBlockedByOutdatedBranch(pb, pr) {
return ErrDisallowedToMerge{
Reason: "The head branch is behind the base branch",
}
return util.ErrorWrap(ErrNotReadyToMerge, "The head branch is behind the base branch")
}
if skipProtectedFilesCheck {
@@ -588,9 +559,7 @@ func CheckPullBranchProtections(ctx context.Context, pr *issues_model.PullReques
}
if pb.MergeBlockedByProtectedFiles(pr.ChangedProtectedFiles) {
return ErrDisallowedToMerge{
Reason: "Changed protected files",
}
return util.ErrorWrap(ErrNotReadyToMerge, "Changed protected files")
}
return nil
@@ -709,7 +678,7 @@ func SetMerged(ctx context.Context, pr *issues_model.PullRequest, mergedCommitID
return false, fmt.Errorf("ChangeIssueStatus: %w", err)
}
// We need to save all of the data used to compute this merge as it may have already been changed by TestPatch. FIXME: need to set some state to prevent TestPatch from running whilst we are merging.
// We need to save all of the data used to compute this merge as it may have already been changed by testPullRequestBranchMergeable. FIXME: need to set some state to prevent testPullRequestBranchMergeable from running whilst we are merging.
if cnt, err := db.GetEngine(ctx).Where("id = ?", pr.ID).
And("has_merged = ?", false).
Cols("has_merged, status, merge_base, merged_commit_id, merger_id, merged_unix, conflicted_files").

View File

@@ -23,7 +23,7 @@ import (
)
type mergeContext struct {
*prContext
*prTmpRepoContext
doer *user_model.User
sig *git.Signature
committer *git.Signature
@@ -68,8 +68,8 @@ func createTemporaryRepoForMerge(ctx context.Context, pr *issues_model.PullReque
}
mergeCtx = &mergeContext{
prContext: prCtx,
doer: doer,
prTmpRepoContext: prCtx,
doer: doer,
}
if expectedHeadCommitID != "" {

View File

@@ -67,9 +67,8 @@ var patchErrorSuffices = []string{
": does not exist in index",
}
// TestPatch will test whether a simple patch will apply
func TestPatch(pr *issues_model.PullRequest) error {
ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("TestPatch: %s", pr))
func testPullRequestBranchMergeable(pr *issues_model.PullRequest) error {
ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("testPullRequestBranchMergeable: %s", pr))
defer finished()
prCtx, cancel, err := createTemporaryRepoForPR(ctx, pr)
@@ -81,10 +80,10 @@ func TestPatch(pr *issues_model.PullRequest) error {
}
defer cancel()
return testPatch(ctx, prCtx, pr)
return testPullRequestTmpRepoBranchMergeable(ctx, prCtx, pr)
}
func testPatch(ctx context.Context, prCtx *prContext, pr *issues_model.PullRequest) error {
func testPullRequestTmpRepoBranchMergeable(ctx context.Context, prCtx *prTmpRepoContext, pr *issues_model.PullRequest) error {
gitRepo, err := git.OpenRepository(ctx, prCtx.tmpBasePath)
if err != nil {
return fmt.Errorf("OpenRepository: %w", err)
@@ -380,7 +379,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo *
return false, nil
}
log.Trace("PullRequest[%d].testPatch (patchPath): %s", pr.ID, patchPath)
log.Trace("PullRequest[%d].testPullRequestTmpRepoBranchMergeable (patchPath): %s", pr.ID, patchPath)
// 4. Read the base branch in to the index of the temporary repository
_, _, err = git.NewCommand("read-tree", "base").RunStdString(gitRepo.Ctx, &git.RunOpts{Dir: tmpBasePath})
@@ -450,7 +449,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo *
scanner := bufio.NewScanner(stderrReader)
for scanner.Scan() {
line := scanner.Text()
log.Trace("PullRequest[%d].testPatch: stderr: %s", pr.ID, line)
log.Trace("PullRequest[%d].testPullRequestTmpRepoBranchMergeable: stderr: %s", pr.ID, line)
if strings.HasPrefix(line, prefix) {
conflict = true
filepath := strings.TrimSpace(strings.Split(line[len(prefix):], ":")[0])

View File

@@ -96,7 +96,7 @@ func NewPullRequest(ctx context.Context, opts *NewPullRequestOptions) error {
}
defer cancel()
if err := testPatch(ctx, prCtx, pr); err != nil {
if err := testPullRequestTmpRepoBranchMergeable(ctx, prCtx, pr); err != nil {
return err
}
@@ -314,12 +314,12 @@ func ChangeTargetBranch(ctx context.Context, pr *issues_model.PullRequest, doer
pr.BaseBranch = targetBranch
// Refresh patch
if err := TestPatch(pr); err != nil {
if err := testPullRequestBranchMergeable(pr); err != nil {
return err
}
// Update target branch, PR diff and status
// This is the same as checkAndUpdateStatus in check service, but also updates base_branch
// This is the same as markPullRequestAsMergeable in check service, but also updates base_branch
if pr.Status == issues_model.PullRequestStatusChecking {
pr.Status = issues_model.PullRequestStatusMergeable
}
@@ -409,7 +409,7 @@ func AddTestPullRequestTask(opts TestPullRequestOptions) {
continue
}
AddToTaskQueue(ctx, pr)
StartPullRequestCheckImmediately(ctx, pr)
comment, err := CreatePushPullComment(ctx, opts.Doer, pr, opts.OldCommitID, opts.NewCommitID)
if err == nil && comment != nil {
notify_service.PullRequestPushCommits(ctx, opts.Doer, pr, comment)
@@ -502,7 +502,7 @@ func AddTestPullRequestTask(opts TestPullRequestOptions) {
log.Error("UpdateCommitDivergence: %v", err)
}
}
AddToTaskQueue(ctx, pr)
StartPullRequestCheckDelayable(ctx, pr)
}
})
}

View File

@@ -28,7 +28,7 @@ const (
stagingBranch = "staging" // this is used for a working branch
)
type prContext struct {
type prTmpRepoContext struct {
context.Context
tmpBasePath string
pr *issues_model.PullRequest
@@ -36,7 +36,7 @@ type prContext struct {
errbuf *strings.Builder // any use should be preceded by a Reset and preferably after use
}
func (ctx *prContext) RunOpts() *git.RunOpts {
func (ctx *prTmpRepoContext) RunOpts() *git.RunOpts {
ctx.outbuf.Reset()
ctx.errbuf.Reset()
return &git.RunOpts{
@@ -48,7 +48,7 @@ func (ctx *prContext) RunOpts() *git.RunOpts {
// createTemporaryRepoForPR creates a temporary repo with "base" for pr.BaseBranch and "tracking" for pr.HeadBranch
// it also create a second base branch called "original_base"
func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) (prCtx *prContext, cancel context.CancelFunc, err error) {
func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) (prCtx *prTmpRepoContext, cancel context.CancelFunc, err error) {
if err := pr.LoadHeadRepo(ctx); err != nil {
log.Error("%-v LoadHeadRepo: %v", pr, err)
return nil, nil, fmt.Errorf("%v LoadHeadRepo: %w", pr, err)
@@ -81,7 +81,7 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest)
}
cancel = cleanup
prCtx = &prContext{
prCtx = &prTmpRepoContext{
Context: ctx,
tmpBasePath: tmpBasePath,
pr: pr,