[Enhancement] Allow admin to merge pr with protected file changes (#12078)

* [Enhancement] Allow admin to merge pr with protected file changes

As tilte, show protected message in diff page and merge box.

Signed-off-by: a1012112796 <1012112796@qq.com>

* remove unused ver

* Update options/locale/locale_en-US.ini

Co-authored-by: Cirno the Strongest <1447794+CirnoT@users.noreply.github.com>

* Add TrN

* Apply suggestions from code review

* fix lint

* Update options/locale/locale_en-US.ini

Co-authored-by: zeripath <art27@cantab.net>

* Apply suggestions from code review

* move pr proteced files check to TestPatch
* Call TestPatch when protected branches settings changed

* Apply review suggestion @CirnoT

* move to service @lunny

* slightly restructure routers/private/hook.go

Adds a lot of comments and simplifies the logic

Signed-off-by: Andrew Thornton <art27@cantab.net>

* placate lint

Signed-off-by: Andrew Thornton <art27@cantab.net>

* skip duplicate protected files check

* fix check logic

* slight refactor of TestPatch

Signed-off-by: Andrew Thornton <art27@cantab.net>

* When checking for protected files changes in TestPatch use the temporary repository

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix introduced issue with hook

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove the check on PR index being greater than 0 as it unnecessary

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: techknowlogick <matti@mdranta.net>
Co-authored-by: Cirno the Strongest <1447794+CirnoT@users.noreply.github.com>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

services/gitdiff/gitdiff.go

@@ -351,6 +351,7 @@ type DiffFile struct {
 	IsSubmodule        bool
 	Sections           []*DiffSection
 	IsIncomplete       bool
+	IsProtected        bool
 }
 
 // GetType returns type of diff file.

services/pull/check.go

@@ -62,7 +62,7 @@ func checkAndUpdateStatus(pr *models.PullRequest) {
 	}
 
 	if !has {
-		if err := pr.UpdateColsIfNotMerged("merge_base", "status", "conflicted_files"); err != nil {
+		if err := pr.UpdateColsIfNotMerged("merge_base", "status", "conflicted_files", "changed_protected_files"); err != nil {
 			log.Error("Update[%d]: %v", pr.ID, err)
 		}
 	}
@@ -228,6 +228,20 @@ func handle(data ...queue.Data) {
 	}
 }
 
+// CheckPrsForBaseBranch check all pulls with bseBrannch
+func CheckPrsForBaseBranch(baseRepo *models.Repository, baseBranchName string) error {
+	prs, err := models.GetUnmergedPullRequestsByBaseInfo(baseRepo.ID, baseBranchName)
+	if err != nil {
+		return err
+	}
+
+	for _, pr := range prs {
+		AddToTaskQueue(pr)
+	}
+
+	return nil
+}
+
 // Init runs the task queue to test all the checking status pull requests
 func Init() error {
 	prQueue = queue.CreateUniqueQueue("pr_patch_checker", handle, "").(queue.UniqueQueue)

services/pull/merge.go

@@ -559,7 +559,7 @@ func IsUserAllowedToMerge(pr *models.PullRequest, p models.Permission, user *mod
 }
 
 // CheckPRReadyToMerge checks whether the PR is ready to be merged (reviews and status checks)
-func CheckPRReadyToMerge(pr *models.PullRequest) (err error) {
+func CheckPRReadyToMerge(pr *models.PullRequest, skipProtectedFilesCheck bool) (err error) {
 	if err = pr.LoadBaseRepo(); err != nil {
 		return fmt.Errorf("LoadBaseRepo: %v", err)
 	}
@@ -598,5 +598,15 @@ func CheckPRReadyToMerge(pr *models.PullRequest) (err error) {
 		}
 	}
 
+	if skipProtectedFilesCheck {
+		return nil
+	}
+
+	if pr.ProtectedBranch.MergeBlockedByProtectedFiles(pr) {
+		return models.ErrNotAllowedToMerge{
+			Reason: "Changed protected files",
+		}
+	}
+
 	return nil
 }

services/pull/patch.go

@@ -18,6 +18,8 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/util"
+
+	"github.com/gobwas/glob"
 )
 
 // DownloadDiffOrPatch will write the patch for the pr to the writer
@@ -66,6 +68,7 @@ func TestPatch(pr *models.PullRequest) error {
 	}
 	defer gitRepo.Close()
 
+	// 1. update merge base
 	pr.MergeBase, err = git.NewCommand("merge-base", "--", "base", "tracking").RunInDir(tmpBasePath)
 	if err != nil {
 		var err2 error
@@ -75,10 +78,32 @@ func TestPatch(pr *models.PullRequest) error {
 		}
 	}
 	pr.MergeBase = strings.TrimSpace(pr.MergeBase)
+
+	// 2. Check for conflicts
+	if conflicts, err := checkConflicts(pr, gitRepo, tmpBasePath); err != nil || conflicts {
+		return err
+	}
+
+	// 3. Check for protected files changes
+	if err = checkPullFilesProtection(pr, gitRepo); err != nil {
+		return fmt.Errorf("pr.CheckPullFilesProtection(): %v", err)
+	}
+
+	if len(pr.ChangedProtectedFiles) > 0 {
+		log.Trace("Found %d protected files changed", len(pr.ChangedProtectedFiles))
+	}
+
+	pr.Status = models.PullRequestStatusMergeable
+
+	return nil
+}
+
+func checkConflicts(pr *models.PullRequest, gitRepo *git.Repository, tmpBasePath string) (bool, error) {
+	// 1. Create a plain patch from head to base
 	tmpPatchFile, err := ioutil.TempFile("", "patch")
 	if err != nil {
 		log.Error("Unable to create temporary patch file! Error: %v", err)
-		return fmt.Errorf("Unable to create temporary patch file! Error: %v", err)
+		return false, fmt.Errorf("Unable to create temporary patch file! Error: %v", err)
 	}
 	defer func() {
 		_ = util.Remove(tmpPatchFile.Name())
@@ -87,38 +112,43 @@ func TestPatch(pr *models.PullRequest) error {
 	if err := gitRepo.GetDiff(pr.MergeBase, "tracking", tmpPatchFile); err != nil {
 		tmpPatchFile.Close()
 		log.Error("Unable to get patch file from %s to %s in %s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.FullName(), err)
-		return fmt.Errorf("Unable to get patch file from %s to %s in %s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.FullName(), err)
+		return false, fmt.Errorf("Unable to get patch file from %s to %s in %s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.FullName(), err)
 	}
 	stat, err := tmpPatchFile.Stat()
 	if err != nil {
 		tmpPatchFile.Close()
-		return fmt.Errorf("Unable to stat patch file: %v", err)
+		return false, fmt.Errorf("Unable to stat patch file: %v", err)
 	}
 	patchPath := tmpPatchFile.Name()
 	tmpPatchFile.Close()
 
+	// 1a. if the size of that patch is 0 - there can be no conflicts!
 	if stat.Size() == 0 {
 		log.Debug("PullRequest[%d]: Patch is empty - ignoring", pr.ID)
 		pr.Status = models.PullRequestStatusMergeable
 		pr.ConflictedFiles = []string{}
-		return nil
+		return false, nil
 	}
 
 	log.Trace("PullRequest[%d].testPatch (patchPath): %s", pr.ID, patchPath)
 
+	// 2. preset the pr.Status as checking (this is not save at present)
 	pr.Status = models.PullRequestStatusChecking
 
+	// 3. Read the base branch in to the index of the temporary repository
 	_, err = git.NewCommand("read-tree", "base").RunInDir(tmpBasePath)
 	if err != nil {
-		return fmt.Errorf("git read-tree %s: %v", pr.BaseBranch, err)
+		return false, fmt.Errorf("git read-tree %s: %v", pr.BaseBranch, err)
 	}
 
+	// 4. Now get the pull request configuration to check if we need to ignore whitespace
 	prUnit, err := pr.BaseRepo.GetUnit(models.UnitTypePullRequests)
 	if err != nil {
-		return err
+		return false, err
 	}
 	prConfig := prUnit.PullRequestsConfig()
 
+	// 5. Prepare the arguments to apply the patch against the index
 	args := []string{"apply", "--check", "--cached"}
 	if prConfig.IgnoreWhitespaceConflicts {
 		args = append(args, "--ignore-whitespace")
@@ -126,26 +156,44 @@ func TestPatch(pr *models.PullRequest) error {
 	args = append(args, patchPath)
 	pr.ConflictedFiles = make([]string, 0, 5)
 
+	// 6. Prep the pipe:
+	//   - Here we could do the equivalent of:
+	//  `git apply --check --cached patch_file > conflicts`
+	//     Then iterate through the conflicts. However, that means storing all the conflicts
+	//     in memory - which is very wasteful.
+	//   - alternatively we can do the equivalent of:
+	//  `git apply --check ... | grep ...`
+	//     meaning we don't store all of the conflicts unnecessarily.
 	stderrReader, stderrWriter, err := os.Pipe()
 	if err != nil {
 		log.Error("Unable to open stderr pipe: %v", err)
-		return fmt.Errorf("Unable to open stderr pipe: %v", err)
+		return false, fmt.Errorf("Unable to open stderr pipe: %v", err)
 	}
 	defer func() {
 		_ = stderrReader.Close()
 		_ = stderrWriter.Close()
 	}()
+
+	// 7. Run the check command
 	conflict := false
 	err = git.NewCommand(args...).
 		RunInDirTimeoutEnvFullPipelineFunc(
 			nil, -1, tmpBasePath,
 			nil, stderrWriter, nil,
 			func(ctx context.Context, cancel context.CancelFunc) error {
+				// Close the writer end of the pipe to begin processing
 				_ = stderrWriter.Close()
+				defer func() {
+					// Close the reader on return to terminate the git command if necessary
+					_ = stderrReader.Close()
+				}()
+
 				const prefix = "error: patch failed:"
 				const errorPrefix = "error: "
+
 				conflictMap := map[string]bool{}
 
+				// Now scan the output from the command
 				scanner := bufio.NewScanner(stderrReader)
 				for scanner.Scan() {
 					line := scanner.Text()
@@ -170,25 +218,111 @@ func TestPatch(pr *models.PullRequest) error {
 						break
 					}
 				}
+
 				if len(conflictMap) > 0 {
 					pr.ConflictedFiles = make([]string, 0, len(conflictMap))
 					for key := range conflictMap {
 						pr.ConflictedFiles = append(pr.ConflictedFiles, key)
 					}
 				}
-				_ = stderrReader.Close()
+
 				return nil
 			})
 
+	// 8. If there is a conflict the `git apply` command will return a non-zero error code - so there will be a positive error.
 	if err != nil {
 		if conflict {
 			pr.Status = models.PullRequestStatusConflict
 			log.Trace("Found %d files conflicted: %v", len(pr.ConflictedFiles), pr.ConflictedFiles)
-			return nil
-		}
-		return fmt.Errorf("git apply --check: %v", err)
-	}
-	pr.Status = models.PullRequestStatusMergeable
 
+			return true, nil
+		}
+		return false, fmt.Errorf("git apply --check: %v", err)
+	}
+	return false, nil
+}
+
+// CheckFileProtection check file Protection
+func CheckFileProtection(oldCommitID, newCommitID string, patterns []glob.Glob, limit int, env []string, repo *git.Repository) ([]string, error) {
+	// 1. If there are no patterns short-circuit and just return nil
+	if len(patterns) == 0 {
+		return nil, nil
+	}
+
+	// 2. Prep the pipe
+	stdoutReader, stdoutWriter, err := os.Pipe()
+	if err != nil {
+		log.Error("Unable to create os.Pipe for %s", repo.Path)
+		return nil, err
+	}
+	defer func() {
+		_ = stdoutReader.Close()
+		_ = stdoutWriter.Close()
+	}()
+
+	changedProtectedFiles := make([]string, 0, limit)
+
+	// 3. Run `git diff --name-only` to get the names of the changed files
+	err = git.NewCommand("diff", "--name-only", oldCommitID, newCommitID).
+		RunInDirTimeoutEnvFullPipelineFunc(env, -1, repo.Path,
+			stdoutWriter, nil, nil,
+			func(ctx context.Context, cancel context.CancelFunc) error {
+				// Close the writer end of the pipe to begin processing
+				_ = stdoutWriter.Close()
+				defer func() {
+					// Close the reader on return to terminate the git command if necessary
+					_ = stdoutReader.Close()
+				}()
+
+				// Now scan the output from the command
+				scanner := bufio.NewScanner(stdoutReader)
+				for scanner.Scan() {
+					path := strings.TrimSpace(scanner.Text())
+					if len(path) == 0 {
+						continue
+					}
+					lpath := strings.ToLower(path)
+					for _, pat := range patterns {
+						if pat.Match(lpath) {
+							changedProtectedFiles = append(changedProtectedFiles, path)
+							break
+						}
+					}
+					if len(changedProtectedFiles) >= limit {
+						break
+					}
+				}
+
+				if len(changedProtectedFiles) > 0 {
+					return models.ErrFilePathProtected{
+						Path: changedProtectedFiles[0],
+					}
+				}
+				return scanner.Err()
+			})
+	// 4. log real errors if there are any...
+	if err != nil && !models.IsErrFilePathProtected(err) {
+		log.Error("Unable to check file protection for commits from %s to %s in %s: %v", oldCommitID, newCommitID, repo.Path, err)
+	}
+
+	return changedProtectedFiles, err
+}
+
+// checkPullFilesProtection check if pr changed protected files and save results
+func checkPullFilesProtection(pr *models.PullRequest, gitRepo *git.Repository) error {
+	if err := pr.LoadProtectedBranch(); err != nil {
+		return err
+	}
+
+	if pr.ProtectedBranch == nil {
+		pr.ChangedProtectedFiles = nil
+		return nil
+	}
+
+	var err error
+	pr.ChangedProtectedFiles, err = CheckFileProtection(pr.MergeBase, "tracking", pr.ProtectedBranch.GetProtectedFilePatterns(), 10, os.Environ(), gitRepo)
+	if err != nil && !models.IsErrFilePathProtected(err) {
+		return err
+	}
 	return nil
 }

services/pull/pull.go

@@ -173,7 +173,7 @@ func ChangeTargetBranch(pr *models.PullRequest, doer *models.User, targetBranch
 	pr.CommitsAhead = divergence.Ahead
 	pr.CommitsBehind = divergence.Behind
 
-	if err := pr.UpdateColsIfNotMerged("merge_base", "status", "conflicted_files", "base_branch", "commits_ahead", "commits_behind"); err != nil {
+	if err := pr.UpdateColsIfNotMerged("merge_base", "status", "conflicted_files", "changed_protected_files", "base_branch", "commits_ahead", "commits_behind"); err != nil {
 		return err
 	}