1
1
mirror of https://github.com/go-gitea/gitea synced 2025-07-22 18:28:37 +00:00

Remove SavePatch and generate patches on the fly (#9302)

* Save patches to temporary files

* Remove SavePatch and generate patches on the fly

* Use ioutil.TempDir

* fixup! Use ioutil.TempDir

* fixup! fixup! Use ioutil.TempDir

* RemoveAll LocalCopyPath() in initIntergrationTest

* Default to status checking on PR creation

* Remove unnecessary set to StatusChecking

* Protect against unable to load repo

* Handle conflicts

* Restore original conflict setting

* In TestPullRequests update status to StatusChecking before running TestPatch
This commit is contained in:
zeripath
2019-12-13 22:21:06 +00:00
committed by Antoine GIRARD
parent 8f16a2c37b
commit 74179d1b5e
16 changed files with 432 additions and 406 deletions

View File

@@ -170,7 +170,7 @@ func TestPullRequests() {
if manuallyMerged(pr) {
continue
}
if err := pr.TestPatch(); err != nil {
if err := TestPatch(pr); err != nil {
log.Error("testPatch: %v", err)
continue
}
@@ -194,7 +194,13 @@ func TestPullRequests() {
continue
} else if manuallyMerged(pr) {
continue
} else if err = pr.TestPatch(); err != nil {
}
pr.Status = models.PullRequestStatusChecking
if err := pr.Update(); err != nil {
log.Error("testPatch[%d]: Unable to update status to Checking Status %v", pr.ID, err)
continue
}
if err = TestPatch(pr); err != nil {
log.Error("testPatch[%d]: %v", pr.ID, err)
continue
}

View File

@@ -68,94 +68,22 @@ func Merge(pr *models.PullRequest, doer *models.User, baseGitRepo *git.Repositor
}()
// Clone base repo.
tmpBasePath, err := models.CreateTemporaryPath("merge")
tmpBasePath, err := createTemporaryRepo(pr)
if err != nil {
log.Error("CreateTemporaryPath: %v", err)
return err
}
defer func() {
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("Merge: RemoveTemporaryPath: %s", err)
}
}()
headRepoPath := pr.HeadRepo.RepoPath()
if err := git.InitRepository(tmpBasePath, false); err != nil {
log.Error("git init tmpBasePath: %v", err)
return err
}
remoteRepoName := "head_repo"
baseBranch := "base"
// Add head repo remote.
addCacheRepo := func(staging, cache string) error {
p := filepath.Join(staging, ".git", "objects", "info", "alternates")
f, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
log.Error("Could not create .git/objects/info/alternates file in %s: %v", staging, err)
return err
}
defer f.Close()
data := filepath.Join(cache, "objects")
if _, err := fmt.Fprintln(f, data); err != nil {
log.Error("Could not write to .git/objects/info/alternates file in %s: %v", staging, err)
return err
}
return nil
}
if err := addCacheRepo(tmpBasePath, baseGitRepo.Path); err != nil {
log.Error("Unable to add base repository to temporary repo [%s -> %s]: %v", pr.BaseRepo.FullName(), tmpBasePath, err)
return fmt.Errorf("Unable to add base repository to temporary repo [%s -> tmpBasePath]: %v", pr.BaseRepo.FullName(), err)
}
trackingBranch := "tracking"
stagingBranch := "staging"
var outbuf, errbuf strings.Builder
if err := git.NewCommand("remote", "add", "-t", pr.BaseBranch, "-m", pr.BaseBranch, "origin", baseGitRepo.Path).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to add base repository as origin [%s -> %s]: %v\n%s\n%s", pr.BaseRepo.FullName(), tmpBasePath, err, outbuf.String(), errbuf.String())
return fmt.Errorf("Unable to add base repository as origin [%s -> tmpBasePath]: %v\n%s\n%s", pr.BaseRepo.FullName(), err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
if err := git.NewCommand("fetch", "origin", "--no-tags", pr.BaseBranch+":"+baseBranch, pr.BaseBranch+":original_"+baseBranch).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to fetch origin base branch [%s:%s -> base, original_base in %s]: %v:\n%s\n%s", pr.BaseRepo.FullName(), pr.BaseBranch, tmpBasePath, err, outbuf.String(), errbuf.String())
return fmt.Errorf("Unable to fetch origin base branch [%s:%s -> base, original_base in tmpBasePath]: %v\n%s\n%s", pr.BaseRepo.FullName(), pr.BaseBranch, err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
if err := git.NewCommand("symbolic-ref", "HEAD", git.BranchPrefix+baseBranch).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to set HEAD as base branch [%s]: %v\n%s\n%s", tmpBasePath, err, outbuf.String(), errbuf.String())
return fmt.Errorf("Unable to set HEAD as base branch [tmpBasePath]: %v\n%s\n%s", err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
if err := addCacheRepo(tmpBasePath, headRepoPath); err != nil {
log.Error("Unable to add head repository to temporary repo [%s -> %s]: %v", pr.HeadRepo.FullName(), tmpBasePath, err)
return fmt.Errorf("Unable to head base repository to temporary repo [%s -> tmpBasePath]: %v", pr.HeadRepo.FullName(), err)
}
if err := git.NewCommand("remote", "add", remoteRepoName, headRepoPath).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to add head repository as head_repo [%s -> %s]: %v\n%s\n%s", pr.HeadRepo.FullName(), tmpBasePath, err, outbuf.String(), errbuf.String())
return fmt.Errorf("Unable to add head repository as head_repo [%s -> tmpBasePath]: %v\n%s\n%s", pr.HeadRepo.FullName(), err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
trackingBranch := "tracking"
// Fetch head branch
if err := git.NewCommand("fetch", "--no-tags", remoteRepoName, pr.HeadBranch+":"+trackingBranch).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to fetch head_repo head branch [%s:%s -> tracking in %s]: %v:\n%s\n%s", pr.HeadRepo.FullName(), pr.HeadBranch, tmpBasePath, err, outbuf.String(), errbuf.String())
return fmt.Errorf("Unable to fetch head_repo head branch [%s:%s -> tracking in tmpBasePath]: %v\n%s\n%s", pr.HeadRepo.FullName(), pr.HeadBranch, err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
stagingBranch := "staging"
// Enable sparse-checkout
sparseCheckoutList, err := getDiffTree(tmpBasePath, baseBranch, trackingBranch)

216
services/pull/patch.go Normal file
View File

@@ -0,0 +1,216 @@
// Copyright 2019 The Gitea Authors.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package pull
import (
"bufio"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
)
// DownloadDiff will write the patch for the pr to the writer
func DownloadDiff(pr *models.PullRequest, w io.Writer, patch bool) error {
return DownloadDiffOrPatch(pr, w, false)
}
// DownloadPatch will write the patch for the pr to the writer
func DownloadPatch(pr *models.PullRequest, w io.Writer, patch bool) error {
return DownloadDiffOrPatch(pr, w, true)
}
// DownloadDiffOrPatch will write the patch for the pr to the writer
func DownloadDiffOrPatch(pr *models.PullRequest, w io.Writer, patch bool) error {
// Clone base repo.
tmpBasePath, err := createTemporaryRepo(pr)
if err != nil {
log.Error("CreateTemporaryPath: %v", err)
return err
}
defer func() {
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("DownloadDiff: RemoveTemporaryPath: %s", err)
}
}()
gitRepo, err := git.OpenRepository(tmpBasePath)
if err != nil {
return fmt.Errorf("OpenRepository: %v", err)
}
defer gitRepo.Close()
pr.MergeBase, err = git.NewCommand("merge-base", "--", "base", "tracking").RunInDir(tmpBasePath)
if err != nil {
pr.MergeBase = "base"
}
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
if err := gitRepo.GetDiffOrPatch(pr.MergeBase, "tracking", w, patch); err != nil {
log.Error("Unable to get patch file from %s to %s in %s/%s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.MustOwner().Name, pr.BaseRepo.Name, err)
return fmt.Errorf("Unable to get patch file from %s to %s in %s/%s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.MustOwner().Name, pr.BaseRepo.Name, err)
}
return nil
}
var patchErrorSuffices = []string{
": already exists in index",
": patch does not apply",
": already exists in working directory",
"unrecognized input",
}
// TestPatch will test whether a simple patch will apply
func TestPatch(pr *models.PullRequest) error {
// Clone base repo.
tmpBasePath, err := createTemporaryRepo(pr)
if err != nil {
log.Error("CreateTemporaryPath: %v", err)
return err
}
defer func() {
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("Merge: RemoveTemporaryPath: %s", err)
}
}()
gitRepo, err := git.OpenRepository(tmpBasePath)
if err != nil {
return fmt.Errorf("OpenRepository: %v", err)
}
defer gitRepo.Close()
pr.MergeBase, err = git.NewCommand("merge-base", "--", "base", "tracking").RunInDir(tmpBasePath)
if err != nil {
var err2 error
pr.MergeBase, err2 = gitRepo.GetRefCommitID(git.BranchPrefix + "base")
if err2 != nil {
return fmt.Errorf("GetMergeBase: %v and can't find commit ID for base: %v", err, err2)
}
}
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
tmpPatchFile, err := ioutil.TempFile("", "patch")
if err != nil {
log.Error("Unable to create temporary patch file! Error: %v", err)
return fmt.Errorf("Unable to create temporary patch file! Error: %v", err)
}
defer func() {
_ = os.Remove(tmpPatchFile.Name())
}()
if err := gitRepo.GetDiff(pr.MergeBase, "tracking", tmpPatchFile); err != nil {
tmpPatchFile.Close()
log.Error("Unable to get patch file from %s to %s in %s/%s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.MustOwner().Name, pr.BaseRepo.Name, err)
return fmt.Errorf("Unable to get patch file from %s to %s in %s/%s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.MustOwner().Name, pr.BaseRepo.Name, err)
}
stat, err := tmpPatchFile.Stat()
if err != nil {
tmpPatchFile.Close()
return fmt.Errorf("Unable to stat patch file: %v", err)
}
patchPath := tmpPatchFile.Name()
tmpPatchFile.Close()
if stat.Size() == 0 {
log.Debug("PullRequest[%d]: Patch is empty - ignoring", pr.ID)
pr.Status = models.PullRequestStatusMergeable
pr.ConflictedFiles = []string{}
return nil
}
log.Trace("PullRequest[%d].testPatch (patchPath): %s", pr.ID, patchPath)
pr.Status = models.PullRequestStatusChecking
_, err = git.NewCommand("read-tree", "base").RunInDir(tmpBasePath)
if err != nil {
return fmt.Errorf("git read-tree %s: %v", pr.BaseBranch, err)
}
prUnit, err := pr.BaseRepo.GetUnit(models.UnitTypePullRequests)
if err != nil {
return err
}
prConfig := prUnit.PullRequestsConfig()
args := []string{"apply", "--check", "--cached"}
if prConfig.IgnoreWhitespaceConflicts {
args = append(args, "--ignore-whitespace")
}
args = append(args, patchPath)
pr.ConflictedFiles = make([]string, 0, 5)
stderrReader, stderrWriter, err := os.Pipe()
if err != nil {
log.Error("Unable to open stderr pipe: %v", err)
return fmt.Errorf("Unable to open stderr pipe: %v", err)
}
defer func() {
_ = stderrReader.Close()
_ = stderrWriter.Close()
}()
conflict := false
err = git.NewCommand(args...).
RunInDirTimeoutEnvFullPipelineFunc(
nil, -1, tmpBasePath,
nil, stderrWriter, nil,
func(ctx context.Context, cancel context.CancelFunc) {
_ = stderrWriter.Close()
const prefix = "error: patch failed:"
const errorPrefix = "error: "
conflictMap := map[string]bool{}
scanner := bufio.NewScanner(stderrReader)
for scanner.Scan() {
line := scanner.Text()
fmt.Printf("%s\n", line)
if strings.HasPrefix(line, prefix) {
conflict = true
filepath := strings.TrimSpace(strings.Split(line[len(prefix):], ":")[0])
conflictMap[filepath] = true
} else if strings.HasPrefix(line, errorPrefix) {
conflict = true
for _, suffix := range patchErrorSuffices {
if strings.HasSuffix(line, suffix) {
filepath := strings.TrimSpace(strings.TrimSuffix(line[len(errorPrefix):], suffix))
if filepath != "" {
conflictMap[filepath] = true
}
break
}
}
}
// only list 10 conflicted files
if len(conflictMap) >= 10 {
break
}
}
if len(conflictMap) > 0 {
pr.ConflictedFiles = make([]string, 0, len(conflictMap))
for key := range conflictMap {
pr.ConflictedFiles = append(pr.ConflictedFiles, key)
}
}
_ = stderrReader.Close()
})
if err != nil {
if conflict {
pr.Status = models.PullRequestStatusConflict
log.Trace("Found %d files conflicted: %v", len(pr.ConflictedFiles), pr.ConflictedFiles)
return nil
}
return fmt.Errorf("git apply --check: %v", err)
}
pr.Status = models.PullRequestStatusMergeable
return nil
}

View File

@@ -15,8 +15,12 @@ import (
)
// NewPullRequest creates new pull request with labels for repository.
func NewPullRequest(repo *models.Repository, pull *models.Issue, labelIDs []int64, uuids []string, pr *models.PullRequest, patch []byte, assigneeIDs []int64) error {
if err := models.NewPullRequest(repo, pull, labelIDs, uuids, pr, patch); err != nil {
func NewPullRequest(repo *models.Repository, pull *models.Issue, labelIDs []int64, uuids []string, pr *models.PullRequest, assigneeIDs []int64) error {
if err := TestPatch(pr); err != nil {
return err
}
if err := models.NewPullRequest(repo, pull, labelIDs, uuids, pr); err != nil {
return err
}
@@ -56,10 +60,7 @@ func checkForInvalidation(requests models.PullRequestList, repoID int64, doer *m
func addHeadRepoTasks(prs []*models.PullRequest) {
for _, pr := range prs {
log.Trace("addHeadRepoTasks[%d]: composing new test task", pr.ID)
if err := pr.UpdatePatch(); err != nil {
log.Error("UpdatePatch: %v", err)
continue
} else if err := pr.PushToBaseRepo(); err != nil {
if err := pr.PushToBaseRepo(); err != nil {
log.Error("PushToBaseRepo: %v", err)
continue
}

152
services/pull/temp_repo.go Normal file
View File

@@ -0,0 +1,152 @@
// Copyright 2019 The Gitea Authors.
// All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package pull
import (
"fmt"
"os"
"path/filepath"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
)
func createTemporaryRepo(pr *models.PullRequest) (string, error) {
if err := pr.GetHeadRepo(); err != nil {
log.Error("GetHeadRepo: %v", err)
return "", fmt.Errorf("GetHeadRepo: %v", err)
} else if pr.HeadRepo == nil {
log.Error("Pr %d HeadRepo %d does not exist", pr.ID, pr.HeadRepoID)
return "", &models.ErrRepoNotExist{
ID: pr.HeadRepoID,
}
} else if err := pr.GetBaseRepo(); err != nil {
log.Error("GetBaseRepo: %v", err)
return "", fmt.Errorf("GetBaseRepo: %v", err)
} else if pr.BaseRepo == nil {
log.Error("Pr %d BaseRepo %d does not exist", pr.ID, pr.BaseRepoID)
return "", &models.ErrRepoNotExist{
ID: pr.BaseRepoID,
}
} else if err := pr.HeadRepo.GetOwner(); err != nil {
log.Error("HeadRepo.GetOwner: %v", err)
return "", fmt.Errorf("HeadRepo.GetOwner: %v", err)
} else if err := pr.BaseRepo.GetOwner(); err != nil {
log.Error("BaseRepo.GetOwner: %v", err)
return "", fmt.Errorf("BaseRepo.GetOwner: %v", err)
}
// Clone base repo.
tmpBasePath, err := models.CreateTemporaryPath("pull")
if err != nil {
log.Error("CreateTemporaryPath: %v", err)
return "", err
}
baseRepoPath := pr.BaseRepo.RepoPath()
headRepoPath := pr.HeadRepo.RepoPath()
if err := git.InitRepository(tmpBasePath, false); err != nil {
log.Error("git init tmpBasePath: %v", err)
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("CreateTempRepo: RemoveTemporaryPath: %s", err)
}
return "", err
}
remoteRepoName := "head_repo"
baseBranch := "base"
// Add head repo remote.
addCacheRepo := func(staging, cache string) error {
p := filepath.Join(staging, ".git", "objects", "info", "alternates")
f, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
log.Error("Could not create .git/objects/info/alternates file in %s: %v", staging, err)
return err
}
defer f.Close()
data := filepath.Join(cache, "objects")
if _, err := fmt.Fprintln(f, data); err != nil {
log.Error("Could not write to .git/objects/info/alternates file in %s: %v", staging, err)
return err
}
return nil
}
if err := addCacheRepo(tmpBasePath, baseRepoPath); err != nil {
log.Error("Unable to add base repository to temporary repo [%s -> %s]: %v", pr.BaseRepo.FullName(), tmpBasePath, err)
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("CreateTempRepo: RemoveTemporaryPath: %s", err)
}
return "", fmt.Errorf("Unable to add base repository to temporary repo [%s -> tmpBasePath]: %v", pr.BaseRepo.FullName(), err)
}
var outbuf, errbuf strings.Builder
if err := git.NewCommand("remote", "add", "-t", pr.BaseBranch, "-m", pr.BaseBranch, "origin", baseRepoPath).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to add base repository as origin [%s -> %s]: %v\n%s\n%s", pr.BaseRepo.FullName(), tmpBasePath, err, outbuf.String(), errbuf.String())
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("CreateTempRepo: RemoveTemporaryPath: %s", err)
}
return "", fmt.Errorf("Unable to add base repository as origin [%s -> tmpBasePath]: %v\n%s\n%s", pr.BaseRepo.FullName(), err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
if err := git.NewCommand("fetch", "origin", "--no-tags", pr.BaseBranch+":"+baseBranch, pr.BaseBranch+":original_"+baseBranch).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to fetch origin base branch [%s:%s -> base, original_base in %s]: %v:\n%s\n%s", pr.BaseRepo.FullName(), pr.BaseBranch, tmpBasePath, err, outbuf.String(), errbuf.String())
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("CreateTempRepo: RemoveTemporaryPath: %s", err)
}
return "", fmt.Errorf("Unable to fetch origin base branch [%s:%s -> base, original_base in tmpBasePath]: %v\n%s\n%s", pr.BaseRepo.FullName(), pr.BaseBranch, err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
if err := git.NewCommand("symbolic-ref", "HEAD", git.BranchPrefix+baseBranch).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to set HEAD as base branch [%s]: %v\n%s\n%s", tmpBasePath, err, outbuf.String(), errbuf.String())
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("CreateTempRepo: RemoveTemporaryPath: %s", err)
}
return "", fmt.Errorf("Unable to set HEAD as base branch [tmpBasePath]: %v\n%s\n%s", err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
if err := addCacheRepo(tmpBasePath, headRepoPath); err != nil {
log.Error("Unable to add head repository to temporary repo [%s -> %s]: %v", pr.HeadRepo.FullName(), tmpBasePath, err)
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("CreateTempRepo: RemoveTemporaryPath: %s", err)
}
return "", fmt.Errorf("Unable to head base repository to temporary repo [%s -> tmpBasePath]: %v", pr.HeadRepo.FullName(), err)
}
if err := git.NewCommand("remote", "add", remoteRepoName, headRepoPath).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to add head repository as head_repo [%s -> %s]: %v\n%s\n%s", pr.HeadRepo.FullName(), tmpBasePath, err, outbuf.String(), errbuf.String())
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("CreateTempRepo: RemoveTemporaryPath: %s", err)
}
return "", fmt.Errorf("Unable to add head repository as head_repo [%s -> tmpBasePath]: %v\n%s\n%s", pr.HeadRepo.FullName(), err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
trackingBranch := "tracking"
// Fetch head branch
if err := git.NewCommand("fetch", "--no-tags", remoteRepoName, pr.HeadBranch+":"+trackingBranch).RunInDirPipeline(tmpBasePath, &outbuf, &errbuf); err != nil {
log.Error("Unable to fetch head_repo head branch [%s:%s -> tracking in %s]: %v:\n%s\n%s", pr.HeadRepo.FullName(), pr.HeadBranch, tmpBasePath, err, outbuf.String(), errbuf.String())
if err := models.RemoveTemporaryPath(tmpBasePath); err != nil {
log.Error("CreateTempRepo: RemoveTemporaryPath: %s", err)
}
return "", fmt.Errorf("Unable to fetch head_repo head branch [%s:%s -> tracking in tmpBasePath]: %v\n%s\n%s", pr.HeadRepo.FullName(), pr.HeadBranch, err, outbuf.String(), errbuf.String())
}
outbuf.Reset()
errbuf.Reset()
return tmpBasePath, nil
}