Move git config/remote to gitrepo package and add global lock to resolve possible conflict when updating repository git config file (#35151)

Partially fix #32018 

`git config` and `git remote` write operations create a temporary file
named `config.lock`. Since these operations are not atomic, they must
not be run in parallel. If two requests attempt to modify the same
repository concurrently—such as during a compare operation—one may fail
due to the presence of an existing `config.lock` file.

In cases where `config.lock` is left behind due to an unexpected program
exit, a global lock mechanism could allow us to safely remove the stale
lock file when a related error is detected. While this behavior is not
yet implemented in this PR, it is planned for a future enhancement.

---------

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

services/mirror/mirror_pull.go

@@ -39,30 +39,27 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
 	}
 
 	remoteName := m.GetRemoteName()
-	repoPath := m.GetRepository(ctx).RepoPath()
+	repo := m.GetRepository(ctx)
 	// Remove old remote
-	_, _, err = git.NewCommand("remote", "rm").AddDynamicArguments(remoteName).RunStdString(ctx, &git.RunOpts{Dir: repoPath})
+	err = gitrepo.GitRemoteRemove(ctx, repo, remoteName)
 	if err != nil && !git.IsRemoteNotExistError(err) {
 		return err
 	}
 
-	cmd := git.NewCommand("remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(addr)
-	_, _, err = cmd.RunStdString(ctx, &git.RunOpts{Dir: repoPath})
+	err = gitrepo.GitRemoteAdd(ctx, repo, remoteName, addr, gitrepo.RemoteOptionMirrorFetch)
 	if err != nil && !git.IsRemoteNotExistError(err) {
 		return err
 	}
 
 	if repo_service.HasWiki(ctx, m.Repo) {
-		wikiPath := m.Repo.WikiPath()
 		wikiRemotePath := repo_module.WikiRemoteURL(ctx, addr)
 		// Remove old remote of wiki
-		_, _, err = git.NewCommand("remote", "rm").AddDynamicArguments(remoteName).RunStdString(ctx, &git.RunOpts{Dir: wikiPath})
+		err = gitrepo.GitRemoteRemove(ctx, repo.WikiStorageRepo(), remoteName)
 		if err != nil && !git.IsRemoteNotExistError(err) {
 			return err
 		}
 
-		cmd = git.NewCommand("remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(wikiRemotePath)
-		_, _, err = cmd.RunStdString(ctx, &git.RunOpts{Dir: wikiPath})
+		err = gitrepo.GitRemoteAdd(ctx, repo.WikiStorageRepo(), remoteName, wikiRemotePath, gitrepo.RemoteOptionMirrorFetch)
 		if err != nil && !git.IsRemoteNotExistError(err) {
 			return err
 		}
@@ -197,25 +194,21 @@ func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult {
 
 func pruneBrokenReferences(ctx context.Context,
 	m *repo_model.Mirror,
-	repoPath string,
 	timeout time.Duration,
 	stdoutBuilder, stderrBuilder *strings.Builder,
 	isWiki bool,
 ) error {
 	wiki := ""
+	var storageRepo gitrepo.Repository = m.Repo
 	if isWiki {
 		wiki = "Wiki "
+		storageRepo = m.Repo.WikiStorageRepo()
 	}
 
 	stderrBuilder.Reset()
 	stdoutBuilder.Reset()
-	pruneErr := git.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).
-		Run(ctx, &git.RunOpts{
-			Timeout: timeout,
-			Dir:     repoPath,
-			Stdout:  stdoutBuilder,
-			Stderr:  stderrBuilder,
-		})
+
+	pruneErr := gitrepo.GitRemotePrune(ctx, storageRepo, m.GetRemoteName(), timeout, stdoutBuilder, stderrBuilder)
 	if pruneErr != nil {
 		stdout := stdoutBuilder.String()
 		stderr := stderrBuilder.String()
@@ -226,7 +219,7 @@ func pruneBrokenReferences(ctx context.Context,
 		stdoutMessage := util.SanitizeCredentialURLs(stdout)
 
 		log.Error("Failed to prune mirror repository %s%-v references:\nStdout: %s\nStderr: %s\nErr: %v", wiki, m.Repo, stdoutMessage, stderrMessage, pruneErr)
-		desc := fmt.Sprintf("Failed to prune mirror repository %s'%s' references: %s", wiki, repoPath, stderrMessage)
+		desc := fmt.Sprintf("Failed to prune mirror repository %s'%s' references: %s", wiki, storageRepo.RelativePath(), stderrMessage)
 		if err := system_model.CreateRepositoryNotice(desc); err != nil {
 			log.Error("CreateRepositoryNotice: %v", err)
 		}
@@ -268,9 +261,9 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
 	}
 	cmd.AddArguments("--tags").AddDynamicArguments(m.GetRemoteName())
 
-	remoteURL, remoteErr := git.GetRemoteURL(ctx, repoPath, m.GetRemoteName())
+	remoteURL, remoteErr := gitrepo.GitRemoteGetURL(ctx, m.Repo, m.GetRemoteName())
 	if remoteErr != nil {
-		log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr)
+		log.Error("SyncMirrors [repo: %-v]: GetRemoteURL Error %v", m.Repo, remoteErr)
 		return nil, false
 	}
 
@@ -298,7 +291,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
 			err = nil
 
 			// Attempt prune
-			pruneErr := pruneBrokenReferences(ctx, m, repoPath, timeout, &stdoutBuilder, &stderrBuilder, false)
+			pruneErr := pruneBrokenReferences(ctx, m, timeout, &stdoutBuilder, &stderrBuilder, false)
 			if pruneErr == nil {
 				// Successful prune - reattempt mirror
 				stderrBuilder.Reset()
@@ -371,13 +364,9 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
 		log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo)
 		stderrBuilder.Reset()
 		stdoutBuilder.Reset()
-		if err := git.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).
-			Run(ctx, &git.RunOpts{
-				Timeout: timeout,
-				Dir:     wikiPath,
-				Stdout:  &stdoutBuilder,
-				Stderr:  &stderrBuilder,
-			}); err != nil {
+
+		if err := gitrepo.GitRemoteUpdatePrune(ctx, m.Repo.WikiStorageRepo(), m.GetRemoteName(),
+			timeout, &stdoutBuilder, &stderrBuilder); err != nil {
 			stdout := stdoutBuilder.String()
 			stderr := stderrBuilder.String()
 
@@ -391,19 +380,14 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
 				err = nil
 
 				// Attempt prune
-				pruneErr := pruneBrokenReferences(ctx, m, repoPath, timeout, &stdoutBuilder, &stderrBuilder, true)
+				pruneErr := pruneBrokenReferences(ctx, m, timeout, &stdoutBuilder, &stderrBuilder, true)
 				if pruneErr == nil {
 					// Successful prune - reattempt mirror
 					stderrBuilder.Reset()
 					stdoutBuilder.Reset()
 
-					if err = git.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).
-						Run(ctx, &git.RunOpts{
-							Timeout: timeout,
-							Dir:     wikiPath,
-							Stdout:  &stdoutBuilder,
-							Stderr:  &stderrBuilder,
-						}); err != nil {
+					if err = gitrepo.GitRemoteUpdatePrune(ctx, m.Repo.WikiStorageRepo(), m.GetRemoteName(),
+						timeout, &stdoutBuilder, &stderrBuilder); err != nil {
 						stdout := stdoutBuilder.String()
 						stderr := stderrBuilder.String()
 						stderrMessage = util.SanitizeCredentialURLs(stderr)

services/mirror/mirror_push.go

@@ -30,28 +30,24 @@ var stripExitStatus = regexp.MustCompile(`exit status \d+ - `)
 
 // AddPushMirrorRemote registers the push mirror remote.
 func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error {
-	addRemoteAndConfig := func(addr, path string) error {
-		cmd := git.NewCommand("remote", "add", "--mirror=push").AddDynamicArguments(m.RemoteName, addr)
-		if _, _, err := cmd.RunStdString(ctx, &git.RunOpts{Dir: path}); err != nil {
+	addRemoteAndConfig := func(storageRepo gitrepo.Repository, addr string) error {
+		if err := gitrepo.GitRemoteAdd(ctx, storageRepo, m.RemoteName, addr, gitrepo.RemoteOptionMirrorPush); err != nil {
 			return err
 		}
-		if _, _, err := git.NewCommand("config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(ctx, &git.RunOpts{Dir: path}); err != nil {
+		if err := gitrepo.GitConfigAdd(ctx, storageRepo, "remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*"); err != nil {
 			return err
 		}
-		if _, _, err := git.NewCommand("config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(ctx, &git.RunOpts{Dir: path}); err != nil {
-			return err
-		}
-		return nil
+		return gitrepo.GitConfigAdd(ctx, storageRepo, "remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*")
 	}
 
-	if err := addRemoteAndConfig(addr, m.Repo.RepoPath()); err != nil {
+	if err := addRemoteAndConfig(m.Repo, addr); err != nil {
 		return err
 	}
 
 	if repo_service.HasWiki(ctx, m.Repo) {
 		wikiRemoteURL := repository.WikiRemoteURL(ctx, addr)
 		if len(wikiRemoteURL) > 0 {
-			if err := addRemoteAndConfig(wikiRemoteURL, m.Repo.WikiPath()); err != nil {
+			if err := addRemoteAndConfig(m.Repo.WikiStorageRepo(), wikiRemoteURL); err != nil {
 				return err
 			}
 		}
@@ -62,15 +58,13 @@ func AddPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr str
 
 // RemovePushMirrorRemote removes the push mirror remote.
 func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error {
-	cmd := git.NewCommand("remote", "rm").AddDynamicArguments(m.RemoteName)
 	_ = m.GetRepository(ctx)
-
-	if _, _, err := cmd.RunStdString(ctx, &git.RunOpts{Dir: m.Repo.RepoPath()}); err != nil {
+	if err := gitrepo.GitRemoteRemove(ctx, m.Repo, m.RemoteName); err != nil {
 		return err
 	}
 
 	if repo_service.HasWiki(ctx, m.Repo) {
-		if _, _, err := cmd.RunStdString(ctx, &git.RunOpts{Dir: m.Repo.WikiPath()}); err != nil {
+		if err := gitrepo.GitRemoteRemove(ctx, m.Repo.WikiStorageRepo(), m.RemoteName); err != nil {
 			// The wiki remote may not exist
 			log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err)
 		}
@@ -129,25 +123,22 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error {
 	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
 
 	performPush := func(repo *repo_model.Repository, isWiki bool) error {
+		var storageRepo gitrepo.Repository = repo
 		path := repo.RepoPath()
 		if isWiki {
+			storageRepo = repo.WikiStorageRepo()
 			path = repo.WikiPath()
 		}
-		remoteURL, err := git.GetRemoteURL(ctx, path, m.RemoteName)
+		remoteURL, err := gitrepo.GitRemoteGetURL(ctx, storageRepo, m.RemoteName)
 		if err != nil {
-			log.Error("GetRemoteAddress(%s) Error %v", path, err)
+			log.Error("GetRemoteURL(%s) Error %v", path, err)
 			return errors.New("Unexpected error")
 		}
 
 		if setting.LFS.StartServer {
 			log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
 
-			var gitRepo *git.Repository
-			if isWiki {
-				gitRepo, err = gitrepo.OpenRepository(ctx, repo.WikiStorageRepo())
-			} else {
-				gitRepo, err = gitrepo.OpenRepository(ctx, repo)
-			}
+			gitRepo, err := gitrepo.OpenRepository(ctx, storageRepo)
 			if err != nil {
 				log.Error("OpenRepository: %v", err)
 				return errors.New("Unexpected error")
@@ -185,14 +176,13 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error {
 	}
 
 	if repo_service.HasWiki(ctx, m.Repo) {
-		_, err := git.GetRemoteAddress(ctx, m.Repo.WikiPath(), m.RemoteName)
-		if err == nil {
+		if _, err := gitrepo.GitRemoteGetURL(ctx, m.Repo.WikiStorageRepo(), m.RemoteName); err == nil {
 			err := performPush(m.Repo, true)
 			if err != nil {
 				return err
 			}
-		} else {
-			log.Trace("Skipping wiki: No remote configured")
+		} else if !errors.Is(err, util.ErrNotExist) {
+			log.Error("GetRemote of wiki failed: %v", err)
 		}
 	}