mirror of
https://github.com/go-gitea/gitea
synced 2024-11-15 22:54:24 +00:00
Reduce integration test overhead (#32475)
In profiling integration tests, I found a couple places where per-test overhead could be reduced: * Avoiding disk IO by synchronizing instead of deleting & copying test Git repository data. This saves ~100ms per test on my machine * When flushing queues in `PrintCurrentTest`, invoke `FlushWithContext` in a parallel. --------- Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
parent
249e67672a
commit
68731c07c5
@ -26,7 +26,7 @@
|
|||||||
fork_id: 0
|
fork_id: 0
|
||||||
is_template: false
|
is_template: false
|
||||||
template_id: 0
|
template_id: 0
|
||||||
size: 8478
|
size: 0
|
||||||
is_fsck_enabled: true
|
is_fsck_enabled: true
|
||||||
close_issues_via_commit_in_any_branch: false
|
close_issues_via_commit_in_any_branch: false
|
||||||
|
|
||||||
|
@ -8,7 +8,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"testing"
|
"testing"
|
||||||
@ -35,27 +34,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...any) (*xorm.Engine, fu
|
|||||||
ourSkip := 2
|
ourSkip := 2
|
||||||
ourSkip += skip
|
ourSkip += skip
|
||||||
deferFn := testlogger.PrintCurrentTest(t, ourSkip)
|
deferFn := testlogger.PrintCurrentTest(t, ourSkip)
|
||||||
assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
|
assert.NoError(t, unittest.SyncDirs(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
|
||||||
assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
|
|
||||||
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
|
|
||||||
if err != nil {
|
|
||||||
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
|
|
||||||
}
|
|
||||||
for _, ownerDir := range ownerDirs {
|
|
||||||
if !ownerDir.Type().IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
|
|
||||||
if err != nil {
|
|
||||||
assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
|
|
||||||
}
|
|
||||||
for _, repoDir := range repoDirs {
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := deleteDB(); err != nil {
|
if err := deleteDB(); err != nil {
|
||||||
t.Errorf("unable to reset database: %v", err)
|
t.Errorf("unable to reset database: %v", err)
|
||||||
@ -123,7 +102,7 @@ func MainTest(m *testing.M) {
|
|||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
giteaBinary += ".exe"
|
giteaBinary += ".exe"
|
||||||
}
|
}
|
||||||
setting.AppPath = path.Join(giteaRoot, giteaBinary)
|
setting.AppPath = filepath.Join(giteaRoot, giteaBinary)
|
||||||
if _, err := os.Stat(setting.AppPath); err != nil {
|
if _, err := os.Stat(setting.AppPath); err != nil {
|
||||||
fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath)
|
fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
@ -131,12 +110,12 @@ func MainTest(m *testing.M) {
|
|||||||
|
|
||||||
giteaConf := os.Getenv("GITEA_CONF")
|
giteaConf := os.Getenv("GITEA_CONF")
|
||||||
if giteaConf == "" {
|
if giteaConf == "" {
|
||||||
giteaConf = path.Join(filepath.Dir(setting.AppPath), "tests/sqlite.ini")
|
giteaConf = filepath.Join(filepath.Dir(setting.AppPath), "tests/sqlite.ini")
|
||||||
fmt.Printf("Environment variable $GITEA_CONF not set - defaulting to %s\n", giteaConf)
|
fmt.Printf("Environment variable $GITEA_CONF not set - defaulting to %s\n", giteaConf)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !path.IsAbs(giteaConf) {
|
if !filepath.IsAbs(giteaConf) {
|
||||||
setting.CustomConf = path.Join(giteaRoot, giteaConf)
|
setting.CustomConf = filepath.Join(giteaRoot, giteaConf)
|
||||||
} else {
|
} else {
|
||||||
setting.CustomConf = giteaConf
|
setting.CustomConf = giteaConf
|
||||||
}
|
}
|
||||||
|
@ -4,10 +4,8 @@
|
|||||||
package unittest
|
package unittest
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
@ -32,67 +30,73 @@ func Copy(src, dest string) error {
|
|||||||
return os.Symlink(target, dest)
|
return os.Symlink(target, dest)
|
||||||
}
|
}
|
||||||
|
|
||||||
sr, err := os.Open(src)
|
return util.CopyFile(src, dest)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer sr.Close()
|
|
||||||
|
|
||||||
dw, err := os.Create(dest)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer dw.Close()
|
|
||||||
|
|
||||||
if _, err = io.Copy(dw, sr); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set back file information.
|
|
||||||
if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return os.Chmod(dest, si.Mode())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyDir copy files recursively from source to target directory.
|
// Sync synchronizes the two files. This is skipped if both files
|
||||||
//
|
// exist and the size, modtime, and mode match.
|
||||||
// The filter accepts a function that process the path info.
|
func Sync(srcPath, destPath string) error {
|
||||||
// and should return true for need to filter.
|
dest, err := os.Stat(destPath)
|
||||||
//
|
if err != nil {
|
||||||
// It returns error when error occurs in underlying functions.
|
if os.IsNotExist(err) {
|
||||||
func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error {
|
return Copy(srcPath, destPath)
|
||||||
// Check if target directory exists.
|
}
|
||||||
if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) {
|
return err
|
||||||
return util.NewAlreadyExistErrorf("file or directory already exists: %s", destPath)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
src, err := os.Stat(srcPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if src.Size() == dest.Size() &&
|
||||||
|
src.ModTime() == dest.ModTime() &&
|
||||||
|
src.Mode() == dest.Mode() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return Copy(srcPath, destPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SyncDirs synchronizes files recursively from source to target directory.
|
||||||
|
// It returns error when error occurs in underlying functions.
|
||||||
|
func SyncDirs(srcPath, destPath string) error {
|
||||||
err := os.MkdirAll(destPath, os.ModePerm)
|
err := os.MkdirAll(destPath, os.ModePerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Gather directory info.
|
// find and delete all untracked files
|
||||||
infos, err := util.StatDir(srcPath, true)
|
destFiles, err := util.StatDir(destPath, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
for _, destFile := range destFiles {
|
||||||
var filter func(filePath string) bool
|
destFilePath := filepath.Join(destPath, destFile)
|
||||||
if len(filters) > 0 {
|
if _, err = os.Stat(filepath.Join(srcPath, destFile)); err != nil {
|
||||||
filter = filters[0]
|
if os.IsNotExist(err) {
|
||||||
|
// if src file does not exist, remove dest file
|
||||||
|
if err = os.RemoveAll(destFilePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, info := range infos {
|
// sync src files to dest
|
||||||
if filter != nil && filter(info) {
|
srcFiles, err := util.StatDir(srcPath, true)
|
||||||
continue
|
if err != nil {
|
||||||
}
|
return err
|
||||||
|
}
|
||||||
curPath := path.Join(destPath, info)
|
for _, srcFile := range srcFiles {
|
||||||
if strings.HasSuffix(info, "/") {
|
destFilePath := filepath.Join(destPath, srcFile)
|
||||||
err = os.MkdirAll(curPath, os.ModePerm)
|
// util.StatDir appends a slash to the directory name
|
||||||
|
if strings.HasSuffix(srcFile, "/") {
|
||||||
|
err = os.MkdirAll(destFilePath, os.ModePerm)
|
||||||
} else {
|
} else {
|
||||||
err = Copy(path.Join(srcPath, info), curPath)
|
err = Sync(filepath.Join(srcPath, srcFile), destFilePath)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -164,35 +164,13 @@ func MainTest(m *testing.M, testOpts ...*TestOptions) {
|
|||||||
if err = storage.Init(); err != nil {
|
if err = storage.Init(); err != nil {
|
||||||
fatalTestError("storage.Init: %v\n", err)
|
fatalTestError("storage.Init: %v\n", err)
|
||||||
}
|
}
|
||||||
if err = util.RemoveAll(repoRootPath); err != nil {
|
if err = SyncDirs(filepath.Join(giteaRoot, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
|
||||||
fatalTestError("util.RemoveAll: %v\n", err)
|
fatalTestError("util.SyncDirs: %v\n", err)
|
||||||
}
|
|
||||||
if err = CopyDir(filepath.Join(giteaRoot, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
|
|
||||||
fatalTestError("util.CopyDir: %v\n", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = git.InitFull(context.Background()); err != nil {
|
if err = git.InitFull(context.Background()); err != nil {
|
||||||
fatalTestError("git.Init: %v\n", err)
|
fatalTestError("git.Init: %v\n", err)
|
||||||
}
|
}
|
||||||
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
|
|
||||||
if err != nil {
|
|
||||||
fatalTestError("unable to read the new repo root: %v\n", err)
|
|
||||||
}
|
|
||||||
for _, ownerDir := range ownerDirs {
|
|
||||||
if !ownerDir.Type().IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
|
|
||||||
if err != nil {
|
|
||||||
fatalTestError("unable to read the new repo root: %v\n", err)
|
|
||||||
}
|
|
||||||
for _, repoDir := range repoDirs {
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(testOpts) > 0 && testOpts[0].SetUp != nil {
|
if len(testOpts) > 0 && testOpts[0].SetUp != nil {
|
||||||
if err := testOpts[0].SetUp(); err != nil {
|
if err := testOpts[0].SetUp(); err != nil {
|
||||||
@ -255,24 +233,7 @@ func PrepareTestDatabase() error {
|
|||||||
// by tests that use the above MainTest(..) function.
|
// by tests that use the above MainTest(..) function.
|
||||||
func PrepareTestEnv(t testing.TB) {
|
func PrepareTestEnv(t testing.TB) {
|
||||||
assert.NoError(t, PrepareTestDatabase())
|
assert.NoError(t, PrepareTestDatabase())
|
||||||
assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
|
|
||||||
metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta")
|
metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta")
|
||||||
assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath))
|
assert.NoError(t, SyncDirs(metaPath, setting.RepoRootPath))
|
||||||
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
for _, ownerDir := range ownerDirs {
|
|
||||||
if !ownerDir.Type().IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
|
|
||||||
assert.NoError(t, err)
|
|
||||||
for _, repoDir := range repoDirs {
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
|
|
||||||
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
base.SetupGiteaRoot() // Makes sure GITEA_ROOT is set
|
base.SetupGiteaRoot() // Makes sure GITEA_ROOT is set
|
||||||
}
|
}
|
||||||
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -21,6 +21,7 @@ import (
|
|||||||
_ "code.gitea.io/gitea/models/activities"
|
_ "code.gitea.io/gitea/models/activities"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
_ "github.com/mattn/go-sqlite3"
|
_ "github.com/mattn/go-sqlite3"
|
||||||
)
|
)
|
||||||
@ -284,15 +285,11 @@ func TestBleveIndexAndSearch(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
|
||||||
idx := bleve.NewIndexer(dir)
|
idx := bleve.NewIndexer(dir)
|
||||||
_, err := idx.Init(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
if idx != nil {
|
|
||||||
idx.Close()
|
|
||||||
}
|
|
||||||
assert.FailNow(t, "Unable to create bleve indexer Error: %v", err)
|
|
||||||
}
|
|
||||||
defer idx.Close()
|
defer idx.Close()
|
||||||
|
|
||||||
|
_, err := idx.Init(context.Background())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
testIndexer("beleve", t, idx)
|
testIndexer("beleve", t, idx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5,6 +5,7 @@ package queue
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -32,6 +33,7 @@ type ManagedWorkerPoolQueue interface {
|
|||||||
|
|
||||||
// FlushWithContext tries to make the handler process all items in the queue synchronously.
|
// FlushWithContext tries to make the handler process all items in the queue synchronously.
|
||||||
// It is for testing purpose only. It's not designed to be used in a cluster.
|
// It is for testing purpose only. It's not designed to be used in a cluster.
|
||||||
|
// Negative timeout means discarding all items in the queue.
|
||||||
FlushWithContext(ctx context.Context, timeout time.Duration) error
|
FlushWithContext(ctx context.Context, timeout time.Duration) error
|
||||||
|
|
||||||
// RemoveAllItems removes all items in the base queue (on-the-fly items are not affected)
|
// RemoveAllItems removes all items in the base queue (on-the-fly items are not affected)
|
||||||
@ -76,15 +78,16 @@ func (m *Manager) ManagedQueues() map[int64]ManagedWorkerPoolQueue {
|
|||||||
|
|
||||||
// FlushAll tries to make all managed queues process all items synchronously, until timeout or the queue is empty.
|
// FlushAll tries to make all managed queues process all items synchronously, until timeout or the queue is empty.
|
||||||
// It is for testing purpose only. It's not designed to be used in a cluster.
|
// It is for testing purpose only. It's not designed to be used in a cluster.
|
||||||
|
// Negative timeout means discarding all items in the queue.
|
||||||
func (m *Manager) FlushAll(ctx context.Context, timeout time.Duration) error {
|
func (m *Manager) FlushAll(ctx context.Context, timeout time.Duration) error {
|
||||||
var finalErr error
|
var finalErrors []error
|
||||||
qs := m.ManagedQueues()
|
qs := m.ManagedQueues()
|
||||||
for _, q := range qs {
|
for _, q := range qs {
|
||||||
if err := q.FlushWithContext(ctx, timeout); err != nil {
|
if err := q.FlushWithContext(ctx, timeout); err != nil {
|
||||||
finalErr = err // TODO: in Go 1.20: errors.Join
|
finalErrors = append(finalErrors, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return finalErr
|
return errors.Join(finalErrors...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateSimpleQueue creates a simple queue from global setting config provider by name
|
// CreateSimpleQueue creates a simple queue from global setting config provider by name
|
||||||
|
@ -197,15 +197,30 @@ func (q *WorkerPoolQueue[T]) doFlush(wg *workerGroup[T], flush flushType) {
|
|||||||
defer log.Debug("Queue %q finishes flushing", q.GetName())
|
defer log.Debug("Queue %q finishes flushing", q.GetName())
|
||||||
|
|
||||||
// stop all workers, and prepare a new worker context to start new workers
|
// stop all workers, and prepare a new worker context to start new workers
|
||||||
|
|
||||||
wg.ctxWorkerCancel()
|
wg.ctxWorkerCancel()
|
||||||
wg.wg.Wait()
|
wg.wg.Wait()
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
close(flush)
|
close(flush.c)
|
||||||
wg.doPrepareWorkerContext()
|
wg.doPrepareWorkerContext()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
if flush.timeout < 0 {
|
||||||
|
// discard everything
|
||||||
|
wg.batchBuffer = nil
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-wg.popItemChan:
|
||||||
|
case <-wg.popItemErr:
|
||||||
|
case <-q.batchChan:
|
||||||
|
case <-q.ctxRun.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// drain the batch channel first
|
// drain the batch channel first
|
||||||
loop:
|
loop:
|
||||||
for {
|
for {
|
||||||
|
@ -42,7 +42,10 @@ type WorkerPoolQueue[T any] struct {
|
|||||||
workerNumMu sync.Mutex
|
workerNumMu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
type flushType chan struct{}
|
type flushType struct {
|
||||||
|
timeout time.Duration
|
||||||
|
c chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
var _ ManagedWorkerPoolQueue = (*WorkerPoolQueue[any])(nil)
|
var _ ManagedWorkerPoolQueue = (*WorkerPoolQueue[any])(nil)
|
||||||
|
|
||||||
@ -104,12 +107,12 @@ func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.
|
|||||||
if timeout > 0 {
|
if timeout > 0 {
|
||||||
after = time.After(timeout)
|
after = time.After(timeout)
|
||||||
}
|
}
|
||||||
c := make(flushType)
|
flush := flushType{timeout: timeout, c: make(chan struct{})}
|
||||||
|
|
||||||
// send flush request
|
// send flush request
|
||||||
// if it blocks, it means that there is a flush in progress or the queue hasn't been started yet
|
// if it blocks, it means that there is a flush in progress or the queue hasn't been started yet
|
||||||
select {
|
select {
|
||||||
case q.flushChan <- c:
|
case q.flushChan <- flush:
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case <-q.ctxRun.Done():
|
case <-q.ctxRun.Done():
|
||||||
@ -120,7 +123,7 @@ func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.
|
|||||||
|
|
||||||
// wait for flush to finish
|
// wait for flush to finish
|
||||||
select {
|
select {
|
||||||
case <-c:
|
case <-flush.c:
|
||||||
return nil
|
return nil
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
|
@ -38,8 +38,8 @@ func TestGetDirectorySize(t *testing.T) {
|
|||||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||||
repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 1)
|
repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 1)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
size, err := getDirectorySize(repo.RepoPath())
|
size, err := getDirectorySize(repo.RepoPath())
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.EqualValues(t, size, repo.Size)
|
repo.Size = 8165 // real size on the disk
|
||||||
|
assert.EqualValues(t, repo.Size, size)
|
||||||
}
|
}
|
||||||
|
@ -118,7 +118,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() {
|
|||||||
_, _ = fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush)
|
_, _ = fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if err := queue.GetManager().FlushAll(context.Background(), time.Minute); err != nil {
|
if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
|
||||||
t.Errorf("Flushing queues failed with error %v", err)
|
t.Errorf("Flushing queues failed with error %v", err)
|
||||||
}
|
}
|
||||||
timer.Stop()
|
timer.Stop()
|
||||||
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -89,7 +89,7 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) {
|
|||||||
|
|
||||||
func TestAdoptRepository(t *testing.T) {
|
func TestAdoptRepository(t *testing.T) {
|
||||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||||
assert.NoError(t, unittest.CopyDir(filepath.Join(setting.RepoRootPath, "user2", "repo1.git"), filepath.Join(setting.RepoRootPath, "user2", "test-adopt.git")))
|
assert.NoError(t, unittest.SyncDirs(filepath.Join(setting.RepoRootPath, "user2", "repo1.git"), filepath.Join(setting.RepoRootPath, "user2", "test-adopt.git")))
|
||||||
user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||||
_, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"})
|
_, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"})
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -1 +0,0 @@
|
|||||||
Unnamed repository; edit this file 'description' to name the repository.
|
|
@ -1,6 +0,0 @@
|
|||||||
# git ls-files --others --exclude-from=.git/info/exclude
|
|
||||||
# Lines that start with '#' are comments.
|
|
||||||
# For a project mostly in C, the following would be a good set of
|
|
||||||
# exclude patterns (uncomment them if you want to use them):
|
|
||||||
# *.[oa]
|
|
||||||
# *~
|
|
@ -400,8 +400,9 @@ func MakeRequest(t testing.TB, rw *RequestWrapper, expectedStatus int) *httptest
|
|||||||
}
|
}
|
||||||
testWebRoutes.ServeHTTP(recorder, req)
|
testWebRoutes.ServeHTTP(recorder, req)
|
||||||
if expectedStatus != NoExpectedStatus {
|
if expectedStatus != NoExpectedStatus {
|
||||||
if !assert.EqualValues(t, expectedStatus, recorder.Code, "Request: %s %s", req.Method, req.URL.String()) {
|
if expectedStatus != recorder.Code {
|
||||||
logUnexpectedResponse(t, recorder)
|
logUnexpectedResponse(t, recorder)
|
||||||
|
require.Equal(t, expectedStatus, recorder.Code, "Request: %s %s", req.Method, req.URL.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return recorder
|
return recorder
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user