2021-11-12 14:36:47 +00:00
|
|
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
2022-11-27 18:20:29 +00:00
|
|
|
// SPDX-License-Identifier: MIT
|
2021-11-12 14:36:47 +00:00
|
|
|
|
|
|
|
package unittest
|
|
|
|
|
|
|
|
import (
|
2021-12-01 07:50:01 +00:00
|
|
|
"context"
|
2021-11-12 14:36:47 +00:00
|
|
|
"fmt"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2023-05-04 03:55:35 +00:00
|
|
|
"strings"
|
2021-11-12 14:36:47 +00:00
|
|
|
"testing"
|
|
|
|
|
|
|
|
"code.gitea.io/gitea/models/db"
|
2023-10-05 01:08:19 +00:00
|
|
|
"code.gitea.io/gitea/models/system"
|
2023-05-04 03:55:35 +00:00
|
|
|
"code.gitea.io/gitea/modules/auth/password/hash"
|
2021-11-12 14:36:47 +00:00
|
|
|
"code.gitea.io/gitea/modules/base"
|
2024-04-25 11:22:32 +00:00
|
|
|
"code.gitea.io/gitea/modules/cache"
|
2022-06-10 01:57:49 +00:00
|
|
|
"code.gitea.io/gitea/modules/git"
|
2024-04-27 12:23:37 +00:00
|
|
|
"code.gitea.io/gitea/modules/log"
|
2021-11-12 14:36:47 +00:00
|
|
|
"code.gitea.io/gitea/modules/setting"
|
2023-10-05 01:08:19 +00:00
|
|
|
"code.gitea.io/gitea/modules/setting/config"
|
2021-11-12 14:36:47 +00:00
|
|
|
"code.gitea.io/gitea/modules/storage"
|
|
|
|
"code.gitea.io/gitea/modules/util"
|
|
|
|
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"xorm.io/xorm"
|
|
|
|
"xorm.io/xorm/names"
|
|
|
|
)
|
|
|
|
|
|
|
|
// giteaRoot a path to the gitea root
|
|
|
|
var (
|
|
|
|
giteaRoot string
|
|
|
|
fixturesDir string
|
|
|
|
)
|
|
|
|
|
|
|
|
// FixturesDir returns the fixture directory
|
|
|
|
func FixturesDir() string {
|
|
|
|
return fixturesDir
|
|
|
|
}
|
|
|
|
|
2023-07-04 18:36:08 +00:00
|
|
|
func fatalTestError(fmtStr string, args ...any) {
|
2021-11-12 14:36:47 +00:00
|
|
|
_, _ = fmt.Fprintf(os.Stderr, fmtStr, args...)
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
|
Refactor path & config system (#25330)
# The problem
There were many "path tricks":
* By default, Gitea uses its program directory as its work path
* Gitea tries to use the "work path" to guess its "custom path" and
"custom conf (app.ini)"
* Users might want to use other directories as work path
* The non-default work path should be passed to Gitea by GITEA_WORK_DIR
or "--work-path"
* But some Gitea processes are started without these values
* The "serv" process started by OpenSSH server
* The CLI sub-commands started by site admin
* The paths are guessed by SetCustomPathAndConf again and again
* The default values of "work path / custom path / custom conf" can be
changed when compiling
# The solution
* Use `InitWorkPathAndCommonConfig` to handle these path tricks, and use
test code to cover its behaviors.
* When Gitea's web server runs, write the WORK_PATH to "app.ini", this
value must be the most correct one, because if this value is not right,
users would find that the web UI doesn't work and then they should be
able to fix it.
* Then all other sub-commands can use the WORK_PATH in app.ini to
initialize their paths.
* By the way, when Gitea starts for git protocol, it shouldn't output
any log, otherwise the git protocol gets broken and client blocks
forever.
The "work path" priority is: WORK_PATH in app.ini > cmd arg --work-path
> env var GITEA_WORK_DIR > builtin default
The "app.ini" searching order is: cmd arg --config > cmd arg "work path
/ custom path" > env var "work path / custom path" > builtin default
## ⚠️ BREAKING
If your instance's "work path / custom path / custom conf" doesn't meet
the requirements (eg: work path must be absolute), Gitea will report a
fatal error and exit. You need to set these values according to the
error log.
----
Close #24818
Close #24222
Close #21606
Close #21498
Close #25107
Close #24981
Maybe close #24503
Replace #23301
Replace #22754
And maybe more
2023-06-21 05:50:26 +00:00
|
|
|
// InitSettings initializes config provider and load common settings for tests
|
2024-02-08 12:31:38 +00:00
|
|
|
func InitSettings() {
|
2024-04-27 12:23:37 +00:00
|
|
|
setting.IsInTesting = true
|
|
|
|
log.OsExiter = func(code int) {
|
|
|
|
if code != 0 {
|
|
|
|
// non-zero exit code (log.Fatal) shouldn't occur during testing, if it happens, show a full stacktrace for more details
|
|
|
|
panic(fmt.Errorf("non-zero exit code during testing: %d", code))
|
|
|
|
}
|
|
|
|
os.Exit(0)
|
|
|
|
}
|
Refactor path & config system (#25330)
# The problem
There were many "path tricks":
* By default, Gitea uses its program directory as its work path
* Gitea tries to use the "work path" to guess its "custom path" and
"custom conf (app.ini)"
* Users might want to use other directories as work path
* The non-default work path should be passed to Gitea by GITEA_WORK_DIR
or "--work-path"
* But some Gitea processes are started without these values
* The "serv" process started by OpenSSH server
* The CLI sub-commands started by site admin
* The paths are guessed by SetCustomPathAndConf again and again
* The default values of "work path / custom path / custom conf" can be
changed when compiling
# The solution
* Use `InitWorkPathAndCommonConfig` to handle these path tricks, and use
test code to cover its behaviors.
* When Gitea's web server runs, write the WORK_PATH to "app.ini", this
value must be the most correct one, because if this value is not right,
users would find that the web UI doesn't work and then they should be
able to fix it.
* Then all other sub-commands can use the WORK_PATH in app.ini to
initialize their paths.
* By the way, when Gitea starts for git protocol, it shouldn't output
any log, otherwise the git protocol gets broken and client blocks
forever.
The "work path" priority is: WORK_PATH in app.ini > cmd arg --work-path
> env var GITEA_WORK_DIR > builtin default
The "app.ini" searching order is: cmd arg --config > cmd arg "work path
/ custom path" > env var "work path / custom path" > builtin default
## ⚠️ BREAKING
If your instance's "work path / custom path / custom conf" doesn't meet
the requirements (eg: work path must be absolute), Gitea will report a
fatal error and exit. You need to set these values according to the
error log.
----
Close #24818
Close #24222
Close #21606
Close #21498
Close #25107
Close #24981
Maybe close #24503
Replace #23301
Replace #22754
And maybe more
2023-06-21 05:50:26 +00:00
|
|
|
if setting.CustomConf == "" {
|
|
|
|
setting.CustomConf = filepath.Join(setting.CustomPath, "conf/app-unittest-tmp.ini")
|
|
|
|
_ = os.Remove(setting.CustomConf)
|
|
|
|
}
|
2024-02-08 12:31:38 +00:00
|
|
|
setting.InitCfgProvider(setting.CustomConf)
|
Refactor path & config system (#25330)
# The problem
There were many "path tricks":
* By default, Gitea uses its program directory as its work path
* Gitea tries to use the "work path" to guess its "custom path" and
"custom conf (app.ini)"
* Users might want to use other directories as work path
* The non-default work path should be passed to Gitea by GITEA_WORK_DIR
or "--work-path"
* But some Gitea processes are started without these values
* The "serv" process started by OpenSSH server
* The CLI sub-commands started by site admin
* The paths are guessed by SetCustomPathAndConf again and again
* The default values of "work path / custom path / custom conf" can be
changed when compiling
# The solution
* Use `InitWorkPathAndCommonConfig` to handle these path tricks, and use
test code to cover its behaviors.
* When Gitea's web server runs, write the WORK_PATH to "app.ini", this
value must be the most correct one, because if this value is not right,
users would find that the web UI doesn't work and then they should be
able to fix it.
* Then all other sub-commands can use the WORK_PATH in app.ini to
initialize their paths.
* By the way, when Gitea starts for git protocol, it shouldn't output
any log, otherwise the git protocol gets broken and client blocks
forever.
The "work path" priority is: WORK_PATH in app.ini > cmd arg --work-path
> env var GITEA_WORK_DIR > builtin default
The "app.ini" searching order is: cmd arg --config > cmd arg "work path
/ custom path" > env var "work path / custom path" > builtin default
## ⚠️ BREAKING
If your instance's "work path / custom path / custom conf" doesn't meet
the requirements (eg: work path must be absolute), Gitea will report a
fatal error and exit. You need to set these values according to the
error log.
----
Close #24818
Close #24222
Close #21606
Close #21498
Close #25107
Close #24981
Maybe close #24503
Replace #23301
Replace #22754
And maybe more
2023-06-21 05:50:26 +00:00
|
|
|
setting.LoadCommonSettings()
|
2023-05-04 03:55:35 +00:00
|
|
|
|
|
|
|
if err := setting.PrepareAppDataPath(); err != nil {
|
2024-04-27 12:23:37 +00:00
|
|
|
log.Fatal("Can not prepare APP_DATA_PATH: %v", err)
|
2023-05-04 03:55:35 +00:00
|
|
|
}
|
|
|
|
// register the dummy hash algorithm function used in the test fixtures
|
|
|
|
_ = hash.Register("dummy", hash.NewDummyHasher)
|
|
|
|
|
|
|
|
setting.PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy")
|
|
|
|
}
|
|
|
|
|
2022-04-14 13:58:21 +00:00
|
|
|
// TestOptions represents test options
|
|
|
|
type TestOptions struct {
|
2023-09-28 01:38:53 +00:00
|
|
|
FixtureFiles []string
|
|
|
|
SetUp func() error // SetUp will be executed before all tests in this package
|
|
|
|
TearDown func() error // TearDown will be executed after all tests in this package
|
2022-04-14 13:58:21 +00:00
|
|
|
}
|
|
|
|
|
2021-11-12 14:36:47 +00:00
|
|
|
// MainTest a reusable TestMain(..) function for unit tests that need to use a
|
|
|
|
// test database. Creates the test database, and sets necessary settings.
|
2023-09-28 01:38:53 +00:00
|
|
|
func MainTest(m *testing.M, testOpts ...*TestOptions) {
|
|
|
|
searchDir, _ := os.Getwd()
|
|
|
|
for searchDir != "" {
|
|
|
|
if _, err := os.Stat(filepath.Join(searchDir, "go.mod")); err == nil {
|
|
|
|
break // The "go.mod" should be the one for Gitea repository
|
|
|
|
}
|
|
|
|
if dir := filepath.Dir(searchDir); dir == searchDir {
|
|
|
|
searchDir = "" // reaches the root of filesystem
|
|
|
|
} else {
|
|
|
|
searchDir = dir
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if searchDir == "" {
|
|
|
|
panic("The tests should run in a Gitea repository, there should be a 'go.mod' in the root")
|
|
|
|
}
|
2021-11-16 08:53:21 +00:00
|
|
|
|
2023-09-28 01:38:53 +00:00
|
|
|
giteaRoot = searchDir
|
|
|
|
setting.CustomPath = filepath.Join(giteaRoot, "custom")
|
|
|
|
InitSettings()
|
2021-11-12 14:36:47 +00:00
|
|
|
|
2023-09-28 01:38:53 +00:00
|
|
|
fixturesDir = filepath.Join(giteaRoot, "models", "fixtures")
|
2021-11-12 14:36:47 +00:00
|
|
|
var opts FixturesOptions
|
2023-09-28 01:38:53 +00:00
|
|
|
if len(testOpts) == 0 || len(testOpts[0].FixtureFiles) == 0 {
|
2021-11-12 14:36:47 +00:00
|
|
|
opts.Dir = fixturesDir
|
|
|
|
} else {
|
2023-09-28 01:38:53 +00:00
|
|
|
for _, f := range testOpts[0].FixtureFiles {
|
2021-11-12 14:36:47 +00:00
|
|
|
if len(f) != 0 {
|
|
|
|
opts.Files = append(opts.Files, filepath.Join(fixturesDir, f))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-28 01:38:53 +00:00
|
|
|
if err := CreateTestEngine(opts); err != nil {
|
2021-11-12 14:36:47 +00:00
|
|
|
fatalTestError("Error creating test engine: %v\n", err)
|
|
|
|
}
|
|
|
|
|
2024-04-25 11:22:32 +00:00
|
|
|
setting.IsInTesting = true
|
2021-11-12 14:36:47 +00:00
|
|
|
setting.AppURL = "https://try.gitea.io/"
|
|
|
|
setting.RunUser = "runuser"
|
2022-02-07 21:56:45 +00:00
|
|
|
setting.SSH.User = "sshuser"
|
|
|
|
setting.SSH.BuiltinServerUser = "builtinuser"
|
2021-11-12 14:36:47 +00:00
|
|
|
setting.SSH.Port = 3000
|
|
|
|
setting.SSH.Domain = "try.gitea.io"
|
2023-03-07 10:51:06 +00:00
|
|
|
setting.Database.Type = "sqlite3"
|
2022-04-09 04:26:48 +00:00
|
|
|
setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master"
|
2021-12-10 19:43:28 +00:00
|
|
|
repoRootPath, err := os.MkdirTemp(os.TempDir(), "repos")
|
2021-11-12 14:36:47 +00:00
|
|
|
if err != nil {
|
|
|
|
fatalTestError("TempDir: %v\n", err)
|
|
|
|
}
|
2021-12-10 19:43:28 +00:00
|
|
|
setting.RepoRootPath = repoRootPath
|
|
|
|
appDataPath, err := os.MkdirTemp(os.TempDir(), "appdata")
|
2021-11-12 14:36:47 +00:00
|
|
|
if err != nil {
|
|
|
|
fatalTestError("TempDir: %v\n", err)
|
|
|
|
}
|
2021-12-10 19:43:28 +00:00
|
|
|
setting.AppDataPath = appDataPath
|
2023-09-28 01:38:53 +00:00
|
|
|
setting.AppWorkPath = giteaRoot
|
|
|
|
setting.StaticRootPath = giteaRoot
|
2022-10-16 23:29:26 +00:00
|
|
|
setting.GravatarSource = "https://secure.gravatar.com/avatar/"
|
|
|
|
|
2021-11-12 14:36:47 +00:00
|
|
|
setting.Attachment.Storage.Path = filepath.Join(setting.AppDataPath, "attachments")
|
|
|
|
|
|
|
|
setting.LFS.Storage.Path = filepath.Join(setting.AppDataPath, "lfs")
|
|
|
|
|
|
|
|
setting.Avatar.Storage.Path = filepath.Join(setting.AppDataPath, "avatars")
|
|
|
|
|
|
|
|
setting.RepoAvatar.Storage.Path = filepath.Join(setting.AppDataPath, "repo-avatars")
|
|
|
|
|
|
|
|
setting.RepoArchive.Storage.Path = filepath.Join(setting.AppDataPath, "repo-archive")
|
|
|
|
|
2022-03-30 08:42:47 +00:00
|
|
|
setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages")
|
|
|
|
|
2023-05-19 13:37:57 +00:00
|
|
|
setting.Actions.LogStorage.Path = filepath.Join(setting.AppDataPath, "actions_log")
|
Implement actions (#21937)
Close #13539.
Co-authored by: @lunny @appleboy @fuxiaohei and others.
Related projects:
- https://gitea.com/gitea/actions-proto-def
- https://gitea.com/gitea/actions-proto-go
- https://gitea.com/gitea/act
- https://gitea.com/gitea/act_runner
### Summary
The target of this PR is to bring a basic implementation of "Actions",
an internal CI/CD system of Gitea. That means even though it has been
merged, the state of the feature is **EXPERIMENTAL**, and please note
that:
- It is disabled by default;
- It shouldn't be used in a production environment currently;
- It shouldn't be used in a public Gitea instance currently;
- Breaking changes may be made before it's stable.
**Please comment on #13539 if you have any different product design
ideas**, all decisions reached there will be adopted here. But in this
PR, we don't talk about **naming, feature-creep or alternatives**.
### ⚠️ Breaking
`gitea-actions` will become a reserved user name. If a user with the
name already exists in the database, it is recommended to rename it.
### Some important reviews
- What is `DEFAULT_ACTIONS_URL` in `app.ini` for?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1055954954
- Why the api for runners is not under the normal `/api/v1` prefix?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1061173592
- Why DBFS?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1061301178
- Why ignore events triggered by `gitea-actions` bot?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1063254103
- Why there's no permission control for actions?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1090229868
### What it looks like
<details>
#### Manage runners
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205870657-c72f590e-2e08-4cd4-be7f-2e0abb299bbf.png">
#### List runs
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205872794-50fde990-2b45-48c1-a178-908e4ec5b627.png">
#### View logs
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205872501-9b7b9000-9542-4991-8f55-18ccdada77c3.png">
</details>
### How to try it
<details>
#### 1. Start Gitea
Clone this branch and [install from
source](https://docs.gitea.io/en-us/install-from-source).
Add additional configurations in `app.ini` to enable Actions:
```ini
[actions]
ENABLED = true
```
Start it.
If all is well, you'll see the management page of runners:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205877365-8e30a780-9b10-4154-b3e8-ee6c3cb35a59.png">
#### 2. Start runner
Clone the [act_runner](https://gitea.com/gitea/act_runner), and follow
the
[README](https://gitea.com/gitea/act_runner/src/branch/main/README.md)
to start it.
If all is well, you'll see a new runner has been added:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205878000-216f5937-e696-470d-b66c-8473987d91c3.png">
#### 3. Enable actions for a repo
Create a new repo or open an existing one, check the `Actions` checkbox
in settings and submit.
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205879705-53e09208-73c0-4b3e-a123-2dcf9aba4b9c.png">
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205879383-23f3d08f-1a85-41dd-a8b3-54e2ee6453e8.png">
If all is well, you'll see a new tab "Actions":
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205881648-a8072d8c-5803-4d76-b8a8-9b2fb49516c1.png">
#### 4. Upload workflow files
Upload some workflow files to `.gitea/workflows/xxx.yaml`, you can
follow the [quickstart](https://docs.github.com/en/actions/quickstart)
of GitHub Actions. Yes, Gitea Actions is compatible with GitHub Actions
in most cases, you can use the same demo:
```yaml
name: GitHub Actions Demo
run-name: ${{ github.actor }} is testing out GitHub Actions 🚀
on: [push]
jobs:
Explore-GitHub-Actions:
runs-on: ubuntu-latest
steps:
- run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
- run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
- name: Check out repository code
uses: actions/checkout@v3
- run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
- run: echo "🖥️ The workflow is now ready to test your code on the runner."
- name: List files in the repository
run: |
ls ${{ github.workspace }}
- run: echo "🍏 This job's status is ${{ job.status }}."
```
If all is well, you'll see a new run in `Actions` tab:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205884473-79a874bc-171b-4aaf-acd5-0241a45c3b53.png">
#### 5. Check the logs of jobs
Click a run and you'll see the logs:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205884800-994b0374-67f7-48ff-be9a-4c53f3141547.png">
#### 6. Go on
You can try more examples in [the
documents](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions)
of GitHub Actions, then you might find a lot of bugs.
Come on, PRs are welcome.
</details>
See also: [Feature Preview: Gitea
Actions](https://blog.gitea.io/2022/12/feature-preview-gitea-actions/)
---------
Co-authored-by: a1012112796 <1012112796@qq.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: ChristopherHX <christopher.homberger@web.de>
Co-authored-by: John Olheiser <john.olheiser@gmail.com>
2023-01-31 01:45:19 +00:00
|
|
|
|
2022-07-08 08:09:07 +00:00
|
|
|
setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home")
|
|
|
|
|
2023-01-14 15:57:10 +00:00
|
|
|
setting.IncomingEmail.ReplyToAddress = "incoming+%{token}@localhost"
|
|
|
|
|
2023-10-05 01:08:19 +00:00
|
|
|
config.SetDynGetter(system.NewDatabaseDynKeyGetter())
|
|
|
|
|
2024-04-25 11:22:32 +00:00
|
|
|
if err = cache.Init(); err != nil {
|
|
|
|
fatalTestError("cache.Init: %v\n", err)
|
|
|
|
}
|
2021-11-12 14:36:47 +00:00
|
|
|
if err = storage.Init(); err != nil {
|
|
|
|
fatalTestError("storage.Init: %v\n", err)
|
|
|
|
}
|
2024-11-14 19:28:46 +00:00
|
|
|
if err = SyncDirs(filepath.Join(giteaRoot, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
|
|
|
|
fatalTestError("util.SyncDirs: %v\n", err)
|
2021-11-12 14:36:47 +00:00
|
|
|
}
|
2022-06-16 15:47:44 +00:00
|
|
|
|
2022-08-09 03:22:24 +00:00
|
|
|
if err = git.InitFull(context.Background()); err != nil {
|
2022-06-10 01:57:49 +00:00
|
|
|
fatalTestError("git.Init: %v\n", err)
|
|
|
|
}
|
2021-12-16 19:01:14 +00:00
|
|
|
|
2023-09-28 01:38:53 +00:00
|
|
|
if len(testOpts) > 0 && testOpts[0].SetUp != nil {
|
|
|
|
if err := testOpts[0].SetUp(); err != nil {
|
2022-04-14 13:58:21 +00:00
|
|
|
fatalTestError("set up failed: %v\n", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-12 14:36:47 +00:00
|
|
|
exitStatus := m.Run()
|
2022-04-14 13:58:21 +00:00
|
|
|
|
2023-09-28 01:38:53 +00:00
|
|
|
if len(testOpts) > 0 && testOpts[0].TearDown != nil {
|
|
|
|
if err := testOpts[0].TearDown(); err != nil {
|
2022-04-14 13:58:21 +00:00
|
|
|
fatalTestError("tear down failed: %v\n", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-10 19:43:28 +00:00
|
|
|
if err = util.RemoveAll(repoRootPath); err != nil {
|
2021-11-12 14:36:47 +00:00
|
|
|
fatalTestError("util.RemoveAll: %v\n", err)
|
|
|
|
}
|
2021-12-10 19:43:28 +00:00
|
|
|
if err = util.RemoveAll(appDataPath); err != nil {
|
2021-11-12 14:36:47 +00:00
|
|
|
fatalTestError("util.RemoveAll: %v\n", err)
|
|
|
|
}
|
|
|
|
os.Exit(exitStatus)
|
|
|
|
}
|
|
|
|
|
|
|
|
// FixturesOptions fixtures needs to be loaded options
|
|
|
|
type FixturesOptions struct {
|
|
|
|
Dir string
|
|
|
|
Files []string
|
|
|
|
}
|
|
|
|
|
|
|
|
// CreateTestEngine creates a memory database and loads the fixture data from fixturesDir
|
|
|
|
func CreateTestEngine(opts FixturesOptions) error {
|
|
|
|
x, err := xorm.NewEngine("sqlite3", "file::memory:?cache=shared&_txlock=immediate")
|
|
|
|
if err != nil {
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 11:49:59 +00:00
|
|
|
if strings.Contains(err.Error(), "unknown driver") {
|
|
|
|
return fmt.Errorf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err)
|
|
|
|
}
|
2021-11-12 14:36:47 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
x.SetMapper(names.GonicMapper{})
|
2021-12-01 07:50:01 +00:00
|
|
|
db.SetDefaultEngine(context.Background(), x)
|
2021-11-12 14:36:47 +00:00
|
|
|
|
|
|
|
if err = db.SyncAllTables(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-01-01 03:31:24 +00:00
|
|
|
switch os.Getenv("GITEA_UNIT_TESTS_LOG_SQL") {
|
2021-11-12 14:36:47 +00:00
|
|
|
case "true", "1":
|
|
|
|
x.ShowSQL(true)
|
|
|
|
}
|
|
|
|
|
|
|
|
return InitFixtures(opts)
|
|
|
|
}
|
|
|
|
|
|
|
|
// PrepareTestDatabase load test fixtures into test database
|
|
|
|
func PrepareTestDatabase() error {
|
|
|
|
return LoadFixtures()
|
|
|
|
}
|
|
|
|
|
|
|
|
// PrepareTestEnv prepares the environment for unit tests. Can only be called
|
|
|
|
// by tests that use the above MainTest(..) function.
|
|
|
|
func PrepareTestEnv(t testing.TB) {
|
|
|
|
assert.NoError(t, PrepareTestDatabase())
|
2022-09-02 19:18:23 +00:00
|
|
|
metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta")
|
2024-11-14 19:28:46 +00:00
|
|
|
assert.NoError(t, SyncDirs(metaPath, setting.RepoRootPath))
|
2021-11-12 14:36:47 +00:00
|
|
|
base.SetupGiteaRoot() // Makes sure GITEA_ROOT is set
|
|
|
|
}
|