mirror of
https://github.com/go-gitea/gitea
synced 2024-10-31 15:24:25 +00:00
Add retry for migration http/https requests (#9019)
* Add retry for migration http/https requests * give the more suitable name for retry configuraion items * fix docs and lint * Only use retryDownloader when setting > 1
This commit is contained in:
parent
6fbfffeeb8
commit
1b7182e5ec
@ -892,3 +892,9 @@ QUEUE_LENGTH = 1000
|
|||||||
; Task queue connection string, available only when `QUEUE_TYPE` is `redis`.
|
; Task queue connection string, available only when `QUEUE_TYPE` is `redis`.
|
||||||
; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`.
|
; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`.
|
||||||
QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
|
QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
|
||||||
|
|
||||||
|
[migrations]
|
||||||
|
; Max attempts per http/https request on migrations.
|
||||||
|
MAX_ATTEMPTS = 3
|
||||||
|
; Backoff time per http/https request retry (seconds)
|
||||||
|
RETRY_BACKOFF = 3
|
@ -585,6 +585,11 @@ Two special environment variables are passed to the render command:
|
|||||||
- `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`.
|
- `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`.
|
||||||
- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`.
|
- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`.
|
||||||
|
|
||||||
|
## Migrations (`migrations`)
|
||||||
|
|
||||||
|
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
|
||||||
|
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
|
||||||
|
|
||||||
## Other (`other`)
|
## Other (`other`)
|
||||||
|
|
||||||
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.
|
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.
|
||||||
|
@ -272,6 +272,11 @@ IS_INPUT_FILE = false
|
|||||||
- `QUEUE_LENGTH`: **1000**: 任务队列长度,当 `QUEUE_TYPE` 为 `channel` 时有效。
|
- `QUEUE_LENGTH`: **1000**: 任务队列长度,当 `QUEUE_TYPE` 为 `channel` 时有效。
|
||||||
- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串,当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码,则可以 `addrs=127.0.0.1:6379 password=123 db=0`。
|
- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串,当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码,则可以 `addrs=127.0.0.1:6379 password=123 db=0`。
|
||||||
|
|
||||||
|
## Migrations (`migrations`)
|
||||||
|
|
||||||
|
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。
|
||||||
|
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。
|
||||||
|
|
||||||
## Other (`other`)
|
## Other (`other`)
|
||||||
|
|
||||||
- `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。
|
- `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。
|
||||||
|
@ -5,7 +5,11 @@
|
|||||||
|
|
||||||
package base
|
package base
|
||||||
|
|
||||||
import "code.gitea.io/gitea/modules/structs"
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/structs"
|
||||||
|
)
|
||||||
|
|
||||||
// Downloader downloads the site repo informations
|
// Downloader downloads the site repo informations
|
||||||
type Downloader interface {
|
type Downloader interface {
|
||||||
@ -25,3 +29,148 @@ type DownloaderFactory interface {
|
|||||||
New(opts MigrateOptions) (Downloader, error)
|
New(opts MigrateOptions) (Downloader, error)
|
||||||
GitServiceType() structs.GitServiceType
|
GitServiceType() structs.GitServiceType
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetryDownloader retry the downloads
|
||||||
|
type RetryDownloader struct {
|
||||||
|
Downloader
|
||||||
|
RetryTimes int // the total execute times
|
||||||
|
RetryDelay int // time to delay seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRetryDownloader creates a retry downloader
|
||||||
|
func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
|
||||||
|
return &RetryDownloader{
|
||||||
|
Downloader: downloader,
|
||||||
|
RetryTimes: retryTimes,
|
||||||
|
RetryDelay: retryDelay,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRepoInfo returns a repository information with retry
|
||||||
|
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
|
||||||
|
var (
|
||||||
|
times = d.RetryTimes
|
||||||
|
repo *Repository
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
for ; times > 0; times-- {
|
||||||
|
if repo, err = d.Downloader.GetRepoInfo(); err == nil {
|
||||||
|
return repo, nil
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second * time.Duration(d.RetryDelay))
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTopics returns a repository's topics with retry
|
||||||
|
func (d *RetryDownloader) GetTopics() ([]string, error) {
|
||||||
|
var (
|
||||||
|
times = d.RetryTimes
|
||||||
|
topics []string
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
for ; times > 0; times-- {
|
||||||
|
if topics, err = d.Downloader.GetTopics(); err == nil {
|
||||||
|
return topics, nil
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second * time.Duration(d.RetryDelay))
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMilestones returns a repository's milestones with retry
|
||||||
|
func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
|
||||||
|
var (
|
||||||
|
times = d.RetryTimes
|
||||||
|
milestones []*Milestone
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
for ; times > 0; times-- {
|
||||||
|
if milestones, err = d.Downloader.GetMilestones(); err == nil {
|
||||||
|
return milestones, nil
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second * time.Duration(d.RetryDelay))
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetReleases returns a repository's releases with retry
|
||||||
|
func (d *RetryDownloader) GetReleases() ([]*Release, error) {
|
||||||
|
var (
|
||||||
|
times = d.RetryTimes
|
||||||
|
releases []*Release
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
for ; times > 0; times-- {
|
||||||
|
if releases, err = d.Downloader.GetReleases(); err == nil {
|
||||||
|
return releases, nil
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second * time.Duration(d.RetryDelay))
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetLabels returns a repository's labels with retry
|
||||||
|
func (d *RetryDownloader) GetLabels() ([]*Label, error) {
|
||||||
|
var (
|
||||||
|
times = d.RetryTimes
|
||||||
|
labels []*Label
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
for ; times > 0; times-- {
|
||||||
|
if labels, err = d.Downloader.GetLabels(); err == nil {
|
||||||
|
return labels, nil
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second * time.Duration(d.RetryDelay))
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetIssues returns a repository's issues with retry
|
||||||
|
func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
|
||||||
|
var (
|
||||||
|
times = d.RetryTimes
|
||||||
|
issues []*Issue
|
||||||
|
isEnd bool
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
for ; times > 0; times-- {
|
||||||
|
if issues, isEnd, err = d.Downloader.GetIssues(page, perPage); err == nil {
|
||||||
|
return issues, isEnd, nil
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second * time.Duration(d.RetryDelay))
|
||||||
|
}
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetComments returns a repository's comments with retry
|
||||||
|
func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
|
||||||
|
var (
|
||||||
|
times = d.RetryTimes
|
||||||
|
comments []*Comment
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
for ; times > 0; times-- {
|
||||||
|
if comments, err = d.Downloader.GetComments(issueNumber); err == nil {
|
||||||
|
return comments, nil
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second * time.Duration(d.RetryDelay))
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetPullRequests returns a repository's pull requests with retry
|
||||||
|
func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, error) {
|
||||||
|
var (
|
||||||
|
times = d.RetryTimes
|
||||||
|
prs []*PullRequest
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
for ; times > 0; times-- {
|
||||||
|
if prs, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
|
||||||
|
return prs, nil
|
||||||
|
}
|
||||||
|
time.Sleep(time.Second * time.Duration(d.RetryDelay))
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
@ -11,6 +11,7 @@ import (
|
|||||||
"code.gitea.io/gitea/models"
|
"code.gitea.io/gitea/models"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/migrations/base"
|
"code.gitea.io/gitea/modules/migrations/base"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -63,6 +64,11 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
|
|||||||
}
|
}
|
||||||
|
|
||||||
uploader.gitServiceType = opts.GitServiceType
|
uploader.gitServiceType = opts.GitServiceType
|
||||||
|
|
||||||
|
if setting.Migrations.MaxAttempts > 1 {
|
||||||
|
downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
|
||||||
|
}
|
||||||
|
|
||||||
if err := migrateRepository(downloader, uploader, opts); err != nil {
|
if err := migrateRepository(downloader, uploader, opts); err != nil {
|
||||||
if err1 := uploader.Rollback(); err1 != nil {
|
if err1 := uploader.Rollback(); err1 != nil {
|
||||||
log.Error("rollback failed: %v", err1)
|
log.Error("rollback failed: %v", err1)
|
||||||
|
22
modules/setting/migrations.go
Normal file
22
modules/setting/migrations.go
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package setting
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Migrations settings
|
||||||
|
Migrations = struct {
|
||||||
|
MaxAttempts int
|
||||||
|
RetryBackoff int
|
||||||
|
}{
|
||||||
|
MaxAttempts: 3,
|
||||||
|
RetryBackoff: 3,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
func newMigrationsService() {
|
||||||
|
sec := Cfg.Section("migrations")
|
||||||
|
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
|
||||||
|
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
|
||||||
|
}
|
@ -1062,6 +1062,7 @@ func NewServices() {
|
|||||||
newRegisterMailService()
|
newRegisterMailService()
|
||||||
newNotifyMailService()
|
newNotifyMailService()
|
||||||
newWebhookService()
|
newWebhookService()
|
||||||
|
newMigrationsService()
|
||||||
newIndexerService()
|
newIndexerService()
|
||||||
newTaskService()
|
newTaskService()
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user