Add proxy settings and support for migration and webhook (#16704)

* Add proxy settings and support for migration and webhook

* Fix default value

* Add newline for example ini

* Add lfs proxy support

* Fix lint

* Follow @zeripath's review

* Fix git clone

* Fix test

* missgin http requests for proxy

* use empty

Co-authored-by: zeripath <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: zeripath <art27@cantab.net>
This commit is contained in:
Lunny Xiao 2021-08-18 21:10:39 +08:00 committed by GitHub
parent 422c30d315
commit f9acad82ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 302 additions and 41 deletions

View File

@ -2127,3 +2127,11 @@ PATH =
;; ;;
;; Minio enabled ssl only available when STORAGE_TYPE is `minio` ;; Minio enabled ssl only available when STORAGE_TYPE is `minio`
;MINIO_USE_SSL = false ;MINIO_USE_SSL = false
;[proxy]
;; Enable the proxy, all requests to external via HTTP will be affected
;PROXY_ENABLED = false
;; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy/no_proxy
;PROXY_URL =
;; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
;PROXY_HOSTS =

View File

@ -549,8 +549,8 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
- `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks. - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
- `SKIP_TLS_VERIFY`: **false**: Allow insecure certification. - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
- `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page. - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy - `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting.
- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. - `PROXY_HOSTS`: **\<empty\>`**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. If not given, will use global proxy setting.
## Mailer (`mailer`) ## Mailer (`mailer`)
@ -950,6 +950,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. - `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored. - `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify
## Mirror (`mirror`) ## Mirror (`mirror`)
@ -1023,6 +1024,19 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`.
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio` - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio` - `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
## Proxy (`proxy`)
- `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy
- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
- `PROXY_HOSTS`: **\<empty\>**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
i.e.
```ini
PROXY_ENABLED = true
PROXY_URL = socks://127.0.0.1:1080
PROXY_HOSTS = *.github.com
```
## Other (`other`) ## Other (`other`)
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.

View File

@ -332,6 +332,7 @@ IS_INPUT_FILE = false
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。 - `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。 - `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918 - `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证
## LFS (`lfs`) ## LFS (`lfs`)
@ -397,6 +398,19 @@ Repository archive 的存储配置。 如果 `STORAGE_TYPE` 为空,则此配
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path ,仅当 `STORAGE_TYPE``minio` 时有效。 - `MINIO_BASE_PATH`: **repo-archive/**: Minio base path ,仅当 `STORAGE_TYPE``minio` 时有效。
- `MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ,仅当 `STORAGE_TYPE``minio` 时有效。 - `MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ,仅当 `STORAGE_TYPE``minio` 时有效。
## Proxy (`proxy`)
- `PROXY_ENABLED`: **false**: 是否启用全局代理。如果为否,则不使用代理,环境变量中的代理也不使用
- `PROXY_URL`: **\<empty\>**: 代理服务器地址,支持 http://, https//, socks://,为空则不启用代理而使用环境变量中的 http_proxy/https_proxy
- `PROXY_HOSTS`: **\<empty\>**: 逗号分隔的多个需要代理的网址,支持 * 号匹配符号, ** 表示匹配所有网站
i.e.
```ini
PROXY_ENABLED = true
PROXY_URL = socks://127.0.0.1:1080
PROXY_HOSTS = *.github.com
```
## Other (`other`) ## Other (`other`)
- `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。

View File

@ -110,24 +110,47 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura
// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout, // RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run. // it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error { func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
if timeout == -1 { return c.RunWithContext(&RunContext{
timeout = defaultCommandExecutionTimeout Env: env,
Timeout: timeout,
Dir: dir,
Stdout: stdout,
Stderr: stderr,
Stdin: stdin,
PipelineFunc: fn,
})
}
// RunContext represents parameters to run the command
type RunContext struct {
Env []string
Timeout time.Duration
Dir string
Stdout, Stderr io.Writer
Stdin io.Reader
PipelineFunc func(context.Context, context.CancelFunc) error
}
// RunWithContext run the command with context
func (c *Command) RunWithContext(rc *RunContext) error {
if rc.Timeout == -1 {
rc.Timeout = defaultCommandExecutionTimeout
} }
if len(dir) == 0 { if len(rc.Dir) == 0 {
log.Debug("%s", c) log.Debug("%s", c)
} else { } else {
log.Debug("%s: %v", dir, c) log.Debug("%s: %v", rc.Dir, c)
} }
ctx, cancel := context.WithTimeout(c.parentContext, timeout) ctx, cancel := context.WithTimeout(c.parentContext, rc.Timeout)
defer cancel() defer cancel()
cmd := exec.CommandContext(ctx, c.name, c.args...) cmd := exec.CommandContext(ctx, c.name, c.args...)
if env == nil { if rc.Env == nil {
cmd.Env = os.Environ() cmd.Env = os.Environ()
} else { } else {
cmd.Env = env cmd.Env = rc.Env
} }
cmd.Env = append( cmd.Env = append(
@ -141,23 +164,23 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.
if goVersionLessThan115 { if goVersionLessThan115 {
cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1") cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
} }
cmd.Dir = dir cmd.Dir = rc.Dir
cmd.Stdout = stdout cmd.Stdout = rc.Stdout
cmd.Stderr = stderr cmd.Stderr = rc.Stderr
cmd.Stdin = stdin cmd.Stdin = rc.Stdin
if err := cmd.Start(); err != nil { if err := cmd.Start(); err != nil {
return err return err
} }
desc := c.desc desc := c.desc
if desc == "" { if desc == "" {
desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir) desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), rc.Dir)
} }
pid := process.GetManager().Add(desc, cancel) pid := process.GetManager().Add(desc, cancel)
defer process.GetManager().Remove(pid) defer process.GetManager().Remove(pid)
if fn != nil { if rc.PipelineFunc != nil {
err := fn(ctx, cancel) err := rc.PipelineFunc(ctx, cancel)
if err != nil { if err != nil {
cancel() cancel()
_ = cmd.Wait() _ = cmd.Wait()

View File

@ -9,11 +9,15 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"io"
"net/url"
"os" "os"
"path" "path"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"code.gitea.io/gitea/modules/proxy"
) )
// GPGSettings represents the default GPG settings for this repository // GPGSettings represents the default GPG settings for this repository
@ -99,12 +103,12 @@ type CloneRepoOptions struct {
} }
// Clone clones original repository to target path. // Clone clones original repository to target path.
func Clone(from, to string, opts CloneRepoOptions) (err error) { func Clone(from, to string, opts CloneRepoOptions) error {
return CloneWithContext(DefaultContext, from, to, opts) return CloneWithContext(DefaultContext, from, to, opts)
} }
// CloneWithContext clones original repository to target path. // CloneWithContext clones original repository to target path.
func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) { func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) error {
cargs := make([]string, len(GlobalCommandArgs)) cargs := make([]string, len(GlobalCommandArgs))
copy(cargs, GlobalCommandArgs) copy(cargs, GlobalCommandArgs)
return CloneWithArgs(ctx, from, to, cargs, opts) return CloneWithArgs(ctx, from, to, cargs, opts)
@ -146,8 +150,24 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo
opts.Timeout = -1 opts.Timeout = -1
} }
_, err = cmd.RunTimeout(opts.Timeout) var envs = os.Environ()
return err u, err := url.Parse(from)
if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) {
if proxy.Match(u.Host) {
envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL()))
}
}
var stderr = new(bytes.Buffer)
if err = cmd.RunWithContext(&RunContext{
Timeout: opts.Timeout,
Env: envs,
Stdout: io.Discard,
Stderr: stderr,
}); err != nil {
return ConcatenateError(err, stderr.String())
}
return nil
} }
// PullRemoteOptions options when pull from remote // PullRemoteOptions options when pull from remote

View File

@ -24,9 +24,9 @@ type Client interface {
} }
// NewClient creates a LFS client // NewClient creates a LFS client
func NewClient(endpoint *url.URL) Client { func NewClient(endpoint *url.URL, skipTLSVerify bool) Client {
if endpoint.Scheme == "file" { if endpoint.Scheme == "file" {
return newFilesystemClient(endpoint) return newFilesystemClient(endpoint)
} }
return newHTTPClient(endpoint) return newHTTPClient(endpoint, skipTLSVerify)
} }

View File

@ -13,10 +13,10 @@ import (
func TestNewClient(t *testing.T) { func TestNewClient(t *testing.T) {
u, _ := url.Parse("file:///test") u, _ := url.Parse("file:///test")
c := NewClient(u) c := NewClient(u, true)
assert.IsType(t, &FilesystemClient{}, c) assert.IsType(t, &FilesystemClient{}, c)
u, _ = url.Parse("https://test.com/lfs") u, _ = url.Parse("https://test.com/lfs")
c = NewClient(u) c = NewClient(u, true)
assert.IsType(t, &HTTPClient{}, c) assert.IsType(t, &HTTPClient{}, c)
} }

View File

@ -7,6 +7,7 @@ package lfs
import ( import (
"bytes" "bytes"
"context" "context"
"crypto/tls"
"errors" "errors"
"fmt" "fmt"
"net/http" "net/http"
@ -15,6 +16,7 @@ import (
"code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/proxy"
) )
const batchSize = 20 const batchSize = 20
@ -32,8 +34,13 @@ func (c *HTTPClient) BatchSize() int {
return batchSize return batchSize
} }
func newHTTPClient(endpoint *url.URL) *HTTPClient { func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient {
hc := &http.Client{} hc := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify},
Proxy: proxy.Proxy(),
},
}
client := &HTTPClient{ client := &HTTPClient{
client: hc, client: hc,

View File

@ -6,6 +6,7 @@ package migrations
import ( import (
"context" "context"
"crypto/tls"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
gitea_sdk "code.gitea.io/sdk/gitea" gitea_sdk "code.gitea.io/sdk/gitea"
@ -87,6 +90,12 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
gitea_sdk.SetToken(token), gitea_sdk.SetToken(token),
gitea_sdk.SetBasicAuth(username, password), gitea_sdk.SetBasicAuth(username, password),
gitea_sdk.SetContext(ctx), gitea_sdk.SetContext(ctx),
gitea_sdk.SetHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}),
) )
if err != nil { if err != nil {
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err)) log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
@ -266,6 +275,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
Created: rel.CreatedAt, Created: rel.CreatedAt,
} }
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for _, asset := range rel.Attachments { for _, asset := range rel.Attachments {
size := int(asset.Size) size := int(asset.Size)
dlCount := int(asset.DownloadCount) dlCount := int(asset.DownloadCount)
@ -282,7 +298,11 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
return nil, err return nil, err
} }
// FIXME: for a private download? // FIXME: for a private download?
resp, err := http.Get(asset.DownloadURL) req, err := http.NewRequest("GET", asset.DownloadURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -7,6 +7,7 @@ package migrations
import ( import (
"context" "context"
"crypto/tls"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
@ -90,7 +93,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
Transport: &http.Transport{ Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) { Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password) req.SetBasicAuth(userName, password)
return nil, nil return proxy.Proxy()(req)
}, },
}, },
} }
@ -269,6 +272,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
r.Published = rel.PublishedAt.Time r.Published = rel.PublishedAt.Time
} }
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for _, asset := range rel.Assets { for _, asset := range rel.Assets {
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
r.Assets = append(r.Assets, &base.ReleaseAsset{ r.Assets = append(r.Assets, &base.ReleaseAsset{
@ -295,7 +305,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
if err != nil { if err != nil {
return nil, err return nil, err
} }
resp, err := http.DefaultClient.Do(req) resp, err := httpClient.Do(req)
err1 := g.RefreshRate() err1 := g.RefreshRate()
if err1 != nil { if err1 != nil {
log.Error("g.client.RateLimits: %s", err1) log.Error("g.client.RateLimits: %s", err1)

View File

@ -6,6 +6,7 @@ package migrations
import ( import (
"context" "context"
"crypto/tls"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"github.com/xanzy/go-gitlab" "github.com/xanzy/go-gitlab"
@ -77,7 +80,12 @@ type GitlabDownloader struct {
// Use either a username/password, personal token entered into the username field, or anonymous/public access // Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access // Note: Public access only allows very basic access
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) { func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL)) gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}))
// Only use basic auth if token is blank and password is NOT // Only use basic auth if token is blank and password is NOT
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage // Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
if token == "" && password != "" { if token == "" && password != "" {
@ -295,6 +303,13 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
PublisherName: rel.Author.Username, PublisherName: rel.Author.Username,
} }
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for k, asset := range rel.Assets.Links { for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, &base.ReleaseAsset{ r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: int64(asset.ID), ID: int64(asset.ID),
@ -313,8 +328,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
return nil, err return nil, err
} }
req = req.WithContext(g.ctx) req = req.WithContext(g.ctx)
resp, err := httpClient.Do(req)
resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -6,6 +6,7 @@ package migrations
import ( import (
"context" "context"
"crypto/tls"
"fmt" "fmt"
"net/http" "net/http"
"net/url" "net/url"
@ -14,6 +15,8 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"github.com/gogs/go-gogs-client" "github.com/gogs/go-gogs-client"
@ -95,9 +98,10 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token,
downloader.userName = token downloader.userName = token
} else { } else {
downloader.transport = &http.Transport{ downloader.transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) { Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password) req.SetBasicAuth(userName, password)
return nil, nil return proxy.Proxy()(req)
}, },
} }

83
modules/proxy/proxy.go Normal file
View File

@ -0,0 +1,83 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package proxy
import (
"net/http"
"net/url"
"os"
"sync"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/gobwas/glob"
)
var (
once sync.Once
hostMatchers []glob.Glob
)
// GetProxyURL returns proxy url
func GetProxyURL() string {
if !setting.Proxy.Enabled {
return ""
}
if setting.Proxy.ProxyURL == "" {
if os.Getenv("http_proxy") != "" {
return os.Getenv("http_proxy")
}
return os.Getenv("https_proxy")
}
return setting.Proxy.ProxyURL
}
// Match return true if url needs to be proxied
func Match(u string) bool {
if !setting.Proxy.Enabled {
return false
}
// enforce do once
Proxy()
for _, v := range hostMatchers {
if v.Match(u) {
return true
}
}
return false
}
// Proxy returns the system proxy
func Proxy() func(req *http.Request) (*url.URL, error) {
if !setting.Proxy.Enabled {
return nil
}
if setting.Proxy.ProxyURL == "" {
return http.ProxyFromEnvironment
}
once.Do(func() {
for _, h := range setting.Proxy.ProxyHosts {
if g, err := glob.Compile(h); err == nil {
hostMatchers = append(hostMatchers, g)
} else {
log.Error("glob.Compile %s failed: %v", h, err)
}
}
})
return func(req *http.Request) (*url.URL, error) {
for _, v := range hostMatchers {
if v.Match(req.URL.Host) {
return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req)
}
}
return http.ProxyFromEnvironment(req)
}
}

View File

@ -126,7 +126,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
if opts.LFS { if opts.LFS {
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil { if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil {
log.Error("Failed to store missing LFS objects for repository: %v", err) log.Error("Failed to store missing LFS objects for repository: %v", err)
} }
} }
@ -316,8 +316,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName
} }
// StoreMissingLfsObjectsInRepository downloads missing LFS objects // StoreMissingLfsObjectsInRepository downloads missing LFS objects
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error { func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
client := lfs.NewClient(endpoint) client := lfs.NewClient(endpoint, skipTLSVerify)
contentStore := lfs.NewContentStore() contentStore := lfs.NewContentStore()
pointerChan := make(chan lfs.PointerBlob) pointerChan := make(chan lfs.PointerBlob)

View File

@ -16,6 +16,7 @@ var (
AllowedDomains []string AllowedDomains []string
BlockedDomains []string BlockedDomains []string
AllowLocalNetworks bool AllowLocalNetworks bool
SkipTLSVerify bool
}{ }{
MaxAttempts: 3, MaxAttempts: 3,
RetryBackoff: 3, RetryBackoff: 3,
@ -37,4 +38,5 @@ func newMigrationsService() {
} }
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false) Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
} }

40
modules/setting/proxy.go Normal file
View File

@ -0,0 +1,40 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"net/url"
"code.gitea.io/gitea/modules/log"
)
var (
// Proxy settings
Proxy = struct {
Enabled bool
ProxyURL string
ProxyURLFixed *url.URL
ProxyHosts []string
}{
Enabled: false,
ProxyURL: "",
ProxyHosts: []string{},
}
)
func newProxyService() {
sec := Cfg.Section("proxy")
Proxy.Enabled = sec.Key("PROXY_ENABLED").MustBool(false)
Proxy.ProxyURL = sec.Key("PROXY_URL").MustString("")
if Proxy.ProxyURL != "" {
var err error
Proxy.ProxyURLFixed, err = url.Parse(Proxy.ProxyURL)
if err != nil {
log.Error("Global PROXY_URL is not valid")
Proxy.ProxyURL = ""
}
}
Proxy.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",")
}

View File

@ -1195,6 +1195,7 @@ func NewServices() {
newMailService() newMailService()
newRegisterMailService() newRegisterMailService()
newNotifyMailService() newNotifyMailService()
newProxyService()
newWebhookService() newWebhookService()
newMigrationsService() newMigrationsService()
newIndexerService() newIndexerService()

View File

@ -196,7 +196,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool)
if m.LFS && setting.LFS.StartServer { if m.LFS && setting.LFS.StartServer {
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint) ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint)
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep, false); err != nil {
log.Error("Failed to synchronize LFS objects for repository: %v", err) log.Error("Failed to synchronize LFS objects for repository: %v", err)
} }
} }

View File

@ -134,7 +134,7 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
defer gitRepo.Close() defer gitRepo.Close()
ep := lfs.DetermineEndpoint(remoteAddr.String(), "") ep := lfs.DetermineEndpoint(remoteAddr.String(), "")
if err := pushAllLFSObjects(ctx, gitRepo, ep); err != nil { if err := pushAllLFSObjects(ctx, gitRepo, ep, false); err != nil {
return util.NewURLSanitizedError(err, remoteAddr, true) return util.NewURLSanitizedError(err, remoteAddr, true)
} }
} }
@ -176,8 +176,8 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
return nil return nil
} }
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL) error { func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
client := lfs.NewClient(endpoint) client := lfs.NewClient(endpoint, skipTLSVerify)
contentStore := lfs.NewContentStore() contentStore := lfs.NewContentStore()
pointerChan := make(chan lfs.PointerBlob) pointerChan := make(chan lfs.PointerBlob)

View File

@ -25,6 +25,7 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"github.com/gobwas/glob" "github.com/gobwas/glob"
) )
@ -260,7 +261,7 @@ var (
func webhookProxy() func(req *http.Request) (*url.URL, error) { func webhookProxy() func(req *http.Request) (*url.URL, error) {
if setting.Webhook.ProxyURL == "" { if setting.Webhook.ProxyURL == "" {
return http.ProxyFromEnvironment return proxy.Proxy()
} }
once.Do(func() { once.Do(func() {