1
1
mirror of https://github.com/go-gitea/gitea synced 2025-07-23 02:38:35 +00:00

Use hostmatcher to replace matchlist, improve security (#17605)

Use hostmacher to replace matchlist.

And we introduce a better DialContext to do a full host/IP check, otherwise the attackers can still bypass the allow/block list by a 302 redirection.
This commit is contained in:
wxiaoguang
2021-11-20 17:34:05 +08:00
committed by GitHub
parent c96be0cd98
commit 013fb73068
33 changed files with 377 additions and 293 deletions

View File

@@ -6,7 +6,6 @@ package migrations
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
@@ -18,8 +17,6 @@ import (
admin_model "code.gitea.io/gitea/models/admin"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
gitea_sdk "code.gitea.io/sdk/gitea"
@@ -90,12 +87,7 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
gitea_sdk.SetToken(token),
gitea_sdk.SetBasicAuth(username, password),
gitea_sdk.SetContext(ctx),
gitea_sdk.SetHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}),
gitea_sdk.SetHTTPClient(NewMigrationHTTPClient()),
)
if err != nil {
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
@@ -275,12 +267,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
Created: rel.CreatedAt,
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
httpClient := NewMigrationHTTPClient()
for _, asset := range rel.Attachments {
size := int(asset.Size)

View File

@@ -125,7 +125,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
Wiki: opts.Wiki,
Releases: opts.Releases, // if didn't get releases, then sync them from tags
MirrorInterval: opts.MirrorInterval,
})
}, NewMigrationHTTPTransport())
g.repo = r
if err != nil {

View File

@@ -7,7 +7,6 @@ package migrations
import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
@@ -19,7 +18,6 @@ import (
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
@@ -100,12 +98,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
)
var client = &http.Client{
Transport: &oauth2.Transport{
Base: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) {
return proxy.Proxy()(req)
},
},
Base: NewMigrationHTTPTransport(),
Source: oauth2.ReuseTokenSource(nil, ts),
},
}
@@ -113,14 +106,13 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
downloader.addClient(client, baseURL)
}
} else {
var transport = NewMigrationHTTPTransport()
transport.Proxy = func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return proxy.Proxy()(req)
}
var client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return proxy.Proxy()(req)
},
},
Transport: transport,
}
downloader.addClient(client, baseURL)
}
@@ -316,12 +308,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
r.Published = rel.PublishedAt.Time
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
httpClient := NewMigrationHTTPClient()
for _, asset := range rel.Assets {
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable

View File

@@ -6,7 +6,6 @@ package migrations
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
@@ -18,8 +17,6 @@ import (
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/xanzy/go-gitlab"
@@ -77,16 +74,11 @@ type GitlabDownloader struct {
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}))
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(NewMigrationHTTPClient()))
// Only use basic auth if token is blank and password is NOT
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
if token == "" && password != "" {
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(NewMigrationHTTPClient()))
}
if err != nil {
@@ -300,12 +292,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
PublisherName: rel.Author.Username,
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
httpClient := NewMigrationHTTPClient()
for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, &base.ReleaseAsset{

View File

@@ -6,7 +6,6 @@ package migrations
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"net/url"
@@ -16,7 +15,6 @@ import (
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/gogs/go-gogs-client"
@@ -97,13 +95,12 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token,
client = gogs.NewClient(baseURL, token)
downloader.userName = token
} else {
downloader.transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return proxy.Proxy()(req)
},
var transport = NewMigrationHTTPTransport()
transport.Proxy = func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return proxy.Proxy()(req)
}
downloader.transport = transport
client = gogs.NewClient(baseURL, "")
client.SetHTTPClient(&http.Client{

View File

@@ -0,0 +1,30 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"crypto/tls"
"net/http"
"code.gitea.io/gitea/modules/hostmatcher"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
)
// NewMigrationHTTPClient returns a HTTP client for migration
func NewMigrationHTTPClient() *http.Client {
return &http.Client{
Transport: NewMigrationHTTPTransport(),
}
}
// NewMigrationHTTPTransport returns a HTTP transport for migration
func NewMigrationHTTPTransport() *http.Transport {
return &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
DialContext: hostmatcher.NewDialContext("migration", allowList, blockList),
}
}

View File

@@ -15,8 +15,8 @@ import (
"code.gitea.io/gitea/models"
admin_model "code.gitea.io/gitea/models/admin"
"code.gitea.io/gitea/modules/hostmatcher"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/matchlist"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@@ -28,8 +28,8 @@ type MigrateOptions = base.MigrateOptions
var (
factories []base.DownloaderFactory
allowList *matchlist.Matchlist
blockList *matchlist.Matchlist
allowList *hostmatcher.HostMatchList
blockList *hostmatcher.HostMatchList
)
// RegisterDownloaderFactory registers a downloader factory
@@ -73,30 +73,35 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
}
host := strings.ToLower(u.Host)
if len(setting.Migrations.AllowedDomains) > 0 {
if !allowList.Match(host) {
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
}
} else {
if blockList.Match(host) {
hostName, _, err := net.SplitHostPort(u.Host)
if err != nil {
// u.Host can be "host" or "host:port"
err = nil //nolint
hostName = u.Host
}
addrList, err := net.LookupIP(hostName)
if err != nil {
return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
}
var ipAllowed bool
var ipBlocked bool
for _, addr := range addrList {
ipAllowed = ipAllowed || allowList.MatchIPAddr(addr)
ipBlocked = ipBlocked || blockList.MatchIPAddr(addr)
}
var blockedError error
if blockList.MatchHostName(hostName) || ipBlocked {
blockedError = &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
}
// if we have an allow-list, check the allow-list first
if !allowList.IsEmpty() {
if !allowList.MatchHostName(hostName) && !ipAllowed {
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
}
}
if !setting.Migrations.AllowLocalNetworks {
addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
if err != nil {
return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
}
for _, addr := range addrList {
if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() {
return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true}
}
}
}
return nil
// otherwise, we always follow the blocked list
return blockedError
}
// MigrateRepository migrate repository according MigrateOptions
@@ -462,16 +467,18 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
// Init migrations service
func Init() error {
var err error
allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
if err != nil {
return fmt.Errorf("init migration allowList domains failed: %v", err)
}
// TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead
blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
if err != nil {
return fmt.Errorf("init migration blockList domains failed: %v", err)
}
blockList = hostmatcher.ParseSimpleMatchList("migrations.BLOCKED_DOMAINS", setting.Migrations.BlockedDomains)
allowList = hostmatcher.ParseSimpleMatchList("migrations.ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS", setting.Migrations.AllowedDomains)
if allowList.IsEmpty() {
// the default policy is that migration module can access external hosts
allowList.AppendBuiltin(hostmatcher.MatchBuiltinExternal)
}
if setting.Migrations.AllowLocalNetworks {
allowList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate)
allowList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback)
}
return nil
}

View File

@@ -21,7 +21,8 @@ func TestMigrateWhiteBlocklist(t *testing.T) {
adminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user1"}).(*models.User)
nonAdminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User)
setting.Migrations.AllowedDomains = []string{"github.com"}
setting.Migrations.AllowedDomains = "github.com"
setting.Migrations.AllowLocalNetworks = false
assert.NoError(t, Init())
err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
@@ -33,8 +34,8 @@ func TestMigrateWhiteBlocklist(t *testing.T) {
err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser)
assert.NoError(t, err)
setting.Migrations.AllowedDomains = []string{}
setting.Migrations.BlockedDomains = []string{"github.com"}
setting.Migrations.AllowedDomains = ""
setting.Migrations.BlockedDomains = "github.com"
assert.NoError(t, Init())
err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
@@ -47,6 +48,7 @@ func TestMigrateWhiteBlocklist(t *testing.T) {
assert.Error(t, err)
setting.Migrations.AllowLocalNetworks = true
assert.NoError(t, Init())
err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
assert.NoError(t, err)

View File

@@ -261,8 +261,9 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool)
if m.LFS && setting.LFS.StartServer {
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint)
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep, false); err != nil {
endpoint := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint)
lfsClient := lfs.NewClient(endpoint, nil)
if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil {
log.Error("Failed to synchronize LFS objects for repository: %v", err)
}
}

View File

@@ -8,7 +8,6 @@ import (
"context"
"errors"
"io"
"net/url"
"regexp"
"time"
@@ -133,8 +132,9 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
}
defer gitRepo.Close()
ep := lfs.DetermineEndpoint(remoteAddr.String(), "")
if err := pushAllLFSObjects(ctx, gitRepo, ep, false); err != nil {
endpoint := lfs.DetermineEndpoint(remoteAddr.String(), "")
lfsClient := lfs.NewClient(endpoint, nil)
if err := pushAllLFSObjects(ctx, gitRepo, lfsClient); err != nil {
return util.NewURLSanitizedError(err, remoteAddr, true)
}
}
@@ -176,8 +176,7 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
return nil
}
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
client := lfs.NewClient(endpoint, skipTLSVerify)
func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, lfsClient lfs.Client) error {
contentStore := lfs.NewContentStore()
pointerChan := make(chan lfs.PointerBlob)
@@ -185,7 +184,7 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u
go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan)
uploadObjects := func(pointers []lfs.Pointer) error {
err := client.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) {
err := lfsClient.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) {
if objectError != nil {
return nil, objectError
}
@@ -219,7 +218,7 @@ func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *u
}
batch = append(batch, pointerBlob.Pointer)
if len(batch) >= client.BatchSize() {
if len(batch) >= lfsClient.BatchSize() {
if err := uploadObjects(batch); err != nil {
return err
}

View File

@@ -13,17 +13,16 @@ import (
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"syscall"
"time"
webhook_model "code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/hostmatcher"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
@@ -31,8 +30,6 @@ import (
"github.com/gobwas/glob"
)
var contextKeyWebhookRequest interface{} = "contextKeyWebhookRequest"
// Deliver deliver hook task
func Deliver(t *webhook_model.HookTask) error {
w, err := webhook_model.GetWebhookByID(t.HookID)
@@ -98,10 +95,10 @@ func Deliver(t *webhook_model.HookTask) error {
return err
}
default:
return fmt.Errorf("Invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
}
default:
return fmt.Errorf("Invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
}
var signatureSHA1 string
@@ -172,10 +169,10 @@ func Deliver(t *webhook_model.HookTask) error {
}()
if setting.DisableWebhooks {
return fmt.Errorf("Webhook task skipped (webhooks disabled): [%d]", t.ID)
return fmt.Errorf("webhook task skipped (webhooks disabled): [%d]", t.ID)
}
resp, err := webhookHTTPClient.Do(req.WithContext(context.WithValue(req.Context(), contextKeyWebhookRequest, req)))
resp, err := webhookHTTPClient.Do(req.WithContext(graceful.GetManager().ShutdownContext()))
if err != nil {
t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
return err
@@ -296,29 +293,18 @@ func webhookProxy() func(req *http.Request) (*url.URL, error) {
func InitDeliverHooks() {
timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
allowedHostListValue := setting.Webhook.AllowedHostList
if allowedHostListValue == "" {
allowedHostListValue = hostmatcher.MatchBuiltinExternal
}
allowedHostMatcher := hostmatcher.ParseHostMatchList("webhook.ALLOWED_HOST_LIST", allowedHostListValue)
webhookHTTPClient = &http.Client{
Timeout: timeout,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify},
Proxy: webhookProxy(),
DialContext: func(ctx context.Context, network, addrOrHost string) (net.Conn, error) {
dialer := net.Dialer{
Timeout: timeout,
Control: func(network, ipAddr string, c syscall.RawConn) error {
// in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here
tcpAddr, err := net.ResolveTCPAddr(network, ipAddr)
req := ctx.Value(contextKeyWebhookRequest).(*http.Request)
if err != nil {
return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err)
}
if !setting.Webhook.AllowedHostList.MatchesHostOrIP(req.Host, tcpAddr.IP) {
return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr)
}
return nil
},
}
return dialer.DialContext(ctx, network, addrOrHost)
},
DialContext: hostmatcher.NewDialContext("webhook", allowedHostMatcher, nil),
},
}