Add Allow-/Block-List for Migrate & Mirrors (#13610)

* add black list and white list support for migrating repositories

* fix fmt

* fix lint

* fix vendor

* fix modules.txt

* clean diff

* specify log message

* use blocklist/allowlist

* allways use lowercase to match url

* Apply allow/block

* Settings: use existing "migrations" section

* convert domains lower case

* dont store unused value

* Block private addresses for migration by default

* fix lint

* use proposed-upstream func to detect private IP addr

* a nit

* add own error for blocked migration, add tests, imprufe api

* fix test

* fix-if-localhost-is-ipv4

* rename error & error message

* rename setting options

* Apply suggestions from code review

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
6543 2020-11-29 01:37:58 +01:00 committed by GitHub
parent 0f14f69e60
commit b2435af9be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 228 additions and 4 deletions

View File

@ -1187,6 +1187,14 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
MAX_ATTEMPTS = 3 MAX_ATTEMPTS = 3
; Backoff time per http/https request retry (seconds) ; Backoff time per http/https request retry (seconds)
RETRY_BACKOFF = 3 RETRY_BACKOFF = 3
; Allowed domains for migrating, default is blank. Blank means everything will be allowed.
; Multiple domains could be separated by commas.
ALLOWED_DOMAINS =
; Blocklist for migrating, default is blank. Multiple domains could be separated by commas.
; When ALLOWED_DOMAINS is not blank, this option will be ignored.
BLOCKED_DOMAINS =
; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default)
ALLOW_LOCALNETWORKS = false
; default storage for attachments, lfs and avatars ; default storage for attachments, lfs and avatars
[storage] [storage]

View File

@ -846,6 +846,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations. - `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds) - `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
## Mirror (`mirror`) ## Mirror (`mirror`)

View File

@ -313,6 +313,9 @@ IS_INPUT_FILE = false
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。 - `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。 - `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
## LFS (`lfs`) ## LFS (`lfs`)

View File

@ -309,6 +309,8 @@ func TestAPIRepoMigrate(t *testing.T) {
{ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden}, {ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden},
{ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated}, {ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated},
{ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden}, {ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden},
{ctxUserID: 2, userID: 3, cloneURL: "https://localhost:3000/user/test_repo.git", repoName: "local-ip", expectedStatus: http.StatusUnprocessableEntity},
{ctxUserID: 2, userID: 3, cloneURL: "https://10.0.0.1/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity},
} }
defer prepareTestEnv(t)() defer prepareTestEnv(t)()
@ -325,8 +327,16 @@ func TestAPIRepoMigrate(t *testing.T) {
if resp.Code == http.StatusUnprocessableEntity { if resp.Code == http.StatusUnprocessableEntity {
respJSON := map[string]string{} respJSON := map[string]string{}
DecodeJSON(t, resp, &respJSON) DecodeJSON(t, resp, &respJSON)
if assert.Equal(t, "Remote visit addressed rate limitation.", respJSON["message"]) { switch respJSON["message"] {
case "Remote visit addressed rate limitation.":
t.Log("test hit github rate limitation") t.Log("test hit github rate limitation")
case "migrate from '10.0.0.1' is not allowed: the host resolve to a private ip address '10.0.0.1'":
assert.EqualValues(t, "private-ip", testCase.repoName)
case "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '::1'",
"migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '127.0.0.1'":
assert.EqualValues(t, "local-ip", testCase.repoName)
default:
t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL)
} }
} else { } else {
assert.EqualValues(t, testCase.expectedStatus, resp.Code) assert.EqualValues(t, testCase.expectedStatus, resp.Code)

View File

@ -1019,6 +1019,29 @@ func IsErrWontSign(err error) bool {
return ok return ok
} }
// ErrMigrationNotAllowed explains why a migration from an url is not allowed
type ErrMigrationNotAllowed struct {
Host string
NotResolvedIP bool
PrivateNet string
}
func (e *ErrMigrationNotAllowed) Error() string {
if e.NotResolvedIP {
return fmt.Sprintf("migrate from '%s' is not allowed: unknown hostname", e.Host)
}
if len(e.PrivateNet) != 0 {
return fmt.Sprintf("migrate from '%s' is not allowed: the host resolve to a private ip address '%s'", e.Host, e.PrivateNet)
}
return fmt.Sprintf("migrate from '%s is not allowed'", e.Host)
}
// IsErrMigrationNotAllowed checks if an error is a ErrMigrationNotAllowed
func IsErrMigrationNotAllowed(err error) bool {
_, ok := err.(*ErrMigrationNotAllowed)
return ok
}
// __________ .__ // __________ .__
// \______ \____________ ____ ____ | |__ // \______ \____________ ____ ____ | |__
// | | _/\_ __ \__ \ / \_/ ___\| | \ // | | _/\_ __ \__ \ / \_/ ___\| | \

View File

@ -0,0 +1,46 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package matchlist
import (
"strings"
"github.com/gobwas/glob"
)
// Matchlist represents a block or allow list
type Matchlist struct {
ruleGlobs []glob.Glob
}
// NewMatchlist creates a new block or allow list
func NewMatchlist(rules ...string) (*Matchlist, error) {
for i := range rules {
rules[i] = strings.ToLower(rules[i])
}
list := Matchlist{
ruleGlobs: make([]glob.Glob, 0, len(rules)),
}
for _, rule := range rules {
rg, err := glob.Compile(rule)
if err != nil {
return nil, err
}
list.ruleGlobs = append(list.ruleGlobs, rg)
}
return &list, nil
}
// Match will matches
func (b *Matchlist) Match(u string) bool {
for _, r := range b.ruleGlobs {
if r.Match(u) {
return true
}
}
return false
}

View File

@ -8,9 +8,13 @@ package migrations
import ( import (
"context" "context"
"fmt" "fmt"
"net"
"net/url"
"strings"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/matchlist"
"code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
) )
@ -20,6 +24,9 @@ type MigrateOptions = base.MigrateOptions
var ( var (
factories []base.DownloaderFactory factories []base.DownloaderFactory
allowList *matchlist.Matchlist
blockList *matchlist.Matchlist
) )
// RegisterDownloaderFactory registers a downloader factory // RegisterDownloaderFactory registers a downloader factory
@ -27,12 +34,49 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) {
factories = append(factories, factory) factories = append(factories, factory)
} }
func isMigrateURLAllowed(remoteURL string) error {
u, err := url.Parse(strings.ToLower(remoteURL))
if err != nil {
return err
}
if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") {
if len(setting.Migrations.AllowedDomains) > 0 {
if !allowList.Match(u.Host) {
return &models.ErrMigrationNotAllowed{Host: u.Host}
}
} else {
if blockList.Match(u.Host) {
return &models.ErrMigrationNotAllowed{Host: u.Host}
}
}
}
if !setting.Migrations.AllowLocalNetworks {
addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
if err != nil {
return &models.ErrMigrationNotAllowed{Host: u.Host, NotResolvedIP: true}
}
for _, addr := range addrList {
if isIPPrivate(addr) || !addr.IsGlobalUnicast() {
return &models.ErrMigrationNotAllowed{Host: u.Host, PrivateNet: addr.String()}
}
}
}
return nil
}
// MigrateRepository migrate repository according MigrateOptions // MigrateRepository migrate repository according MigrateOptions
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
err := isMigrateURLAllowed(opts.CloneAddr)
if err != nil {
return nil, err
}
var ( var (
downloader base.Downloader downloader base.Downloader
uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
err error
) )
for _, factory := range factories { for _, factory := range factories {
@ -308,3 +352,32 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
return nil return nil
} }
// Init migrations service
func Init() error {
var err error
allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
if err != nil {
return fmt.Errorf("init migration allowList domains failed: %v", err)
}
blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
if err != nil {
return fmt.Errorf("init migration blockList domains failed: %v", err)
}
return nil
}
// isIPPrivate reports whether ip is a private address, according to
// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses).
// from https://github.com/golang/go/pull/42793
// TODO remove if https://github.com/golang/go/issues/29146 got resolved
func isIPPrivate(ip net.IP) bool {
if ip4 := ip.To4(); ip4 != nil {
return ip4[0] == 10 ||
(ip4[0] == 172 && ip4[1]&0xf0 == 16) ||
(ip4[0] == 192 && ip4[1] == 168)
}
return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc
}

View File

@ -0,0 +1,34 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"testing"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
)
func TestMigrateWhiteBlocklist(t *testing.T) {
setting.Migrations.AllowedDomains = []string{"github.com"}
assert.NoError(t, Init())
err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git")
assert.Error(t, err)
err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git")
assert.NoError(t, err)
setting.Migrations.AllowedDomains = []string{}
setting.Migrations.BlockedDomains = []string{"github.com"}
assert.NoError(t, Init())
err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git")
assert.NoError(t, err)
err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git")
assert.Error(t, err)
}

View File

@ -4,11 +4,18 @@
package setting package setting
import (
"strings"
)
var ( var (
// Migrations settings // Migrations settings
Migrations = struct { Migrations = struct {
MaxAttempts int MaxAttempts int
RetryBackoff int RetryBackoff int
AllowedDomains []string
BlockedDomains []string
AllowLocalNetworks bool
}{ }{
MaxAttempts: 3, MaxAttempts: 3,
RetryBackoff: 3, RetryBackoff: 3,
@ -19,4 +26,15 @@ func newMigrationsService() {
sec := Cfg.Section("migrations") sec := Cfg.Section("migrations")
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",")
for i := range Migrations.AllowedDomains {
Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i])
}
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",")
for i := range Migrations.BlockedDomains {
Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i])
}
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
} }

View File

@ -212,6 +212,8 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name)) ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name))
case models.IsErrNamePatternNotAllowed(err): case models.IsErrNamePatternNotAllowed(err):
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern)) ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern))
case models.IsErrMigrationNotAllowed(err):
ctx.Error(http.StatusUnprocessableEntity, "", err)
default: default:
err = util.URLSanitizedError(err, remoteAddr) err = util.URLSanitizedError(err, remoteAddr)
if strings.Contains(err.Error(), "Authentication failed") || if strings.Contains(err.Error(), "Authentication failed") ||

View File

@ -24,6 +24,7 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/external" "code.gitea.io/gitea/modules/markup/external"
repo_migrations "code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
@ -201,6 +202,9 @@ func GlobalInit(ctx context.Context) {
if err := task.Init(); err != nil { if err := task.Init(); err != nil {
log.Fatal("Failed to initialize task scheduler: %v", err) log.Fatal("Failed to initialize task scheduler: %v", err)
} }
if err := repo_migrations.Init(); err != nil {
log.Fatal("Failed to initialize repository migrations: %v", err)
}
eventsource.GetManager().Init() eventsource.GetManager().Init()
if setting.EnableSQLite3 { if setting.EnableSQLite3 {