mirror of
https://github.com/go-gitea/gitea
synced 2025-07-23 02:38:35 +00:00
Azure blob storage support (#30995)
This PR implemented object storages(LFS/Packages/Attachments and etc.) for Azure Blob Storage. It depends on azure official golang SDK and can support both the azure blob storage cloud service and azurite mock server. Replace #25458 Fix #22527 - [x] CI Tests - [x] integration test, MSSQL integration tests will now based on azureblob - [x] unit test - [x] CLI Migrate Storage - [x] Documentation for configuration added ------ TODO (other PRs): - [ ] Improve performance of `blob download`. --------- Co-authored-by: yp05327 <576951401@qq.com>
This commit is contained in:
Lunny Xiao
GitHub
@@ -18,11 +18,14 @@ const (
|
||||
LocalStorageType StorageType = "local"
|
||||
// MinioStorageType is the type descriptor for minio storage
|
||||
MinioStorageType StorageType = "minio"
|
||||
// AzureBlobStorageType is the type descriptor for azure blob storage
|
||||
AzureBlobStorageType StorageType = "azureblob"
|
||||
)
|
||||
|
||||
var storageTypes = []StorageType{
|
||||
LocalStorageType,
|
||||
MinioStorageType,
|
||||
AzureBlobStorageType,
|
||||
}
|
||||
|
||||
// IsValidStorageType returns true if the given storage type is valid
|
||||
@@ -50,25 +53,55 @@ type MinioStorageConfig struct {
|
||||
BucketLookUpType string `ini:"MINIO_BUCKET_LOOKUP_TYPE" json:",omitempty"`
|
||||
}
|
||||
|
||||
func (cfg *MinioStorageConfig) ToShadow() {
|
||||
if cfg.AccessKeyID != "" {
|
||||
cfg.AccessKeyID = "******"
|
||||
}
|
||||
if cfg.SecretAccessKey != "" {
|
||||
cfg.SecretAccessKey = "******"
|
||||
}
|
||||
}
|
||||
|
||||
// MinioStorageConfig represents the configuration for a minio storage
|
||||
type AzureBlobStorageConfig struct {
|
||||
Endpoint string `ini:"AZURE_BLOB_ENDPOINT" json:",omitempty"`
|
||||
AccountName string `ini:"AZURE_BLOB_ACCOUNT_NAME" json:",omitempty"`
|
||||
AccountKey string `ini:"AZURE_BLOB_ACCOUNT_KEY" json:",omitempty"`
|
||||
Container string `ini:"AZURE_BLOB_CONTAINER" json:",omitempty"`
|
||||
BasePath string `ini:"AZURE_BLOB_BASE_PATH" json:",omitempty"`
|
||||
ServeDirect bool `ini:"SERVE_DIRECT"`
|
||||
}
|
||||
|
||||
func (cfg *AzureBlobStorageConfig) ToShadow() {
|
||||
if cfg.AccountKey != "" {
|
||||
cfg.AccountKey = "******"
|
||||
}
|
||||
if cfg.AccountName != "" {
|
||||
cfg.AccountName = "******"
|
||||
}
|
||||
}
|
||||
|
||||
// Storage represents configuration of storages
|
||||
type Storage struct {
|
||||
Type StorageType // local or minio
|
||||
Path string `json:",omitempty"` // for local type
|
||||
TemporaryPath string `json:",omitempty"`
|
||||
MinioConfig MinioStorageConfig // for minio type
|
||||
Type StorageType // local or minio or azureblob
|
||||
Path string `json:",omitempty"` // for local type
|
||||
TemporaryPath string `json:",omitempty"`
|
||||
MinioConfig MinioStorageConfig // for minio type
|
||||
AzureBlobConfig AzureBlobStorageConfig // for azureblob type
|
||||
}
|
||||
|
||||
func (storage *Storage) ToShadowCopy() Storage {
|
||||
shadowStorage := *storage
|
||||
if shadowStorage.MinioConfig.AccessKeyID != "" {
|
||||
shadowStorage.MinioConfig.AccessKeyID = "******"
|
||||
}
|
||||
if shadowStorage.MinioConfig.SecretAccessKey != "" {
|
||||
shadowStorage.MinioConfig.SecretAccessKey = "******"
|
||||
}
|
||||
shadowStorage.MinioConfig.ToShadow()
|
||||
shadowStorage.AzureBlobConfig.ToShadow()
|
||||
return shadowStorage
|
||||
}
|
||||
|
||||
func (storage *Storage) ServeDirect() bool {
|
||||
return (storage.Type == MinioStorageType && storage.MinioConfig.ServeDirect) ||
|
||||
(storage.Type == AzureBlobStorageType && storage.AzureBlobConfig.ServeDirect)
|
||||
}
|
||||
|
||||
const storageSectionName = "storage"
|
||||
|
||||
func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
|
||||
@@ -84,6 +117,10 @@ func getDefaultStorageSection(rootCfg ConfigProvider) ConfigSection {
|
||||
storageSec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
|
||||
storageSec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
|
||||
storageSec.Key("MINIO_BUCKET_LOOKUP_TYPE").MustString("auto")
|
||||
storageSec.Key("AZURE_BLOB_ENDPOINT").MustString("")
|
||||
storageSec.Key("AZURE_BLOB_ACCOUNT_NAME").MustString("")
|
||||
storageSec.Key("AZURE_BLOB_ACCOUNT_KEY").MustString("")
|
||||
storageSec.Key("AZURE_BLOB_CONTAINER").MustString("gitea")
|
||||
return storageSec
|
||||
}
|
||||
|
||||
@@ -107,6 +144,8 @@ func getStorage(rootCfg ConfigProvider, name, typ string, sec ConfigSection) (*S
|
||||
return getStorageForLocal(targetSec, overrideSec, tp, name)
|
||||
case string(MinioStorageType):
|
||||
return getStorageForMinio(targetSec, overrideSec, tp, name)
|
||||
case string(AzureBlobStorageType):
|
||||
return getStorageForAzureBlob(targetSec, overrideSec, tp, name)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage type %q", targetType)
|
||||
}
|
||||
@@ -247,7 +286,7 @@ func getStorageForLocal(targetSec, overrideSec ConfigSection, tp targetSecType,
|
||||
return &storage, nil
|
||||
}
|
||||
|
||||
func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) {
|
||||
func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl
|
||||
var storage Storage
|
||||
storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String())
|
||||
if err := targetSec.MapTo(&storage.MinioConfig); err != nil {
|
||||
@@ -275,3 +314,32 @@ func getStorageForMinio(targetSec, overrideSec ConfigSection, tp targetSecType,
|
||||
}
|
||||
return &storage, nil
|
||||
}
|
||||
|
||||
func getStorageForAzureBlob(targetSec, overrideSec ConfigSection, tp targetSecType, name string) (*Storage, error) { //nolint:dupl
|
||||
var storage Storage
|
||||
storage.Type = StorageType(targetSec.Key("STORAGE_TYPE").String())
|
||||
if err := targetSec.MapTo(&storage.AzureBlobConfig); err != nil {
|
||||
return nil, fmt.Errorf("map azure blob config failed: %v", err)
|
||||
}
|
||||
|
||||
var defaultPath string
|
||||
if storage.AzureBlobConfig.BasePath != "" {
|
||||
if tp == targetSecIsStorage || tp == targetSecIsDefault {
|
||||
defaultPath = strings.TrimSuffix(storage.AzureBlobConfig.BasePath, "/") + "/" + name + "/"
|
||||
} else {
|
||||
defaultPath = storage.AzureBlobConfig.BasePath
|
||||
}
|
||||
}
|
||||
if defaultPath == "" {
|
||||
defaultPath = name + "/"
|
||||
}
|
||||
|
||||
if overrideSec != nil {
|
||||
storage.AzureBlobConfig.ServeDirect = ConfigSectionKeyBool(overrideSec, "SERVE_DIRECT", storage.AzureBlobConfig.ServeDirect)
|
||||
storage.AzureBlobConfig.BasePath = ConfigSectionKeyString(overrideSec, "AZURE_BLOB_BASE_PATH", defaultPath)
|
||||
storage.AzureBlobConfig.Container = ConfigSectionKeyString(overrideSec, "AZURE_BLOB_CONTAINER", storage.AzureBlobConfig.Container)
|
||||
} else {
|
||||
storage.AzureBlobConfig.BasePath = defaultPath
|
||||
}
|
||||
return &storage, nil
|
||||
}
|
||||
|
||||
@@ -97,6 +97,44 @@ STORAGE_TYPE = minio
|
||||
assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.MinioConfig.BasePath)
|
||||
}
|
||||
|
||||
func Test_getStorageInheritStorageTypeAzureBlob(t *testing.T) {
|
||||
iniStr := `
|
||||
[storage]
|
||||
STORAGE_TYPE = azureblob
|
||||
`
|
||||
cfg, err := NewConfigProviderFromData(iniStr)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.NoError(t, loadPackagesFrom(cfg))
|
||||
assert.EqualValues(t, "azureblob", Packages.Storage.Type)
|
||||
assert.EqualValues(t, "gitea", Packages.Storage.AzureBlobConfig.Container)
|
||||
assert.EqualValues(t, "packages/", Packages.Storage.AzureBlobConfig.BasePath)
|
||||
|
||||
assert.NoError(t, loadRepoArchiveFrom(cfg))
|
||||
assert.EqualValues(t, "azureblob", RepoArchive.Storage.Type)
|
||||
assert.EqualValues(t, "gitea", RepoArchive.Storage.AzureBlobConfig.Container)
|
||||
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath)
|
||||
|
||||
assert.NoError(t, loadActionsFrom(cfg))
|
||||
assert.EqualValues(t, "azureblob", Actions.LogStorage.Type)
|
||||
assert.EqualValues(t, "gitea", Actions.LogStorage.AzureBlobConfig.Container)
|
||||
assert.EqualValues(t, "actions_log/", Actions.LogStorage.AzureBlobConfig.BasePath)
|
||||
|
||||
assert.EqualValues(t, "azureblob", Actions.ArtifactStorage.Type)
|
||||
assert.EqualValues(t, "gitea", Actions.ArtifactStorage.AzureBlobConfig.Container)
|
||||
assert.EqualValues(t, "actions_artifacts/", Actions.ArtifactStorage.AzureBlobConfig.BasePath)
|
||||
|
||||
assert.NoError(t, loadAvatarsFrom(cfg))
|
||||
assert.EqualValues(t, "azureblob", Avatar.Storage.Type)
|
||||
assert.EqualValues(t, "gitea", Avatar.Storage.AzureBlobConfig.Container)
|
||||
assert.EqualValues(t, "avatars/", Avatar.Storage.AzureBlobConfig.BasePath)
|
||||
|
||||
assert.NoError(t, loadRepoAvatarFrom(cfg))
|
||||
assert.EqualValues(t, "azureblob", RepoAvatar.Storage.Type)
|
||||
assert.EqualValues(t, "gitea", RepoAvatar.Storage.AzureBlobConfig.Container)
|
||||
assert.EqualValues(t, "repo-avatars/", RepoAvatar.Storage.AzureBlobConfig.BasePath)
|
||||
}
|
||||
|
||||
type testLocalStoragePathCase struct {
|
||||
loader func(rootCfg ConfigProvider) error
|
||||
storagePtr **Storage
|
||||
@@ -465,3 +503,77 @@ MINIO_BASE_PATH = /lfs
|
||||
assert.EqualValues(t, true, LFS.Storage.MinioConfig.UseSSL)
|
||||
assert.EqualValues(t, "/lfs", LFS.Storage.MinioConfig.BasePath)
|
||||
}
|
||||
|
||||
func Test_getStorageConfiguration29(t *testing.T) {
|
||||
cfg, err := NewConfigProviderFromData(`
|
||||
[repo-archive]
|
||||
STORAGE_TYPE = azureblob
|
||||
AZURE_BLOB_ACCOUNT_NAME = my_account_name
|
||||
AZURE_BLOB_ACCOUNT_KEY = my_account_key
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
// assert.Error(t, loadRepoArchiveFrom(cfg))
|
||||
// FIXME: this should return error but now ini package's MapTo() doesn't check type
|
||||
assert.NoError(t, loadRepoArchiveFrom(cfg))
|
||||
}
|
||||
|
||||
func Test_getStorageConfiguration30(t *testing.T) {
|
||||
cfg, err := NewConfigProviderFromData(`
|
||||
[storage.repo-archive]
|
||||
STORAGE_TYPE = azureblob
|
||||
AZURE_BLOB_ACCOUNT_NAME = my_account_name
|
||||
AZURE_BLOB_ACCOUNT_KEY = my_account_key
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, loadRepoArchiveFrom(cfg))
|
||||
assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName)
|
||||
assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey)
|
||||
assert.EqualValues(t, "repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath)
|
||||
}
|
||||
|
||||
func Test_getStorageConfiguration31(t *testing.T) {
|
||||
cfg, err := NewConfigProviderFromData(`
|
||||
[storage]
|
||||
STORAGE_TYPE = azureblob
|
||||
AZURE_BLOB_ACCOUNT_NAME = my_account_name
|
||||
AZURE_BLOB_ACCOUNT_KEY = my_account_key
|
||||
AZURE_BLOB_BASE_PATH = /prefix
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, loadRepoArchiveFrom(cfg))
|
||||
assert.EqualValues(t, "my_account_name", RepoArchive.Storage.AzureBlobConfig.AccountName)
|
||||
assert.EqualValues(t, "my_account_key", RepoArchive.Storage.AzureBlobConfig.AccountKey)
|
||||
assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.AzureBlobConfig.BasePath)
|
||||
|
||||
cfg, err = NewConfigProviderFromData(`
|
||||
[storage]
|
||||
STORAGE_TYPE = azureblob
|
||||
AZURE_BLOB_ACCOUNT_NAME = my_account_name
|
||||
AZURE_BLOB_ACCOUNT_KEY = my_account_key
|
||||
AZURE_BLOB_BASE_PATH = /prefix
|
||||
|
||||
[lfs]
|
||||
AZURE_BLOB_BASE_PATH = /lfs
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, loadLFSFrom(cfg))
|
||||
assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName)
|
||||
assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey)
|
||||
assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath)
|
||||
|
||||
cfg, err = NewConfigProviderFromData(`
|
||||
[storage]
|
||||
STORAGE_TYPE = azureblob
|
||||
AZURE_BLOB_ACCOUNT_NAME = my_account_name
|
||||
AZURE_BLOB_ACCOUNT_KEY = my_account_key
|
||||
AZURE_BLOB_BASE_PATH = /prefix
|
||||
|
||||
[storage.lfs]
|
||||
AZURE_BLOB_BASE_PATH = /lfs
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, loadLFSFrom(cfg))
|
||||
assert.EqualValues(t, "my_account_name", LFS.Storage.AzureBlobConfig.AccountName)
|
||||
assert.EqualValues(t, "my_account_key", LFS.Storage.AzureBlobConfig.AccountKey)
|
||||
assert.EqualValues(t, "/lfs", LFS.Storage.AzureBlobConfig.BasePath)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user