From 02259a0f3aaa257daf97fb1ec5f5c83c937a2602 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 1 Nov 2020 23:12:50 +0800 Subject: [PATCH] Storage configuration support `[storage]` (#13314) (#13379) * Fix minio bug * Add tests for storage configuration * Change the Seek flag to keep compitable minio? * Fix test when first-byte-pos of all ranges is greater than the resource length Co-authored-by: techknowlogick Co-authored-by: techknowlogick --- integrations/integration_test.go | 29 ++++++++++++++++++----------- integrations/lfs_getobject_test.go | 3 ++- integrations/mysql.ini.tmpl | 26 ++++++++++++-------------- modules/lfs/content_store.go | 23 ++++++++++++++++++++++- modules/lfs/server.go | 8 ++++++-- modules/setting/storage.go | 6 ++---- 6 files changed, 62 insertions(+), 33 deletions(-) diff --git a/integrations/integration_test.go b/integrations/integration_test.go index 3942d54410..13a1bac370 100644 --- a/integrations/integration_test.go +++ b/integrations/integration_test.go @@ -11,7 +11,6 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" "net/http/cookiejar" "net/http/httptest" @@ -27,8 +26,10 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers" "code.gitea.io/gitea/routers/routes" @@ -59,6 +60,8 @@ func NewNilResponseRecorder() *NilResponseRecorder { } func TestMain(m *testing.M) { + defer log.Close() + managerCtx, cancel := context.WithCancel(context.Background()) graceful.InitManager(managerCtx) defer cancel() @@ -142,6 +145,10 @@ func initIntegrationTest() { util.RemoveAll(models.LocalCopyPath()) setting.CheckLFSVersion() setting.InitDBConfig() + if err := storage.Init(); err != nil { + fmt.Printf("Init storage failed: %v", err) + os.Exit(1) + } switch { case setting.Database.UseMySQL: @@ -149,27 +156,27 @@ func initIntegrationTest() { setting.Database.User, setting.Database.Passwd, setting.Database.Host)) defer db.Close() if err != nil { - log.Fatalf("sql.Open: %v", err) + log.Fatal("sql.Open: %v", err) } if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil { - log.Fatalf("db.Exec: %v", err) + log.Fatal("db.Exec: %v", err) } case setting.Database.UsePostgreSQL: db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode)) defer db.Close() if err != nil { - log.Fatalf("sql.Open: %v", err) + log.Fatal("sql.Open: %v", err) } dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name)) if err != nil { - log.Fatalf("db.Query: %v", err) + log.Fatal("db.Query: %v", err) } defer dbrows.Close() if !dbrows.Next() { if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil { - log.Fatalf("db.Exec: CREATE DATABASE: %v", err) + log.Fatal("db.Exec: CREATE DATABASE: %v", err) } } // Check if we need to setup a specific schema @@ -183,18 +190,18 @@ func initIntegrationTest() { // This is a different db object; requires a different Close() defer db.Close() if err != nil { - log.Fatalf("sql.Open: %v", err) + log.Fatal("sql.Open: %v", err) } schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema)) if err != nil { - log.Fatalf("db.Query: %v", err) + log.Fatal("db.Query: %v", err) } defer schrows.Close() if !schrows.Next() { // Create and setup a DB schema if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil { - log.Fatalf("db.Exec: CREATE SCHEMA: %v", err) + log.Fatal("db.Exec: CREATE SCHEMA: %v", err) } } @@ -203,10 +210,10 @@ func initIntegrationTest() { db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, "master", setting.Database.User, setting.Database.Passwd)) if err != nil { - log.Fatalf("sql.Open: %v", err) + log.Fatal("sql.Open: %v", err) } if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil { - log.Fatalf("db.Exec: %v", err) + log.Fatal("db.Exec: %v", err) } defer db.Close() } diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index 431c7ed9e8..180182dd42 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -78,6 +78,7 @@ func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, exp } } } + resp := session.MakeRequest(t, req, expectedStatus) return resp @@ -210,7 +211,7 @@ func TestGetLFSRange(t *testing.T) { {"bytes=0-10", "123456789\n", http.StatusPartialContent}, // end-range bigger than length-1 is ignored {"bytes=0-11", "123456789\n", http.StatusPartialContent}, - {"bytes=11-", "", http.StatusPartialContent}, + {"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable}, // incorrect header value cause whole header to be ignored {"bytes=-", "123456789\n", http.StatusOK}, {"foobar", "123456789\n", http.StatusOK}, diff --git a/integrations/mysql.ini.tmpl b/integrations/mysql.ini.tmpl index b546748d17..db1051e62a 100644 --- a/integrations/mysql.ini.tmpl +++ b/integrations/mysql.ini.tmpl @@ -45,19 +45,21 @@ START_SSH_SERVER = true OFFLINE_MODE = false LFS_START_SERVER = true -LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w -LFS_STORE_TYPE = minio -LFS_SERVE_DIRECT = false -LFS_MINIO_ENDPOINT = minio:9000 -LFS_MINIO_ACCESS_KEY_ID = 123456 -LFS_MINIO_SECRET_ACCESS_KEY = 12345678 -LFS_MINIO_BUCKET = gitea -LFS_MINIO_LOCATION = us-east-1 -LFS_MINIO_BASE_PATH = lfs/ -LFS_MINIO_USE_SSL = false + +[lfs] +MINIO_BASE_PATH = lfs/ [attachment] +MINIO_BASE_PATH = attachments/ + +[avatars] +MINIO_BASE_PATH = avatars/ + +[repo-avatars] +MINIO_BASE_PATH = repo-avatars/ + +[storage] STORAGE_TYPE = minio SERVE_DIRECT = false MINIO_ENDPOINT = minio:9000 @@ -65,7 +67,6 @@ MINIO_ACCESS_KEY_ID = 123456 MINIO_SECRET_ACCESS_KEY = 12345678 MINIO_BUCKET = gitea MINIO_LOCATION = us-east-1 -MINIO_BASE_PATH = attachments/ MINIO_USE_SSL = false [mailer] @@ -88,9 +89,6 @@ ENABLE_NOTIFY_MAIL = true DISABLE_GRAVATAR = false ENABLE_FEDERATED_AVATAR = false -AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/avatars -REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars - [session] PROVIDER = file PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index cf0a05d644..247191a1bf 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "io" "os" @@ -21,6 +22,21 @@ var ( errSizeMismatch = errors.New("Content size does not match") ) +// ErrRangeNotSatisfiable represents an error which request range is not satisfiable. +type ErrRangeNotSatisfiable struct { + FromByte int64 +} + +func (err ErrRangeNotSatisfiable) Error() string { + return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte) +} + +// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable +func IsErrRangeNotSatisfiable(err error) bool { + _, ok := err.(ErrRangeNotSatisfiable) + return ok +} + // ContentStore provides a simple file system based storage. type ContentStore struct { storage.ObjectStorage @@ -35,7 +51,12 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC return nil, err } if fromByte > 0 { - _, err = f.Seek(fromByte, os.SEEK_CUR) + if fromByte >= meta.Size { + return nil, ErrRangeNotSatisfiable{ + FromByte: fromByte, + } + } + _, err = f.Seek(fromByte, io.SeekStart) if err != nil { log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err) } diff --git a/modules/lfs/server.go b/modules/lfs/server.go index 2801f8410c..b093213643 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -191,8 +191,12 @@ func getContentHandler(ctx *context.Context) { contentStore := &ContentStore{ObjectStorage: storage.LFS} content, err := contentStore.Get(meta, fromByte) if err != nil { - // Errors are logged in contentStore.Get - writeStatus(ctx, 404) + if IsErrRangeNotSatisfiable(err) { + writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable) + } else { + // Errors are logged in contentStore.Get + writeStatus(ctx, 404) + } return } defer content.Close() diff --git a/modules/setting/storage.go b/modules/setting/storage.go index e743d6c20c..27788da1ff 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -32,14 +32,12 @@ func (s *Storage) MapTo(v interface{}) error { } func getStorage(name, typ string, overrides ...*ini.Section) Storage { - sectionName := "storage" - if len(name) > 0 { - sectionName = sectionName + "." + typ - } + const sectionName = "storage" sec := Cfg.Section(sectionName) if len(overrides) == 0 { overrides = []*ini.Section{ + Cfg.Section(sectionName + "." + typ), Cfg.Section(sectionName + "." + name), } }