Azure blob storage support (#30995)

This PR implemented object storages(LFS/Packages/Attachments and etc.)
for Azure Blob Storage. It depends on azure official golang SDK and can
support both the azure blob storage cloud service and azurite mock
server.

Replace #25458
Fix #22527 

- [x] CI Tests
- [x] integration test, MSSQL integration tests will now based on
azureblob
  - [x] unit test 
- [x] CLI Migrate Storage
- [x] Documentation for configuration added

------

TODO (other PRs):
- [ ] Improve performance of `blob download`.

---------

Co-authored-by: yp05327 <576951401@qq.com>

routers/api/actions/artifacts.go

@@ -428,7 +428,7 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
 	var items []downloadArtifactResponseItem
 	for _, artifact := range artifacts {
 		var downloadURL string
-		if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
+		if setting.Actions.ArtifactStorage.ServeDirect() {
 			u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName)
 			if err != nil && !errors.Is(err, storage.ErrURLNotSupported) {
 				log.Error("Error getting serve direct url: %v", err)

routers/api/actions/artifacts_chunks.go

@@ -55,7 +55,7 @@ func saveUploadChunkBase(st storage.ObjectStorage, ctx *ArtifactContext,
 		}
 	}
 	if writtenSize != contentSize {
-		checkErr = errors.Join(checkErr, fmt.Errorf("contentSize not match body size"))
+		checkErr = errors.Join(checkErr, fmt.Errorf("writtenSize %d not match contentSize %d", writtenSize, contentSize))
 	}
 	if checkErr != nil {
 		if err := st.Delete(storagePath); err != nil {

routers/api/actions/artifactsv4.go

@@ -448,7 +448,7 @@ func (r *artifactV4Routes) getSignedArtifactURL(ctx *ArtifactContext) {
 
 	respData := GetSignedArtifactURLResponse{}
 
-	if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
+	if setting.Actions.ArtifactStorage.ServeDirect() {
 		u, err := storage.ActionsArtifacts.URL(artifact.StoragePath, artifact.ArtifactPath)
 		if u != nil && err == nil {
 			respData.SignedUrl = u.String()

routers/api/v1/repo/file.go

@@ -201,7 +201,7 @@ func GetRawFileOrLFS(ctx *context.APIContext) {
 		return
 	}
 
-	if setting.LFS.Storage.MinioConfig.ServeDirect {
+	if setting.LFS.Storage.ServeDirect() {
 		// If we have a signed url (S3, object storage), redirect to this directly.
 		u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
 		if u != nil && err == nil {
@@ -326,7 +326,7 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model.
 		archiver.CommitID, archiver.CommitID))
 
 	rPath := archiver.RelativePath()
-	if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
+	if setting.RepoArchive.Storage.ServeDirect() {
 		// If we have a signed url (S3, object storage), redirect to this directly.
 		u, err := storage.RepoArchives.URL(rPath, downloadName)
 		if u != nil && err == nil {

routers/web/base.go

@@ -23,7 +23,7 @@ func storageHandler(storageSetting *setting.Storage, prefix string, objStore sto
 	prefix = strings.Trim(prefix, "/")
 	funcInfo := routing.GetFuncInfo(storageHandler, prefix)
 
-	if storageSetting.MinioConfig.ServeDirect {
+	if storageSetting.ServeDirect() {
 		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 			if req.Method != "GET" && req.Method != "HEAD" {
 				http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)

routers/web/repo/actions/view.go

@@ -625,7 +625,7 @@ func ArtifactsDownloadView(ctx *context_module.Context) {
 	// The v4 backend enshures ContentEncoding is set to "application/zip", which is not the case for the old backend
 	if len(artifacts) == 1 && artifacts[0].ArtifactName+".zip" == artifacts[0].ArtifactPath && artifacts[0].ContentEncoding == "application/zip" {
 		art := artifacts[0]
-		if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
+		if setting.Actions.ArtifactStorage.ServeDirect() {
 			u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath)
 			if u != nil && err == nil {
 				ctx.Redirect(u.String())

routers/web/repo/attachment.go

@@ -127,7 +127,7 @@ func ServeAttachment(ctx *context.Context, uuid string) {
 		return
 	}
 
-	if setting.Attachment.Storage.MinioConfig.ServeDirect {
+	if setting.Attachment.Storage.ServeDirect() {
 		// If we have a signed url (S3, object storage), redirect to this directly.
 		u, err := storage.Attachments.URL(attach.RelativePath(), attach.Name)
 

routers/web/repo/download.go

@@ -53,8 +53,8 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim
 			return nil
 		}
 
-		if setting.LFS.Storage.MinioConfig.ServeDirect {
-			// If we have a signed url (S3, object storage), redirect to this directly.
+		if setting.LFS.Storage.ServeDirect() {
+			// If we have a signed url (S3, object storage, blob storage), redirect to this directly.
 			u, err := storage.LFS.URL(pointer.RelativePath(), blob.Name())
 			if u != nil && err == nil {
 				ctx.Redirect(u.String())

routers/web/repo/repo.go

@@ -491,7 +491,7 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep
 		archiver.CommitID, archiver.CommitID))
 
 	rPath := archiver.RelativePath()
-	if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
+	if setting.RepoArchive.Storage.ServeDirect() {
 		// If we have a signed url (S3, object storage), redirect to this directly.
 		u, err := storage.RepoArchives.URL(rPath, downloadName)
 		if u != nil && err == nil {