From a807031a3093964387171a5bb07cc2c9181f0444 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 6 Sep 2021 22:46:20 +0800 Subject: [PATCH] Fix storage Iterate bug and Add storage doctor to delete garbage attachments (#16971) * Fix storage Iterate bug and Add storage doctor to delete garbage attachments * Close object when used --- cmd/doctor.go | 1 - models/attachment.go | 5 +++ modules/doctor/storage.go | 76 +++++++++++++++++++++++++++++++++++++++ modules/storage/minio.go | 4 +-- 4 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 modules/doctor/storage.go diff --git a/cmd/doctor.go b/cmd/doctor.go index 0152aebe39..2e79275434 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -124,7 +124,6 @@ func runRecreateTable(ctx *cli.Context) error { } func runDoctor(ctx *cli.Context) error { - // Silence the default loggers log.DelNamedLogger("console") log.DelNamedLogger(log.DEFAULT) diff --git a/models/attachment.go b/models/attachment.go index e12609f810..4e0ccba5a0 100644 --- a/models/attachment.go +++ b/models/attachment.go @@ -144,6 +144,11 @@ func GetAttachmentByUUID(uuid string) (*Attachment, error) { return getAttachmentByUUID(x, uuid) } +// ExistAttachmentsByUUID returns true if attachment is exist by given UUID +func ExistAttachmentsByUUID(uuid string) (bool, error) { + return x.Where("`uuid`=?", uuid).Exist(new(Attachment)) +} + // GetAttachmentByReleaseIDFileName returns attachment by given releaseId and fileName. func GetAttachmentByReleaseIDFileName(releaseID int64, fileName string) (*Attachment, error) { return getAttachmentByReleaseIDFileName(x, releaseID, fileName) diff --git a/modules/doctor/storage.go b/modules/doctor/storage.go new file mode 100644 index 0000000000..cb06d66204 --- /dev/null +++ b/modules/doctor/storage.go @@ -0,0 +1,76 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" +) + +func checkAttachmentStorageFiles(logger log.Logger, autofix bool) error { + var total, garbageNum int + var deletePaths []string + if err := storage.Attachments.IterateObjects(func(p string, obj storage.Object) error { + defer obj.Close() + + total++ + stat, err := obj.Stat() + if err != nil { + return err + } + exist, err := models.ExistAttachmentsByUUID(stat.Name()) + if err != nil { + return err + } + if !exist { + garbageNum++ + if autofix { + deletePaths = append(deletePaths, p) + } + } + return nil + }); err != nil { + logger.Error("storage.Attachments.IterateObjects failed: %v", err) + return err + } + + if garbageNum > 0 { + if autofix { + var deletedNum int + for _, p := range deletePaths { + if err := storage.Attachments.Delete(p); err != nil { + log.Error("Delete attachment %s failed: %v", p, err) + } else { + deletedNum++ + } + } + logger.Info("%d missed information attachment detected, %d deleted.", garbageNum, deletedNum) + } else { + logger.Warn("Checked %d attachment, %d missed information.", total, garbageNum) + } + } + return nil +} + +func checkStorageFiles(logger log.Logger, autofix bool) error { + if err := storage.Init(); err != nil { + logger.Error("storage.Init failed: %v", err) + return err + } + return checkAttachmentStorageFiles(logger, autofix) +} + +func init() { + Register(&Check{ + Title: "Check if there is garbage storage files", + Name: "storages", + IsDefault: false, + Run: checkStorageFiles, + AbortIfFailed: false, + SkipDatabaseInitialization: false, + Priority: 1, + }) +} diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 724445c0ab..f78ba6aa27 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -151,7 +151,7 @@ type minioFileInfo struct { } func (m minioFileInfo) Name() string { - return m.ObjectInfo.Key + return path.Base(m.ObjectInfo.Key) } func (m minioFileInfo) Size() int64 { @@ -219,7 +219,7 @@ func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) er } if err := func(object *minio.Object, fn func(path string, obj Object) error) error { defer object.Close() - return fn(strings.TrimPrefix(m.basePath, mObjInfo.Key), &minioObject{object}) + return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object}) }(object, fn); err != nil { return convertMinioErr(err) }