1
1
mirror of https://github.com/go-gitea/gitea synced 2024-12-23 09:04:26 +00:00
gitea/models/lfs.go
wxiaoguang 385dc6a992
Allow admin to associate missing LFS objects for repositories (#18143)
This PR reworked the Find pointer files feature in Settings -> LFS page.

When a LFS object is missing from database but exists in LFS content store, admin can associate it to the repository by clicking the Associate button.

This PR is not perfect (because the LFS module itself should be improved too), it's just a nice-to-have feature to help users recover their LFS repositories (eg: database was lost / table was truncated)
2022-01-01 17:05:31 +08:00

248 lines
7.1 KiB
Go

// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package models
import (
"context"
"errors"
"fmt"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/builder"
)
// LFSMetaObject stores metadata for LFS tracked files.
type LFSMetaObject struct {
ID int64 `xorm:"pk autoincr"`
lfs.Pointer `xorm:"extends"`
RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
Existing bool `xorm:"-"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
}
func init() {
db.RegisterModel(new(LFSMetaObject))
}
// LFSTokenResponse defines the JSON structure in which the JWT token is stored.
// This structure is fetched via SSH and passed by the Git LFS client to the server
// endpoint for authorization.
type LFSTokenResponse struct {
Header map[string]string `json:"header"`
Href string `json:"href"`
}
// ErrLFSObjectNotExist is returned from lfs models functions in order
// to differentiate between database and missing object errors.
var ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist")
// NewLFSMetaObject stores a given populated LFSMetaObject structure in the database
// if it is not already present.
func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
var err error
ctx, committer, err := db.TxContext()
if err != nil {
return nil, err
}
defer committer.Close()
has, err := db.GetByBean(ctx, m)
if err != nil {
return nil, err
}
if has {
m.Existing = true
return m, committer.Commit()
}
if err = db.Insert(ctx, m); err != nil {
return nil, err
}
return m, committer.Commit()
}
// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error. If the error is nil,
// the returned pointer is a valid LFSMetaObject.
func GetLFSMetaObjectByOid(repoID int64, oid string) (*LFSMetaObject, error) {
if len(oid) == 0 {
return nil, ErrLFSObjectNotExist
}
m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repoID}
has, err := db.GetEngine(db.DefaultContext).Get(m)
if err != nil {
return nil, err
} else if !has {
return nil, ErrLFSObjectNotExist
}
return m, nil
}
// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error.
func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) {
if len(oid) == 0 {
return 0, ErrLFSObjectNotExist
}
ctx, committer, err := db.TxContext()
if err != nil {
return 0, err
}
defer committer.Close()
m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repoID}
if _, err := db.DeleteByBean(ctx, m); err != nil {
return -1, err
}
count, err := db.CountByBean(ctx, &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
if err != nil {
return count, err
}
return count, committer.Commit()
}
// GetLFSMetaObjects returns all LFSMetaObjects associated with a repository
func GetLFSMetaObjects(repoID int64, page, pageSize int) ([]*LFSMetaObject, error) {
sess := db.GetEngine(db.DefaultContext)
if page >= 0 && pageSize > 0 {
start := 0
if page > 0 {
start = (page - 1) * pageSize
}
sess.Limit(pageSize, start)
}
lfsObjects := make([]*LFSMetaObject, 0, pageSize)
return lfsObjects, sess.Find(&lfsObjects, &LFSMetaObject{RepositoryID: repoID})
}
// CountLFSMetaObjects returns a count of all LFSMetaObjects associated with a repository
func CountLFSMetaObjects(repoID int64) (int64, error) {
return db.GetEngine(db.DefaultContext).Count(&LFSMetaObject{RepositoryID: repoID})
}
// LFSObjectAccessible checks if a provided Oid is accessible to the user
func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) {
if user.IsAdmin {
count, err := db.GetEngine(db.DefaultContext).Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
return count > 0, err
}
cond := accessibleRepositoryCondition(user)
count, err := db.GetEngine(db.DefaultContext).Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
return count > 0, err
}
// LFSObjectIsAssociated checks if a provided Oid is associated
func LFSObjectIsAssociated(oid string) (bool, error) {
return db.GetEngine(db.DefaultContext).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
}
// LFSAutoAssociate auto associates accessible LFSMetaObjects
func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int64) error {
ctx, committer, err := db.TxContext()
if err != nil {
return err
}
defer committer.Close()
sess := db.GetEngine(ctx)
oids := make([]interface{}, len(metas))
oidMap := make(map[string]*LFSMetaObject, len(metas))
for i, meta := range metas {
oids[i] = meta.Oid
oidMap[meta.Oid] = meta
}
if !user.IsAdmin {
newMetas := make([]*LFSMetaObject, 0, len(metas))
cond := builder.In(
"`lfs_meta_object`.repository_id",
builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user)),
)
err = sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas)
if err != nil {
return err
}
if len(newMetas) != len(oidMap) {
return fmt.Errorf("unable collect all LFS objects from database, expected %d, actually %d", len(oidMap), len(newMetas))
}
for i := range newMetas {
newMetas[i].Size = oidMap[newMetas[i].Oid].Size
newMetas[i].RepositoryID = repoID
}
if err = db.Insert(ctx, newMetas); err != nil {
return err
}
} else {
// admin can associate any LFS object to any repository, and we do not care about errors (eg: duplicated unique key),
// even if error occurs, it won't hurt users and won't make things worse
for i := range metas {
_, err = sess.Insert(&LFSMetaObject{
Pointer: lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size},
RepositoryID: repoID,
})
if err != nil {
log.Warn("failed to insert LFS meta object into database, err=%v", err)
}
}
}
return committer.Commit()
}
// IterateLFS iterates lfs object
func IterateLFS(f func(mo *LFSMetaObject) error) error {
var start int
const batchSize = 100
var e = db.GetEngine(db.DefaultContext)
for {
mos := make([]*LFSMetaObject, 0, batchSize)
if err := e.Limit(batchSize, start).Find(&mos); err != nil {
return err
}
if len(mos) == 0 {
return nil
}
start += len(mos)
for _, mo := range mos {
if err := f(mo); err != nil {
return err
}
}
}
}
// CopyLFS copies LFS data from one repo to another
func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
var lfsObjects []*LFSMetaObject
if err := db.GetEngine(ctx).Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil {
return err
}
for _, v := range lfsObjects {
v.ID = 0
v.RepositoryID = newRepo.ID
if _, err := db.GetEngine(ctx).Insert(v); err != nil {
return err
}
}
return nil
}