From a95247b7dfb89d574605a768d96f1f1576b216ca Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 13 Dec 2022 06:06:23 -0500 Subject: [PATCH 01/16] Fix autofilled text visibility in dark mode (#22088) Fixes #22087 --- web_src/less/_base.less | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/web_src/less/_base.less b/web_src/less/_base.less index 022afcea92..697bc0ee74 100644 --- a/web_src/less/_base.less +++ b/web_src/less/_base.less @@ -1354,6 +1354,12 @@ a.ui.card:hover, border-color: var(--color-error-border) !important; } +// A fix for text visibility issue in Chrome autofill in dark mode. +// It's a problem from Formatic UI, and this rule overrides it. +.ui.form .field.field input:-webkit-autofill { + -webkit-text-fill-color: var(--color-black) !important; +} + .ui.loading.loading.input > i.icon svg { visibility: hidden; } From 5aa854693fc6dbd77a83cb89c88e45a06413a697 Mon Sep 17 00:00:00 2001 From: Yakov5776 Date: Tue, 13 Dec 2022 19:20:36 -0500 Subject: [PATCH 02/16] Fix markdown typo of an extra backtick in docs (#22123) Fixes a visual bug in docs which is caused by a typo of an extra backtick. --- docs/content/doc/installation/from-source.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/installation/from-source.en-us.md b/docs/content/doc/installation/from-source.en-us.md index 01a5e1eaee..4394d19203 100644 --- a/docs/content/doc/installation/from-source.en-us.md +++ b/docs/content/doc/installation/from-source.en-us.md @@ -145,7 +145,7 @@ launched manually from command line, it can be killed by pressing `Ctrl + C`. Gitea will search for a number of things from the _`CustomPath`_. By default this is the `custom/` directory in the current working directory when running Gitea. It will also -look for its configuration file _`CustomConf`_ in _`CustomPath`_/conf/app.ini`, and will use the +look for its configuration file _`CustomConf`_ in `$(CustomPath)/conf/app.ini`, and will use the current working directory as the relative base path _`AppWorkPath`_ for a number configurable values. Finally the static files will be served from _`StaticRootPath`_ which defaults to the _`AppWorkPath`_. From 4fb2006ca1cba98fe3851901982a3ecab252cfe3 Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 14 Dec 2022 06:15:11 +0000 Subject: [PATCH 03/16] Make gitea work using cmd.exe again (#22073) Gitea will attempt to lookup its location using LookPath however, this fails on cmd.exe if gitea is in the current working directory. exec.LookPath will return an exec.ErrDot error which we can test for and then simply using filepath.Abs(os.Args[0]) to absolute gitea against the current working directory. Fix #22063 Signed-off-by: Andrew Thornton Signed-off-by: Andrew Thornton Co-authored-by: Lunny Xiao Co-authored-by: techknowlogick --- modules/setting/setting.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 9f2f0933d4..47e0ae2cda 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -465,6 +465,13 @@ func getAppPath() (string, error) { appPath, err = exec.LookPath(os.Args[0]) } + if err != nil { + // FIXME: Once we switch to go 1.19 use !errors.Is(err, exec.ErrDot) + if !strings.Contains(err.Error(), "cannot run executable found relative to current directory") { + return "", err + } + appPath, err = filepath.Abs(os.Args[0]) + } if err != nil { return "", err } From 07461e18d33c78de2d6ae028f8ddba72f971db81 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 14 Dec 2022 10:16:01 +0100 Subject: [PATCH 04/16] Fix condition for is_internal (#22095) depends on #22094 Fixes https://codeberg.org/forgejo/forgejo/issues/77 The old logic did not consider `is_internal`. Co-authored-by: Lunny Xiao Co-authored-by: techknowlogick --- models/packages/package_version.go | 7 ++++++- routers/api/packages/npm/npm.go | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/models/packages/package_version.go b/models/packages/package_version.go index a2a8a45d8f..928f9d47d6 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -302,9 +302,14 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P cond := opts.toConds(). And(builder.Expr("pv2.id IS NULL")) + joinCond := builder.Expr("package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))") + if !opts.IsInternal.IsNone() { + joinCond = joinCond.And(builder.Eq{"pv2.is_internal": opts.IsInternal.IsTrue()}) + } + sess := db.GetEngine(ctx). Table("package_version"). - Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))"). + Join("LEFT", "package_version pv2", joinCond). Join("INNER", "package", "package.id = package_version.package_id"). Where(cond) diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 4306d5ba36..1a09cb6f36 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -405,8 +405,9 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo func PackageSearch(ctx *context.Context) { pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ - OwnerID: ctx.Package.Owner.ID, - Type: packages_model.TypeNpm, + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeNpm, + IsInternal: util.OptionalBoolFalse, Name: packages_model.SearchValue{ ExactMatch: false, Value: ctx.FormTrim("text"), From 1b32ed014a61216b75d80643df53e120606ca763 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 14 Dec 2022 18:11:11 +0800 Subject: [PATCH 05/16] remove duplicated read file code (#22042) Merge the duplicated read file code as one function in reading text file and readme file. --- routers/web/repo/view.go | 211 ++++++++++++++++----------------------- 1 file changed, 88 insertions(+), 123 deletions(-) diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index fa4eb6d61f..f139a971fc 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -241,18 +241,19 @@ func findReadmeFile(ctx *context.Context, entries git.Entries, treeLink string) return readmeFile, readmeTreelink } -func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelink string) { - ctx.Data["RawFileLink"] = "" - ctx.Data["ReadmeInList"] = true - ctx.Data["ReadmeExist"] = true - ctx.Data["FileIsSymlink"] = readmeFile.isSymlink +type fileInfo struct { + isTextFile bool + isLFSFile bool + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType +} - dataRc, err := readmeFile.blob.DataAsync() +func getFileReader(repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { + dataRc, err := blob.DataAsync() if err != nil { - ctx.ServerError("Data", err) - return + return nil, nil, nil, err } - defer dataRc.Close() buf := make([]byte, 1024) n, _ := util.ReadAtMost(dataRc, buf) @@ -261,67 +262,75 @@ func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelin st := typesniffer.DetectContentType(buf) isTextFile := st.IsText() - ctx.Data["FileIsText"] = isTextFile - ctx.Data["FileName"] = readmeFile.name - fileSize := int64(0) - isLFSFile := false - ctx.Data["IsLFSFile"] = false - // FIXME: what happens when README file is an image? - if isTextFile && setting.LFS.StartServer { - pointer, _ := lfs.ReadPointerFromBuffer(buf) - if pointer.IsValid() { - meta, err := git_model.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid) - if err != nil && err != git_model.ErrLFSObjectNotExist { - ctx.ServerError("GetLFSMetaObject", err) - return - } - if meta != nil { - ctx.Data["IsLFSFile"] = true - isLFSFile = true - - // OK read the lfs object - var err error - dataRc, err = lfs.ReadMetaObject(pointer) - if err != nil { - ctx.ServerError("ReadMetaObject", err) - return - } - defer dataRc.Close() - - buf = make([]byte, 1024) - n, err = util.ReadAtMost(dataRc, buf) - if err != nil { - ctx.ServerError("Data", err) - return - } - buf = buf[:n] - - st = typesniffer.DetectContentType(buf) - isTextFile = st.IsText() - ctx.Data["IsTextFile"] = isTextFile - - fileSize = meta.Size - ctx.Data["FileSize"] = meta.Size - filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) - ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.HTMLURL(), url.PathEscape(meta.Oid), url.PathEscape(filenameBase64)) - } - } + if !isTextFile || !setting.LFS.StartServer { + return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil } - if !isTextFile { + pointer, _ := lfs.ReadPointerFromBuffer(buf) + if !pointer.IsValid() { // fallback to plain file + return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + } + + meta, err := git_model.GetLFSMetaObjectByOid(repoID, pointer.Oid) + if err != git_model.ErrLFSObjectNotExist { // fallback to plain file + return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + } + + dataRc.Close() + if err != nil { + return nil, nil, nil, err + } + + dataRc, err = lfs.ReadMetaObject(pointer) + if err != nil { + return nil, nil, nil, err + } + + buf = make([]byte, 1024) + n, err = util.ReadAtMost(dataRc, buf) + if err != nil { + dataRc.Close() + return nil, nil, nil, err + } + buf = buf[:n] + + st = typesniffer.DetectContentType(buf) + + return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil +} + +func renderReadmeFile(ctx *context.Context, readmeFile *namedBlob, readmeTreelink string) { + ctx.Data["RawFileLink"] = "" + ctx.Data["ReadmeInList"] = true + ctx.Data["ReadmeExist"] = true + ctx.Data["FileIsSymlink"] = readmeFile.isSymlink + + buf, dataRc, fInfo, err := getFileReader(ctx.Repo.Repository.ID, readmeFile.blob) + if err != nil { + ctx.ServerError("getFileReader", err) + return + } + defer dataRc.Close() + + ctx.Data["FileIsText"] = fInfo.isTextFile + ctx.Data["FileName"] = readmeFile.name + ctx.Data["IsLFSFile"] = fInfo.isLFSFile + + if fInfo.isLFSFile { + filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) + ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.HTMLURL(), url.PathEscape(fInfo.lfsMeta.Oid), url.PathEscape(filenameBase64)) + } + + if !fInfo.isTextFile { return } - if !isLFSFile { - fileSize = readmeFile.blob.Size() - } - - if fileSize >= setting.UI.MaxDisplayFileSize { + if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { // Pretend that this is a normal text file to display 'This file is too large to be shown' ctx.Data["IsFileTooLarge"] = true ctx.Data["IsTextFile"] = true - ctx.Data["FileSize"] = fileSize + ctx.Data["FileSize"] = fInfo.fileSize return } @@ -362,16 +371,14 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["IsViewFile"] = true ctx.Data["HideRepoInfo"] = true blob := entry.Blob() - dataRc, err := blob.DataAsync() + buf, dataRc, fInfo, err := getFileReader(ctx.Repo.Repository.ID, blob) if err != nil { - ctx.ServerError("DataAsync", err) + ctx.ServerError("getFileReader", err) return } defer dataRc.Close() ctx.Data["Title"] = ctx.Tr("repo.file.title", ctx.Repo.Repository.Name+"/"+path.Base(ctx.Repo.TreePath), ctx.Repo.RefName) - - fileSize := blob.Size() ctx.Data["FileIsSymlink"] = entry.IsLink() ctx.Data["FileName"] = blob.Name() ctx.Data["RawFileLink"] = rawLink + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) @@ -381,69 +388,27 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["FileError"] = editorconfigErr } - buf := make([]byte, 1024) - n, _ := util.ReadAtMost(dataRc, buf) - buf = buf[:n] - - st := typesniffer.DetectContentType(buf) - isTextFile := st.IsText() - - isLFSFile := false isDisplayingSource := ctx.FormString("display") == "source" isDisplayingRendered := !isDisplayingSource - // Check for LFS meta file - if isTextFile && setting.LFS.StartServer { - pointer, _ := lfs.ReadPointerFromBuffer(buf) - if pointer.IsValid() { - meta, err := git_model.GetLFSMetaObjectByOid(ctx.Repo.Repository.ID, pointer.Oid) - if err != nil && err != git_model.ErrLFSObjectNotExist { - ctx.ServerError("GetLFSMetaObject", err) - return - } - if meta != nil { - isLFSFile = true - - // OK read the lfs object - var err error - dataRc, err = lfs.ReadMetaObject(pointer) - if err != nil { - ctx.ServerError("ReadMetaObject", err) - return - } - defer dataRc.Close() - - buf = make([]byte, 1024) - n, err = util.ReadAtMost(dataRc, buf) - if err != nil { - ctx.ServerError("Data", err) - return - } - buf = buf[:n] - - st = typesniffer.DetectContentType(buf) - isTextFile = st.IsText() - - fileSize = meta.Size - ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) - } - } + if fInfo.isLFSFile { + ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) } - isRepresentableAsText := st.IsRepresentableAsText() + isRepresentableAsText := fInfo.st.IsRepresentableAsText() if !isRepresentableAsText { // If we can't show plain text, always try to render. isDisplayingSource = false isDisplayingRendered = true } - ctx.Data["IsLFSFile"] = isLFSFile - ctx.Data["FileSize"] = fileSize - ctx.Data["IsTextFile"] = isTextFile + ctx.Data["IsLFSFile"] = fInfo.isLFSFile + ctx.Data["FileSize"] = fInfo.fileSize + ctx.Data["IsTextFile"] = fInfo.isTextFile ctx.Data["IsRepresentableAsText"] = isRepresentableAsText ctx.Data["IsDisplayingSource"] = isDisplayingSource ctx.Data["IsDisplayingRendered"] = isDisplayingRendered - isTextSource := isTextFile || isDisplayingSource + isTextSource := fInfo.isTextFile || isDisplayingSource ctx.Data["IsTextSource"] = isTextSource if isTextSource { ctx.Data["CanCopyContent"] = true @@ -468,7 +433,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } // Assume file is not editable first. - if isLFSFile { + if fInfo.isLFSFile { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files") } else if !isRepresentableAsText { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files") @@ -476,13 +441,13 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st switch { case isRepresentableAsText: - if st.IsSvgImage() { + if fInfo.st.IsSvgImage() { ctx.Data["IsImageFile"] = true ctx.Data["CanCopyContent"] = true ctx.Data["HasSourceRenderedToggle"] = true } - if fileSize >= setting.UI.MaxDisplayFileSize { + if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { ctx.Data["IsFileTooLarge"] = true break } @@ -589,7 +554,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["FileContent"] = fileContent ctx.Data["LineEscapeStatus"] = statuses } - if !isLFSFile { + if !fInfo.isLFSFile { if ctx.Repo.CanEnableEditor(ctx.Doer) { if lfsLock != nil && lfsLock.OwnerID != ctx.Doer.ID { ctx.Data["CanEditFile"] = false @@ -605,17 +570,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } } - case st.IsPDF(): + case fInfo.st.IsPDF(): ctx.Data["IsPDFFile"] = true - case st.IsVideo(): + case fInfo.st.IsVideo(): ctx.Data["IsVideoFile"] = true - case st.IsAudio(): + case fInfo.st.IsAudio(): ctx.Data["IsAudioFile"] = true - case st.IsImage() && (setting.UI.SVG.Enabled || !st.IsSvgImage()): + case fInfo.st.IsImage() && (setting.UI.SVG.Enabled || !fInfo.st.IsSvgImage()): ctx.Data["IsImageFile"] = true ctx.Data["CanCopyContent"] = true default: - if fileSize >= setting.UI.MaxDisplayFileSize { + if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { ctx.Data["IsFileTooLarge"] = true break } From cdc43454a49f97b663f885e18d0adc12253597a3 Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 14 Dec 2022 13:45:33 +0000 Subject: [PATCH 06/16] Correctly handle moved files in apply patch (#22118) Moved files in a patch will result in git apply returning: ``` error: {filename}: No such file or directory ``` This wasn't handled by the git apply patch code. This PR adds handling for this. Fix #22083 Signed-off-by: Andrew Thornton Co-authored-by: techknowlogick --- services/pull/patch.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/pull/patch.go b/services/pull/patch.go index 809b75e6b4..e0da410c4d 100644 --- a/services/pull/patch.go +++ b/services/pull/patch.go @@ -53,6 +53,8 @@ var patchErrorSuffices = []string{ ": patch does not apply", ": already exists in working directory", "unrecognized input", + ": No such file or directory", + ": does not exist in index", } // TestPatch will test whether a simple patch will apply @@ -416,6 +418,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * scanner := bufio.NewScanner(stderrReader) for scanner.Scan() { line := scanner.Text() + log.Trace("PullRequest[%d].testPatch: stderr: %s", pr.ID, line) if strings.HasPrefix(line, prefix) { conflict = true filepath := strings.TrimSpace(strings.Split(line[len(prefix):], ":")[0]) From ae971974048aef78f67e3ffe16c9ff0537f27493 Mon Sep 17 00:00:00 2001 From: Nathaniel Sabanski Date: Wed, 14 Dec 2022 13:14:50 -0800 Subject: [PATCH 07/16] Fixed Project view .board-column height for tall screens. (#22108) This bug occurs because we are calculating `.board-column` height strictly off of `vh`, when the layout header is of static height. BEFORE https://user-images.githubusercontent.com/24665/206991060-372c24e3-986e-4fc6-9fc8-aab8b4ef09bb.mp4 AFTER https://user-images.githubusercontent.com/24665/206991070-91b7cbab-d807-4016-8696-e43bdaf8a7ff.mp4 --- web_src/less/features/projects.less | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web_src/less/features/projects.less b/web_src/less/features/projects.less index 21514688b6..b0f674060a 100644 --- a/web_src/less/features/projects.less +++ b/web_src/less/features/projects.less @@ -12,7 +12,8 @@ margin: 0 .5rem !important; padding: .5rem !important; width: 320px; - height: 60vh; + height: calc(100vh - 450px); + min-height: 60vh; overflow-y: scroll; flex: 0 0 auto; overflow: visible; From 3243dbe1a9e3ff7031f243c72232bcc31bbdec75 Mon Sep 17 00:00:00 2001 From: silentcodeg <105681448+silentcodeg@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:41:38 +0100 Subject: [PATCH 08/16] remove silentcode from MAINTAINERS (#22143) Signed-off-by: silentcode --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index d383b8b164..98830cbb94 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -44,7 +44,6 @@ Janis Estelmann (@KN4CK3R) Steven Kriegler (@justusbunsi) Jimmy Praet (@jpraet) Leon Hofmeister (@delvh) -silentcode (@silentcodeg) Wim (@42wim) xinyu (@penlinux) Jason Song (@wolfogre) From 651fe4bb7dda16dee48b31ee964493a05e979c78 Mon Sep 17 00:00:00 2001 From: zeripath Date: Thu, 15 Dec 2022 20:44:16 +0000 Subject: [PATCH 09/16] Add doctor command for full GC of LFS (#21978) The recent PR adding orphaned checks to the LFS storage is not sufficient to completely GC LFS, as it is possible for LFSMetaObjects to remain associated with repos but still need to be garbage collected. Imagine a situation where a branch is uploaded containing LFS files but that branch is later completely deleted. The LFSMetaObjects will remain associated with the Repository but the Repository will no longer contain any pointers to the object. This PR adds a second doctor command to perform a full GC. Signed-off-by: Andrew Thornton --- models/git/lfs.go | 54 ++++++++++++++++++ modules/doctor/lfs.go | 37 ++++++++++++ services/cron/tasks_basic.go | 2 +- services/repository/check.go | 86 +++++++++++++++------------- services/repository/lfs.go | 105 +++++++++++++++++++++++++++++++++++ 5 files changed, 245 insertions(+), 39 deletions(-) create mode 100644 modules/doctor/lfs.go create mode 100644 services/repository/lfs.go diff --git a/models/git/lfs.go b/models/git/lfs.go index a86e84c050..8d418b928d 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -6,6 +6,7 @@ package git import ( "context" "fmt" + "time" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/perm" @@ -14,6 +15,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -180,6 +182,12 @@ func GetLFSMetaObjectByOid(repoID int64, oid string) (*LFSMetaObject, error) { // RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID. // It may return ErrLFSObjectNotExist or a database error. func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) { + return RemoveLFSMetaObjectByOidFn(repoID, oid, nil) +} + +// RemoveLFSMetaObjectByOidFn removes a LFSMetaObject entry from database by its OID. +// It may return ErrLFSObjectNotExist or a database error. It will run Fn with the current count within the transaction +func RemoveLFSMetaObjectByOidFn(repoID int64, oid string, fn func(count int64) error) (int64, error) { if len(oid) == 0 { return 0, ErrLFSObjectNotExist } @@ -200,6 +208,12 @@ func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) { return count, err } + if fn != nil { + if err := fn(count); err != nil { + return count, err + } + } + return count, committer.Commit() } @@ -319,3 +333,43 @@ func GetRepoLFSSize(ctx context.Context, repoID int64) (int64, error) { } return lfsSize, nil } + +type IterateLFSMetaObjectsForRepoOptions struct { + OlderThan time.Time +} + +// IterateLFSMetaObjectsForRepo provides a iterator for LFSMetaObjects per Repo +func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(context.Context, *LFSMetaObject, int64) error, opts *IterateLFSMetaObjectsForRepoOptions) error { + var start int + batchSize := setting.Database.IterateBufferSize + engine := db.GetEngine(ctx) + type CountLFSMetaObject struct { + Count int64 + LFSMetaObject + } + + for { + beans := make([]*CountLFSMetaObject, 0, batchSize) + // SELECT `lfs_meta_object`.*, COUNT(`l1`.id) as `count` FROM lfs_meta_object INNER JOIN lfs_meta_object AS l1 ON l1.oid = lfs_meta_object.oid WHERE lfs_meta_object.repository_id = ? GROUP BY lfs_meta_object.id + sess := engine.Select("`lfs_meta_object`.*, COUNT(`l1`.oid) AS `count`"). + Join("INNER", "`lfs_meta_object` AS l1", "`lfs_meta_object`.oid = `l1`.oid"). + Where("`lfs_meta_object`.repository_id = ?", repoID) + if !opts.OlderThan.IsZero() { + sess.And("`lfs_meta_object`.created_unix < ?", opts.OlderThan) + } + sess.GroupBy("`lfs_meta_object`.id") + if err := sess.Limit(batchSize, start).Find(&beans); err != nil { + return err + } + if len(beans) == 0 { + return nil + } + start += len(beans) + + for _, bean := range beans { + if err := f(ctx, &bean.LFSMetaObject, bean.Count); err != nil { + return err + } + } + } +} diff --git a/modules/doctor/lfs.go b/modules/doctor/lfs.go new file mode 100644 index 0000000000..410ed5a9a5 --- /dev/null +++ b/modules/doctor/lfs.go @@ -0,0 +1,37 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package doctor + +import ( + "context" + "fmt" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/repository" +) + +func init() { + Register(&Check{ + Title: "Garbage collect LFS", + Name: "gc-lfs", + IsDefault: false, + Run: garbageCollectLFSCheck, + AbortIfFailed: false, + SkipDatabaseInitialization: false, + Priority: 1, + }) +} + +func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool) error { + if !setting.LFS.StartServer { + return fmt.Errorf("LFS support is disabled") + } + + if err := repository.GarbageCollectLFSMetaObjects(ctx, logger, autofix); err != nil { + return err + } + + return checkStorage(&checkStorageOptions{LFS: true})(ctx, logger, autofix) +} diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index acf3896b71..05aef6623d 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -63,7 +63,7 @@ func registerRepoHealthCheck() { for _, arg := range rhcConfig.Args { args = append(args, git.CmdArg(arg)) } - return repo_service.GitFsck(ctx, rhcConfig.Timeout, args) + return repo_service.GitFsckRepos(ctx, rhcConfig.Timeout, args) }) } diff --git a/services/repository/check.go b/services/repository/check.go index 6e29dc93d1..293cb04d38 100644 --- a/services/repository/check.go +++ b/services/repository/check.go @@ -22,8 +22,8 @@ import ( "xorm.io/builder" ) -// GitFsck calls 'git fsck' to check repository health. -func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) error { +// GitFsckRepos calls 'git fsck' to check repository health. +func GitFsckRepos(ctx context.Context, timeout time.Duration, args []git.CmdArg) error { log.Trace("Doing: GitFsck") if err := db.Iterate( @@ -35,15 +35,7 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro return db.ErrCancelledf("before fsck of %s", repo.FullName()) default: } - log.Trace("Running health check on repository %v", repo) - repoPath := repo.RepoPath() - if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil { - log.Warn("Failed to health check repository (%v): %v", repo, err) - if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - } - return nil + return GitFsckRepo(ctx, repo, timeout, args) }, ); err != nil { log.Trace("Error: GitFsck: %v", err) @@ -54,6 +46,19 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro return nil } +// GitFsckRepo calls 'git fsck' to check an individual repository's health. +func GitFsckRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args []git.CmdArg) error { + log.Trace("Running health check on repository %-v", repo) + repoPath := repo.RepoPath() + if err := git.Fsck(ctx, repoPath, timeout, args...); err != nil { + log.Warn("Failed to health check repository (%-v): %v", repo, err) + if err = system_model.CreateRepositoryNotice("Failed to health check repository (%s): %v", repo.FullName(), err); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } + return nil +} + // GitGcRepos calls 'git gc' to remove unnecessary files and optimize the local repository func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg) error { log.Trace("Doing: GitGcRepos") @@ -68,33 +73,7 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg) return db.ErrCancelledf("before GC of %s", repo.FullName()) default: } - log.Trace("Running git gc on %v", repo) - command := git.NewCommand(ctx, args...). - SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())) - var stdout string - var err error - stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) - - if err != nil { - log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err) - desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) - if err = system_model.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) - } - - // Now update the size of the repository - if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { - log.Error("Updating size as part of garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err) - desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) - if err = system_model.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) - } - - return nil + return GitGcRepo(ctx, repo, timeout, args) }, ); err != nil { return err @@ -104,6 +83,37 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg) return nil } +// GitGcRepo calls 'git gc' to remove unnecessary files and optimize the local repository +func GitGcRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args []git.CmdArg) error { + log.Trace("Running git gc on %-v", repo) + command := git.NewCommand(ctx, args...). + SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())) + var stdout string + var err error + stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) + + if err != nil { + log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err) + desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) + if err = system_model.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return fmt.Errorf("Repository garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) + } + + // Now update the size of the repository + if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Updating size as part of garbage collection failed for %-v. Stdout: %s\nError: %v", repo, stdout, err) + desc := fmt.Sprintf("Updating size as part of garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) + if err = system_model.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return fmt.Errorf("Updating size as part of garbage collection failed in repo: %s: Error: %w", repo.FullName(), err) + } + + return nil +} + func gatherMissingRepoRecords(ctx context.Context) ([]*repo_model.Repository, error) { repos := make([]*repo_model.Repository, 0, 10) if err := db.Iterate( diff --git a/services/repository/lfs.go b/services/repository/lfs.go new file mode 100644 index 0000000000..0e88d359a8 --- /dev/null +++ b/services/repository/lfs.go @@ -0,0 +1,105 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "time" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + + "xorm.io/builder" +) + +func GarbageCollectLFSMetaObjects(ctx context.Context, logger log.Logger, autofix bool) error { + log.Trace("Doing: GarbageCollectLFSMetaObjects") + + if err := db.Iterate( + ctx, + builder.And(builder.Gt{"id": 0}), + func(ctx context.Context, repo *repo_model.Repository) error { + return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, logger, autofix) + }, + ); err != nil { + return err + } + + log.Trace("Finished: GarbageCollectLFSMetaObjects") + return nil +} + +func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, logger log.Logger, autofix bool) error { + if logger != nil { + logger.Info("Checking %-v", repo) + } + total, orphaned, collected, deleted := 0, 0, 0, 0 + if logger != nil { + defer func() { + if orphaned == 0 { + logger.Info("Found %d total LFSMetaObjects in %-v", total, repo) + } else if !autofix { + logger.Info("Found %d/%d orphaned LFSMetaObjects in %-v", orphaned, total, repo) + } else { + logger.Info("Collected %d/%d orphaned/%d total LFSMetaObjects in %-v. %d removed from storage.", collected, orphaned, total, repo, deleted) + } + }() + } + + gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) + if err != nil { + log.Error("Unable to open git repository %-v: %v", repo, err) + return err + } + defer gitRepo.Close() + + store := lfs.NewContentStore() + + return git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject, count int64) error { + total++ + pointerSha := git.ComputeBlobHash([]byte(metaObject.Pointer.StringContent())) + + if gitRepo.IsObjectExist(pointerSha.String()) { + return nil + } + orphaned++ + + if !autofix { + return nil + } + // Non-existent pointer file + _, err = git_model.RemoveLFSMetaObjectByOidFn(repo.ID, metaObject.Oid, func(count int64) error { + if count > 0 { + return nil + } + + if err := store.Delete(metaObject.RelativePath()); err != nil { + log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err) + } + deleted++ + return nil + }) + if err != nil { + return fmt.Errorf("unable to remove meta-object %s in %s: %w", metaObject.Oid, repo.FullName(), err) + } + collected++ + + return nil + }, &git_model.IterateLFSMetaObjectsForRepoOptions{ + // Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload + // and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby + // an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid + // changes in new branches that might lead to lfs objects becoming temporarily unassociated with git + // objects. + // + // It is likely that a week is potentially excessive but it should definitely be enough that any + // unassociated LFS object is genuinely unassociated. + OlderThan: time.Now().Add(-24 * 7 * time.Hour), + }) +} From 84285a1169787c54ec2c583af83bcfb9182f00fa Mon Sep 17 00:00:00 2001 From: Christian Ullrich Date: Fri, 16 Dec 2022 09:58:56 +0100 Subject: [PATCH 10/16] Do not list active repositories as unadopted (#22034) This fixes a bug where, when searching unadopted repositories, active repositories will be listed as well. This is because the size of the array of repository names to check is larger by one than the `IterateBufferSize`. For an `IterateBufferSize` of 50, the original code will pass 51 repository names but set the query to `LIMIT 50`. If all repositories in the query are active (i.e. not unadopted) one of them will be omitted from the result. Due to the `ORDER BY` clause it will be the oldest (or least recently modified) one. Bug found in 1.17.3. Co-authored-by: zeripath Co-authored-by: Lunny Xiao --- services/repository/adopt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 828068e8ec..93eeb56456 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -337,7 +337,7 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in } repoNamesToCheck = append(repoNamesToCheck, name) - if len(repoNamesToCheck) > setting.Database.IterateBufferSize { + if len(repoNamesToCheck) >= setting.Database.IterateBufferSize { if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil { return err } From 84001467bd893c0ccaa7e4c4d0de5d69b7461496 Mon Sep 17 00:00:00 2001 From: Xinyu Zhou Date: Fri, 16 Dec 2022 21:11:03 +0800 Subject: [PATCH 11/16] Update username (#22147) update my email & username --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 98830cbb94..74196d4bd8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -45,6 +45,6 @@ Steven Kriegler (@justusbunsi) Jimmy Praet (@jpraet) Leon Hofmeister (@delvh) Wim (@42wim) -xinyu (@penlinux) +Xinyu Zhou (@xin-u) Jason Song (@wolfogre) Yarden Shoham (@yardenshoham) From c4c4151f7d7a6aa18b354ae45c6ed93570d5de77 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 17 Dec 2022 02:58:57 +0100 Subject: [PATCH 12/16] Fix margin and alignment in dashboard repolist (#22120) Seems this has recently regressed, previously, there was a significant whitespace between icon and text, but it seems to be gone, so I added the margin and also vertically aligned the icon because it was slightly misaligned. Before: Screenshot 2022-12-13 at 20 03 51 After: image Co-authored-by: KN4CK3R --- templates/user/dashboard/repolist.tmpl | 6 +++--- web_src/less/_dashboard.less | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/templates/user/dashboard/repolist.tmpl b/templates/user/dashboard/repolist.tmpl index 620cc322f0..9d0ea01ec5 100644 --- a/templates/user/dashboard/repolist.tmpl +++ b/templates/user/dashboard/repolist.tmpl @@ -128,9 +128,9 @@