Add package registry cleanup rules (#21658)

Fixes #20514
Fixes #20766
Fixes #20631

This PR adds Cleanup Rules for the package registry. This allows to
delete unneeded packages automatically. Cleanup rules can be set up from
the user or org settings.
Please have a look at the documentation because I'm not a native english
speaker.

Rule Form

![grafik](https://user-images.githubusercontent.com/1666336/199330792-c13918a6-e196-4e71-9f53-18554515edca.png)

Rule List

![grafik](https://user-images.githubusercontent.com/1666336/199331261-5f6878e8-a80c-4985-800d-ebb3524b1a8d.png)

Rule Preview

![grafik](https://user-images.githubusercontent.com/1666336/199330917-c95e4017-cf64-4142-a3e4-af18c4f127c3.png)

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

services/forms/package_form.go

@@ -0,0 +1,31 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package forms
+
+import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/web/middleware"
+
+	"gitea.com/go-chi/binding"
+)
+
+type PackageCleanupRuleForm struct {
+	ID            int64
+	Enabled       bool
+	Type          string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"`
+	KeepCount     int    `binding:"In(0,1,5,10,25,50,100)"`
+	KeepPattern   string `binding:"RegexPattern"`
+	RemoveDays    int    `binding:"In(0,7,14,30,60,90,180)"`
+	RemovePattern string `binding:"RegexPattern"`
+	MatchFullName bool
+	Action        string `binding:"Required;In(save,remove)"`
+}
+
+func (f *PackageCleanupRuleForm) Validate(req *http.Request, errs binding.Errors) binding.Errors {
+	ctx := context.GetContext(req)
+	return middleware.Validate(errs, ctx.Data, f, ctx.Locale)
+}

services/packages/container/cleanup.go

@@ -6,13 +6,12 @@ package container
 
 import (
 	"context"
-	"strings"
 	"time"
 
 	packages_model "code.gitea.io/gitea/models/packages"
 	container_model "code.gitea.io/gitea/models/packages/container"
-	user_model "code.gitea.io/gitea/models/user"
 	container_module "code.gitea.io/gitea/modules/packages/container"
+	"code.gitea.io/gitea/modules/packages/container/oci"
 	"code.gitea.io/gitea/modules/util"
 )
 
@@ -82,24 +81,30 @@ func cleanupExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) e
 	return nil
 }
 
-// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
-func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
-	ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
-	if err != nil {
-		return err
+func ShouldBeSkipped(ctx context.Context, pcr *packages_model.PackageCleanupRule, p *packages_model.Package, pv *packages_model.PackageVersion) (bool, error) {
+	// Always skip the "latest" tag
+	if pv.LowerVersion == "latest" {
+		return true, nil
 	}
 
-	newOwnerName = strings.ToLower(newOwnerName)
-
-	for _, p := range ps {
-		if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
-			return err
+	// Check if the version is a digest (or untagged)
+	if oci.Digest(pv.LowerVersion).Validate() {
+		// Check if there is another manifest referencing this version
+		has, err := packages_model.ExistVersion(ctx, &packages_model.PackageSearchOptions{
+			PackageID: p.ID,
+			Properties: map[string]string{
+				container_module.PropertyManifestReference: pv.LowerVersion,
+			},
+		})
+		if err != nil {
+			return false, err
 		}
 
-		if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
-			return err
+		// Skip it if the version is referenced
+		if has {
+			return true, nil
 		}
 	}
 
-	return nil
+	return false, nil
 }

services/packages/container/common.go

@@ -0,0 +1,36 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package container
+
+import (
+	"context"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	user_model "code.gitea.io/gitea/models/user"
+	container_module "code.gitea.io/gitea/modules/packages/container"
+)
+
+// UpdateRepositoryNames updates the repository name property for all packages of the specific owner
+func UpdateRepositoryNames(ctx context.Context, owner *user_model.User, newOwnerName string) error {
+	ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeContainer)
+	if err != nil {
+		return err
+	}
+
+	newOwnerName = strings.ToLower(newOwnerName)
+
+	for _, p := range ps {
+		if err := packages_model.DeletePropertyByName(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository); err != nil {
+			return err
+		}
+
+		if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, newOwnerName+"/"+p.LowerName); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

services/packages/packages.go

@@ -443,13 +443,80 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro
 }
 
 // Cleanup removes expired package data
-func Cleanup(unused context.Context, olderThan time.Duration) error {
-	ctx, committer, err := db.TxContext(db.DefaultContext)
+func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
+	ctx, committer, err := db.TxContext(taskCtx)
 	if err != nil {
 		return err
 	}
 	defer committer.Close()
 
+	err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
+		select {
+		case <-taskCtx.Done():
+			return db.ErrCancelledf("While processing package cleanup rules")
+		default:
+		}
+
+		if err := pcr.CompiledPattern(); err != nil {
+			return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
+		}
+
+		olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
+
+		packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
+		if err != nil {
+			return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
+		}
+
+		for _, p := range packages {
+			pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
+				PackageID:  p.ID,
+				IsInternal: util.OptionalBoolFalse,
+				Sort:       packages_model.SortCreatedDesc,
+				Paginator:  db.NewAbsoluteListOptions(pcr.KeepCount, 200),
+			})
+			if err != nil {
+				return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
+			}
+			for _, pv := range pvs {
+				if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
+				} else if skip {
+					log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+
+				toMatch := pv.LowerVersion
+				if pcr.MatchFullName {
+					toMatch = p.LowerName + "/" + pv.LowerVersion
+				}
+
+				if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
+					log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+				if pv.CreatedUnix.AsLocalTime().After(olderThan) {
+					log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+				if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
+					log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+
+				log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
+
+				if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
 	if err := container_service.Cleanup(ctx, olderThan); err != nil {
 		return err
 	}