Add Arch package registry (#32692)

Close #25037
Close #31037

This PR adds a Arch package registry usable with pacman.

![grafik](https://github.com/user-attachments/assets/81cdb0c2-02f9-4733-bee2-e48af6b45224)

Rewrite of #25396 and #31037. You can follow [this
tutorial](https://wiki.archlinux.org/title/Creating_packages) to build a
package for testing.

Docs PR: https://gitea.com/gitea/docs/pulls/111

Co-authored-by: [d1nch8g@ion.lc](mailto:d1nch8g@ion.lc)
Co-authored-by: @ExplodingDragon

---------

Co-authored-by: dancheg97 <dancheg97@fmnx.su>
Co-authored-by: dragon <ExplodingFKL@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

services/forms/package_form.go

@@ -15,7 +15,7 @@ import (
 type PackageCleanupRuleForm struct {
 	ID            int64
 	Enabled       bool
-	Type          string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
+	Type          string `binding:"Required;In(alpine,arch,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
 	KeepCount     int    `binding:"In(0,1,5,10,25,50,100)"`
 	KeepPattern   string `binding:"RegexPattern"`
 	RemoveDays    int    `binding:"In(0,7,14,30,60,90,180)"`

services/packages/alpine/repository.go

@@ -72,7 +72,7 @@ func GetOrCreateKeyPair(ctx context.Context, ownerID int64) (string, string, err
 	return priv, pub, nil
 }
 
-// BuildAllRepositoryFiles (re)builds all repository files for every available distributions, components and architectures
+// BuildAllRepositoryFiles (re)builds all repository files for every available branches, repositories and architectures
 func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
 	pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
 	if err != nil {

services/packages/arch/repository.go

@@ -0,0 +1,401 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package arch
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	arch_model "code.gitea.io/gitea/models/packages/arch"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/container"
+	"code.gitea.io/gitea/modules/globallock"
+	"code.gitea.io/gitea/modules/json"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	arch_module "code.gitea.io/gitea/modules/packages/arch"
+	"code.gitea.io/gitea/modules/util"
+	packages_service "code.gitea.io/gitea/services/packages"
+
+	"github.com/keybase/go-crypto/openpgp"
+	"github.com/keybase/go-crypto/openpgp/armor"
+	"github.com/keybase/go-crypto/openpgp/packet"
+)
+
+const (
+	IndexArchiveFilename = "packages.db"
+)
+
+func AquireRegistryLock(ctx context.Context, ownerID int64) (globallock.ReleaseFunc, error) {
+	return globallock.Lock(ctx, fmt.Sprintf("packages_arch_%d", ownerID))
+}
+
+// GetOrCreateRepositoryVersion gets or creates the internal repository package
+// The Arch registry needs multiple index files which are stored in this package.
+func GetOrCreateRepositoryVersion(ctx context.Context, ownerID int64) (*packages_model.PackageVersion, error) {
+	return packages_service.GetOrCreateInternalPackageVersion(ctx, ownerID, packages_model.TypeArch, arch_module.RepositoryPackage, arch_module.RepositoryVersion)
+}
+
+// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository files
+func GetOrCreateKeyPair(ctx context.Context, ownerID int64) (string, string, error) {
+	priv, err := user_model.GetSetting(ctx, ownerID, arch_module.SettingKeyPrivate)
+	if err != nil && !errors.Is(err, util.ErrNotExist) {
+		return "", "", err
+	}
+
+	pub, err := user_model.GetSetting(ctx, ownerID, arch_module.SettingKeyPublic)
+	if err != nil && !errors.Is(err, util.ErrNotExist) {
+		return "", "", err
+	}
+
+	if priv == "" || pub == "" {
+		priv, pub, err = generateKeypair()
+		if err != nil {
+			return "", "", err
+		}
+
+		if err := user_model.SetUserSetting(ctx, ownerID, arch_module.SettingKeyPrivate, priv); err != nil {
+			return "", "", err
+		}
+
+		if err := user_model.SetUserSetting(ctx, ownerID, arch_module.SettingKeyPublic, pub); err != nil {
+			return "", "", err
+		}
+	}
+
+	return priv, pub, nil
+}
+
+func generateKeypair() (string, string, error) {
+	e, err := openpgp.NewEntity("", "Arch Registry", "", nil)
+	if err != nil {
+		return "", "", err
+	}
+
+	var priv strings.Builder
+	var pub strings.Builder
+
+	w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil)
+	if err != nil {
+		return "", "", err
+	}
+	if err := e.SerializePrivate(w, nil); err != nil {
+		return "", "", err
+	}
+	w.Close()
+
+	w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil)
+	if err != nil {
+		return "", "", err
+	}
+	if err := e.Serialize(w); err != nil {
+		return "", "", err
+	}
+	w.Close()
+
+	return priv.String(), pub.String(), nil
+}
+
+func SignData(ctx context.Context, ownerID int64, r io.Reader) ([]byte, error) {
+	priv, _, err := GetOrCreateKeyPair(ctx, ownerID)
+	if err != nil {
+		return nil, err
+	}
+
+	block, err := armor.Decode(strings.NewReader(priv))
+	if err != nil {
+		return nil, err
+	}
+
+	e, err := openpgp.ReadEntity(packet.NewReader(block.Body))
+	if err != nil {
+		return nil, err
+	}
+
+	buf := &bytes.Buffer{}
+	if err := openpgp.DetachSign(buf, e, r, nil); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+// BuildAllRepositoryFiles (re)builds all repository files for every available repositories and architectures
+func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
+	pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
+	if err != nil {
+		return err
+	}
+
+	// 1. Delete all existing repository files
+	pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
+	if err != nil {
+		return err
+	}
+
+	for _, pf := range pfs {
+		if err := packages_service.DeletePackageFile(ctx, pf); err != nil {
+			return err
+		}
+	}
+
+	// 2. (Re)Build repository files for existing packages
+	repositories, err := arch_model.GetRepositories(ctx, ownerID)
+	if err != nil {
+		return err
+	}
+	for _, repository := range repositories {
+		architectures, err := arch_model.GetArchitectures(ctx, ownerID, repository)
+		if err != nil {
+			return err
+		}
+		for _, architecture := range architectures {
+			if err := buildPackagesIndex(ctx, ownerID, pv, repository, architecture); err != nil {
+				return fmt.Errorf("failed to build repository files [%s/%s]: %w", repository, architecture, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// BuildSpecificRepositoryFiles builds index files for the repository
+func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, repository, architecture string) error {
+	pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
+	if err != nil {
+		return err
+	}
+
+	architectures := container.SetOf(architecture)
+	if architecture == arch_module.AnyArch {
+		// Update all other architectures too when updating the any index
+		additionalArchitectures, err := arch_model.GetArchitectures(ctx, ownerID, repository)
+		if err != nil {
+			return err
+		}
+		architectures.AddMultiple(additionalArchitectures...)
+	}
+
+	for architecture := range architectures {
+		if err := buildPackagesIndex(ctx, ownerID, pv, repository, architecture); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func searchPackageFiles(ctx context.Context, ownerID int64, repository, architecture string) ([]*packages_model.PackageFile, error) {
+	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+		OwnerID:     ownerID,
+		PackageType: packages_model.TypeArch,
+		Query:       "%.pkg.tar.%",
+		Properties: map[string]string{
+			arch_module.PropertyRepository:   repository,
+			arch_module.PropertyArchitecture: architecture,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return pfs, nil
+}
+
+func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, repository, architecture string) error {
+	pfs, err := searchPackageFiles(ctx, ownerID, repository, architecture)
+	if err != nil {
+		return err
+	}
+	if architecture != arch_module.AnyArch {
+		// Add all any packages too
+		anyarchFiles, err := searchPackageFiles(ctx, ownerID, repository, arch_module.AnyArch)
+		if err != nil {
+			return err
+		}
+		pfs = append(pfs, anyarchFiles...)
+	}
+
+	// Delete the package indices if there are no packages
+	if len(pfs) == 0 {
+		pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, IndexArchiveFilename, fmt.Sprintf("%s|%s", repository, architecture))
+		if err != nil && !errors.Is(err, util.ErrNotExist) {
+			return err
+		} else if pf == nil {
+			return nil
+		}
+
+		return packages_service.DeletePackageFile(ctx, pf)
+	}
+
+	indexContent, _ := packages_module.NewHashedBuffer()
+	defer indexContent.Close()
+
+	gw := gzip.NewWriter(indexContent)
+	tw := tar.NewWriter(gw)
+
+	cache := make(map[int64]*packages_model.Package)
+
+	for _, pf := range pfs {
+		opts := &entryOptions{
+			File: pf,
+		}
+
+		opts.Version, err = packages_model.GetVersionByID(ctx, pf.VersionID)
+		if err != nil {
+			return err
+		}
+		if err := json.Unmarshal([]byte(opts.Version.MetadataJSON), &opts.VersionMetadata); err != nil {
+			return err
+		}
+		opts.Package = cache[opts.Version.PackageID]
+		if opts.Package == nil {
+			opts.Package, err = packages_model.GetPackageByID(ctx, opts.Version.PackageID)
+			if err != nil {
+				return err
+			}
+			cache[opts.Package.ID] = opts.Package
+		}
+		opts.Blob, err = packages_model.GetBlobByID(ctx, pf.BlobID)
+		if err != nil {
+			return err
+		}
+
+		sig, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertySignature)
+		if err != nil {
+			return err
+		}
+		if len(sig) == 0 {
+			return util.ErrNotExist
+		}
+		opts.Signature = sig[0].Value
+
+		meta, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyMetadata)
+		if err != nil {
+			return err
+		}
+		if len(meta) == 0 {
+			return util.ErrNotExist
+		}
+		if err := json.Unmarshal([]byte(meta[0].Value), &opts.FileMetadata); err != nil {
+			return err
+		}
+
+		if err := writeFiles(tw, opts); err != nil {
+			return err
+		}
+		if err := writeDescription(tw, opts); err != nil {
+			return err
+		}
+	}
+
+	tw.Close()
+	gw.Close()
+
+	signature, err := SignData(ctx, ownerID, indexContent)
+	if err != nil {
+		return err
+	}
+
+	if _, err := indexContent.Seek(0, io.SeekStart); err != nil {
+		return err
+	}
+
+	_, err = packages_service.AddFileToPackageVersionInternal(
+		ctx,
+		repoVersion,
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename:     IndexArchiveFilename,
+				CompositeKey: fmt.Sprintf("%s|%s", repository, architecture),
+			},
+			Creator:           user_model.NewGhostUser(),
+			Data:              indexContent,
+			IsLead:            false,
+			OverwriteExisting: true,
+			Properties: map[string]string{
+				arch_module.PropertyRepository:   repository,
+				arch_module.PropertyArchitecture: architecture,
+				arch_module.PropertySignature:    base64.StdEncoding.EncodeToString(signature),
+			},
+		},
+	)
+	return err
+}
+
+type entryOptions struct {
+	Package         *packages_model.Package
+	Version         *packages_model.PackageVersion
+	VersionMetadata *arch_module.VersionMetadata
+	File            *packages_model.PackageFile
+	FileMetadata    *arch_module.FileMetadata
+	Blob            *packages_model.PackageBlob
+	Signature       string
+}
+
+type keyValue struct {
+	Key   string
+	Value string
+}
+
+func writeFiles(tw *tar.Writer, opts *entryOptions) error {
+	return writeFields(tw, fmt.Sprintf("%s-%s/files", opts.Package.Name, opts.Version.Version), []keyValue{
+		{"FILES", strings.Join(opts.FileMetadata.Files, "\n")},
+	})
+}
+
+// https://gitlab.archlinux.org/pacman/pacman/-/blob/master/lib/libalpm/be_sync.c#L562
+func writeDescription(tw *tar.Writer, opts *entryOptions) error {
+	return writeFields(tw, fmt.Sprintf("%s-%s/desc", opts.Package.Name, opts.Version.Version), []keyValue{
+		{"FILENAME", opts.File.Name},
+		{"MD5SUM", opts.Blob.HashMD5},
+		{"SHA256SUM", opts.Blob.HashSHA256},
+		{"PGPSIG", opts.Signature},
+		{"CSIZE", fmt.Sprintf("%d", opts.Blob.Size)},
+		{"ISIZE", fmt.Sprintf("%d", opts.FileMetadata.InstalledSize)},
+		{"NAME", opts.Package.Name},
+		{"BASE", opts.FileMetadata.Base},
+		{"ARCH", opts.FileMetadata.Architecture},
+		{"VERSION", opts.Version.Version},
+		{"DESC", opts.VersionMetadata.Description},
+		{"URL", opts.VersionMetadata.ProjectURL},
+		{"LICENSE", strings.Join(opts.VersionMetadata.Licenses, "\n")},
+		{"GROUPS", strings.Join(opts.FileMetadata.Groups, "\n")},
+		{"BUILDDATE", fmt.Sprintf("%d", opts.FileMetadata.BuildDate)},
+		{"PACKAGER", opts.FileMetadata.Packager},
+		{"PROVIDES", strings.Join(opts.FileMetadata.Provides, "\n")},
+		{"DEPENDS", strings.Join(opts.FileMetadata.Depends, "\n")},
+		{"OPTDEPENDS", strings.Join(opts.FileMetadata.OptDepends, "\n")},
+		{"MAKEDEPENDS", strings.Join(opts.FileMetadata.MakeDepends, "\n")},
+		{"CHECKDEPENDS", strings.Join(opts.FileMetadata.CheckDepends, "\n")},
+		{"XDATA", strings.Join(opts.FileMetadata.XData, "\n")},
+	})
+}
+
+func writeFields(tw *tar.Writer, filename string, fields []keyValue) error {
+	buf := &bytes.Buffer{}
+	for _, kv := range fields {
+		if kv.Value == "" {
+			continue
+		}
+		fmt.Fprintf(buf, "%%%s%%\n%s\n\n", kv.Key, kv.Value)
+	}
+
+	if err := tw.WriteHeader(&tar.Header{
+		Name: filename,
+		Size: int64(buf.Len()),
+		Mode: int64(os.ModePerm),
+	}); err != nil {
+		return err
+	}
+
+	_, err := io.Copy(tw, buf)
+	return err
+}

services/packages/cleanup/cleanup.go

@@ -16,6 +16,7 @@ import (
 	packages_module "code.gitea.io/gitea/modules/packages"
 	packages_service "code.gitea.io/gitea/services/packages"
 	alpine_service "code.gitea.io/gitea/services/packages/alpine"
+	arch_service "code.gitea.io/gitea/services/packages/arch"
 	cargo_service "code.gitea.io/gitea/services/packages/cargo"
 	container_service "code.gitea.io/gitea/services/packages/container"
 	debian_service "code.gitea.io/gitea/services/packages/debian"
@@ -120,18 +121,29 @@ func ExecuteCleanupRules(outerCtx context.Context) error {
 		}
 
 		if anyVersionDeleted {
-			if pcr.Type == packages_model.TypeDebian {
+			switch pcr.Type {
+			case packages_model.TypeDebian:
 				if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
 					return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
 				}
-			} else if pcr.Type == packages_model.TypeAlpine {
+			case packages_model.TypeAlpine:
 				if err := alpine_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
 					return fmt.Errorf("CleanupRule [%d]: alpine.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
 				}
-			} else if pcr.Type == packages_model.TypeRpm {
+			case packages_model.TypeRpm:
 				if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
 					return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
 				}
+			case packages_model.TypeArch:
+				release, err := arch_service.AquireRegistryLock(ctx, pcr.OwnerID)
+				if err != nil {
+					return err
+				}
+				defer release()
+
+				if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+				}
 			}
 		}
 		return nil

services/packages/packages.go

@@ -355,6 +355,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
 	switch packageType {
 	case packages_model.TypeAlpine:
 		typeSpecificSize = setting.Packages.LimitSizeAlpine
+	case packages_model.TypeArch:
+		typeSpecificSize = setting.Packages.LimitSizeArch
 	case packages_model.TypeCargo:
 		typeSpecificSize = setting.Packages.LimitSizeCargo
 	case packages_model.TypeChef: