Add Arch package registry (#32692)

Close #25037
Close #31037

This PR adds a Arch package registry usable with pacman.

![grafik](https://github.com/user-attachments/assets/81cdb0c2-02f9-4733-bee2-e48af6b45224)

Rewrite of #25396 and #31037. You can follow [this
tutorial](https://wiki.archlinux.org/title/Creating_packages) to build a
package for testing.

Docs PR: https://gitea.com/gitea/docs/pulls/111

Co-authored-by: [d1nch8g@ion.lc](mailto:d1nch8g@ion.lc)
Co-authored-by: @ExplodingDragon

---------

Co-authored-by: dancheg97 <dancheg97@fmnx.su>
Co-authored-by: dragon <ExplodingFKL@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

routers/api/packages/api.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/web"
 	"code.gitea.io/gitea/routers/api/packages/alpine"
+	"code.gitea.io/gitea/routers/api/packages/arch"
 	"code.gitea.io/gitea/routers/api/packages/cargo"
 	"code.gitea.io/gitea/routers/api/packages/chef"
 	"code.gitea.io/gitea/routers/api/packages/composer"
@@ -135,6 +136,49 @@ func CommonRoutes() *web.Router {
 				})
 			})
 		}, reqPackageAccess(perm.AccessModeRead))
+		r.Group("/arch", func() {
+			r.Methods("HEAD,GET", "/repository.key", arch.GetRepositoryKey)
+
+			r.Methods("HEAD,GET,PUT,DELETE", "*", func(ctx *context.Context) {
+				path := strings.Trim(ctx.PathParam("*"), "/")
+
+				if ctx.Req.Method == "PUT" {
+					reqPackageAccess(perm.AccessModeWrite)(ctx)
+					if ctx.Written() {
+						return
+					}
+					ctx.SetPathParam("repository", path)
+					arch.UploadPackageFile(ctx)
+					return
+				}
+
+				pathFields := strings.Split(path, "/")
+				pathFieldsLen := len(pathFields)
+
+				if (ctx.Req.Method == "HEAD" || ctx.Req.Method == "GET") && pathFieldsLen >= 2 {
+					ctx.SetPathParam("repository", strings.Join(pathFields[:pathFieldsLen-2], "/"))
+					ctx.SetPathParam("architecture", pathFields[pathFieldsLen-2])
+					ctx.SetPathParam("filename", pathFields[pathFieldsLen-1])
+					arch.GetPackageOrRepositoryFile(ctx)
+					return
+				}
+
+				if ctx.Req.Method == "DELETE" && pathFieldsLen >= 3 {
+					reqPackageAccess(perm.AccessModeWrite)(ctx)
+					if ctx.Written() {
+						return
+					}
+					ctx.SetPathParam("repository", strings.Join(pathFields[:pathFieldsLen-3], "/"))
+					ctx.SetPathParam("name", pathFields[pathFieldsLen-3])
+					ctx.SetPathParam("version", pathFields[pathFieldsLen-2])
+					ctx.SetPathParam("architecture", pathFields[pathFieldsLen-1])
+					arch.DeletePackageVersion(ctx)
+					return
+				}
+
+				ctx.Status(http.StatusNotFound)
+			})
+		}, reqPackageAccess(perm.AccessModeRead))
 		r.Group("/cargo", func() {
 			r.Group("/api/v1/crates", func() {
 				r.Get("", cargo.SearchPackages)

routers/api/packages/arch/arch.go

@@ -0,0 +1,306 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package arch
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/modules/json"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	arch_module "code.gitea.io/gitea/modules/packages/arch"
+	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/routers/api/packages/helper"
+	"code.gitea.io/gitea/services/context"
+	packages_service "code.gitea.io/gitea/services/packages"
+	arch_service "code.gitea.io/gitea/services/packages/arch"
+)
+
+func apiError(ctx *context.Context, status int, obj any) {
+	helper.LogAndProcessError(ctx, status, obj, func(message string) {
+		ctx.PlainText(status, message)
+	})
+}
+
+func GetRepositoryKey(ctx *context.Context) {
+	_, pub, err := arch_service.GetOrCreateKeyPair(ctx, ctx.Package.Owner.ID)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.ServeContent(strings.NewReader(pub), &context.ServeHeaderOptions{
+		ContentType: "application/pgp-keys",
+	})
+}
+
+func UploadPackageFile(ctx *context.Context) {
+	repository := strings.TrimSpace(ctx.PathParam("repository"))
+
+	upload, needToClose, err := ctx.UploadStream()
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if needToClose {
+		defer upload.Close()
+	}
+
+	buf, err := packages_module.CreateHashedBufferFromReader(upload)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer buf.Close()
+
+	pck, err := arch_module.ParsePackage(buf)
+	if err != nil {
+		if errors.Is(err, util.ErrInvalidArgument) || err == io.EOF {
+			apiError(ctx, http.StatusBadRequest, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	if _, err := buf.Seek(0, io.SeekStart); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	fileMetadataRaw, err := json.Marshal(pck.FileMetadata)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	signature, err := arch_service.SignData(ctx, ctx.Package.Owner.ID, buf)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	if _, err := buf.Seek(0, io.SeekStart); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	release, err := arch_service.AquireRegistryLock(ctx, ctx.Package.Owner.ID)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer release()
+
+	// Search for duplicates with different file compression
+	has, err := packages_model.HasFiles(ctx, &packages_model.PackageFileSearchOptions{
+		OwnerID:     ctx.Package.Owner.ID,
+		PackageType: packages_model.TypeArch,
+		Query:       fmt.Sprintf("%s-%s-%s.pkg.tar.%%", pck.Name, pck.Version, pck.FileMetadata.Architecture),
+		Properties: map[string]string{
+			arch_module.PropertyRepository:   repository,
+			arch_module.PropertyArchitecture: pck.FileMetadata.Architecture,
+		},
+	})
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if has {
+		apiError(ctx, http.StatusConflict, packages_model.ErrDuplicatePackageFile)
+		return
+	}
+
+	_, _, err = packages_service.CreatePackageOrAddFileToExisting(
+		ctx,
+		&packages_service.PackageCreationInfo{
+			PackageInfo: packages_service.PackageInfo{
+				Owner:       ctx.Package.Owner,
+				PackageType: packages_model.TypeArch,
+				Name:        pck.Name,
+				Version:     pck.Version,
+			},
+			Creator:  ctx.Doer,
+			Metadata: pck.VersionMetadata,
+		},
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename:     fmt.Sprintf("%s-%s-%s.pkg.tar.%s", pck.Name, pck.Version, pck.FileMetadata.Architecture, pck.FileCompressionExtension),
+				CompositeKey: fmt.Sprintf("%s|%s", repository, pck.FileMetadata.Architecture),
+			},
+			Creator: ctx.Doer,
+			Data:    buf,
+			IsLead:  true,
+			Properties: map[string]string{
+				arch_module.PropertyRepository:   repository,
+				arch_module.PropertyArchitecture: pck.FileMetadata.Architecture,
+				arch_module.PropertyMetadata:     string(fileMetadataRaw),
+				arch_module.PropertySignature:    base64.StdEncoding.EncodeToString(signature),
+			},
+		},
+	)
+	if err != nil {
+		switch err {
+		case packages_model.ErrDuplicatePackageVersion, packages_model.ErrDuplicatePackageFile:
+			apiError(ctx, http.StatusConflict, err)
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	if err := arch_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, repository, pck.FileMetadata.Architecture); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.Status(http.StatusCreated)
+}
+
+func GetPackageOrRepositoryFile(ctx *context.Context) {
+	repository := ctx.PathParam("repository")
+	architecture := ctx.PathParam("architecture")
+	filename := ctx.PathParam("filename")
+	filenameOrig := filename
+
+	isSignature := strings.HasSuffix(filename, ".sig")
+	if isSignature {
+		filename = filename[:len(filename)-len(".sig")]
+	}
+
+	opts := &packages_model.PackageFileSearchOptions{
+		OwnerID:      ctx.Package.Owner.ID,
+		PackageType:  packages_model.TypeArch,
+		Query:        filename,
+		CompositeKey: fmt.Sprintf("%s|%s", repository, architecture),
+	}
+
+	if strings.HasSuffix(filename, ".db.tar.gz") || strings.HasSuffix(filename, ".files.tar.gz") || strings.HasSuffix(filename, ".files") || strings.HasSuffix(filename, ".db") {
+		// The requested filename is based on the user-defined repository name.
+		// Normalize everything to "packages.db".
+		opts.Query = arch_service.IndexArchiveFilename
+
+		pv, err := arch_service.GetOrCreateRepositoryVersion(ctx, ctx.Package.Owner.ID)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+		opts.VersionID = pv.ID
+	}
+
+	pfs, _, err := packages_model.SearchFiles(ctx, opts)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(pfs) == 0 {
+		// Try again with architecture 'any'
+		if architecture == arch_module.AnyArch {
+			apiError(ctx, http.StatusNotFound, nil)
+			return
+		}
+
+		opts.CompositeKey = fmt.Sprintf("%s|%s", repository, arch_module.AnyArch)
+		if pfs, _, err = packages_model.SearchFiles(ctx, opts); err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+	}
+	if len(pfs) != 1 {
+		apiError(ctx, http.StatusNotFound, nil)
+		return
+	}
+
+	if isSignature {
+		pfps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pfs[0].ID, arch_module.PropertySignature)
+		if err != nil || len(pfps) == 0 {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+
+		data, err := base64.StdEncoding.DecodeString(pfps[0].Value)
+		if err != nil {
+			apiError(ctx, http.StatusInternalServerError, err)
+			return
+		}
+
+		ctx.ServeContent(bytes.NewReader(data), &context.ServeHeaderOptions{
+			Filename: filenameOrig,
+		})
+		return
+	}
+
+	s, u, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0])
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	helper.ServePackageFile(ctx, s, u, pf)
+}
+
+func DeletePackageVersion(ctx *context.Context) {
+	repository := ctx.PathParam("repository")
+	architecture := ctx.PathParam("architecture")
+	name := ctx.PathParam("name")
+	version := ctx.PathParam("version")
+
+	release, err := arch_service.AquireRegistryLock(ctx, ctx.Package.Owner.ID)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer release()
+
+	pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeArch, name, version)
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+		VersionID:    pv.ID,
+		CompositeKey: fmt.Sprintf("%s|%s", repository, architecture),
+	})
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(pfs) != 1 {
+		apiError(ctx, http.StatusNotFound, nil)
+		return
+	}
+
+	if err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx, ctx.Doer, pfs[0]); err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	if err := arch_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, repository, architecture); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.Status(http.StatusNoContent)
+}

routers/web/user/package.go

@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/optional"
 	alpine_module "code.gitea.io/gitea/modules/packages/alpine"
+	arch_module "code.gitea.io/gitea/modules/packages/arch"
 	debian_module "code.gitea.io/gitea/modules/packages/debian"
 	rpm_module "code.gitea.io/gitea/modules/packages/rpm"
 	"code.gitea.io/gitea/modules/setting"
@@ -178,13 +179,13 @@ func ViewPackageVersion(ctx *context.Context) {
 	ctx.Data["IsPackagesPage"] = true
 	ctx.Data["PackageDescriptor"] = pd
 
+	registryHostURL, err := url.Parse(httplib.GuessCurrentHostURL(ctx))
+	if err != nil {
+		registryHostURL, _ = url.Parse(setting.AppURL)
+	}
+	ctx.Data["PackageRegistryHost"] = registryHostURL.Host
+
 	switch pd.Package.Type {
-	case packages_model.TypeContainer:
-		registryAppURL, err := url.Parse(httplib.GuessCurrentAppURL(ctx))
-		if err != nil {
-			registryAppURL, _ = url.Parse(setting.AppURL)
-		}
-		ctx.Data["RegistryHost"] = registryAppURL.Host
 	case packages_model.TypeAlpine:
 		branches := make(container.Set[string])
 		repositories := make(container.Set[string])
@@ -204,6 +205,23 @@ func ViewPackageVersion(ctx *context.Context) {
 		}
 
 		ctx.Data["Branches"] = util.Sorted(branches.Values())
+		ctx.Data["Repositories"] = util.Sorted(repositories.Values())
+		ctx.Data["Architectures"] = util.Sorted(architectures.Values())
+	case packages_model.TypeArch:
+		repositories := make(container.Set[string])
+		architectures := make(container.Set[string])
+
+		for _, f := range pd.Files {
+			for _, pp := range f.Properties {
+				switch pp.Name {
+				case arch_module.PropertyRepository:
+					repositories.Add(pp.Value)
+				case arch_module.PropertyArchitecture:
+					architectures.Add(pp.Value)
+				}
+			}
+		}
+
 		ctx.Data["Repositories"] = util.Sorted(repositories.Values())
 		ctx.Data["Architectures"] = util.Sorted(architectures.Values())
 	case packages_model.TypeDebian:
@@ -249,7 +267,6 @@ func ViewPackageVersion(ctx *context.Context) {
 	var (
 		total int64
 		pvs   []*packages_model.PackageVersion
-		err   error
 	)
 	switch pd.Package.Type {
 	case packages_model.TypeContainer: