Add Alpine package registry (#23714)

This PR adds an Alpine package registry. You can follow [this
tutorial](https://wiki.alpinelinux.org/wiki/Creating_an_Alpine_package)
to build a *.apk package for testing.

This functionality is similar to the Debian registry (#22854) and
therefore shares some methods. I marked this PR as blocked because it
should be merged after #22854.


![grafik](https://user-images.githubusercontent.com/1666336/227779595-b76163aa-eea1-4a79-9583-775c24ad74e8.png)

---------

Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Giteabot <teabot@gitea.io>

services/auth/source/oauth2/jwtsigningkey.go

@@ -23,7 +23,6 @@ import (
 	"code.gitea.io/gitea/modules/util"
 
 	"github.com/golang-jwt/jwt/v4"
-	"github.com/minio/sha256-simd"
 )
 
 // ErrInvalidAlgorithmType represents an invalid algorithm error.
@@ -82,7 +81,7 @@ type rsaSingingKey struct {
 }
 
 func newRSASingingKey(signingMethod jwt.SigningMethod, key *rsa.PrivateKey) (rsaSingingKey, error) {
-	kid, err := createPublicKeyFingerprint(key.Public().(*rsa.PublicKey))
+	kid, err := util.CreatePublicKeyFingerprint(key.Public().(*rsa.PublicKey))
 	if err != nil {
 		return rsaSingingKey{}, err
 	}
@@ -133,7 +132,7 @@ type eddsaSigningKey struct {
 }
 
 func newEdDSASingingKey(signingMethod jwt.SigningMethod, key ed25519.PrivateKey) (eddsaSigningKey, error) {
-	kid, err := createPublicKeyFingerprint(key.Public().(ed25519.PublicKey))
+	kid, err := util.CreatePublicKeyFingerprint(key.Public().(ed25519.PublicKey))
 	if err != nil {
 		return eddsaSigningKey{}, err
 	}
@@ -184,7 +183,7 @@ type ecdsaSingingKey struct {
 }
 
 func newECDSASingingKey(signingMethod jwt.SigningMethod, key *ecdsa.PrivateKey) (ecdsaSingingKey, error) {
-	kid, err := createPublicKeyFingerprint(key.Public().(*ecdsa.PublicKey))
+	kid, err := util.CreatePublicKeyFingerprint(key.Public().(*ecdsa.PublicKey))
 	if err != nil {
 		return ecdsaSingingKey{}, err
 	}
@@ -229,19 +228,6 @@ func (key ecdsaSingingKey) PreProcessToken(token *jwt.Token) {
 	token.Header["kid"] = key.id
 }
 
-// createPublicKeyFingerprint creates a fingerprint of the given key.
-// The fingerprint is the sha256 sum of the PKIX structure of the key.
-func createPublicKeyFingerprint(key interface{}) ([]byte, error) {
-	bytes, err := x509.MarshalPKIXPublicKey(key)
-	if err != nil {
-		return nil, err
-	}
-
-	checksum := sha256.Sum256(bytes)
-
-	return checksum[:], nil
-}
-
 // CreateJWTSigningKey creates a signing key from an algorithm / key pair.
 func CreateJWTSigningKey(algorithm string, key interface{}) (JWTSigningKey, error) {
 	var signingMethod jwt.SigningMethod

services/forms/package_form.go

@@ -15,7 +15,7 @@ import (
 type PackageCleanupRuleForm struct {
 	ID            int64
 	Enabled       bool
-	Type          string `binding:"Required;In(cargo,chef,composer,conan,conda,container,debian,generic,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
+	Type          string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,debian,generic,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
 	KeepCount     int    `binding:"In(0,1,5,10,25,50,100)"`
 	KeepPattern   string `binding:"RegexPattern"`
 	RemoveDays    int    `binding:"In(0,7,14,30,60,90,180)"`

services/packages/alpine/repository.go

@@ -0,0 +1,328 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package alpine
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	alpine_model "code.gitea.io/gitea/models/packages/alpine"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/json"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	alpine_module "code.gitea.io/gitea/modules/packages/alpine"
+	"code.gitea.io/gitea/modules/util"
+	packages_service "code.gitea.io/gitea/services/packages"
+)
+
+const IndexFilename = "APKINDEX.tar.gz"
+
+// GetOrCreateRepositoryVersion gets or creates the internal repository package
+// The Alpine registry needs multiple index files which are stored in this package.
+func GetOrCreateRepositoryVersion(ownerID int64) (*packages_model.PackageVersion, error) {
+	return packages_service.GetOrCreateInternalPackageVersion(ownerID, packages_model.TypeAlpine, alpine_module.RepositoryPackage, alpine_module.RepositoryVersion)
+}
+
+// GetOrCreateKeyPair gets or creates the RSA keys used to sign repository files
+func GetOrCreateKeyPair(ownerID int64) (string, string, error) {
+	priv, err := user_model.GetSetting(ownerID, alpine_module.SettingKeyPrivate)
+	if err != nil && !errors.Is(err, util.ErrNotExist) {
+		return "", "", err
+	}
+
+	pub, err := user_model.GetSetting(ownerID, alpine_module.SettingKeyPublic)
+	if err != nil && !errors.Is(err, util.ErrNotExist) {
+		return "", "", err
+	}
+
+	if priv == "" || pub == "" {
+		priv, pub, err = util.GenerateKeyPair(4096)
+		if err != nil {
+			return "", "", err
+		}
+
+		if err := user_model.SetUserSetting(ownerID, alpine_module.SettingKeyPrivate, priv); err != nil {
+			return "", "", err
+		}
+
+		if err := user_model.SetUserSetting(ownerID, alpine_module.SettingKeyPublic, pub); err != nil {
+			return "", "", err
+		}
+	}
+
+	return priv, pub, nil
+}
+
+// BuildAllRepositoryFiles (re)builds all repository files for every available distributions, components and architectures
+func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
+	pv, err := GetOrCreateRepositoryVersion(ownerID)
+	if err != nil {
+		return err
+	}
+
+	// 1. Delete all existing repository files
+	pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
+	if err != nil {
+		return err
+	}
+
+	for _, pf := range pfs {
+		if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+			return err
+		}
+		if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+			return err
+		}
+	}
+
+	// 2. (Re)Build repository files for existing packages
+	branches, err := alpine_model.GetBranches(ctx, ownerID)
+	if err != nil {
+		return err
+	}
+	for _, branch := range branches {
+		repositories, err := alpine_model.GetRepositories(ctx, ownerID, branch)
+		if err != nil {
+			return err
+		}
+		for _, repository := range repositories {
+			architectures, err := alpine_model.GetArchitectures(ctx, ownerID, repository)
+			if err != nil {
+				return err
+			}
+			for _, architecture := range architectures {
+				if err := buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture); err != nil {
+					return fmt.Errorf("failed to build repository files [%s/%s/%s]: %w", branch, repository, architecture, err)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// BuildSpecificRepositoryFiles builds index files for the repository
+func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, branch, repository, architecture string) error {
+	pv, err := GetOrCreateRepositoryVersion(ownerID)
+	if err != nil {
+		return err
+	}
+
+	return buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture)
+}
+
+type packageData struct {
+	Package         *packages_model.Package
+	Version         *packages_model.PackageVersion
+	Blob            *packages_model.PackageBlob
+	VersionMetadata *alpine_module.VersionMetadata
+	FileMetadata    *alpine_module.FileMetadata
+}
+
+type packageCache = map[*packages_model.PackageFile]*packageData
+
+// https://wiki.alpinelinux.org/wiki/Apk_spec#APKINDEX_Format
+func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, branch, repository, architecture string) error {
+	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+		OwnerID:     ownerID,
+		PackageType: packages_model.TypeAlpine,
+		Query:       "%.apk",
+		Properties: map[string]string{
+			alpine_module.PropertyBranch:       branch,
+			alpine_module.PropertyRepository:   repository,
+			alpine_module.PropertyArchitecture: architecture,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	// Delete the package indices if there are no packages
+	if len(pfs) == 0 {
+		pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, IndexFilename, fmt.Sprintf("%s|%s|%s", branch, repository, architecture))
+		if err != nil && !errors.Is(err, util.ErrNotExist) {
+			return err
+		}
+
+		if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+			return err
+		}
+		return packages_model.DeleteFileByID(ctx, pf.ID)
+	}
+
+	// Cache data needed for all repository files
+	cache := make(packageCache)
+	for _, pf := range pfs {
+		pv, err := packages_model.GetVersionByID(ctx, pf.VersionID)
+		if err != nil {
+			return err
+		}
+		p, err := packages_model.GetPackageByID(ctx, pv.PackageID)
+		if err != nil {
+			return err
+		}
+		pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
+		if err != nil {
+			return err
+		}
+		pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, alpine_module.PropertyMetadata)
+		if err != nil {
+			return err
+		}
+
+		pd := &packageData{
+			Package: p,
+			Version: pv,
+			Blob:    pb,
+		}
+
+		if err := json.Unmarshal([]byte(pv.MetadataJSON), &pd.VersionMetadata); err != nil {
+			return err
+		}
+		if len(pps) > 0 {
+			if err := json.Unmarshal([]byte(pps[0].Value), &pd.FileMetadata); err != nil {
+				return err
+			}
+		}
+
+		cache[pf] = pd
+	}
+
+	var buf bytes.Buffer
+	for _, pf := range pfs {
+		pd := cache[pf]
+
+		fmt.Fprintf(&buf, "C:%s\n", pd.FileMetadata.Checksum)
+		fmt.Fprintf(&buf, "P:%s\n", pd.Package.Name)
+		fmt.Fprintf(&buf, "V:%s\n", pd.Version.Version)
+		fmt.Fprintf(&buf, "A:%s\n", pd.FileMetadata.Architecture)
+		if pd.VersionMetadata.Description != "" {
+			fmt.Fprintf(&buf, "T:%s\n", pd.VersionMetadata.Description)
+		}
+		if pd.VersionMetadata.ProjectURL != "" {
+			fmt.Fprintf(&buf, "U:%s\n", pd.VersionMetadata.ProjectURL)
+		}
+		if pd.VersionMetadata.License != "" {
+			fmt.Fprintf(&buf, "L:%s\n", pd.VersionMetadata.License)
+		}
+		fmt.Fprintf(&buf, "S:%d\n", pd.Blob.Size)
+		fmt.Fprintf(&buf, "I:%d\n", pd.FileMetadata.Size)
+		fmt.Fprintf(&buf, "o:%s\n", pd.FileMetadata.Origin)
+		fmt.Fprintf(&buf, "m:%s\n", pd.VersionMetadata.Maintainer)
+		fmt.Fprintf(&buf, "t:%d\n", pd.FileMetadata.BuildDate)
+		if pd.FileMetadata.CommitHash != "" {
+			fmt.Fprintf(&buf, "c:%s\n", pd.FileMetadata.CommitHash)
+		}
+		if len(pd.FileMetadata.Dependencies) > 0 {
+			fmt.Fprintf(&buf, "D:%s\n", strings.Join(pd.FileMetadata.Dependencies, " "))
+		}
+		if len(pd.FileMetadata.Provides) > 0 {
+			fmt.Fprintf(&buf, "p:%s\n", strings.Join(pd.FileMetadata.Provides, " "))
+		}
+		fmt.Fprint(&buf, "\n")
+	}
+
+	unsignedIndexContent, _ := packages_module.NewHashedBuffer()
+	h := sha1.New()
+
+	if err := writeGzipStream(io.MultiWriter(unsignedIndexContent, h), "APKINDEX", buf.Bytes(), true); err != nil {
+		return err
+	}
+
+	priv, _, err := GetOrCreateKeyPair(ownerID)
+	if err != nil {
+		return err
+	}
+
+	privPem, _ := pem.Decode([]byte(priv))
+	if privPem == nil {
+		return fmt.Errorf("failed to decode private key pem")
+	}
+
+	privKey, err := x509.ParsePKCS1PrivateKey(privPem.Bytes)
+	if err != nil {
+		return err
+	}
+
+	sign, err := rsa.SignPKCS1v15(rand.Reader, privKey, crypto.SHA1, h.Sum(nil))
+	if err != nil {
+		return err
+	}
+
+	owner, err := user_model.GetUserByID(ctx, ownerID)
+	if err != nil {
+		return err
+	}
+
+	fingerprint, err := util.CreatePublicKeyFingerprint(&privKey.PublicKey)
+	if err != nil {
+		return err
+	}
+
+	signedIndexContent, _ := packages_module.NewHashedBuffer()
+
+	if err := writeGzipStream(
+		signedIndexContent,
+		fmt.Sprintf(".SIGN.RSA.%s@%s.rsa.pub", owner.LowerName, hex.EncodeToString(fingerprint)),
+		sign,
+		false,
+	); err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(signedIndexContent, unsignedIndexContent); err != nil {
+		return err
+	}
+
+	_, err = packages_service.AddFileToPackageVersionInternal(
+		repoVersion,
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename:     IndexFilename,
+				CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture),
+			},
+			Creator:           user_model.NewGhostUser(),
+			Data:              signedIndexContent,
+			IsLead:            false,
+			OverwriteExisting: true,
+		},
+	)
+	return err
+}
+
+func writeGzipStream(w io.Writer, filename string, content []byte, addTarEnd bool) error {
+	zw := gzip.NewWriter(w)
+	defer zw.Close()
+
+	tw := tar.NewWriter(zw)
+	if addTarEnd {
+		defer tw.Close()
+	}
+	hdr := &tar.Header{
+		Name: filename,
+		Mode: 0o600,
+		Size: int64(len(content)),
+	}
+	if err := tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+	if _, err := tw.Write(content); err != nil {
+		return err
+	}
+	return nil
+}

services/packages/packages.go

@@ -351,6 +351,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
 
 	var typeSpecificSize int64
 	switch packageType {
+	case packages_model.TypeAlpine:
+		typeSpecificSize = setting.Packages.LimitSizeAlpine
 	case packages_model.TypeCargo:
 		typeSpecificSize = setting.Packages.LimitSizeCargo
 	case packages_model.TypeChef:
@@ -486,6 +488,47 @@ func RemovePackageVersion(doer *user_model.User, pv *packages_model.PackageVersi
 	return nil
 }
 
+// RemovePackageFileAndVersionIfUnreferenced deletes the package file and the version if there are no referenced files afterwards
+func RemovePackageFileAndVersionIfUnreferenced(doer *user_model.User, pf *packages_model.PackageFile) error {
+	var pd *packages_model.PackageDescriptor
+
+	if err := db.WithTx(db.DefaultContext, func(ctx context.Context) error {
+		if err := DeletePackageFile(ctx, pf); err != nil {
+			return err
+		}
+
+		has, err := packages_model.HasVersionFileReferences(ctx, pf.VersionID)
+		if err != nil {
+			return err
+		}
+		if !has {
+			pv, err := packages_model.GetVersionByID(ctx, pf.VersionID)
+			if err != nil {
+				return err
+			}
+
+			pd, err = packages_model.GetPackageDescriptor(ctx, pv)
+			if err != nil {
+				return err
+			}
+
+			if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	if pd != nil {
+		notification.NotifyPackageDelete(db.DefaultContext, doer, pd)
+	}
+
+	return nil
+}
+
 // DeletePackageVersionAndReferences deletes the package version and its properties and files
 func DeletePackageVersionAndReferences(ctx context.Context, pv *packages_model.PackageVersion) error {
 	if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil {