Refactor container and UI (#34736)

routers/api/packages/api.go

@@ -701,18 +701,18 @@ func ContainerRoutes() *web.Router {
 	r.Get("/_catalog", container.ReqContainerAccess, container.GetRepositoryList)
 	r.Group("/{username}", func() {
 		r.PathGroup("/*", func(g *web.RouterPathGroup) {
-			g.MatchPath("POST", "/<image:*>/blobs/uploads", reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName, container.InitiateUploadBlob)
-			g.MatchPath("GET", "/<image:*>/tags/list", container.VerifyImageName, container.GetTagList)
+			g.MatchPath("POST", "/<image:*>/blobs/uploads", reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName, container.PostBlobsUploads)
+			g.MatchPath("GET", "/<image:*>/tags/list", container.VerifyImageName, container.GetTagsList)
 			g.MatchPath("GET,PATCH,PUT,DELETE", `/<image:*>/blobs/uploads/<uuid:[-.=\w]+>`, reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName, func(ctx *context.Context) {
 				switch ctx.Req.Method {
 				case http.MethodGet:
-					container.GetUploadBlob(ctx)
+					container.GetBlobsUpload(ctx)
 				case http.MethodPatch:
-					container.UploadBlob(ctx)
+					container.PatchBlobsUpload(ctx)
 				case http.MethodPut:
-					container.EndUploadBlob(ctx)
+					container.PutBlobsUpload(ctx)
 				default: /* DELETE */
-					container.CancelUploadBlob(ctx)
+					container.DeleteBlobsUpload(ctx)
 				}
 			})
 			g.MatchPath("HEAD", `/<image:*>/blobs/<digest>`, container.VerifyImageName, container.HeadBlob)
@@ -721,7 +721,7 @@ func ContainerRoutes() *web.Router {
 
 			g.MatchPath("HEAD", `/<image:*>/manifests/<reference>`, container.VerifyImageName, container.HeadManifest)
 			g.MatchPath("GET", `/<image:*>/manifests/<reference>`, container.VerifyImageName, container.GetManifest)
-			g.MatchPath("PUT", `/<image:*>/manifests/<reference>`, container.VerifyImageName, reqPackageAccess(perm.AccessModeWrite), container.UploadManifest)
+			g.MatchPath("PUT", `/<image:*>/manifests/<reference>`, container.VerifyImageName, reqPackageAccess(perm.AccessModeWrite), container.PutManifest)
 			g.MatchPath("DELETE", `/<image:*>/manifests/<reference>`, container.VerifyImageName, reqPackageAccess(perm.AccessModeWrite), container.DeleteManifest)
 		})
 	}, container.ReqContainerAccess, context.UserAssignmentWeb(), context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead))

routers/api/packages/container/blob.go

@@ -20,6 +20,8 @@ import (
 	container_module "code.gitea.io/gitea/modules/packages/container"
 	"code.gitea.io/gitea/modules/util"
 	packages_service "code.gitea.io/gitea/services/packages"
+
+	"github.com/opencontainers/go-digest"
 )
 
 // saveAsPackageBlob creates a package blob from an upload
@@ -175,7 +177,7 @@ func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, p
 	return nil
 }
 
-func deleteBlob(ctx context.Context, ownerID int64, image, digest string) error {
+func deleteBlob(ctx context.Context, ownerID int64, image string, digest digest.Digest) error {
 	releaser, err := globallock.Lock(ctx, containerPkgName(ownerID, image))
 	if err != nil {
 		return err
@@ -186,7 +188,7 @@ func deleteBlob(ctx context.Context, ownerID int64, image, digest string) error
 		pfds, err := container_model.GetContainerBlobs(ctx, &container_model.BlobSearchOptions{
 			OwnerID: ownerID,
 			Image:   image,
-			Digest:  digest,
+			Digest:  string(digest),
 		})
 		if err != nil {
 			return err

routers/api/packages/container/container.go

@@ -231,7 +231,7 @@ func GetRepositoryList(ctx *context.Context) {
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#mounting-a-blob-from-another-repository
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#single-post
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks
-func InitiateUploadBlob(ctx *context.Context) {
+func PostBlobsUploads(ctx *context.Context) {
 	image := ctx.PathParam("image")
 
 	mount := ctx.FormTrim("mount")
@@ -319,7 +319,7 @@ func InitiateUploadBlob(ctx *context.Context) {
 }
 
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks
-func GetUploadBlob(ctx *context.Context) {
+func GetBlobsUpload(ctx *context.Context) {
 	uuid := ctx.PathParam("uuid")
 
 	upload, err := packages_model.GetBlobUploadByID(ctx, uuid)
@@ -345,7 +345,7 @@ func GetUploadBlob(ctx *context.Context) {
 
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#single-post
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks
-func UploadBlob(ctx *context.Context) {
+func PatchBlobsUpload(ctx *context.Context) {
 	image := ctx.PathParam("image")
 
 	uploader, err := container_service.NewBlobUploader(ctx, ctx.PathParam("uuid"))
@@ -393,7 +393,7 @@ func UploadBlob(ctx *context.Context) {
 }
 
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-a-blob-in-chunks
-func EndUploadBlob(ctx *context.Context) {
+func PutBlobsUpload(ctx *context.Context) {
 	image := ctx.PathParam("image")
 
 	digest := ctx.FormTrim("digest")
@@ -462,7 +462,7 @@ func EndUploadBlob(ctx *context.Context) {
 }
 
 // https://docs.docker.com/registry/spec/api/#delete-blob-upload
-func CancelUploadBlob(ctx *context.Context) {
+func DeleteBlobsUpload(ctx *context.Context) {
 	uuid := ctx.PathParam("uuid")
 
 	_, err := packages_model.GetBlobUploadByID(ctx, uuid)
@@ -486,16 +486,15 @@ func CancelUploadBlob(ctx *context.Context) {
 }
 
 func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescriptor, error) {
-	d := ctx.PathParam("digest")
-
-	if digest.Digest(d).Validate() != nil {
+	d := digest.Digest(ctx.PathParam("digest"))
+	if d.Validate() != nil {
 		return nil, container_model.ErrContainerBlobNotExist
 	}
 
 	return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
 		OwnerID: ctx.Package.Owner.ID,
 		Image:   ctx.PathParam("image"),
-		Digest:  d,
+		Digest:  string(d),
 	})
 }
 
@@ -535,9 +534,8 @@ func GetBlob(ctx *context.Context) {
 
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-blobs
 func DeleteBlob(ctx *context.Context) {
-	d := ctx.PathParam("digest")
-
-	if digest.Digest(d).Validate() != nil {
+	d := digest.Digest(ctx.PathParam("digest"))
+	if d.Validate() != nil {
 		apiErrorDefined(ctx, errBlobUnknown)
 		return
 	}
@@ -553,7 +551,7 @@ func DeleteBlob(ctx *context.Context) {
 }
 
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#pushing-manifests
-func UploadManifest(ctx *context.Context) {
+func PutManifest(ctx *context.Context) {
 	reference := ctx.PathParam("reference")
 
 	mci := &manifestCreationInfo{
@@ -609,18 +607,18 @@ func UploadManifest(ctx *context.Context) {
 }
 
 func getBlobSearchOptionsFromContext(ctx *context.Context) (*container_model.BlobSearchOptions, error) {
-	reference := ctx.PathParam("reference")
-
 	opts := &container_model.BlobSearchOptions{
 		OwnerID:    ctx.Package.Owner.ID,
 		Image:      ctx.PathParam("image"),
 		IsManifest: true,
 	}
 
-	if digest.Digest(reference).Validate() == nil {
-		opts.Digest = reference
+	reference := ctx.PathParam("reference")
+	if d := digest.Digest(reference); d.Validate() == nil {
+		opts.Digest = string(d)
 	} else if referencePattern.MatchString(reference) {
 		opts.Tag = reference
+		opts.OnlyLead = true
 	} else {
 		return nil, container_model.ErrContainerBlobNotExist
 	}
@@ -737,7 +735,7 @@ func serveBlob(ctx *context.Context, pfd *packages_model.PackageFileDescriptor)
 }
 
 // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#content-discovery
-func GetTagList(ctx *context.Context) {
+func GetTagsList(ctx *context.Context) {
 	image := ctx.PathParam("image")
 
 	if _, err := packages_model.GetPackageByName(ctx, ctx.Package.Owner.ID, packages_model.TypeContainer, image); err != nil {

routers/api/packages/container/manifest.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/models/db"
 	packages_model "code.gitea.io/gitea/models/packages"
@@ -23,19 +24,19 @@ import (
 	notify_service "code.gitea.io/gitea/services/notify"
 	packages_service "code.gitea.io/gitea/services/packages"
 
-	digest "github.com/opencontainers/go-digest"
+	"github.com/opencontainers/go-digest"
 	oci "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-func isValidMediaType(mt string) bool {
+func isMediaTypeValid(mt string) bool {
 	return strings.HasPrefix(mt, "application/vnd.docker.") || strings.HasPrefix(mt, "application/vnd.oci.")
 }
 
-func isImageManifestMediaType(mt string) bool {
+func isMediaTypeImageManifest(mt string) bool {
 	return strings.EqualFold(mt, oci.MediaTypeImageManifest) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.v2+json")
 }
 
-func isImageIndexMediaType(mt string) bool {
+func isMediaTypeImageIndex(mt string) bool {
 	return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json")
 }
 
@@ -64,22 +65,22 @@ func processManifest(ctx context.Context, mci *manifestCreationInfo, buf *packag
 		return "", err
 	}
 
-	if !isValidMediaType(mci.MediaType) {
+	if !isMediaTypeValid(mci.MediaType) {
 		mci.MediaType = index.MediaType
-		if !isValidMediaType(mci.MediaType) {
+		if !isMediaTypeValid(mci.MediaType) {
 			return "", errManifestInvalid.WithMessage("MediaType not recognized")
 		}
 	}
 
-	if isImageManifestMediaType(mci.MediaType) {
-		return processImageManifest(ctx, mci, buf)
-	} else if isImageIndexMediaType(mci.MediaType) {
-		return processImageManifestIndex(ctx, mci, buf)
+	if isMediaTypeImageManifest(mci.MediaType) {
+		return processOciImageManifest(ctx, mci, buf)
+	} else if isMediaTypeImageIndex(mci.MediaType) {
+		return processOciImageIndex(ctx, mci, buf)
 	}
 	return "", errManifestInvalid
 }
 
-func processImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
+func processOciImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
 	manifestDigest := ""
 
 	err := func() error {
@@ -156,7 +157,7 @@ func processImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *p
 		}
 
 		for _, ref := range blobReferences {
-			if err := createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil {
+			if _, err = createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil {
 				return err
 			}
 		}
@@ -196,7 +197,7 @@ func processImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *p
 	return manifestDigest, nil
 }
 
-func processImageManifestIndex(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
+func processOciImageIndex(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
 	manifestDigest := ""
 
 	err := func() error {
@@ -221,7 +222,7 @@ func processImageManifestIndex(ctx context.Context, mci *manifestCreationInfo, b
 		}
 
 		for _, manifest := range index.Manifests {
-			if !isImageManifestMediaType(manifest.MediaType) {
+			if !isMediaTypeImageManifest(manifest.MediaType) {
 				return errManifestInvalid
 			}
 
@@ -349,24 +350,31 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met
 		LowerVersion: strings.ToLower(mci.Reference),
 		MetadataJSON: string(metadataJSON),
 	}
-	var pv *packages_model.PackageVersion
-	if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
+	pv, err := packages_model.GetOrInsertVersion(ctx, _pv)
+	if err != nil {
 		if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
 			log.Error("Error inserting package: %v", err)
 			return nil, err
 		}
 
-		if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
-			return nil, err
-		}
-
-		// keep download count on overwrite
-		_pv.DownloadCount = pv.DownloadCount
-
-		if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
-			if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
-				log.Error("Error inserting package: %v", err)
-				return nil, err
+		if isMediaTypeImageIndex(mci.MediaType) {
+			if pv.CreatedUnix.AsTime().Before(time.Now().Add(-24 * time.Hour)) {
+				if err = packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
+					return nil, err
+				}
+				// keep download count on overwriting
+				_pv.DownloadCount = pv.DownloadCount
+				if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
+					if !errors.Is(err, packages_model.ErrDuplicatePackageVersion) {
+						log.Error("Error inserting package: %v", err)
+						return nil, err
+					}
+				}
+			} else {
+				err = packages_model.UpdateVersion(ctx, &packages_model.PackageVersion{ID: pv.ID, MetadataJSON: _pv.MetadataJSON})
+				if err != nil {
+					return nil, err
+				}
 			}
 		}
 	}
@@ -376,14 +384,23 @@ func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, met
 	}
 
 	if mci.IsTagged {
-		if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil {
-			log.Error("Error setting package version property: %v", err)
+		if err = packages_model.InsertOrUpdateProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil {
 			return nil, err
 		}
+	} else {
+		props, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged)
+		if err != nil {
+			return nil, err
+		}
+		for _, prop := range props {
+			if err = packages_model.DeletePropertyByID(ctx, prop.ID); err != nil {
+				return nil, err
+			}
+		}
 	}
+
 	for _, manifest := range metadata.Manifests {
-		if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil {
-			log.Error("Error setting package version property: %v", err)
+		if err = packages_model.InsertOrUpdateProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil {
 			return nil, err
 		}
 	}
@@ -400,9 +417,9 @@ type blobReference struct {
 	IsLead       bool
 }
 
-func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) error {
+func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) (*packages_model.PackageFile, error) {
 	if ref.File.Blob.Size != ref.ExpectedSize {
-		return errSizeInvalid
+		return nil, errSizeInvalid
 	}
 
 	if ref.Name == "" {
@@ -410,20 +427,21 @@ func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *package
 	}
 
 	pf := &packages_model.PackageFile{
-		VersionID: pv.ID,
-		BlobID:    ref.File.Blob.ID,
-		Name:      ref.Name,
-		LowerName: ref.Name,
-		IsLead:    ref.IsLead,
+		VersionID:    pv.ID,
+		BlobID:       ref.File.Blob.ID,
+		Name:         ref.Name,
+		LowerName:    ref.Name,
+		CompositeKey: string(ref.Digest),
+		IsLead:       ref.IsLead,
 	}
 	var err error
 	if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
 		if errors.Is(err, packages_model.ErrDuplicatePackageFile) {
 			// Skip this blob because the manifest contains the same filesystem layer multiple times.
-			return nil
+			return pf, nil
 		}
 		log.Error("Error inserting package file: %v", err)
-		return err
+		return nil, err
 	}
 
 	props := map[string]string{
@@ -433,18 +451,18 @@ func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *package
 	for name, value := range props {
 		if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil {
 			log.Error("Error setting package file property: %v", err)
-			return err
+			return nil, err
 		}
 	}
 
-	// Remove the file from the blob upload version
+	// Remove the ref file (old file) from the blob upload version
 	if uploadVersion != nil && ref.File.File != nil && uploadVersion.ID == ref.File.File.VersionID {
 		if err := packages_service.DeletePackageFile(ctx, ref.File.File); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
-	return nil
+	return pf, nil
 }
 
 func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (*packages_model.PackageBlob, bool, string, error) {
@@ -471,7 +489,7 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack
 	}
 
 	manifestDigest := digestFromHashSummer(buf)
-	err = createFileFromBlobReference(ctx, pv, nil, &blobReference{
+	pf, err := createFileFromBlobReference(ctx, pv, nil, &blobReference{
 		Digest:       digest.Digest(manifestDigest),
 		MediaType:    mci.MediaType,
 		Name:         container_model.ManifestFilename,
@@ -479,6 +497,26 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack
 		ExpectedSize: pb.Size,
 		IsLead:       true,
 	})
+	if err != nil {
+		return nil, false, "", err
+	}
 
+	oldManifestFiles, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+		OwnerID:     mci.Owner.ID,
+		PackageType: packages_model.TypeContainer,
+		VersionID:   pv.ID,
+		Query:       container_model.ManifestFilename,
+	})
+	if err != nil {
+		return nil, false, "", err
+	}
+	for _, oldManifestFile := range oldManifestFiles {
+		if oldManifestFile.ID != pf.ID && oldManifestFile.IsLead {
+			err = packages_model.UpdateFile(ctx, &packages_model.PackageFile{ID: oldManifestFile.ID, IsLead: false}, []string{"is_lead"})
+			if err != nil {
+				return nil, false, "", err
+			}
+		}
+	}
 	return pb, !exists, manifestDigest, err
 }