From bafb80f80d5505b03e5994d1ea6e2dab10052fe1 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 17 Apr 2024 17:30:41 +0200 Subject: [PATCH] Support nuspec manifest download for nuget packages (#28921) Support downloading nuget nuspec manifest[^1]. This is useful for renovate because it uses this api to find the corresponding repository - Store nuspec along with nupkg on upload - allow downloading nuspec - add doctor command to add missing nuspec files [^1]: https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec --------- Co-authored-by: KN4CK3R --- modules/packages/nuget/metadata.go | 21 +++--- routers/api/packages/nuget/nuget.go | 32 +++++++- tests/integration/api_packages_nuget_test.go | 77 ++++++++++++++------ 3 files changed, 95 insertions(+), 35 deletions(-) diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 6769c514cc..1e98ddffde 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -48,10 +48,11 @@ const maxNuspecFileSize = 3 * 1024 * 1024 // Package represents a Nuget package type Package struct { - PackageType PackageType - ID string - Version string - Metadata *Metadata + PackageType PackageType + ID string + Version string + Metadata *Metadata + NuspecContent *bytes.Buffer } // Metadata represents the metadata of a Nuget package @@ -138,8 +139,9 @@ func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) { // ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { + var nuspecBuf bytes.Buffer var p nuspecPackage - if err := xml.NewDecoder(r).Decode(&p); err != nil { + if err := xml.NewDecoder(io.TeeReader(r, &nuspecBuf)).Decode(&p); err != nil { return nil, err } @@ -212,10 +214,11 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { } } return &Package{ - PackageType: packageType, - ID: p.Metadata.ID, - Version: toNormalizedVersion(v), - Metadata: m, + PackageType: packageType, + ID: p.Metadata.ID, + Version: toNormalizedVersion(v), + Metadata: m, + NuspecContent: &nuspecBuf, }, nil } diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index c28bc6c9d9..09156ece6b 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -388,7 +388,8 @@ func EnumeratePackageVersionsV3(ctx *context.Context) { ctx.JSON(http.StatusOK, resp) } -// https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg +// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec +// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg func DownloadPackageFile(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") @@ -431,7 +432,7 @@ func UploadPackage(ctx *context.Context) { return } - _, _, err := packages_service.CreatePackageAndAddFile( + pv, _, err := packages_service.CreatePackageAndAddFile( ctx, &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ @@ -465,6 +466,33 @@ func UploadPackage(ctx *context.Context) { return } + nuspecBuf, err := packages_module.CreateHashedBufferFromReaderWithSize(np.NuspecContent, np.NuspecContent.Len()) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer nuspecBuf.Close() + + _, err = packages_service.AddFileToPackageVersionInternal( + ctx, + pv, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.nuspec", np.ID)), + }, + Data: nuspecBuf, + }, + ) + if err != nil { + switch err { + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + ctx.Status(http.StatusCreated) } diff --git a/tests/integration/api_packages_nuget_test.go b/tests/integration/api_packages_nuget_test.go index 20dafd5cc7..83947ff967 100644 --- a/tests/integration/api_packages_nuget_test.go +++ b/tests/integration/api_packages_nuget_test.go @@ -90,29 +90,33 @@ func TestPackageNuGet(t *testing.T) { symbolFilename := "test.pdb" symbolID := "d910bb6948bd4c6cb40155bcf52c3c94" - createPackage := func(id, version string) io.Reader { + createNuspec := func(id, version string) string { + return ` + + + ` + id + ` + ` + version + ` + ` + packageAuthors + ` + ` + packageDescription + ` + + + + + + +` + } + + createPackage := func(id, version string) *bytes.Buffer { var buf bytes.Buffer archive := zip.NewWriter(&buf) w, _ := archive.Create("package.nuspec") - w.Write([]byte(` - - - ` + id + ` - ` + version + ` - ` + packageAuthors + ` - ` + packageDescription + ` - - - - - - - `)) + w.Write([]byte(createNuspec(id, version))) archive.Close() return &buf } - content, _ := io.ReadAll(createPackage(packageName, packageVersion)) + content := createPackage(packageName, packageVersion).Bytes() url := fmt.Sprintf("/api/packages/%s/nuget", user.Name) @@ -224,7 +228,7 @@ func TestPackageNuGet(t *testing.T) { pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet) assert.NoError(t, err) - assert.Len(t, pvs, 1) + assert.Len(t, pvs, 1, "Should have one version") pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) @@ -235,13 +239,21 @@ func TestPackageNuGet(t *testing.T) { pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) - assert.Len(t, pfs, 1) - assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name) - assert.True(t, pfs[0].IsLead) + assert.Len(t, pfs, 2, "Should have 2 files: nuget and nuspec") + for _, pf := range pfs { + switch pf.Name { + case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion): + assert.True(t, pf.IsLead) - pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) - assert.NoError(t, err) - assert.Equal(t, int64(len(content)), pb.Size) + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) + case fmt.Sprintf("%s.nuspec", packageName): + assert.False(t, pf.IsLead) + default: + assert.Fail(t, "unexpected filename: %v", pf.Name) + } + } req = NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)). AddBasicAuth(user.Name) @@ -302,16 +314,27 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) - assert.Len(t, pfs, 3) + assert.Len(t, pfs, 4, "Should have 4 files: nupkg, snupkg, nuspec and pdb") for _, pf := range pfs { switch pf.Name { case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion): + assert.True(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(412), pb.Size) case fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion): assert.False(t, pf.IsLead) pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) assert.NoError(t, err) assert.Equal(t, int64(616), pb.Size) + case fmt.Sprintf("%s.nuspec", packageName): + assert.False(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(427), pb.Size) case symbolFilename: assert.False(t, pf.IsLead) @@ -353,6 +376,12 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) assert.Equal(t, content, resp.Body.Bytes()) + req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.nuspec", url, packageName, packageVersion, packageName)). + AddBasicAuth(user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, createNuspec(packageName, packageVersion), resp.Body.String()) + checkDownloadCount(1) req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)).