From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- modules/packages/nuget/metadata.go | 239 ++++++++++++++++++++++++ modules/packages/nuget/metadata_test.go | 188 +++++++++++++++++++ modules/packages/nuget/symbol_extractor.go | 186 ++++++++++++++++++ modules/packages/nuget/symbol_extractor_test.go | 82 ++++++++ 4 files changed, 695 insertions(+) create mode 100644 modules/packages/nuget/metadata.go create mode 100644 modules/packages/nuget/metadata_test.go create mode 100644 modules/packages/nuget/symbol_extractor.go create mode 100644 modules/packages/nuget/symbol_extractor_test.go (limited to 'modules/packages/nuget') diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go new file mode 100644 index 0000000..1e98ddf --- /dev/null +++ b/modules/packages/nuget/metadata.go @@ -0,0 +1,239 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "path/filepath" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +var ( + // ErrMissingNuspecFile indicates a missing Nuspec file + ErrMissingNuspecFile = util.NewInvalidArgumentErrorf("Nuspec file is missing") + // ErrNuspecFileTooLarge indicates a Nuspec file which is too large + ErrNuspecFileTooLarge = util.NewInvalidArgumentErrorf("Nuspec file is too large") + // ErrNuspecInvalidID indicates an invalid id in the Nuspec file + ErrNuspecInvalidID = util.NewInvalidArgumentErrorf("Nuspec file contains an invalid id") + // ErrNuspecInvalidVersion indicates an invalid version in the Nuspec file + ErrNuspecInvalidVersion = util.NewInvalidArgumentErrorf("Nuspec file contains an invalid version") +) + +// PackageType specifies the package type the metadata describes +type PackageType int + +const ( + // DependencyPackage represents a package (*.nupkg) + DependencyPackage PackageType = iota + 1 + // SymbolsPackage represents a symbol package (*.snupkg) + SymbolsPackage + + PropertySymbolID = "nuget.symbol.id" +) + +var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`) + +const maxNuspecFileSize = 3 * 1024 * 1024 + +// Package represents a Nuget package +type Package struct { + PackageType PackageType + ID string + Version string + Metadata *Metadata + NuspecContent *bytes.Buffer +} + +// Metadata represents the metadata of a Nuget package +type Metadata struct { + Description string `json:"description,omitempty"` + ReleaseNotes string `json:"release_notes,omitempty"` + Readme string `json:"readme,omitempty"` + Authors string `json:"authors,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + RequireLicenseAcceptance bool `json:"require_license_acceptance"` + Dependencies map[string][]Dependency `json:"dependencies,omitempty"` +} + +// Dependency represents a dependency of a Nuget package +type Dependency struct { + ID string `json:"id"` + Version string `json:"version"` +} + +// https://learn.microsoft.com/en-us/nuget/reference/nuspec +type nuspecPackage struct { + Metadata struct { + ID string `xml:"id"` + Version string `xml:"version"` + Authors string `xml:"authors"` + RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + ProjectURL string `xml:"projectUrl"` + Description string `xml:"description"` + ReleaseNotes string `xml:"releaseNotes"` + Readme string `xml:"readme"` + PackageTypes struct { + PackageType []struct { + Name string `xml:"name,attr"` + } `xml:"packageType"` + } `xml:"packageTypes"` + Repository struct { + URL string `xml:"url,attr"` + } `xml:"repository"` + Dependencies struct { + Dependency []struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + Exclude string `xml:"exclude,attr"` + } `xml:"dependency"` + Group []struct { + TargetFramework string `xml:"targetFramework,attr"` + Dependency []struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + Exclude string `xml:"exclude,attr"` + } `xml:"dependency"` + } `xml:"group"` + } `xml:"dependencies"` + } `xml:"metadata"` +} + +// ParsePackageMetaData parses the metadata of a Nuget package file +func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range archive.File { + if filepath.Dir(file.Name) != "." { + continue + } + if strings.HasSuffix(strings.ToLower(file.Name), ".nuspec") { + if file.UncompressedSize64 > maxNuspecFileSize { + return nil, ErrNuspecFileTooLarge + } + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseNuspecMetaData(archive, f) + } + } + return nil, ErrMissingNuspecFile +} + +// ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package +func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { + var nuspecBuf bytes.Buffer + var p nuspecPackage + if err := xml.NewDecoder(io.TeeReader(r, &nuspecBuf)).Decode(&p); err != nil { + return nil, err + } + + if !idmatch.MatchString(p.Metadata.ID) { + return nil, ErrNuspecInvalidID + } + + v, err := version.NewSemver(p.Metadata.Version) + if err != nil { + return nil, ErrNuspecInvalidVersion + } + + if !validation.IsValidURL(p.Metadata.ProjectURL) { + p.Metadata.ProjectURL = "" + } + + packageType := DependencyPackage + for _, pt := range p.Metadata.PackageTypes.PackageType { + if pt.Name == "SymbolsPackage" { + packageType = SymbolsPackage + break + } + } + + m := &Metadata{ + Description: p.Metadata.Description, + ReleaseNotes: p.Metadata.ReleaseNotes, + Authors: p.Metadata.Authors, + ProjectURL: p.Metadata.ProjectURL, + RepositoryURL: p.Metadata.Repository.URL, + RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, + Dependencies: make(map[string][]Dependency), + } + + if p.Metadata.Readme != "" { + f, err := archive.Open(p.Metadata.Readme) + if err == nil { + buf, _ := io.ReadAll(f) + m.Readme = string(buf) + _ = f.Close() + } + } + + if len(p.Metadata.Dependencies.Dependency) > 0 { + deps := make([]Dependency, 0, len(p.Metadata.Dependencies.Dependency)) + for _, dep := range p.Metadata.Dependencies.Dependency { + if dep.ID == "" || dep.Version == "" { + continue + } + deps = append(deps, Dependency{ + ID: dep.ID, + Version: dep.Version, + }) + } + m.Dependencies[""] = deps + } + for _, group := range p.Metadata.Dependencies.Group { + deps := make([]Dependency, 0, len(group.Dependency)) + for _, dep := range group.Dependency { + if dep.ID == "" || dep.Version == "" { + continue + } + deps = append(deps, Dependency{ + ID: dep.ID, + Version: dep.Version, + }) + } + if len(deps) > 0 { + m.Dependencies[group.TargetFramework] = deps + } + } + return &Package{ + PackageType: packageType, + ID: p.Metadata.ID, + Version: toNormalizedVersion(v), + Metadata: m, + NuspecContent: &nuspecBuf, + }, nil +} + +// https://learn.microsoft.com/en-us/nuget/concepts/package-versioning#normalized-version-numbers +// https://github.com/NuGet/NuGet.Client/blob/dccbd304b11103e08b97abf4cf4bcc1499d9235a/src/NuGet.Core/NuGet.Versioning/VersionFormatter.cs#L121 +func toNormalizedVersion(v *version.Version) string { + var buf bytes.Buffer + segments := v.Segments64() + fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) + if len(segments) > 3 && segments[3] > 0 { + fmt.Fprintf(&buf, ".%d", segments[3]) + } + pre := v.Prerelease() + if pre != "" { + fmt.Fprint(&buf, "-", pre) + } + return buf.String() +} diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go new file mode 100644 index 0000000..ecce052 --- /dev/null +++ b/modules/packages/nuget/metadata_test.go @@ -0,0 +1,188 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package nuget + +import ( + "archive/zip" + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + id = "System.Gitea" + semver = "1.0.1" + authors = "Gitea Authors" + projectURL = "https://gitea.io" + description = "Package Description" + releaseNotes = "Package Release Notes" + readme = "Readme" + repositoryURL = "https://gitea.io/gitea/gitea" + targetFramework = ".NETStandard2.1" + dependencyID = "System.Text.Json" + dependencyVersion = "5.0.0" +) + +const nuspecContent = ` + + + ` + id + ` + ` + semver + ` + ` + authors + ` + true + ` + projectURL + ` + ` + description + ` + ` + releaseNotes + ` + + README.md + + + + + + +` + +const symbolsNuspecContent = ` + + + ` + id + ` + ` + semver + ` + ` + description + ` + + + + + + + +` + +func TestParsePackageMetaData(t *testing.T) { + createArchive := func(files map[string]string) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + for name, content := range files { + w, _ := archive.Create(name) + w.Write([]byte(content)) + } + archive.Close() + return buf.Bytes() + } + + t.Run("MissingNuspecFile", func(t *testing.T) { + data := createArchive(map[string]string{"dummy.txt": ""}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.ErrorIs(t, err, ErrMissingNuspecFile) + }) + + t.Run("MissingNuspecFileInRoot", func(t *testing.T) { + data := createArchive(map[string]string{"sub/package.nuspec": ""}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.ErrorIs(t, err, ErrMissingNuspecFile) + }) + + t.Run("InvalidNuspecFile", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": ""}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.Error(t, err) + }) + + t.Run("InvalidPackageId", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": ` + + + `}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.ErrorIs(t, err, ErrNuspecInvalidID) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": ` + + + ` + id + ` + + `}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.ErrorIs(t, err, ErrNuspecInvalidVersion) + }) + + t.Run("MissingReadme", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": nuspecContent}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, np) + assert.Empty(t, np.Metadata.Readme) + }) + + t.Run("Dependency Package", func(t *testing.T) { + data := createArchive(map[string]string{ + "package.nuspec": nuspecContent, + "README.md": readme, + }) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, np) + assert.Equal(t, DependencyPackage, np.PackageType) + + assert.Equal(t, id, np.ID) + assert.Equal(t, semver, np.Version) + assert.Equal(t, authors, np.Metadata.Authors) + assert.Equal(t, projectURL, np.Metadata.ProjectURL) + assert.Equal(t, description, np.Metadata.Description) + assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) + assert.Equal(t, readme, np.Metadata.Readme) + assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL) + assert.Len(t, np.Metadata.Dependencies, 1) + assert.Contains(t, np.Metadata.Dependencies, targetFramework) + deps := np.Metadata.Dependencies[targetFramework] + assert.Len(t, deps, 1) + assert.Equal(t, dependencyID, deps[0].ID) + assert.Equal(t, dependencyVersion, deps[0].Version) + + t.Run("NormalizedVersion", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": ` + + + test + 1.04.5.2.5-rc.1+metadata + + `}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, np) + assert.Equal(t, "1.4.5.2-rc.1", np.Version) + }) + }) + + t.Run("Symbols Package", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": symbolsNuspecContent}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, np) + assert.Equal(t, SymbolsPackage, np.PackageType) + + assert.Equal(t, id, np.ID) + assert.Equal(t, semver, np.Version) + assert.Equal(t, description, np.Metadata.Description) + assert.Empty(t, np.Metadata.Dependencies) + }) +} diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go new file mode 100644 index 0000000..81bf037 --- /dev/null +++ b/modules/packages/nuget/symbol_extractor.go @@ -0,0 +1,186 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/binary" + "fmt" + "io" + "path" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util" +) + +var ( + ErrMissingPdbFiles = util.NewInvalidArgumentErrorf("package does not contain PDB files") + ErrInvalidFiles = util.NewInvalidArgumentErrorf("package contains invalid files") + ErrInvalidPdbMagicNumber = util.NewInvalidArgumentErrorf("invalid Portable PDB magic number") + ErrMissingPdbStream = util.NewInvalidArgumentErrorf("missing PDB stream") +) + +type PortablePdb struct { + Name string + ID string + Content *packages.HashedBuffer +} + +type PortablePdbList []*PortablePdb + +func (l PortablePdbList) Close() { + for _, pdb := range l { + pdb.Content.Close() + } +} + +// ExtractPortablePdb extracts PDB files from a .snupkg file +func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + var pdbs PortablePdbList + + err = func() error { + for _, file := range archive.File { + if strings.HasSuffix(file.Name, "/") { + continue + } + ext := strings.ToLower(filepath.Ext(file.Name)) + + switch ext { + case ".nuspec", ".xml", ".psmdcp", ".rels", ".p7s": + continue + case ".pdb": + f, err := archive.Open(file.Name) + if err != nil { + return err + } + + buf, err := packages.CreateHashedBufferFromReader(f) + + f.Close() + + if err != nil { + return err + } + + id, err := ParseDebugHeaderID(buf) + if err != nil { + buf.Close() + return fmt.Errorf("Invalid PDB file: %w", err) + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + buf.Close() + return err + } + + pdbs = append(pdbs, &PortablePdb{ + Name: path.Base(file.Name), + ID: id, + Content: buf, + }) + default: + return ErrInvalidFiles + } + } + return nil + }() + if err != nil { + pdbs.Close() + return nil, err + } + + if len(pdbs) == 0 { + return nil, ErrMissingPdbFiles + } + + return pdbs, nil +} + +// ParseDebugHeaderID TODO +func ParseDebugHeaderID(r io.ReadSeeker) (string, error) { + var magic uint32 + if err := binary.Read(r, binary.LittleEndian, &magic); err != nil { + return "", err + } + if magic != 0x424A5342 { + return "", ErrInvalidPdbMagicNumber + } + + if _, err := r.Seek(8, io.SeekCurrent); err != nil { + return "", err + } + + var versionStringSize int32 + if err := binary.Read(r, binary.LittleEndian, &versionStringSize); err != nil { + return "", err + } + if _, err := r.Seek(int64(versionStringSize), io.SeekCurrent); err != nil { + return "", err + } + if _, err := r.Seek(2, io.SeekCurrent); err != nil { + return "", err + } + + var streamCount int16 + if err := binary.Read(r, binary.LittleEndian, &streamCount); err != nil { + return "", err + } + + read4ByteAlignedString := func(r io.Reader) (string, error) { + b := make([]byte, 4) + var buf bytes.Buffer + for { + if _, err := r.Read(b); err != nil { + return "", err + } + if i := bytes.IndexByte(b, 0); i != -1 { + buf.Write(b[:i]) + return buf.String(), nil + } + buf.Write(b) + } + } + + for i := 0; i < int(streamCount); i++ { + var offset uint32 + if err := binary.Read(r, binary.LittleEndian, &offset); err != nil { + return "", err + } + if _, err := r.Seek(4, io.SeekCurrent); err != nil { + return "", err + } + name, err := read4ByteAlignedString(r) + if err != nil { + return "", err + } + + if name == "#Pdb" { + if _, err := r.Seek(int64(offset), io.SeekStart); err != nil { + return "", err + } + + b := make([]byte, 16) + if _, err := r.Read(b); err != nil { + return "", err + } + + data1 := binary.LittleEndian.Uint32(b[0:4]) + data2 := binary.LittleEndian.Uint16(b[4:6]) + data3 := binary.LittleEndian.Uint16(b[6:8]) + data4 := b[8:16] + + return fmt.Sprintf("%08x%04x%04x%04x%012x", data1, data2, data3, data4[:2], data4[2:]), nil + } + } + + return "", ErrMissingPdbStream +} diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go new file mode 100644 index 0000000..b767ed0 --- /dev/null +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const pdbContent = `QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAABgB8AAAAWAAAACNQZGIAAAAA1AAAAAgBAAAj +fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB +AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` + +func TestExtractPortablePdb(t *testing.T) { + createArchive := func(name string, content []byte) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(name) + w.Write(content) + archive.Close() + return buf.Bytes() + } + + t.Run("MissingPdbFiles", func(t *testing.T) { + var buf bytes.Buffer + zip.NewWriter(&buf).Close() + + pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.ErrorIs(t, err, ErrMissingPdbFiles) + assert.Empty(t, pdbs) + }) + + t.Run("InvalidFiles", func(t *testing.T) { + data := createArchive("sub/test.bin", []byte{}) + + pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data))) + require.ErrorIs(t, err, ErrInvalidFiles) + assert.Empty(t, pdbs) + }) + + t.Run("Valid", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(pdbContent) + data := createArchive("test.pdb", b) + + pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.Len(t, pdbs, 1) + assert.Equal(t, "test.pdb", pdbs[0].Name) + assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", pdbs[0].ID) + pdbs.Close() + }) +} + +func TestParseDebugHeaderID(t *testing.T) { + t.Run("InvalidPdbMagicNumber", func(t *testing.T) { + id, err := ParseDebugHeaderID(bytes.NewReader([]byte{0, 0, 0, 0})) + require.ErrorIs(t, err, ErrInvalidPdbMagicNumber) + assert.Empty(t, id) + }) + + t.Run("MissingPdbStream", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(`QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAAAQB8AAAAWAAAACNVUwA=`) + + id, err := ParseDebugHeaderID(bytes.NewReader(b)) + require.ErrorIs(t, err, ErrMissingPdbStream) + assert.Empty(t, id) + }) + + t.Run("Valid", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(pdbContent) + + id, err := ParseDebugHeaderID(bytes.NewReader(b)) + require.NoError(t, err) + assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", id) + }) +} -- cgit v1.2.3