diff options
Diffstat (limited to 'modules/packages')
56 files changed, 8355 insertions, 0 deletions
diff --git a/modules/packages/alpine/metadata.go b/modules/packages/alpine/metadata.go new file mode 100644 index 0000000..582c426 --- /dev/null +++ b/modules/packages/alpine/metadata.go @@ -0,0 +1,242 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package alpine + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "crypto/sha1" + "encoding/base64" + "io" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" +) + +var ( + ErrMissingPKGINFOFile = util.NewInvalidArgumentErrorf("PKGINFO file is missing") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") +) + +const ( + PropertyMetadata = "alpine.metadata" + PropertyBranch = "alpine.branch" + PropertyRepository = "alpine.repository" + PropertyArchitecture = "alpine.architecture" + + SettingKeyPrivate = "alpine.key.private" + SettingKeyPublic = "alpine.key.public" + + RepositoryPackage = "_alpine" + RepositoryVersion = "_repository" +) + +// https://wiki.alpinelinux.org/wiki/Apk_spec + +// Package represents an Alpine package +type Package struct { + Name string + Version string + VersionMetadata VersionMetadata + FileMetadata FileMetadata +} + +// Metadata of an Alpine package +type VersionMetadata struct { + Description string `json:"description,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Maintainer string `json:"maintainer,omitempty"` +} + +type FileMetadata struct { + Checksum string `json:"checksum"` + Packager string `json:"packager,omitempty"` + BuildDate int64 `json:"build_date,omitempty"` + Size int64 `json:"size,omitempty"` + Architecture string `json:"architecture,omitempty"` + Origin string `json:"origin,omitempty"` + CommitHash string `json:"commit_hash,omitempty"` + InstallIf string `json:"install_if,omitempty"` + Provides []string `json:"provides,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` + ProviderPriority int64 `json:"provider_priority,omitempty"` +} + +// ParsePackage parses the Alpine package file +func ParsePackage(r io.Reader) (*Package, error) { + // Alpine packages are concated .tar.gz streams. Usually the first stream contains the package metadata. + + br := bufio.NewReader(r) // needed for gzip Multistream + + h := sha1.New() + + gzr, err := gzip.NewReader(&teeByteReader{br, h}) + if err != nil { + return nil, err + } + defer gzr.Close() + + for { + gzr.Multistream(false) + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Name == ".PKGINFO" { + p, err := ParsePackageInfo(tr) + if err != nil { + return nil, err + } + + // drain the reader + for { + if _, err := tr.Next(); err != nil { + break + } + } + + p.FileMetadata.Checksum = "Q1" + base64.StdEncoding.EncodeToString(h.Sum(nil)) + + return p, nil + } + } + + h = sha1.New() + + err = gzr.Reset(&teeByteReader{br, h}) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + + return nil, ErrMissingPKGINFOFile +} + +// ParsePackageInfo parses a PKGINFO file to retrieve the metadata of an Alpine package +func ParsePackageInfo(r io.Reader) (*Package, error) { + p := &Package{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "#") { + continue + } + + i := strings.IndexRune(line, '=') + if i == -1 { + continue + } + + key := strings.TrimSpace(line[:i]) + value := strings.TrimSpace(line[i+1:]) + + switch key { + case "pkgname": + p.Name = value + case "pkgver": + p.Version = value + case "pkgdesc": + p.VersionMetadata.Description = value + case "url": + p.VersionMetadata.ProjectURL = value + case "builddate": + n, err := strconv.ParseInt(value, 10, 64) + if err == nil { + p.FileMetadata.BuildDate = n + } + case "size": + n, err := strconv.ParseInt(value, 10, 64) + if err == nil { + p.FileMetadata.Size = n + } + case "arch": + p.FileMetadata.Architecture = value + case "origin": + p.FileMetadata.Origin = value + case "commit": + p.FileMetadata.CommitHash = value + case "maintainer": + p.VersionMetadata.Maintainer = value + case "packager": + p.FileMetadata.Packager = value + case "license": + p.VersionMetadata.License = value + case "install_if": + p.FileMetadata.InstallIf = value + case "provides": + if value != "" { + p.FileMetadata.Provides = append(p.FileMetadata.Provides, value) + } + case "depend": + if value != "" { + p.FileMetadata.Dependencies = append(p.FileMetadata.Dependencies, value) + } + case "provider_priority": + n, err := strconv.ParseInt(value, 10, 64) + if err == nil { + p.FileMetadata.ProviderPriority = n + } + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + + if p.Name == "" { + return nil, ErrInvalidName + } + + if p.Version == "" { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + p.VersionMetadata.ProjectURL = "" + } + + return p, nil +} + +// Same as io.TeeReader but implements io.ByteReader +type teeByteReader struct { + r *bufio.Reader + w io.Writer +} + +func (t *teeByteReader) Read(p []byte) (int, error) { + n, err := t.r.Read(p) + if n > 0 { + if n, err := t.w.Write(p[:n]); err != nil { + return n, err + } + } + return n, err +} + +func (t *teeByteReader) ReadByte() (byte, error) { + b, err := t.r.ReadByte() + if err == nil { + if _, err := t.w.Write([]byte{b}); err != nil { + return 0, err + } + } + return b, err +} diff --git a/modules/packages/alpine/metadata_test.go b/modules/packages/alpine/metadata_test.go new file mode 100644 index 0000000..8167b49 --- /dev/null +++ b/modules/packages/alpine/metadata_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package alpine + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + packageDescription = "Package Description" + packageProjectURL = "https://gitea.io" + packageMaintainer = "KN4CK3R <dummy@gitea.io>" +) + +func createPKGINFOContent(name, version string) []byte { + return []byte(`pkgname = ` + name + ` +pkgver = ` + version + ` +pkgdesc = ` + packageDescription + ` +url = ` + packageProjectURL + ` +# comment +builddate = 1678834800 +packager = Gitea <pack@ag.er> +size = 123456 +arch = aarch64 +origin = origin +commit = 1111e709613fbc979651b09ac2bc27c6591a9999 +maintainer = ` + packageMaintainer + ` +license = MIT +depend = common +install_if = value +depend = gitea +provides = common +provides = gitea`) +} + +func TestParsePackage(t *testing.T) { + createPackage := func(name string, content []byte) io.Reader { + names := []string{"first.stream", name} + contents := [][]byte{{0}, content} + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + + for i := range names { + if i != 0 { + zw.Close() + zw.Reset(&buf) + } + + tw := tar.NewWriter(zw) + hdr := &tar.Header{ + Name: names[i], + Mode: 0o600, + Size: int64(len(contents[i])), + } + tw.WriteHeader(hdr) + tw.Write(contents[i]) + tw.Close() + } + + zw.Close() + + return &buf + } + + t.Run("MissingPKGINFOFile", func(t *testing.T) { + data := createPackage("dummy.txt", []byte{}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + require.ErrorIs(t, err, ErrMissingPKGINFOFile) + }) + + t.Run("InvalidPKGINFOFile", func(t *testing.T) { + data := createPackage(".PKGINFO", []byte{}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + require.ErrorIs(t, err, ErrInvalidName) + }) + + t.Run("Valid", func(t *testing.T) { + data := createPackage(".PKGINFO", createPKGINFOContent(packageName, packageVersion)) + + p, err := ParsePackage(data) + require.NoError(t, err) + assert.NotNil(t, p) + + assert.Equal(t, "Q1SRYURM5+uQDqfHSwTnNIOIuuDVQ=", p.FileMetadata.Checksum) + }) +} + +func TestParsePackageInfo(t *testing.T) { + t.Run("InvalidName", func(t *testing.T) { + data := createPKGINFOContent("", packageVersion) + + p, err := ParsePackageInfo(bytes.NewReader(data)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidName) + }) + + t.Run("InvalidVersion", func(t *testing.T) { + data := createPKGINFOContent(packageName, "") + + p, err := ParsePackageInfo(bytes.NewReader(data)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidVersion) + }) + + t.Run("Valid", func(t *testing.T) { + data := createPKGINFOContent(packageName, packageVersion) + + p, err := ParsePackageInfo(bytes.NewReader(data)) + require.NoError(t, err) + assert.NotNil(t, p) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, packageDescription, p.VersionMetadata.Description) + assert.Equal(t, packageMaintainer, p.VersionMetadata.Maintainer) + assert.Equal(t, packageProjectURL, p.VersionMetadata.ProjectURL) + assert.Equal(t, "MIT", p.VersionMetadata.License) + assert.Empty(t, p.FileMetadata.Checksum) + assert.Equal(t, "Gitea <pack@ag.er>", p.FileMetadata.Packager) + assert.EqualValues(t, 1678834800, p.FileMetadata.BuildDate) + assert.EqualValues(t, 123456, p.FileMetadata.Size) + assert.Equal(t, "aarch64", p.FileMetadata.Architecture) + assert.Equal(t, "origin", p.FileMetadata.Origin) + assert.Equal(t, "1111e709613fbc979651b09ac2bc27c6591a9999", p.FileMetadata.CommitHash) + assert.Equal(t, "value", p.FileMetadata.InstallIf) + assert.ElementsMatch(t, []string{"common", "gitea"}, p.FileMetadata.Provides) + assert.ElementsMatch(t, []string{"common", "gitea"}, p.FileMetadata.Dependencies) + }) +} diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go new file mode 100644 index 0000000..6cdde75 --- /dev/null +++ b/modules/packages/arch/metadata.go @@ -0,0 +1,341 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bufio" + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/mholt/archiver/v3" +) + +// Arch Linux Packages +// https://man.archlinux.org/man/PKGBUILD.5 + +const ( + PropertyDescription = "arch.description" + PropertyArch = "arch.architecture" + PropertyDistribution = "arch.distribution" + + SettingKeyPrivate = "arch.key.private" + SettingKeyPublic = "arch.key.public" + + RepositoryPackage = "_arch" + RepositoryVersion = "_repository" +) + +var ( + reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) + reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?(:.*)?$`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?$`) + + magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} + magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} + magicGZ = []byte{0x1F, 0x8B} +) + +type Package struct { + Name string `json:"name"` + Version string `json:"version"` // Includes version, release and epoch + CompressType string `json:"compress_type"` + VersionMetadata VersionMetadata + FileMetadata FileMetadata +} + +// Arch package metadata related to specific version. +// Version metadata the same across different architectures and distributions. +type VersionMetadata struct { + Base string `json:"base"` + Description string `json:"description"` + ProjectURL string `json:"project_url"` + Groups []string `json:"groups,omitempty"` + Provides []string `json:"provides,omitempty"` + License []string `json:"license,omitempty"` + Depends []string `json:"depends,omitempty"` + OptDepends []string `json:"opt_depends,omitempty"` + MakeDepends []string `json:"make_depends,omitempty"` + CheckDepends []string `json:"check_depends,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Replaces []string `json:"replaces,omitempty"` + Backup []string `json:"backup,omitempty"` + XData []string `json:"xdata,omitempty"` +} + +// FileMetadata Metadata related to specific package file. +// This metadata might vary for different architecture and distribution. +type FileMetadata struct { + CompressedSize int64 `json:"compressed_size"` + InstalledSize int64 `json:"installed_size"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + BuildDate int64 `json:"build_date"` + Packager string `json:"packager"` + Arch string `json:"arch"` + PgpSigned string `json:"pgp"` +} + +// ParsePackage Function that receives arch package archive data and returns it's metadata. +func ParsePackage(r *packages.HashedBuffer) (*Package, error) { + md5, _, sha256, _ := r.Sums() + _, err := r.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + header := make([]byte, 5) + _, err = r.Read(header) + if err != nil { + return nil, err + } + _, err = r.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + var tarball archiver.Reader + var tarballType string + if bytes.Equal(header[:len(magicZSTD)], magicZSTD) { + tarballType = "zst" + tarball = archiver.NewTarZstd() + } else if bytes.Equal(header[:len(magicXZ)], magicXZ) { + tarballType = "xz" + tarball = archiver.NewTarXz() + } else if bytes.Equal(header[:len(magicGZ)], magicGZ) { + tarballType = "gz" + tarball = archiver.NewTarGz() + } else { + return nil, errors.New("not supported compression") + } + err = tarball.Open(r, 0) + if err != nil { + return nil, err + } + defer tarball.Close() + + var pkg *Package + var mTree bool + + for { + f, err := tarball.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + switch f.Name() { + case ".PKGINFO": + pkg, err = ParsePackageInfo(tarballType, f) + if err != nil { + _ = f.Close() + return nil, err + } + case ".MTREE": + mTree = true + } + _ = f.Close() + } + + if pkg == nil { + return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found") + } + + if !mTree { + return nil, util.NewInvalidArgumentErrorf(".MTREE file not found") + } + + pkg.FileMetadata.CompressedSize = r.Size() + pkg.FileMetadata.MD5 = hex.EncodeToString(md5) + pkg.FileMetadata.SHA256 = hex.EncodeToString(sha256) + + return pkg, nil +} + +// ParsePackageInfo Function that accepts reader for .PKGINFO file from package archive, +// validates all field according to PKGBUILD spec and returns package. +func ParsePackageInfo(compressType string, r io.Reader) (*Package, error) { + p := &Package{ + CompressType: compressType, + } + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "#") { + continue + } + + key, value, find := strings.Cut(line, "=") + if !find { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch key { + case "pkgname": + p.Name = value + case "pkgbase": + p.VersionMetadata.Base = value + case "pkgver": + p.Version = value + case "pkgdesc": + p.VersionMetadata.Description = value + case "url": + p.VersionMetadata.ProjectURL = value + case "packager": + p.FileMetadata.Packager = value + case "arch": + p.FileMetadata.Arch = value + case "provides": + p.VersionMetadata.Provides = append(p.VersionMetadata.Provides, value) + case "license": + p.VersionMetadata.License = append(p.VersionMetadata.License, value) + case "depend": + p.VersionMetadata.Depends = append(p.VersionMetadata.Depends, value) + case "optdepend": + p.VersionMetadata.OptDepends = append(p.VersionMetadata.OptDepends, value) + case "makedepend": + p.VersionMetadata.MakeDepends = append(p.VersionMetadata.MakeDepends, value) + case "checkdepend": + p.VersionMetadata.CheckDepends = append(p.VersionMetadata.CheckDepends, value) + case "backup": + p.VersionMetadata.Backup = append(p.VersionMetadata.Backup, value) + case "group": + p.VersionMetadata.Groups = append(p.VersionMetadata.Groups, value) + case "conflict": + p.VersionMetadata.Conflicts = append(p.VersionMetadata.Conflicts, value) + case "replaces": + p.VersionMetadata.Replaces = append(p.VersionMetadata.Replaces, value) + case "xdata": + p.VersionMetadata.XData = append(p.VersionMetadata.XData, value) + case "builddate": + bd, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + p.FileMetadata.BuildDate = bd + case "size": + is, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + p.FileMetadata.InstalledSize = is + default: + return nil, util.NewInvalidArgumentErrorf("property is not supported %s", key) + } + } + + return p, errors.Join(scanner.Err(), ValidatePackageSpec(p)) +} + +// ValidatePackageSpec Arch package validation according to PKGBUILD specification. +func ValidatePackageSpec(p *Package) error { + if !reName.MatchString(p.Name) { + return util.NewInvalidArgumentErrorf("invalid package name") + } + if !reName.MatchString(p.VersionMetadata.Base) { + return util.NewInvalidArgumentErrorf("invalid package base") + } + if !reVer.MatchString(p.Version) { + return util.NewInvalidArgumentErrorf("invalid package version") + } + if p.FileMetadata.Arch == "" { + return util.NewInvalidArgumentErrorf("architecture should be specified") + } + if p.VersionMetadata.ProjectURL != "" { + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + return util.NewInvalidArgumentErrorf("invalid project URL") + } + } + for _, checkDepend := range p.VersionMetadata.CheckDepends { + if !rePkgVer.MatchString(checkDepend) { + return util.NewInvalidArgumentErrorf("invalid check dependency: %s", checkDepend) + } + } + for _, depend := range p.VersionMetadata.Depends { + if !rePkgVer.MatchString(depend) { + return util.NewInvalidArgumentErrorf("invalid dependency: %s", depend) + } + } + for _, makeDepend := range p.VersionMetadata.MakeDepends { + if !rePkgVer.MatchString(makeDepend) { + return util.NewInvalidArgumentErrorf("invalid make dependency: %s", makeDepend) + } + } + for _, provide := range p.VersionMetadata.Provides { + if !rePkgVer.MatchString(provide) { + return util.NewInvalidArgumentErrorf("invalid provides: %s", provide) + } + } + for _, conflict := range p.VersionMetadata.Conflicts { + if !rePkgVer.MatchString(conflict) { + return util.NewInvalidArgumentErrorf("invalid conflicts: %s", conflict) + } + } + for _, replace := range p.VersionMetadata.Replaces { + if !rePkgVer.MatchString(replace) { + return util.NewInvalidArgumentErrorf("invalid replaces: %s", replace) + } + } + for _, optDepend := range p.VersionMetadata.OptDepends { + if !reOptDep.MatchString(optDepend) { + return util.NewInvalidArgumentErrorf("invalid optional dependency: %s", optDepend) + } + } + for _, b := range p.VersionMetadata.Backup { + if strings.HasPrefix(b, "/") { + return util.NewInvalidArgumentErrorf("backup file contains leading forward slash") + } + } + return nil +} + +// Desc Create pacman package description file. +func (p *Package) Desc() string { + entries := []string{ + "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.%s", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType), + "NAME", p.Name, + "BASE", p.VersionMetadata.Base, + "VERSION", p.Version, + "DESC", p.VersionMetadata.Description, + "GROUPS", strings.Join(p.VersionMetadata.Groups, "\n"), + "CSIZE", fmt.Sprintf("%d", p.FileMetadata.CompressedSize), + "ISIZE", fmt.Sprintf("%d", p.FileMetadata.InstalledSize), + "MD5SUM", p.FileMetadata.MD5, + "SHA256SUM", p.FileMetadata.SHA256, + "PGPSIG", p.FileMetadata.PgpSigned, + "URL", p.VersionMetadata.ProjectURL, + "LICENSE", strings.Join(p.VersionMetadata.License, "\n"), + "ARCH", p.FileMetadata.Arch, + "BUILDDATE", fmt.Sprintf("%d", p.FileMetadata.BuildDate), + "PACKAGER", p.FileMetadata.Packager, + "REPLACES", strings.Join(p.VersionMetadata.Replaces, "\n"), + "CONFLICTS", strings.Join(p.VersionMetadata.Conflicts, "\n"), + "PROVIDES", strings.Join(p.VersionMetadata.Provides, "\n"), + "DEPENDS", strings.Join(p.VersionMetadata.Depends, "\n"), + "OPTDEPENDS", strings.Join(p.VersionMetadata.OptDepends, "\n"), + "MAKEDEPENDS", strings.Join(p.VersionMetadata.MakeDepends, "\n"), + "CHECKDEPENDS", strings.Join(p.VersionMetadata.CheckDepends, "\n"), + } + + var buf bytes.Buffer + for i := 0; i < len(entries); i += 2 { + if entries[i+1] != "" { + _, _ = fmt.Fprintf(&buf, "%%%s%%\n%s\n\n", entries[i], entries[i+1]) + } + } + return buf.String() +} diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go new file mode 100644 index 0000000..ddb35ca --- /dev/null +++ b/modules/packages/arch/metadata_test.go @@ -0,0 +1,447 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bytes" + "errors" + "os" + "strings" + "testing" + "testing/fstest" + "time" + + "code.gitea.io/gitea/modules/packages" + + "github.com/mholt/archiver/v3" + "github.com/stretchr/testify/require" +) + +func TestParsePackage(t *testing.T) { + // Minimal PKGINFO contents and test FS + const PKGINFO = `pkgname = a +pkgbase = b +pkgver = 1-2 +arch = x86_64 +` + fs := fstest.MapFS{ + "pkginfo": &fstest.MapFile{ + Data: []byte(PKGINFO), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + "mtree": &fstest.MapFile{ + Data: []byte("data"), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + } + + // Test .PKGINFO file + pinf, err := fs.Stat("pkginfo") + require.NoError(t, err) + + pfile, err := fs.Open("pkginfo") + require.NoError(t, err) + + parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO") + require.NoError(t, err) + + // Test .MTREE file + minf, err := fs.Stat("mtree") + require.NoError(t, err) + + mfile, err := fs.Open("mtree") + require.NoError(t, err) + + marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE") + require.NoError(t, err) + + t.Run("normal archive", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + require.NoError(t, errors.Join(pfile.Close(), err)) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: minf, + CustomName: marcname, + }, + ReadCloser: mfile, + }) + require.NoError(t, errors.Join(mfile.Close(), archive.Close(), err)) + + reader, err := packages.CreateHashedBufferFromReader(&buf) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + _, err = ParsePackage(reader) + + require.NoError(t, err) + }) + + t.Run("missing .PKGINFO", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + require.NoError(t, archive.Close()) + + reader, err := packages.CreateHashedBufferFromReader(&buf) + require.NoError(t, err) + + defer reader.Close() + _, err = ParsePackage(reader) + + require.Error(t, err) + require.Contains(t, err.Error(), ".PKGINFO file not found") + }) + + t.Run("missing .MTREE", func(t *testing.T) { + var buf bytes.Buffer + + pfile, err := fs.Open("pkginfo") + require.NoError(t, err) + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + require.NoError(t, errors.Join(pfile.Close(), archive.Close(), err)) + reader, err := packages.CreateHashedBufferFromReader(&buf) + require.NoError(t, err) + + defer reader.Close() + _, err = ParsePackage(reader) + + require.Error(t, err) + require.Contains(t, err.Error(), ".MTREE file not found") + }) +} + +func TestParsePackageInfo(t *testing.T) { + const PKGINFO = `# Generated by makepkg 6.0.2 +# using fakeroot version 1.31 +pkgname = a +pkgbase = b +pkgver = 1-2 +pkgdesc = comment +url = https://example.com/ +group = group +builddate = 3 +packager = Name Surname <login@example.com> +size = 5 +arch = x86_64 +license = BSD +provides = pvd +depend = smth +optdepend = hex +checkdepend = ola +makedepend = cmake +backup = usr/bin/paket1 +` + p, err := ParsePackageInfo("zst", strings.NewReader(PKGINFO)) + require.NoError(t, err) + require.Equal(t, Package{ + CompressType: "zst", + Name: "a", + Version: "1-2", + VersionMetadata: VersionMetadata{ + Base: "b", + Description: "comment", + ProjectURL: "https://example.com/", + Groups: []string{"group"}, + Provides: []string{"pvd"}, + License: []string{"BSD"}, + Depends: []string{"smth"}, + OptDepends: []string{"hex"}, + MakeDepends: []string{"cmake"}, + CheckDepends: []string{"ola"}, + Backup: []string{"usr/bin/paket1"}, + }, + FileMetadata: FileMetadata{ + InstalledSize: 5, + BuildDate: 3, + Packager: "Name Surname <login@example.com>", + Arch: "x86_64", + }, + }, *p) +} + +func TestValidatePackageSpec(t *testing.T) { + newpkg := func() Package { + return Package{ + Name: "abc", + Version: "1-1", + VersionMetadata: VersionMetadata{ + Base: "ghx", + Description: "whoami", + ProjectURL: "https://example.com/", + Groups: []string{"gnome"}, + Provides: []string{"abc", "def"}, + License: []string{"GPL"}, + Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"}, + OptDepends: []string{"git", "libgcc=1.0", "gzip>1.0", "gz>=1.0", "lz<1.0", "gzip<=1.0", "zstd>1.0:foo bar<test>"}, + MakeDepends: []string{"chrom"}, + CheckDepends: []string{"bariy"}, + Backup: []string{"etc/pacman.d/filo"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 1, + InstalledSize: 2, + SHA256: "def", + BuildDate: 3, + Packager: "smon", + Arch: "x86_64", + }, + } + } + + t.Run("valid package", func(t *testing.T) { + p := newpkg() + + err := ValidatePackageSpec(&p) + + require.NoError(t, err) + }) + + t.Run("invalid package name", func(t *testing.T) { + p := newpkg() + p.Name = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package name") + }) + + t.Run("invalid package base", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "una-luna?" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.Version = "una-luna" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package version") + }) + + t.Run("missing architecture", func(t *testing.T) { + p := newpkg() + p.FileMetadata.Arch = "" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "architecture should be specified") + }) + + t.Run("invalid URL", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.ProjectURL = "http%%$#" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid project URL") + }) + + t.Run("invalid check dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.CheckDepends = []string{"Err^_^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid check dependency") + }) + + t.Run("invalid dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Depends = []string{"^^abc"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid dependency") + }) + + t.Run("invalid make dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.MakeDepends = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid make dependency") + }) + + t.Run("invalid provides", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Provides = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid provides") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.OptDepends = []string{"^m^:MM"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid optional dependency") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Backup = []string{"/ola/cola"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "backup file contains leading forward slash") + }) +} + +func TestDescString(t *testing.T) { + const pkgdesc = `%FILENAME% +zstd-1.5.5-1-x86_64.pkg.tar.zst + +%NAME% +zstd + +%BASE% +zstd + +%VERSION% +1.5.5-1 + +%DESC% +Zstandard - Fast real-time compression algorithm + +%GROUPS% +dummy1 +dummy2 + +%CSIZE% +401 + +%ISIZE% +1500453 + +%MD5SUM% +5016660ef3d9aa148a7b72a08d3df1b2 + +%SHA256SUM% +9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd + +%URL% +https://facebook.github.io/zstd/ + +%LICENSE% +BSD +GPL2 + +%ARCH% +x86_64 + +%BUILDDATE% +1681646714 + +%PACKAGER% +Jelle van der Waa <jelle@archlinux.org> + +%PROVIDES% +libzstd.so=1-64 + +%DEPENDS% +glibc +gcc-libs +zlib +xz +lz4 + +%OPTDEPENDS% +dummy3 +dummy4 + +%MAKEDEPENDS% +cmake +gtest +ninja + +%CHECKDEPENDS% +dummy5 +dummy6 + +` + + md := &Package{ + CompressType: "zst", + Name: "zstd", + Version: "1.5.5-1", + VersionMetadata: VersionMetadata{ + Base: "zstd", + Description: "Zstandard - Fast real-time compression algorithm", + ProjectURL: "https://facebook.github.io/zstd/", + Groups: []string{"dummy1", "dummy2"}, + Provides: []string{"libzstd.so=1-64"}, + License: []string{"BSD", "GPL2"}, + Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}, + OptDepends: []string{"dummy3", "dummy4"}, + MakeDepends: []string{"cmake", "gtest", "ninja"}, + CheckDepends: []string{"dummy5", "dummy6"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 401, + InstalledSize: 1500453, + MD5: "5016660ef3d9aa148a7b72a08d3df1b2", + SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd", + BuildDate: 1681646714, + Packager: "Jelle van der Waa <jelle@archlinux.org>", + Arch: "x86_64", + }, + } + require.Equal(t, pkgdesc, md.Desc()) +} diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go new file mode 100644 index 0000000..36cd44d --- /dev/null +++ b/modules/packages/cargo/parser.go @@ -0,0 +1,169 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "encoding/binary" + "errors" + "io" + "regexp" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +const PropertyYanked = "cargo.yanked" + +var ( + ErrInvalidName = errors.New("package name is invalid") + ErrInvalidVersion = errors.New("package version is invalid") +) + +// Package represents a Cargo package +type Package struct { + Name string + Version string + Metadata *Metadata + Content io.Reader + ContentSize int64 +} + +// Metadata represents the metadata of a Cargo package +type Metadata struct { + Dependencies []*Dependency `json:"dependencies,omitempty"` + Features map[string][]string `json:"features,omitempty"` + Authors []string `json:"authors,omitempty"` + Description string `json:"description,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Readme string `json:"readme,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Categories []string `json:"categories,omitempty"` + License string `json:"license,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Links string `json:"links,omitempty"` +} + +type Dependency struct { + Name string `json:"name"` + Req string `json:"req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry"` + Package *string `json:"package"` +} + +var nameMatch = regexp.MustCompile(`\A[a-zA-Z][a-zA-Z0-9-_]{0,63}\z`) + +// ParsePackage reads the metadata and content of a package +func ParsePackage(r io.Reader) (*Package, error) { + var size uint32 + if err := binary.Read(r, binary.LittleEndian, &size); err != nil { + return nil, err + } + + p, err := parsePackage(io.LimitReader(r, int64(size))) + if err != nil { + return nil, err + } + + if err := binary.Read(r, binary.LittleEndian, &size); err != nil { + return nil, err + } + + p.Content = io.LimitReader(r, int64(size)) + p.ContentSize = int64(size) + + return p, nil +} + +func parsePackage(r io.Reader) (*Package, error) { + var meta struct { + Name string `json:"name"` + Vers string `json:"vers"` + Deps []struct { + Name string `json:"name"` + VersionReq string `json:"version_req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry"` + ExplicitNameInToml string `json:"explicit_name_in_toml"` + } `json:"deps"` + Features map[string][]string `json:"features"` + Authors []string `json:"authors"` + Description string `json:"description"` + Documentation string `json:"documentation"` + Homepage string `json:"homepage"` + Readme string `json:"readme"` + ReadmeFile string `json:"readme_file"` + Keywords []string `json:"keywords"` + Categories []string `json:"categories"` + License string `json:"license"` + LicenseFile string `json:"license_file"` + Repository string `json:"repository"` + Links string `json:"links"` + } + if err := json.NewDecoder(r).Decode(&meta); err != nil { + return nil, err + } + + if !nameMatch.MatchString(meta.Name) { + return nil, ErrInvalidName + } + + if _, err := version.NewSemver(meta.Vers); err != nil { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(meta.Homepage) { + meta.Homepage = "" + } + if !validation.IsValidURL(meta.Documentation) { + meta.Documentation = "" + } + if !validation.IsValidURL(meta.Repository) { + meta.Repository = "" + } + + dependencies := make([]*Dependency, 0, len(meta.Deps)) + for _, dep := range meta.Deps { + dependencies = append(dependencies, &Dependency{ + Name: dep.Name, + Req: dep.VersionReq, + Features: dep.Features, + Optional: dep.Optional, + DefaultFeatures: dep.DefaultFeatures, + Target: dep.Target, + Kind: dep.Kind, + Registry: dep.Registry, + }) + } + + return &Package{ + Name: meta.Name, + Version: meta.Vers, + Metadata: &Metadata{ + Dependencies: dependencies, + Features: meta.Features, + Authors: meta.Authors, + Description: meta.Description, + DocumentationURL: meta.Documentation, + ProjectURL: meta.Homepage, + Readme: meta.Readme, + Keywords: meta.Keywords, + Categories: meta.Categories, + License: meta.License, + RepositoryURL: meta.Repository, + Links: meta.Links, + }, + }, nil +} diff --git a/modules/packages/cargo/parser_test.go b/modules/packages/cargo/parser_test.go new file mode 100644 index 0000000..4b357cb --- /dev/null +++ b/modules/packages/cargo/parser_test.go @@ -0,0 +1,87 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cargo + +import ( + "bytes" + "encoding/binary" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + description = "Package Description" + author = "KN4CK3R" + homepage = "https://gitea.io/" + license = "MIT" +) + +func TestParsePackage(t *testing.T) { + createPackage := func(name, version string) io.Reader { + metadata := `{ + "name":"` + name + `", + "vers":"` + version + `", + "description":"` + description + `", + "authors": ["` + author + `"], + "deps":[ + { + "name":"dep", + "version_req":"1.0" + } + ], + "homepage":"` + homepage + `", + "license":"` + license + `" +}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("test") + return &buf + } + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "0test", "-test", "_test", strings.Repeat("a", 65)} { + data := createPackage(name, "1.0.0") + + cp, err := ParsePackage(data) + assert.Nil(t, cp) + require.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1.", "-1.0", "1.0.0/1"} { + data := createPackage("test", version) + + cp, err := ParsePackage(data) + assert.Nil(t, cp) + require.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + data := createPackage("test", "1.0.0") + + cp, err := ParsePackage(data) + assert.NotNil(t, cp) + require.NoError(t, err) + + assert.Equal(t, "test", cp.Name) + assert.Equal(t, "1.0.0", cp.Version) + assert.Equal(t, description, cp.Metadata.Description) + assert.Equal(t, []string{author}, cp.Metadata.Authors) + assert.Len(t, cp.Metadata.Dependencies, 1) + assert.Equal(t, "dep", cp.Metadata.Dependencies[0].Name) + assert.Equal(t, homepage, cp.Metadata.ProjectURL) + assert.Equal(t, license, cp.Metadata.License) + content, _ := io.ReadAll(cp.Content) + assert.Equal(t, "test", string(content)) + }) +} diff --git a/modules/packages/chef/metadata.go b/modules/packages/chef/metadata.go new file mode 100644 index 0000000..a1c9187 --- /dev/null +++ b/modules/packages/chef/metadata.go @@ -0,0 +1,134 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package chef + +import ( + "archive/tar" + "compress/gzip" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" +) + +const ( + KeyBits = 4096 + SettingPublicPem = "chef.public_pem" +) + +var ( + ErrMissingMetadataFile = util.NewInvalidArgumentErrorf("metadata.json file is missing") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") + + namePattern = regexp.MustCompile(`\A\S+\z`) + versionPattern = regexp.MustCompile(`\A\d+\.\d+(?:\.\d+)?\z`) +) + +// Package represents a Chef package +type Package struct { + Name string + Version string + Metadata *Metadata +} + +// Metadata represents the metadata of a Chef package +type Metadata struct { + Description string `json:"description,omitempty"` + LongDescription string `json:"long_description,omitempty"` + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` +} + +type chefMetadata struct { + Name string `json:"name"` + Description string `json:"description"` + LongDescription string `json:"long_description"` + Maintainer string `json:"maintainer"` + MaintainerEmail string `json:"maintainer_email"` + License string `json:"license"` + Platforms map[string]string `json:"platforms"` + Dependencies map[string]string `json:"dependencies"` + Providing map[string]string `json:"providing"` + Recipes map[string]string `json:"recipes"` + Version string `json:"version"` + SourceURL string `json:"source_url"` + IssuesURL string `json:"issues_url"` + Privacy bool `json:"privacy"` + ChefVersions [][]string `json:"chef_versions"` + Gems [][]string `json:"gems"` + EagerLoadLibraries bool `json:"eager_load_libraries"` +} + +// ParsePackage parses the Chef package file +func ParsePackage(r io.Reader) (*Package, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if strings.Count(hd.Name, "/") != 1 { + continue + } + + if hd.FileInfo().Name() == "metadata.json" { + return ParseChefMetadata(tr) + } + } + + return nil, ErrMissingMetadataFile +} + +// ParseChefMetadata parses a metadata.json file to retrieve the metadata of a Chef package +func ParseChefMetadata(r io.Reader) (*Package, error) { + var cm chefMetadata + if err := json.NewDecoder(r).Decode(&cm); err != nil { + return nil, err + } + + if !namePattern.MatchString(cm.Name) { + return nil, ErrInvalidName + } + + if !versionPattern.MatchString(cm.Version) { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(cm.SourceURL) { + cm.SourceURL = "" + } + + return &Package{ + Name: cm.Name, + Version: cm.Version, + Metadata: &Metadata{ + Description: cm.Description, + LongDescription: cm.LongDescription, + Author: cm.Maintainer, + License: cm.License, + RepositoryURL: cm.SourceURL, + Dependencies: cm.Dependencies, + }, + }, nil +} diff --git a/modules/packages/chef/metadata_test.go b/modules/packages/chef/metadata_test.go new file mode 100644 index 0000000..8784c62 --- /dev/null +++ b/modules/packages/chef/metadata_test.go @@ -0,0 +1,93 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package chef + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + packageAuthor = "KN4CK3R" + packageDescription = "Package Description" + packageRepositoryURL = "https://gitea.io/gitea/gitea" +) + +func TestParsePackage(t *testing.T) { + t.Run("MissingMetadataFile", func(t *testing.T) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + tw.Close() + zw.Close() + + p, err := ParsePackage(&buf) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrMissingMetadataFile) + }) + + t.Run("Valid", func(t *testing.T) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + + content := `{"name":"` + packageName + `","version":"` + packageVersion + `"}` + + hdr := &tar.Header{ + Name: packageName + "/metadata.json", + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write([]byte(content)) + + tw.Close() + zw.Close() + + p, err := ParsePackage(&buf) + require.NoError(t, err) + assert.NotNil(t, p) + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.NotNil(t, p.Metadata) + }) +} + +func TestParseChefMetadata(t *testing.T) { + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{" test", "test "} { + p, err := ParseChefMetadata(strings.NewReader(`{"name":"` + name + `","version":"1.0.0"}`)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"1", "1.2.3.4", "1.0.0 "} { + p, err := ParseChefMetadata(strings.NewReader(`{"name":"test","version":"` + version + `"}`)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + p, err := ParseChefMetadata(strings.NewReader(`{"name":"` + packageName + `","version":"` + packageVersion + `","description":"` + packageDescription + `","maintainer":"` + packageAuthor + `","source_url":"` + packageRepositoryURL + `"}`)) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, packageDescription, p.Metadata.Description) + assert.Equal(t, packageAuthor, p.Metadata.Author) + assert.Equal(t, packageRepositoryURL, p.Metadata.RepositoryURL) + }) +} diff --git a/modules/packages/composer/metadata.go b/modules/packages/composer/metadata.go new file mode 100644 index 0000000..6035eae --- /dev/null +++ b/modules/packages/composer/metadata.go @@ -0,0 +1,187 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package composer + +import ( + "archive/zip" + "io" + "path" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +// TypeProperty is the name of the property for Composer package types +const TypeProperty = "composer.type" + +var ( + // ErrMissingComposerFile indicates a missing composer.json file + ErrMissingComposerFile = util.NewInvalidArgumentErrorf("composer.json file is missing") + // ErrInvalidName indicates an invalid package name + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + // ErrInvalidVersion indicates an invalid package version + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") +) + +// Package represents a Composer package +type Package struct { + Name string + Version string + Type string + Metadata *Metadata +} + +// https://getcomposer.org/doc/04-schema.md + +// Metadata represents the metadata of a Composer package +type Metadata struct { + Description string `json:"description,omitempty"` + Readme string `json:"readme,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Comments Comments `json:"_comments,omitempty"` + Homepage string `json:"homepage,omitempty"` + License Licenses `json:"license,omitempty"` + Authors []Author `json:"authors,omitempty"` + Bin []string `json:"bin,omitempty"` + Autoload map[string]any `json:"autoload,omitempty"` + AutoloadDev map[string]any `json:"autoload-dev,omitempty"` + Extra map[string]any `json:"extra,omitempty"` + Require map[string]string `json:"require,omitempty"` + RequireDev map[string]string `json:"require-dev,omitempty"` + Suggest map[string]string `json:"suggest,omitempty"` + Provide map[string]string `json:"provide,omitempty"` +} + +// Licenses represents the licenses of a Composer package +type Licenses []string + +// UnmarshalJSON reads from a string or array +func (l *Licenses) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + *l = Licenses{value} + case '[': + values := make([]string, 0, 5) + if err := json.Unmarshal(data, &values); err != nil { + return err + } + *l = Licenses(values) + } + return nil +} + +// Comments represents the comments of a Composer package +type Comments []string + +// UnmarshalJSON reads from a string or array +func (c *Comments) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + *c = Comments{value} + case '[': + values := make([]string, 0, 5) + if err := json.Unmarshal(data, &values); err != nil { + return err + } + *c = Comments(values) + } + return nil +} + +// Author represents an author +type Author struct { + Name string `json:"name,omitempty"` + Email string `json:"email,omitempty"` + Homepage string `json:"homepage,omitempty"` +} + +var nameMatch = regexp.MustCompile(`\A[a-z0-9]([_\.-]?[a-z0-9]+)*/[a-z0-9](([_\.]?|-{0,2})[a-z0-9]+)*\z`) + +// ParsePackage parses the metadata of a Composer package file +func ParsePackage(r io.ReaderAt, size int64) (*Package, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range archive.File { + if strings.Count(file.Name, "/") > 1 { + continue + } + if strings.HasSuffix(strings.ToLower(file.Name), "composer.json") { + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseComposerFile(archive, path.Dir(file.Name), f) + } + } + return nil, ErrMissingComposerFile +} + +// ParseComposerFile parses a composer.json file to retrieve the metadata of a Composer package +func ParseComposerFile(archive *zip.Reader, pathPrefix string, r io.Reader) (*Package, error) { + var cj struct { + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + Metadata + } + if err := json.NewDecoder(r).Decode(&cj); err != nil { + return nil, err + } + + if !nameMatch.MatchString(cj.Name) { + return nil, ErrInvalidName + } + + if cj.Version != "" { + if _, err := version.NewSemver(cj.Version); err != nil { + return nil, ErrInvalidVersion + } + } + + if !validation.IsValidURL(cj.Homepage) { + cj.Homepage = "" + } + + if cj.Type == "" { + cj.Type = "library" + } + + if cj.Readme == "" { + cj.Readme = "README.md" + } + f, err := archive.Open(path.Join(pathPrefix, cj.Readme)) + if err == nil { + // 10kb limit for readme content + buf, _ := io.ReadAll(io.LimitReader(f, 10*1024)) + cj.Readme = string(buf) + _ = f.Close() + } else { + cj.Readme = "" + } + + return &Package{ + Name: cj.Name, + Version: cj.Version, + Type: cj.Type, + Metadata: &cj.Metadata, + }, nil +} diff --git a/modules/packages/composer/metadata_test.go b/modules/packages/composer/metadata_test.go new file mode 100644 index 0000000..2bdb239 --- /dev/null +++ b/modules/packages/composer/metadata_test.go @@ -0,0 +1,154 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package composer + +import ( + "archive/zip" + "bytes" + "strings" + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + name = "gitea/composer-package" + description = "Package Description" + readme = "Package Readme" + comments = "Package Comment" + packageType = "composer-plugin" + author = "Gitea Authors" + email = "no.reply@gitea.io" + homepage = "https://gitea.io" + license = "MIT" +) + +const composerContent = `{ + "name": "` + name + `", + "description": "` + description + `", + "type": "` + packageType + `", + "license": "` + license + `", + "authors": [ + { + "name": "` + author + `", + "email": "` + email + `" + } + ], + "homepage": "` + homepage + `", + "autoload": { + "psr-4": {"Gitea\\ComposerPackage\\": "src/"} + }, + "require": { + "php": ">=7.2 || ^8.0" + }, + "_comments": "` + comments + `" +}` + +func TestLicenseUnmarshal(t *testing.T) { + var l Licenses + require.NoError(t, json.NewDecoder(strings.NewReader(`["MIT"]`)).Decode(&l)) + assert.Len(t, l, 1) + assert.Equal(t, "MIT", l[0]) + require.NoError(t, json.NewDecoder(strings.NewReader(`"MIT"`)).Decode(&l)) + assert.Len(t, l, 1) + assert.Equal(t, "MIT", l[0]) +} + +func TestCommentsUnmarshal(t *testing.T) { + var c Comments + require.NoError(t, json.NewDecoder(strings.NewReader(`["comment"]`)).Decode(&c)) + assert.Len(t, c, 1) + assert.Equal(t, "comment", c[0]) + require.NoError(t, json.NewDecoder(strings.NewReader(`"comment"`)).Decode(&c)) + assert.Len(t, c, 1) + assert.Equal(t, "comment", c[0]) +} + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string]string) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + for name, content := range files { + w, _ := archive.Create(name) + w.Write([]byte(content)) + } + archive.Close() + return buf.Bytes() + } + + t.Run("MissingComposerFile", func(t *testing.T) { + data := createArchive(map[string]string{"dummy.txt": ""}) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + require.ErrorIs(t, err, ErrMissingComposerFile) + }) + + t.Run("MissingComposerFileInRoot", func(t *testing.T) { + data := createArchive(map[string]string{"sub/sub/composer.json": ""}) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + require.ErrorIs(t, err, ErrMissingComposerFile) + }) + + t.Run("InvalidComposerFile", func(t *testing.T) { + data := createArchive(map[string]string{"composer.json": ""}) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + require.Error(t, err) + }) + + t.Run("InvalidPackageName", func(t *testing.T) { + data := createArchive(map[string]string{"composer.json": "{}"}) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + require.ErrorIs(t, err, ErrInvalidName) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + data := createArchive(map[string]string{"composer.json": `{"name": "gitea/composer-package", "version": "1.a.3"}`}) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, cp) + require.ErrorIs(t, err, ErrInvalidVersion) + }) + + t.Run("InvalidReadmePath", func(t *testing.T) { + data := createArchive(map[string]string{"composer.json": `{"name": "gitea/composer-package", "readme": "sub/README.md"}`}) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, cp) + + assert.Empty(t, cp.Metadata.Readme) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive(map[string]string{"composer.json": composerContent, "README.md": readme}) + + cp, err := ParsePackage(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, cp) + + assert.Equal(t, name, cp.Name) + assert.Empty(t, cp.Version) + assert.Equal(t, description, cp.Metadata.Description) + assert.Equal(t, readme, cp.Metadata.Readme) + assert.Len(t, cp.Metadata.Comments, 1) + assert.Equal(t, comments, cp.Metadata.Comments[0]) + assert.Len(t, cp.Metadata.Authors, 1) + assert.Equal(t, author, cp.Metadata.Authors[0].Name) + assert.Equal(t, email, cp.Metadata.Authors[0].Email) + assert.Equal(t, homepage, cp.Metadata.Homepage) + assert.Equal(t, packageType, cp.Type) + assert.Len(t, cp.Metadata.License, 1) + assert.Equal(t, license, cp.Metadata.License[0]) + }) +} diff --git a/modules/packages/conan/conanfile_parser.go b/modules/packages/conan/conanfile_parser.go new file mode 100644 index 0000000..c47b242 --- /dev/null +++ b/modules/packages/conan/conanfile_parser.go @@ -0,0 +1,67 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conan + +import ( + "io" + "regexp" + "strings" +) + +var ( + patternAuthor = compilePattern("author") + patternHomepage = compilePattern("homepage") + patternURL = compilePattern("url") + patternLicense = compilePattern("license") + patternDescription = compilePattern("description") + patternTopics = regexp.MustCompile(`(?im)^\s*topics\s*=\s*\((.+)\)`) + patternTopicList = regexp.MustCompile(`\s*['"](.+?)['"]\s*,?`) +) + +func compilePattern(name string) *regexp.Regexp { + return regexp.MustCompile(`(?im)^\s*` + name + `\s*=\s*['"\(](.+)['"\)]`) +} + +func ParseConanfile(r io.Reader) (*Metadata, error) { + buf, err := io.ReadAll(io.LimitReader(r, 1<<20)) + if err != nil { + return nil, err + } + + metadata := &Metadata{} + + m := patternAuthor.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.Author = string(m[1]) + } + m = patternHomepage.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.ProjectURL = string(m[1]) + } + m = patternURL.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.RepositoryURL = string(m[1]) + } + m = patternLicense.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.License = strings.ReplaceAll(strings.ReplaceAll(string(m[1]), "'", ""), "\"", "") + } + m = patternDescription.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + metadata.Description = string(m[1]) + } + m = patternTopics.FindSubmatch(buf) + if len(m) > 1 && len(m[1]) > 0 { + m2 := patternTopicList.FindAllSubmatch(m[1], -1) + if len(m2) > 0 { + metadata.Keywords = make([]string, 0, len(m2)) + for _, g := range m2 { + if len(g) > 1 { + metadata.Keywords = append(metadata.Keywords, string(g[1])) + } + } + } + } + return metadata, nil +} diff --git a/modules/packages/conan/conanfile_parser_test.go b/modules/packages/conan/conanfile_parser_test.go new file mode 100644 index 0000000..fe867fb --- /dev/null +++ b/modules/packages/conan/conanfile_parser_test.go @@ -0,0 +1,51 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conan + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + name = "ConanPackage" + version = "1.2" + license = "MIT" + author = "Gitea <info@gitea.io>" + homepage = "https://gitea.io/" + url = "https://gitea.com/" + description = "Description of ConanPackage" + topic1 = "gitea" + topic2 = "conan" + contentConanfile = `from conans import ConanFile, CMake, tools + +class ConanPackageConan(ConanFile): + name = "` + name + `" + version = "` + version + `" + license = "` + license + `" + author = "` + author + `" + homepage = "` + homepage + `" + url = "` + url + `" + description = "` + description + `" + topics = ("` + topic1 + `", "` + topic2 + `") + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + generators = "cmake" +` +) + +func TestParseConanfile(t *testing.T) { + metadata, err := ParseConanfile(strings.NewReader(contentConanfile)) + require.NoError(t, err) + assert.Equal(t, license, metadata.License) + assert.Equal(t, author, metadata.Author) + assert.Equal(t, homepage, metadata.ProjectURL) + assert.Equal(t, url, metadata.RepositoryURL) + assert.Equal(t, description, metadata.Description) + assert.Equal(t, []string{topic1, topic2}, metadata.Keywords) +} diff --git a/modules/packages/conan/conaninfo_parser.go b/modules/packages/conan/conaninfo_parser.go new file mode 100644 index 0000000..de11dbe --- /dev/null +++ b/modules/packages/conan/conaninfo_parser.go @@ -0,0 +1,123 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conan + +import ( + "bufio" + "io" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +// Conaninfo represents infos of a Conan package +type Conaninfo struct { + Settings map[string]string `json:"settings"` + FullSettings map[string]string `json:"full_settings"` + Requires []string `json:"requires"` + FullRequires []string `json:"full_requires"` + Options map[string]string `json:"options"` + FullOptions map[string]string `json:"full_options"` + RecipeHash string `json:"recipe_hash"` + Environment map[string][]string `json:"environment"` +} + +func ParseConaninfo(r io.Reader) (*Conaninfo, error) { + sections, err := readSections(io.LimitReader(r, 1<<20)) + if err != nil { + return nil, err + } + + info := &Conaninfo{} + for section, lines := range sections { + if len(lines) == 0 { + continue + } + switch section { + case "settings": + info.Settings = toMap(lines) + case "full_settings": + info.FullSettings = toMap(lines) + case "options": + info.Options = toMap(lines) + case "full_options": + info.FullOptions = toMap(lines) + case "requires": + info.Requires = lines + case "full_requires": + info.FullRequires = lines + case "recipe_hash": + info.RecipeHash = lines[0] + case "env": + info.Environment = toMapArray(lines) + } + } + return info, nil +} + +func readSections(r io.Reader) (map[string][]string, error) { + sections := make(map[string][]string) + + section := "" + lines := make([]string, 0, 5) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + if section != "" { + sections[section] = lines + } + section = line[1 : len(line)-1] + lines = make([]string, 0, 5) + continue + } + if section != "" { + if line != "" { + lines = append(lines, line) + } + continue + } + if line != "" { + return nil, util.NewInvalidArgumentErrorf("invalid conaninfo.txt") + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + if section != "" { + sections[section] = lines + } + return sections, nil +} + +func toMap(lines []string) map[string]string { + result := make(map[string]string) + for _, line := range lines { + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + continue + } + result[parts[0]] = parts[1] + } + return result +} + +func toMapArray(lines []string) map[string][]string { + result := make(map[string][]string) + for _, line := range lines { + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { + continue + } + var items []string + if strings.HasPrefix(parts[1], "[") && strings.HasSuffix(parts[1], "]") { + items = strings.Split(parts[1], ",") + } else { + items = []string{parts[1]} + } + result[parts[0]] = items + } + return result +} diff --git a/modules/packages/conan/conaninfo_parser_test.go b/modules/packages/conan/conaninfo_parser_test.go new file mode 100644 index 0000000..dfb1836 --- /dev/null +++ b/modules/packages/conan/conaninfo_parser_test.go @@ -0,0 +1,85 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conan + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + settingsKey = "arch" + settingsValue = "x84_64" + optionsKey = "shared" + optionsValue = "False" + requires = "fmt/7.1.3" + hash = "74714915a51073acb548ca1ce29afbac" + envKey = "CC" + envValue = "gcc-10" + + contentConaninfo = `[settings] + ` + settingsKey + `=` + settingsValue + ` + +[requires] + ` + requires + ` + +[options] + ` + optionsKey + `=` + optionsValue + ` + +[full_settings] + ` + settingsKey + `=` + settingsValue + ` + +[full_requires] + ` + requires + ` + +[full_options] + ` + optionsKey + `=` + optionsValue + ` + +[recipe_hash] + ` + hash + ` + +[env] +` + envKey + `=` + envValue + ` + +` +) + +func TestParseConaninfo(t *testing.T) { + info, err := ParseConaninfo(strings.NewReader(contentConaninfo)) + assert.NotNil(t, info) + require.NoError(t, err) + assert.Equal( + t, + map[string]string{ + settingsKey: settingsValue, + }, + info.Settings, + ) + assert.Equal(t, info.Settings, info.FullSettings) + assert.Equal( + t, + map[string]string{ + optionsKey: optionsValue, + }, + info.Options, + ) + assert.Equal(t, info.Options, info.FullOptions) + assert.Equal( + t, + []string{requires}, + info.Requires, + ) + assert.Equal(t, info.Requires, info.FullRequires) + assert.Equal(t, hash, info.RecipeHash) + assert.Equal( + t, + map[string][]string{ + envKey: {envValue}, + }, + info.Environment, + ) +} diff --git a/modules/packages/conan/metadata.go b/modules/packages/conan/metadata.go new file mode 100644 index 0000000..256a376 --- /dev/null +++ b/modules/packages/conan/metadata.go @@ -0,0 +1,23 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conan + +const ( + PropertyRecipeUser = "conan.recipe.user" + PropertyRecipeChannel = "conan.recipe.channel" + PropertyRecipeRevision = "conan.recipe.revision" + PropertyPackageReference = "conan.package.reference" + PropertyPackageRevision = "conan.package.revision" + PropertyPackageInfo = "conan.package.info" +) + +// Metadata represents the metadata of a Conan package +type Metadata struct { + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Description string `json:"description,omitempty"` + Keywords []string `json:"keywords,omitempty"` +} diff --git a/modules/packages/conan/reference.go b/modules/packages/conan/reference.go new file mode 100644 index 0000000..58f268b --- /dev/null +++ b/modules/packages/conan/reference.go @@ -0,0 +1,155 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conan + +import ( + "fmt" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +const ( + // taken from https://github.com/conan-io/conan/blob/develop/conans/model/ref.py + minChars = 2 + maxChars = 51 + + // DefaultRevision if no revision is specified + DefaultRevision = "0" +) + +var ( + namePattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{%d,%d}$`, minChars-1, maxChars-1)) + revisionPattern = regexp.MustCompile(fmt.Sprintf(`^[a-zA-Z0-9]{1,%d}$`, maxChars)) + + ErrValidation = util.NewInvalidArgumentErrorf("could not validate one or more reference fields") +) + +// RecipeReference represents a recipe <Name>/<Version>@<User>/<Channel>#<Revision> +type RecipeReference struct { + Name string + Version string + User string + Channel string + Revision string +} + +func NewRecipeReference(name, version, user, channel, revision string) (*RecipeReference, error) { + log.Trace("Conan Recipe: %s/%s(@%s/%s(#%s))", name, version, user, channel, revision) + + if user == "_" { + user = "" + } + if channel == "_" { + channel = "" + } + + if (user != "" && channel == "") || (user == "" && channel != "") { + return nil, ErrValidation + } + + if !namePattern.MatchString(name) { + return nil, ErrValidation + } + + v := strings.TrimSpace(version) + if v == "" { + return nil, ErrValidation + } + if user != "" && !namePattern.MatchString(user) { + return nil, ErrValidation + } + if channel != "" && !namePattern.MatchString(channel) { + return nil, ErrValidation + } + if revision != "" && !revisionPattern.MatchString(revision) { + return nil, ErrValidation + } + + return &RecipeReference{name, v, user, channel, revision}, nil +} + +func (r *RecipeReference) RevisionOrDefault() string { + if r.Revision == "" { + return DefaultRevision + } + return r.Revision +} + +func (r *RecipeReference) String() string { + rev := "" + if r.Revision != "" { + rev = "#" + r.Revision + } + if r.User == "" || r.Channel == "" { + return fmt.Sprintf("%s/%s%s", r.Name, r.Version, rev) + } + return fmt.Sprintf("%s/%s@%s/%s%s", r.Name, r.Version, r.User, r.Channel, rev) +} + +func (r *RecipeReference) LinkName() string { + user := r.User + if user == "" { + user = "_" + } + channel := r.Channel + if channel == "" { + channel = "_" + } + return fmt.Sprintf("%s/%s/%s/%s/%s", r.Name, r.Version, user, channel, r.RevisionOrDefault()) +} + +func (r *RecipeReference) WithRevision(revision string) *RecipeReference { + return &RecipeReference{r.Name, r.Version, r.User, r.Channel, revision} +} + +// AsKey builds the additional key for the package file +func (r *RecipeReference) AsKey() string { + return fmt.Sprintf("%s|%s|%s", r.User, r.Channel, r.RevisionOrDefault()) +} + +// PackageReference represents a package of a recipe <Name>/<Version>@<User>/<Channel>#<Revision> <Reference>#<Revision> +type PackageReference struct { + Recipe *RecipeReference + Reference string + Revision string +} + +func NewPackageReference(recipe *RecipeReference, reference, revision string) (*PackageReference, error) { + log.Trace("Conan Package: %v %s(#%s)", recipe, reference, revision) + + if recipe == nil { + return nil, ErrValidation + } + if reference == "" || !revisionPattern.MatchString(reference) { + return nil, ErrValidation + } + if revision != "" && !revisionPattern.MatchString(revision) { + return nil, ErrValidation + } + + return &PackageReference{recipe, reference, revision}, nil +} + +func (r *PackageReference) RevisionOrDefault() string { + if r.Revision == "" { + return DefaultRevision + } + return r.Revision +} + +func (r *PackageReference) LinkName() string { + return fmt.Sprintf("%s/%s", r.Reference, r.RevisionOrDefault()) +} + +func (r *PackageReference) WithRevision(revision string) *PackageReference { + return &PackageReference{r.Recipe, r.Reference, revision} +} + +// AsKey builds the additional key for the package file +func (r *PackageReference) AsKey() string { + return fmt.Sprintf("%s|%s|%s|%s|%s", r.Recipe.User, r.Recipe.Channel, r.Recipe.RevisionOrDefault(), r.Reference, r.RevisionOrDefault()) +} diff --git a/modules/packages/conan/reference_test.go b/modules/packages/conan/reference_test.go new file mode 100644 index 0000000..7d39bd8 --- /dev/null +++ b/modules/packages/conan/reference_test.go @@ -0,0 +1,148 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conan + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewRecipeReference(t *testing.T) { + cases := []struct { + Name string + Version string + User string + Channel string + Revision string + IsValid bool + }{ + {"", "", "", "", "", false}, + {"name", "", "", "", "", false}, + {"", "1.0", "", "", "", false}, + {"", "", "user", "", "", false}, + {"", "", "", "channel", "", false}, + {"", "", "", "", "0", false}, + {"name", "1.0", "", "", "", true}, + {"name", "1.0", "user", "", "", false}, + {"name", "1.0", "", "channel", "", false}, + {"name", "1.0", "user", "channel", "", true}, + {"name", "1.0", "_", "", "", true}, + {"name", "1.0", "", "_", "", true}, + {"name", "1.0", "_", "_", "", true}, + {"name", "1.0", "_", "_", "0", true}, + {"name", "1.0", "", "", "0", true}, + {"name", "1.0.0q", "", "", "0", true}, + {"name", "1.0", "", "", "000000000000000000000000000000000000000000000000000000000000", false}, + } + + for i, c := range cases { + rref, err := NewRecipeReference(c.Name, c.Version, c.User, c.Channel, c.Revision) + if c.IsValid { + require.NoError(t, err, "case %d, should be invalid", i) + assert.NotNil(t, rref, "case %d, should not be nil", i) + } else { + require.Error(t, err, "case %d, should be valid", i) + } + } +} + +func TestRecipeReferenceRevisionOrDefault(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + require.NoError(t, err) + assert.Equal(t, DefaultRevision, rref.RevisionOrDefault()) + + rref, err = NewRecipeReference("name", "1.0", "", "", DefaultRevision) + require.NoError(t, err) + assert.Equal(t, DefaultRevision, rref.RevisionOrDefault()) + + rref, err = NewRecipeReference("name", "1.0", "", "", "Az09") + require.NoError(t, err) + assert.Equal(t, "Az09", rref.RevisionOrDefault()) +} + +func TestRecipeReferenceString(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + require.NoError(t, err) + assert.Equal(t, "name/1.0", rref.String()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "") + require.NoError(t, err) + assert.Equal(t, "name/1.0@user/channel", rref.String()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09") + require.NoError(t, err) + assert.Equal(t, "name/1.0@user/channel#Az09", rref.String()) +} + +func TestRecipeReferenceLinkName(t *testing.T) { + rref, err := NewRecipeReference("name", "1.0", "", "", "") + require.NoError(t, err) + assert.Equal(t, "name/1.0/_/_/0", rref.LinkName()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "") + require.NoError(t, err) + assert.Equal(t, "name/1.0/user/channel/0", rref.LinkName()) + + rref, err = NewRecipeReference("name", "1.0", "user", "channel", "Az09") + require.NoError(t, err) + assert.Equal(t, "name/1.0/user/channel/Az09", rref.LinkName()) +} + +func TestNewPackageReference(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + cases := []struct { + Recipe *RecipeReference + Reference string + Revision string + IsValid bool + }{ + {nil, "", "", false}, + {rref, "", "", false}, + {nil, "aZ09", "", false}, + {rref, "aZ09", "", true}, + {rref, "", "Az09", false}, + {rref, "aZ09", "Az09", true}, + } + + for i, c := range cases { + pref, err := NewPackageReference(c.Recipe, c.Reference, c.Revision) + if c.IsValid { + require.NoError(t, err, "case %d, should be invalid", i) + assert.NotNil(t, pref, "case %d, should not be nil", i) + } else { + require.Error(t, err, "case %d, should be valid", i) + } + } +} + +func TestPackageReferenceRevisionOrDefault(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + pref, err := NewPackageReference(rref, "ref", "") + require.NoError(t, err) + assert.Equal(t, DefaultRevision, pref.RevisionOrDefault()) + + pref, err = NewPackageReference(rref, "ref", DefaultRevision) + require.NoError(t, err) + assert.Equal(t, DefaultRevision, pref.RevisionOrDefault()) + + pref, err = NewPackageReference(rref, "ref", "Az09") + require.NoError(t, err) + assert.Equal(t, "Az09", pref.RevisionOrDefault()) +} + +func TestPackageReferenceLinkName(t *testing.T) { + rref, _ := NewRecipeReference("name", "1.0", "", "", "") + + pref, err := NewPackageReference(rref, "ref", "") + require.NoError(t, err) + assert.Equal(t, "ref/0", pref.LinkName()) + + pref, err = NewPackageReference(rref, "ref", "Az09") + require.NoError(t, err) + assert.Equal(t, "ref/Az09", pref.LinkName()) +} diff --git a/modules/packages/conda/metadata.go b/modules/packages/conda/metadata.go new file mode 100644 index 0000000..76ba95e --- /dev/null +++ b/modules/packages/conda/metadata.go @@ -0,0 +1,242 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conda + +import ( + "archive/tar" + "archive/zip" + "compress/bzip2" + "io" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + "code.gitea.io/gitea/modules/zstd" +) + +var ( + ErrInvalidStructure = util.SilentWrap{Message: "package structure is invalid", Err: util.ErrInvalidArgument} + ErrInvalidName = util.SilentWrap{Message: "package name is invalid", Err: util.ErrInvalidArgument} + ErrInvalidVersion = util.SilentWrap{Message: "package version is invalid", Err: util.ErrInvalidArgument} +) + +const ( + PropertyName = "conda.name" + PropertyChannel = "conda.channel" + PropertySubdir = "conda.subdir" + PropertyMetadata = "conda.metadata" +) + +// Package represents a Conda package +type Package struct { + Name string + Version string + Subdir string + VersionMetadata *VersionMetadata + FileMetadata *FileMetadata +} + +// VersionMetadata represents the metadata of a Conda package +type VersionMetadata struct { + Description string `json:"description,omitempty"` + Summary string `json:"summary,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + License string `json:"license,omitempty"` + LicenseFamily string `json:"license_family,omitempty"` +} + +// FileMetadata represents the metadata of a Conda package file +type FileMetadata struct { + IsCondaPackage bool `json:"is_conda"` + Architecture string `json:"architecture,omitempty"` + NoArch string `json:"noarch,omitempty"` + Build string `json:"build,omitempty"` + BuildNumber int64 `json:"build_number,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` + Platform string `json:"platform,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` +} + +type index struct { + Name string `json:"name"` + Version string `json:"version"` + Architecture string `json:"arch"` + NoArch string `json:"noarch"` + Build string `json:"build"` + BuildNumber int64 `json:"build_number"` + Dependencies []string `json:"depends"` + License string `json:"license"` + LicenseFamily string `json:"license_family"` + Platform string `json:"platform"` + Subdir string `json:"subdir"` + Timestamp int64 `json:"timestamp"` +} + +type about struct { + Description string `json:"description"` + Summary string `json:"summary"` + ProjectURL string `json:"home"` + RepositoryURL string `json:"dev_url"` + DocumentationURL string `json:"doc_url"` +} + +type ReaderAndReaderAt interface { + io.Reader + io.ReaderAt +} + +// ParsePackageBZ2 parses the Conda package file compressed with bzip2 +func ParsePackageBZ2(r io.Reader) (*Package, error) { + gzr := bzip2.NewReader(r) + + return parsePackageTar(gzr) +} + +// ParsePackageConda parses the Conda package file compressed with zip and zstd +func ParsePackageConda(r io.ReaderAt, size int64) (*Package, error) { + zr, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range zr.File { + if strings.HasPrefix(file.Name, "info-") && strings.HasSuffix(file.Name, ".tar.zst") { + f, err := zr.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + dec, err := zstd.NewReader(f) + if err != nil { + return nil, err + } + defer dec.Close() + + p, err := parsePackageTar(dec) + if p != nil { + p.FileMetadata.IsCondaPackage = true + } + return p, err + } + } + + return nil, ErrInvalidStructure +} + +func parsePackageTar(r io.Reader) (*Package, error) { + var i *index + var a *about + + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + + if hdr.Name == "info/index.json" { + if err := json.NewDecoder(tr).Decode(&i); err != nil { + return nil, err + } + + if !checkName(i.Name) { + return nil, ErrInvalidName + } + + if !checkVersion(i.Version) { + return nil, ErrInvalidVersion + } + + if a != nil { + break // stop loop if both files were found + } + } else if hdr.Name == "info/about.json" { + if err := json.NewDecoder(tr).Decode(&a); err != nil { + return nil, err + } + + if !validation.IsValidURL(a.ProjectURL) { + a.ProjectURL = "" + } + if !validation.IsValidURL(a.RepositoryURL) { + a.RepositoryURL = "" + } + if !validation.IsValidURL(a.DocumentationURL) { + a.DocumentationURL = "" + } + + if i != nil { + break // stop loop if both files were found + } + } + } + + if i == nil { + return nil, ErrInvalidStructure + } + if a == nil { + a = &about{} + } + + return &Package{ + Name: i.Name, + Version: i.Version, + Subdir: i.Subdir, + VersionMetadata: &VersionMetadata{ + License: i.License, + LicenseFamily: i.LicenseFamily, + Description: a.Description, + Summary: a.Summary, + ProjectURL: a.ProjectURL, + RepositoryURL: a.RepositoryURL, + DocumentationURL: a.DocumentationURL, + }, + FileMetadata: &FileMetadata{ + Architecture: i.Architecture, + NoArch: i.NoArch, + Build: i.Build, + BuildNumber: i.BuildNumber, + Dependencies: i.Dependencies, + Platform: i.Platform, + Timestamp: i.Timestamp, + }, + }, nil +} + +// https://github.com/conda/conda-build/blob/db9a728a9e4e6cfc895637ca3221117970fc2663/conda_build/metadata.py#L1393 +func checkName(name string) bool { + if name == "" { + return false + } + if name != strings.ToLower(name) { + return false + } + return !checkBadCharacters(name, "!") +} + +// https://github.com/conda/conda-build/blob/db9a728a9e4e6cfc895637ca3221117970fc2663/conda_build/metadata.py#L1403 +func checkVersion(version string) bool { + if version == "" { + return false + } + return !checkBadCharacters(version, "-") +} + +func checkBadCharacters(s, additional string) bool { + if strings.ContainsAny(s, "=@#$%^&*:;\"'\\|<>?/ ") { + return true + } + return strings.ContainsAny(s, additional) +} diff --git a/modules/packages/conda/metadata_test.go b/modules/packages/conda/metadata_test.go new file mode 100644 index 0000000..25b0295 --- /dev/null +++ b/modules/packages/conda/metadata_test.go @@ -0,0 +1,152 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conda + +import ( + "archive/tar" + "archive/zip" + "bytes" + "io" + "testing" + + "code.gitea.io/gitea/modules/zstd" + + "github.com/dsnet/compress/bzip2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + description = "Package Description" + projectURL = "https://gitea.com" + repositoryURL = "https://gitea.com/gitea/gitea" + documentationURL = "https://docs.gitea.com" +) + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) *bytes.Buffer { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for filename, content := range files { + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + } + tw.Close() + return &buf + } + + t.Run("MissingIndexFile", func(t *testing.T) { + buf := createArchive(map[string][]byte{"dummy.txt": {}}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidStructure) + }) + + t.Run("MissingAboutFile", func(t *testing.T) { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"name","version":"1.0"}`)}) + + p, err := parsePackageTar(buf) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, "name", p.Name) + assert.Equal(t, "1.0", p.Version) + assert.Empty(t, p.VersionMetadata.ProjectURL) + }) + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "name!", "nAMe"} { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"` + name + `","version":"1.0"}`)}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1.0-2"} { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"name","version":"` + version + `"}`)}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + buf := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `","subdir":"linux-64"}`), + "info/about.json": []byte(`{"description":"` + description + `","dev_url":"` + repositoryURL + `","doc_url":"` + documentationURL + `","home":"` + projectURL + `"}`), + }) + + p, err := parsePackageTar(buf) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, "linux-64", p.Subdir) + assert.Equal(t, description, p.VersionMetadata.Description) + assert.Equal(t, projectURL, p.VersionMetadata.ProjectURL) + assert.Equal(t, repositoryURL, p.VersionMetadata.RepositoryURL) + assert.Equal(t, documentationURL, p.VersionMetadata.DocumentationURL) + }) + + t.Run(".tar.bz2", func(t *testing.T) { + tarArchive := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `"}`), + }) + + var buf bytes.Buffer + bw, _ := bzip2.NewWriter(&buf, nil) + io.Copy(bw, tarArchive) + bw.Close() + + br := bytes.NewReader(buf.Bytes()) + + p, err := ParsePackageBZ2(br) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.False(t, p.FileMetadata.IsCondaPackage) + }) + + t.Run(".conda", func(t *testing.T) { + tarArchive := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `"}`), + }) + + var infoBuf bytes.Buffer + zsw, _ := zstd.NewWriter(&infoBuf) + io.Copy(zsw, tarArchive) + zsw.Close() + + var buf bytes.Buffer + zpw := zip.NewWriter(&buf) + w, _ := zpw.Create("info-x.tar.zst") + w.Write(infoBuf.Bytes()) + zpw.Close() + + br := bytes.NewReader(buf.Bytes()) + + p, err := ParsePackageConda(br, int64(br.Len())) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.True(t, p.FileMetadata.IsCondaPackage) + }) +} diff --git a/modules/packages/container/helm/helm.go b/modules/packages/container/helm/helm.go new file mode 100644 index 0000000..6981d43 --- /dev/null +++ b/modules/packages/container/helm/helm.go @@ -0,0 +1,55 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package helm + +// https://github.com/helm/helm/blob/main/pkg/chart/ + +const ConfigMediaType = "application/vnd.cncf.helm.config.v1+json" + +// Maintainer describes a Chart maintainer. +type Maintainer struct { + // Name is a user name or organization name + Name string `json:"name,omitempty"` + // Email is an optional email address to contact the named maintainer + Email string `json:"email,omitempty"` + // URL is an optional URL to an address for the named maintainer + URL string `json:"url,omitempty"` +} + +// Metadata for a Chart file. This models the structure of a Chart.yaml file. +type Metadata struct { + // The name of the chart. Required. + Name string `json:"name,omitempty"` + // The URL to a relevant project page, git repo, or contact person + Home string `json:"home,omitempty"` + // Source is the URL to the source code of this chart + Sources []string `json:"sources,omitempty"` + // A SemVer 2 conformant version string of the chart. Required. + Version string `json:"version,omitempty"` + // A one-sentence description of the chart + Description string `json:"description,omitempty"` + // A list of string keywords + Keywords []string `json:"keywords,omitempty"` + // A list of name and URL/email address combinations for the maintainer(s) + Maintainers []*Maintainer `json:"maintainers,omitempty"` + // The URL to an icon file. + Icon string `json:"icon,omitempty"` + // The API Version of this chart. Required. + APIVersion string `json:"apiVersion,omitempty"` + // The condition to check to enable chart + Condition string `json:"condition,omitempty"` + // The tags to check to enable chart + Tags string `json:"tags,omitempty"` + // The version of the application enclosed inside of this chart. + AppVersion string `json:"appVersion,omitempty"` + // Whether or not this chart is deprecated + Deprecated bool `json:"deprecated,omitempty"` + // Annotations are additional mappings uninterpreted by Helm, + // made available for inspection by other applications. + Annotations map[string]string `json:"annotations,omitempty"` + // KubeVersion is a SemVer constraint specifying the version of Kubernetes required. + KubeVersion string `json:"kubeVersion,omitempty"` + // Specifies the chart type: application or library + Type string `json:"type,omitempty"` +} diff --git a/modules/packages/container/metadata.go b/modules/packages/container/metadata.go new file mode 100644 index 0000000..2a41fb9 --- /dev/null +++ b/modules/packages/container/metadata.go @@ -0,0 +1,166 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package container + +import ( + "fmt" + "io" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/container/helm" + "code.gitea.io/gitea/modules/validation" + + oci "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + PropertyRepository = "container.repository" + PropertyDigest = "container.digest" + PropertyMediaType = "container.mediatype" + PropertyManifestTagged = "container.manifest.tagged" + PropertyManifestReference = "container.manifest.reference" + + DefaultPlatform = "linux/amd64" + + labelLicenses = "org.opencontainers.image.licenses" + labelURL = "org.opencontainers.image.url" + labelSource = "org.opencontainers.image.source" + labelDocumentation = "org.opencontainers.image.documentation" + labelDescription = "org.opencontainers.image.description" + labelAuthors = "org.opencontainers.image.authors" +) + +type ImageType string + +const ( + TypeOCI ImageType = "oci" + TypeHelm ImageType = "helm" +) + +// Name gets the name of the image type +func (it ImageType) Name() string { + switch it { + case TypeHelm: + return "Helm Chart" + default: + return "OCI / Docker" + } +} + +// Metadata represents the metadata of a Container package +type Metadata struct { + Type ImageType `json:"type"` + IsTagged bool `json:"is_tagged"` + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ImageLayers []string `json:"layer_creation,omitempty"` + Manifests []*Manifest `json:"manifests,omitempty"` +} + +type Manifest struct { + Platform string `json:"platform"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +// ParseImageConfig parses the metadata of an image config +func ParseImageConfig(mt string, r io.Reader) (*Metadata, error) { + if strings.EqualFold(mt, helm.ConfigMediaType) { + return parseHelmConfig(r) + } + + // fallback to OCI Image Config + return parseOCIImageConfig(r) +} + +func parseOCIImageConfig(r io.Reader) (*Metadata, error) { + var image oci.Image + if err := json.NewDecoder(r).Decode(&image); err != nil { + return nil, err + } + + platform := DefaultPlatform + if image.OS != "" && image.Architecture != "" { + platform = fmt.Sprintf("%s/%s", image.OS, image.Architecture) + if image.Variant != "" { + platform = fmt.Sprintf("%s/%s", platform, image.Variant) + } + } + + imageLayers := make([]string, 0, len(image.History)) + for _, history := range image.History { + cmd := history.CreatedBy + if i := strings.Index(cmd, "#(nop) "); i != -1 { + cmd = strings.TrimSpace(cmd[i+7:]) + } + if cmd != "" { + imageLayers = append(imageLayers, cmd) + } + } + + metadata := &Metadata{ + Type: TypeOCI, + Platform: platform, + Licenses: image.Config.Labels[labelLicenses], + ProjectURL: image.Config.Labels[labelURL], + RepositoryURL: image.Config.Labels[labelSource], + DocumentationURL: image.Config.Labels[labelDocumentation], + Description: image.Config.Labels[labelDescription], + Labels: image.Config.Labels, + ImageLayers: imageLayers, + } + + if authors, ok := image.Config.Labels[labelAuthors]; ok { + metadata.Authors = []string{authors} + } + + if !validation.IsValidURL(metadata.ProjectURL) { + metadata.ProjectURL = "" + } + if !validation.IsValidURL(metadata.RepositoryURL) { + metadata.RepositoryURL = "" + } + if !validation.IsValidURL(metadata.DocumentationURL) { + metadata.DocumentationURL = "" + } + + return metadata, nil +} + +func parseHelmConfig(r io.Reader) (*Metadata, error) { + var config helm.Metadata + if err := json.NewDecoder(r).Decode(&config); err != nil { + return nil, err + } + + metadata := &Metadata{ + Type: TypeHelm, + Description: config.Description, + ProjectURL: config.Home, + } + + if len(config.Maintainers) > 0 { + authors := make([]string, 0, len(config.Maintainers)) + for _, maintainer := range config.Maintainers { + authors = append(authors, maintainer.Name) + } + metadata.Authors = authors + } + + if len(config.Sources) > 0 && validation.IsValidURL(config.Sources[0]) { + metadata.RepositoryURL = config.Sources[0] + } + if !validation.IsValidURL(metadata.ProjectURL) { + metadata.ProjectURL = "" + } + + return metadata, nil +} diff --git a/modules/packages/container/metadata_test.go b/modules/packages/container/metadata_test.go new file mode 100644 index 0000000..930cf48 --- /dev/null +++ b/modules/packages/container/metadata_test.go @@ -0,0 +1,62 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package container + +import ( + "strings" + "testing" + + "code.gitea.io/gitea/modules/packages/container/helm" + + oci "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseImageConfig(t *testing.T) { + description := "Image Description" + author := "Gitea" + license := "MIT" + projectURL := "https://gitea.com" + repositoryURL := "https://gitea.com/gitea" + documentationURL := "https://docs.gitea.com" + + configOCI := `{"config": {"labels": {"` + labelAuthors + `": "` + author + `", "` + labelLicenses + `": "` + license + `", "` + labelURL + `": "` + projectURL + `", "` + labelSource + `": "` + repositoryURL + `", "` + labelDocumentation + `": "` + documentationURL + `", "` + labelDescription + `": "` + description + `"}}, "history": [{"created_by": "do it 1"}, {"created_by": "dummy #(nop) do it 2"}]}` + + metadata, err := ParseImageConfig(oci.MediaTypeImageManifest, strings.NewReader(configOCI)) + require.NoError(t, err) + + assert.Equal(t, TypeOCI, metadata.Type) + assert.Equal(t, description, metadata.Description) + assert.ElementsMatch(t, []string{author}, metadata.Authors) + assert.Equal(t, license, metadata.Licenses) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) + assert.Equal(t, documentationURL, metadata.DocumentationURL) + assert.ElementsMatch(t, []string{"do it 1", "do it 2"}, metadata.ImageLayers) + assert.Equal( + t, + map[string]string{ + labelAuthors: author, + labelLicenses: license, + labelURL: projectURL, + labelSource: repositoryURL, + labelDocumentation: documentationURL, + labelDescription: description, + }, + metadata.Labels, + ) + assert.Empty(t, metadata.Manifests) + + configHelm := `{"description":"` + description + `", "home": "` + projectURL + `", "sources": ["` + repositoryURL + `"], "maintainers":[{"name":"` + author + `"}]}` + + metadata, err = ParseImageConfig(helm.ConfigMediaType, strings.NewReader(configHelm)) + require.NoError(t, err) + + assert.Equal(t, TypeHelm, metadata.Type) + assert.Equal(t, description, metadata.Description) + assert.ElementsMatch(t, []string{author}, metadata.Authors) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) +} diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go new file mode 100644 index 0000000..da93e6c --- /dev/null +++ b/modules/packages/content_store.go @@ -0,0 +1,75 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package packages + +import ( + "io" + "net/url" + "path" + "strings" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/util" +) + +// BlobHash256Key is the key to address a blob content +type BlobHash256Key string + +// ContentStore is a wrapper around ObjectStorage +type ContentStore struct { + store storage.ObjectStorage +} + +// NewContentStore creates the default package store +func NewContentStore() *ContentStore { + contentStore := &ContentStore{storage.Packages} + return contentStore +} + +// Get gets a package blob +func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { + return s.store.Open(KeyToRelativePath(key)) +} + +func (s *ContentStore) ShouldServeDirect() bool { + return setting.Packages.Storage.MinioConfig.ServeDirect +} + +func (s *ContentStore) GetServeDirectURL(key BlobHash256Key, filename string) (*url.URL, error) { + return s.store.URL(KeyToRelativePath(key), filename) +} + +// FIXME: Workaround to be removed in v1.20 +// https://github.com/go-gitea/gitea/issues/19586 +func (s *ContentStore) Has(key BlobHash256Key) error { + _, err := s.store.Stat(KeyToRelativePath(key)) + return err +} + +// Save stores a package blob +func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error { + _, err := s.store.Save(KeyToRelativePath(key), r, size) + return err +} + +// Delete deletes a package blob +func (s *ContentStore) Delete(key BlobHash256Key) error { + return s.store.Delete(KeyToRelativePath(key)) +} + +// KeyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000... +func KeyToRelativePath(key BlobHash256Key) string { + return path.Join(string(key)[0:2], string(key)[2:4], string(key)) +} + +// RelativePathToKey converts a relative path aa/bb/aabb000000... to the sha256 key aabb000000... +func RelativePathToKey(relativePath string) (BlobHash256Key, error) { + parts := strings.SplitN(relativePath, "/", 3) + if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 || len(parts[2]) < 4 || parts[0]+parts[1] != parts[2][0:4] { + return "", util.ErrInvalidArgument + } + + return BlobHash256Key(parts[2]), nil +} diff --git a/modules/packages/cran/metadata.go b/modules/packages/cran/metadata.go new file mode 100644 index 0000000..0b0bfb0 --- /dev/null +++ b/modules/packages/cran/metadata.go @@ -0,0 +1,242 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cran + +import ( + "archive/tar" + "archive/zip" + "bufio" + "compress/gzip" + "io" + "path" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +const ( + PropertyType = "cran.type" + PropertyPlatform = "cran.platform" + PropertyRVersion = "cran.rvserion" + + TypeSource = "source" + TypeBinary = "binary" +) + +var ( + ErrMissingDescriptionFile = util.NewInvalidArgumentErrorf("DESCRIPTION file is missing") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") +) + +var ( + fieldPattern = regexp.MustCompile(`\A\S+:`) + namePattern = regexp.MustCompile(`\A[a-zA-Z][a-zA-Z0-9\.]*[a-zA-Z0-9]\z`) + versionPattern = regexp.MustCompile(`\A[0-9]+(?:[.\-][0-9]+){1,3}\z`) + authorReplacePattern = regexp.MustCompile(`[\[\(].+?[\]\)]`) +) + +// Package represents a CRAN package +type Package struct { + Name string + Version string + FileExtension string + Metadata *Metadata +} + +// Metadata represents the metadata of a CRAN package +type Metadata struct { + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + ProjectURL []string `json:"project_url,omitempty"` + License string `json:"license,omitempty"` + Authors []string `json:"authors,omitempty"` + Depends []string `json:"depends,omitempty"` + Imports []string `json:"imports,omitempty"` + Suggests []string `json:"suggests,omitempty"` + LinkingTo []string `json:"linking_to,omitempty"` + NeedsCompilation bool `json:"needs_compilation"` +} + +type ReaderReaderAt interface { + io.Reader + io.ReaderAt +} + +// ParsePackage reads the package metadata from a CRAN package +// .zip and .tar.gz/.tgz files are supported. +func ParsePackage(r ReaderReaderAt, size int64) (*Package, error) { + magicBytes := make([]byte, 2) + if _, err := r.ReadAt(magicBytes, 0); err != nil { + return nil, err + } + + if magicBytes[0] == 0x1F && magicBytes[1] == 0x8B { + return parsePackageTarGz(r) + } + return parsePackageZip(r, size) +} + +func parsePackageTarGz(r io.Reader) (*Package, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if strings.Count(hd.Name, "/") > 1 { + continue + } + + if path.Base(hd.Name) == "DESCRIPTION" { + p, err := ParseDescription(tr) + if p != nil { + p.FileExtension = ".tar.gz" + } + return p, err + } + } + + return nil, ErrMissingDescriptionFile +} + +func parsePackageZip(r io.ReaderAt, size int64) (*Package, error) { + zr, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range zr.File { + if strings.Count(file.Name, "/") > 1 { + continue + } + + if path.Base(file.Name) == "DESCRIPTION" { + f, err := zr.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + p, err := ParseDescription(f) + if p != nil { + p.FileExtension = ".zip" + } + return p, err + } + } + + return nil, ErrMissingDescriptionFile +} + +// ParseDescription parses a DESCRIPTION file to retrieve the metadata of a CRAN package +func ParseDescription(r io.Reader) (*Package, error) { + p := &Package{ + Metadata: &Metadata{}, + } + + scanner := bufio.NewScanner(r) + + var b strings.Builder + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + if !fieldPattern.MatchString(line) { + b.WriteRune(' ') + b.WriteString(line) + continue + } + + if err := setField(p, b.String()); err != nil { + return nil, err + } + + b.Reset() + b.WriteString(line) + } + + if err := setField(p, b.String()); err != nil { + return nil, err + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return p, nil +} + +func setField(p *Package, data string) error { + if data == "" { + return nil + } + + parts := strings.SplitN(data, ":", 2) + if len(parts) != 2 { + return nil + } + + name := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + switch name { + case "Package": + if !namePattern.MatchString(value) { + return ErrInvalidName + } + p.Name = value + case "Version": + if !versionPattern.MatchString(value) { + return ErrInvalidVersion + } + p.Version = value + case "Title": + p.Metadata.Title = value + case "Description": + p.Metadata.Description = value + case "URL": + p.Metadata.ProjectURL = splitAndTrim(value) + case "License": + p.Metadata.License = value + case "Author": + p.Metadata.Authors = splitAndTrim(authorReplacePattern.ReplaceAllString(value, "")) + case "Depends": + p.Metadata.Depends = splitAndTrim(value) + case "Imports": + p.Metadata.Imports = splitAndTrim(value) + case "Suggests": + p.Metadata.Suggests = splitAndTrim(value) + case "LinkingTo": + p.Metadata.LinkingTo = splitAndTrim(value) + case "NeedsCompilation": + p.Metadata.NeedsCompilation = value == "yes" + } + + return nil +} + +func splitAndTrim(s string) []string { + items := strings.Split(s, ", ") + for i := range items { + items[i] = strings.TrimSpace(items[i]) + } + return items +} diff --git a/modules/packages/cran/metadata_test.go b/modules/packages/cran/metadata_test.go new file mode 100644 index 0000000..3287380 --- /dev/null +++ b/modules/packages/cran/metadata_test.go @@ -0,0 +1,153 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cran + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + author = "KN4CK3R" + description = "Package Description" + projectURL = "https://gitea.io" + license = "GPL (>= 2)" +) + +func createDescription(name, version string) *bytes.Buffer { + var buf bytes.Buffer + fmt.Fprintln(&buf, "Package:", name) + fmt.Fprintln(&buf, "Version:", version) + fmt.Fprintln(&buf, "Description:", "Package\n\n Description") + fmt.Fprintln(&buf, "URL:", projectURL) + fmt.Fprintln(&buf, "Imports: abc,\n123") + fmt.Fprintln(&buf, "NeedsCompilation: yes") + fmt.Fprintln(&buf, "License:", license) + fmt.Fprintln(&buf, "Author:", author) + return &buf +} + +func TestParsePackage(t *testing.T) { + t.Run(".tar.gz", func(t *testing.T) { + createArchive := func(filename string, content []byte) *bytes.Reader { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + tw.Close() + gw.Close() + return bytes.NewReader(buf.Bytes()) + } + + t.Run("MissingDescriptionFile", func(t *testing.T) { + buf := createArchive( + "dummy.txt", + []byte{}, + ) + + p, err := ParsePackage(buf, buf.Size()) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrMissingDescriptionFile) + }) + + t.Run("Valid", func(t *testing.T) { + buf := createArchive( + "package/DESCRIPTION", + createDescription(packageName, packageVersion).Bytes(), + ) + + p, err := ParsePackage(buf, buf.Size()) + + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + }) + }) + + t.Run(".zip", func(t *testing.T) { + createArchive := func(filename string, content []byte) *bytes.Reader { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(filename) + w.Write(content) + archive.Close() + return bytes.NewReader(buf.Bytes()) + } + + t.Run("MissingDescriptionFile", func(t *testing.T) { + buf := createArchive( + "dummy.txt", + []byte{}, + ) + + p, err := ParsePackage(buf, buf.Size()) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrMissingDescriptionFile) + }) + + t.Run("Valid", func(t *testing.T) { + buf := createArchive( + "package/DESCRIPTION", + createDescription(packageName, packageVersion).Bytes(), + ) + + p, err := ParsePackage(buf, buf.Size()) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + }) + }) +} + +func TestParseDescription(t *testing.T) { + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"123abc", "ab-cd", "ab cd", "ab/cd"} { + p, err := ParseDescription(createDescription(name, packageVersion)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"1", "1 0", "1.2.3.4.5", "1-2-3-4-5", "1.", "1.0.", "1-", "1-0-"} { + p, err := ParseDescription(createDescription(packageName, version)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + p, err := ParseDescription(createDescription(packageName, packageVersion)) + require.NoError(t, err) + assert.NotNil(t, p) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, description, p.Metadata.Description) + assert.ElementsMatch(t, []string{projectURL}, p.Metadata.ProjectURL) + assert.ElementsMatch(t, []string{author}, p.Metadata.Authors) + assert.Equal(t, license, p.Metadata.License) + assert.ElementsMatch(t, []string{"abc", "123"}, p.Metadata.Imports) + assert.True(t, p.Metadata.NeedsCompilation) + }) +} diff --git a/modules/packages/debian/metadata.go b/modules/packages/debian/metadata.go new file mode 100644 index 0000000..e76db63 --- /dev/null +++ b/modules/packages/debian/metadata.go @@ -0,0 +1,221 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package debian + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "io" + "net/mail" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + "code.gitea.io/gitea/modules/zstd" + + "github.com/blakesmith/ar" + "github.com/ulikunitz/xz" +) + +const ( + PropertyDistribution = "debian.distribution" + PropertyComponent = "debian.component" + PropertyArchitecture = "debian.architecture" + PropertyControl = "debian.control" + PropertyRepositoryIncludeInRelease = "debian.repository.include_in_release" + + SettingKeyPrivate = "debian.key.private" + SettingKeyPublic = "debian.key.public" + + RepositoryPackage = "_debian" + RepositoryVersion = "_repository" + + controlTar = "control.tar" +) + +var ( + ErrMissingControlFile = util.NewInvalidArgumentErrorf("control file is missing") + ErrUnsupportedCompression = util.NewInvalidArgumentErrorf("unsupported compression algorithm") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") + ErrInvalidArchitecture = util.NewInvalidArgumentErrorf("package architecture is invalid") + + // https://www.debian.org/doc/debian-policy/ch-controlfields.html#source + namePattern = regexp.MustCompile(`\A[a-z0-9][a-z0-9+-.]+\z`) + // https://www.debian.org/doc/debian-policy/ch-controlfields.html#version + versionPattern = regexp.MustCompile(`\A(?:[0-9]:)?[a-zA-Z0-9.+~]+(?:-[a-zA-Z0-9.+-~]+)?\z`) +) + +type Package struct { + Name string + Version string + Architecture string + Control string + Metadata *Metadata +} + +type Metadata struct { + Maintainer string `json:"maintainer,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Description string `json:"description,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` +} + +// ParsePackage parses the Debian package file +// https://manpages.debian.org/bullseye/dpkg-dev/deb.5.en.html +func ParsePackage(r io.Reader) (*Package, error) { + arr := ar.NewReader(r) + + for { + hd, err := arr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if strings.HasPrefix(hd.Name, controlTar) { + var inner io.Reader + // https://man7.org/linux/man-pages/man5/deb-split.5.html#FORMAT + // The file names might contain a trailing slash (since dpkg 1.15.6). + switch strings.TrimSuffix(hd.Name[len(controlTar):], "/") { + case "": + inner = arr + case ".gz": + gzr, err := gzip.NewReader(arr) + if err != nil { + return nil, err + } + defer gzr.Close() + + inner = gzr + case ".xz": + xzr, err := xz.NewReader(arr) + if err != nil { + return nil, err + } + + inner = xzr + case ".zst": + zr, err := zstd.NewReader(arr) + if err != nil { + return nil, err + } + defer zr.Close() + + inner = zr + default: + return nil, ErrUnsupportedCompression + } + + tr := tar.NewReader(inner) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if hd.FileInfo().Name() == "control" { + return ParseControlFile(tr) + } + } + } + } + + return nil, ErrMissingControlFile +} + +// ParseControlFile parses a Debian control file to retrieve the metadata +func ParseControlFile(r io.Reader) (*Package, error) { + p := &Package{ + Metadata: &Metadata{}, + } + + key := "" + var depends strings.Builder + var control strings.Builder + + s := bufio.NewScanner(io.TeeReader(r, &control)) + for s.Scan() { + line := s.Text() + + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + + if line[0] == ' ' || line[0] == '\t' { + switch key { + case "Description": + p.Metadata.Description += line + case "Depends": + depends.WriteString(trimmed) + } + } else { + parts := strings.SplitN(trimmed, ":", 2) + if len(parts) < 2 { + continue + } + + key = parts[0] + value := strings.TrimSpace(parts[1]) + switch key { + case "Package": + p.Name = value + case "Version": + p.Version = value + case "Architecture": + p.Architecture = value + case "Maintainer": + a, err := mail.ParseAddress(value) + if err != nil || a.Name == "" { + p.Metadata.Maintainer = value + } else { + p.Metadata.Maintainer = a.Name + } + case "Description": + p.Metadata.Description = value + case "Depends": + depends.WriteString(value) + case "Homepage": + if validation.IsValidURL(value) { + p.Metadata.ProjectURL = value + } + } + } + } + if err := s.Err(); err != nil { + return nil, err + } + + if !namePattern.MatchString(p.Name) { + return nil, ErrInvalidName + } + if !versionPattern.MatchString(p.Version) { + return nil, ErrInvalidVersion + } + if p.Architecture == "" { + return nil, ErrInvalidArchitecture + } + + dependencies := strings.Split(depends.String(), ",") + for i := range dependencies { + dependencies[i] = strings.TrimSpace(dependencies[i]) + } + p.Metadata.Dependencies = dependencies + + p.Control = strings.TrimSpace(control.String()) + + return p, nil +} diff --git a/modules/packages/debian/metadata_test.go b/modules/packages/debian/metadata_test.go new file mode 100644 index 0000000..6f6c469 --- /dev/null +++ b/modules/packages/debian/metadata_test.go @@ -0,0 +1,187 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package debian + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "testing" + + "code.gitea.io/gitea/modules/zstd" + + "github.com/blakesmith/ar" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/ulikunitz/xz" +) + +const ( + packageName = "gitea" + packageVersion = "0:1.0.1-te~st" + packageArchitecture = "amd64" + packageAuthor = "KN4CK3R" + description = "Description with multiple lines." + projectURL = "https://gitea.io" +) + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) io.Reader { + var buf bytes.Buffer + aw := ar.NewWriter(&buf) + aw.WriteGlobalHeader() + for filename, content := range files { + hdr := &ar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + aw.WriteHeader(hdr) + aw.Write(content) + } + return &buf + } + + t.Run("MissingControlFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"dummy.txt": {}}) + + p, err := ParsePackage(data) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrMissingControlFile) + }) + + t.Run("Compression", func(t *testing.T) { + t.Run("Unsupported", func(t *testing.T) { + data := createArchive(map[string][]byte{"control.tar.foo": {}}) + + p, err := ParsePackage(data) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrUnsupportedCompression) + }) + + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + tw.WriteHeader(&tar.Header{ + Name: "control", + Mode: 0o600, + Size: 50, + }) + tw.Write([]byte("Package: gitea\nVersion: 1.0.0\nArchitecture: amd64\n")) + tw.Close() + + cases := []struct { + Extension string + WriterFactory func(io.Writer) io.WriteCloser + }{ + { + Extension: "", + WriterFactory: func(w io.Writer) io.WriteCloser { + return nopCloser{w} + }, + }, + { + Extension: ".gz", + WriterFactory: func(w io.Writer) io.WriteCloser { + return gzip.NewWriter(w) + }, + }, + { + Extension: ".xz", + WriterFactory: func(w io.Writer) io.WriteCloser { + xw, _ := xz.NewWriter(w) + return xw + }, + }, + { + Extension: ".zst", + WriterFactory: func(w io.Writer) io.WriteCloser { + zw, _ := zstd.NewWriter(w) + return zw + }, + }, + } + + for _, c := range cases { + t.Run(c.Extension, func(t *testing.T) { + var cbuf bytes.Buffer + w := c.WriterFactory(&cbuf) + w.Write(buf.Bytes()) + w.Close() + + data := createArchive(map[string][]byte{"control.tar" + c.Extension: cbuf.Bytes()}) + + p, err := ParsePackage(data) + assert.NotNil(t, p) + require.NoError(t, err) + assert.Equal(t, "gitea", p.Name) + + t.Run("TrailingSlash", func(t *testing.T) { + data := createArchive(map[string][]byte{"control.tar" + c.Extension + "/": cbuf.Bytes()}) + + p, err := ParsePackage(data) + assert.NotNil(t, p) + require.NoError(t, err) + assert.Equal(t, "gitea", p.Name) + }) + }) + } + }) +} + +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { + return nil +} + +func TestParseControlFile(t *testing.T) { + buildContent := func(name, version, architecture string) *bytes.Buffer { + var buf bytes.Buffer + buf.WriteString("Package: " + name + "\nVersion: " + version + "\nArchitecture: " + architecture + "\nMaintainer: " + packageAuthor + " <kn4ck3r@gitea.io>\nHomepage: " + projectURL + "\nDepends: a,\n b\nDescription: Description\n with multiple\n lines.") + return &buf + } + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "-cd"} { + p, err := ParseControlFile(buildContent(name, packageVersion, packageArchitecture)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1-", ":1.0", "1_0"} { + p, err := ParseControlFile(buildContent(packageName, version, packageArchitecture)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("InvalidArchitecture", func(t *testing.T) { + p, err := ParseControlFile(buildContent(packageName, packageVersion, "")) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidArchitecture) + }) + + t.Run("Valid", func(t *testing.T) { + content := buildContent(packageName, packageVersion, packageArchitecture) + full := content.String() + + p, err := ParseControlFile(content) + require.NoError(t, err) + assert.NotNil(t, p) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, packageArchitecture, p.Architecture) + assert.Equal(t, description, p.Metadata.Description) + assert.Equal(t, projectURL, p.Metadata.ProjectURL) + assert.Equal(t, packageAuthor, p.Metadata.Maintainer) + assert.Equal(t, []string{"a", "b"}, p.Metadata.Dependencies) + assert.Equal(t, full, p.Control) + }) +} diff --git a/modules/packages/goproxy/metadata.go b/modules/packages/goproxy/metadata.go new file mode 100644 index 0000000..40f7d20 --- /dev/null +++ b/modules/packages/goproxy/metadata.go @@ -0,0 +1,94 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package goproxy + +import ( + "archive/zip" + "fmt" + "io" + "path" + "strings" + + "code.gitea.io/gitea/modules/util" +) + +const ( + PropertyGoMod = "go.mod" + + maxGoModFileSize = 16 * 1024 * 1024 // https://go.dev/ref/mod#zip-path-size-constraints +) + +var ( + ErrInvalidStructure = util.NewInvalidArgumentErrorf("package has invalid structure") + ErrGoModFileTooLarge = util.NewInvalidArgumentErrorf("go.mod file is too large") +) + +type Package struct { + Name string + Version string + GoMod string +} + +// ParsePackage parses the Go package file +// https://go.dev/ref/mod#zip-files +func ParsePackage(r io.ReaderAt, size int64) (*Package, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + var p *Package + + for _, file := range archive.File { + nameAndVersion := path.Dir(file.Name) + + parts := strings.SplitN(nameAndVersion, "@", 2) + if len(parts) != 2 { + continue + } + + versionParts := strings.SplitN(parts[1], "/", 2) + + if p == nil { + p = &Package{ + Name: strings.TrimSuffix(nameAndVersion, "@"+parts[1]), + Version: versionParts[0], + } + } + + if len(versionParts) > 1 { + // files are expected in the "root" folder + continue + } + + if path.Base(file.Name) == "go.mod" { + if file.UncompressedSize64 > maxGoModFileSize { + return nil, ErrGoModFileTooLarge + } + + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + bytes, err := io.ReadAll(&io.LimitedReader{R: f, N: maxGoModFileSize}) + if err != nil { + return nil, err + } + + p.GoMod = string(bytes) + + return p, nil + } + } + + if p == nil { + return nil, ErrInvalidStructure + } + + p.GoMod = fmt.Sprintf("module %s", p.Name) + + return p, nil +} diff --git a/modules/packages/goproxy/metadata_test.go b/modules/packages/goproxy/metadata_test.go new file mode 100644 index 0000000..3a47f10 --- /dev/null +++ b/modules/packages/goproxy/metadata_test.go @@ -0,0 +1,76 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package goproxy + +import ( + "archive/zip" + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + packageName = "gitea.com/go-gitea/gitea" + packageVersion = "v0.0.1" +) + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) *bytes.Reader { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, content := range files { + w, _ := zw.Create(name) + w.Write(content) + } + zw.Close() + return bytes.NewReader(buf.Bytes()) + } + + t.Run("EmptyPackage", func(t *testing.T) { + data := createArchive(nil) + + p, err := ParsePackage(data, int64(data.Len())) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidStructure) + }) + + t.Run("InvalidNameOrVersionStructure", func(t *testing.T) { + data := createArchive(map[string][]byte{ + packageName + "/" + packageVersion + "/go.mod": {}, + }) + + p, err := ParsePackage(data, int64(data.Len())) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidStructure) + }) + + t.Run("GoModFileInWrongDirectory", func(t *testing.T) { + data := createArchive(map[string][]byte{ + packageName + "@" + packageVersion + "/subdir/go.mod": {}, + }) + + p, err := ParsePackage(data, int64(data.Len())) + assert.NotNil(t, p) + require.NoError(t, err) + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, "module gitea.com/go-gitea/gitea", p.GoMod) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive(map[string][]byte{ + packageName + "@" + packageVersion + "/subdir/go.mod": []byte("invalid"), + packageName + "@" + packageVersion + "/go.mod": []byte("valid"), + }) + + p, err := ParsePackage(data, int64(data.Len())) + assert.NotNil(t, p) + require.NoError(t, err) + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, "valid", p.GoMod) + }) +} diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go new file mode 100644 index 0000000..4ab45ed --- /dev/null +++ b/modules/packages/hashed_buffer.go @@ -0,0 +1,81 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package packages + +import ( + "io" + + "code.gitea.io/gitea/modules/util/filebuffer" +) + +// HashedSizeReader provide methods to read, sum hashes and a Size method +type HashedSizeReader interface { + io.Reader + HashSummer + Size() int64 +} + +// HashedBuffer is buffer which calculates multiple checksums +type HashedBuffer struct { + *filebuffer.FileBackedBuffer + + hash *MultiHasher + + combinedWriter io.Writer +} + +const DefaultMemorySize = 32 * 1024 * 1024 + +// NewHashedBuffer creates a hashed buffer with the default memory size +func NewHashedBuffer() (*HashedBuffer, error) { + return NewHashedBufferWithSize(DefaultMemorySize) +} + +// NewHashedBufferWithSize creates a hashed buffer with a specific memory size +func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { + b, err := filebuffer.New(maxMemorySize) + if err != nil { + return nil, err + } + + hash := NewMultiHasher() + + combinedWriter := io.MultiWriter(b, hash) + + return &HashedBuffer{ + b, + hash, + combinedWriter, + }, nil +} + +// CreateHashedBufferFromReader creates a hashed buffer with the default memory size and copies the provided reader data into it. +func CreateHashedBufferFromReader(r io.Reader) (*HashedBuffer, error) { + return CreateHashedBufferFromReaderWithSize(r, DefaultMemorySize) +} + +// CreateHashedBufferFromReaderWithSize creates a hashed buffer and copies the provided reader data into it. +func CreateHashedBufferFromReaderWithSize(r io.Reader, maxMemorySize int) (*HashedBuffer, error) { + b, err := NewHashedBufferWithSize(maxMemorySize) + if err != nil { + return nil, err + } + + _, err = io.Copy(b, r) + if err != nil { + return nil, err + } + + return b, nil +} + +// Write implements io.Writer +func (b *HashedBuffer) Write(p []byte) (int, error) { + return b.combinedWriter.Write(p) +} + +// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data +func (b *HashedBuffer) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { + return b.hash.Sums() +} diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go new file mode 100644 index 0000000..ed5267c --- /dev/null +++ b/modules/packages/hashed_buffer_test.go @@ -0,0 +1,47 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package packages + +import ( + "encoding/hex" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHashedBuffer(t *testing.T) { + cases := []struct { + MaxMemorySize int + Data string + HashMD5 string + HashSHA1 string + HashSHA256 string + HashSHA512 string + }{ + {5, "test", "098f6bcd4621d373cade4e832627b4f6", "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", "ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff"}, + {5, "testtest", "05a671c66aefea124cc08b76ea6d30bb", "51abb9636078defbf888d8457a7c76f85c8f114c", "37268335dd6931045bdcdf92623ff819a64244b53d0e746d438797349d4da578", "125d6d03b32c84d492747f79cf0bf6e179d287f341384eb5d6d3197525ad6be8e6df0116032935698f99a09e265073d1d6c32c274591bf1d0a20ad67cba921bc"}, + } + + for _, c := range cases { + buf, err := CreateHashedBufferFromReaderWithSize(strings.NewReader(c.Data), c.MaxMemorySize) + require.NoError(t, err) + + assert.EqualValues(t, len(c.Data), buf.Size()) + + data, err := io.ReadAll(buf) + require.NoError(t, err) + assert.Equal(t, c.Data, string(data)) + + hashMD5, hashSHA1, hashSHA256, hashSHA512 := buf.Sums() + assert.Equal(t, c.HashMD5, hex.EncodeToString(hashMD5)) + assert.Equal(t, c.HashSHA1, hex.EncodeToString(hashSHA1)) + assert.Equal(t, c.HashSHA256, hex.EncodeToString(hashSHA256)) + assert.Equal(t, c.HashSHA512, hex.EncodeToString(hashSHA512)) + + require.NoError(t, buf.Close()) + } +} diff --git a/modules/packages/helm/metadata.go b/modules/packages/helm/metadata.go new file mode 100644 index 0000000..421fc5e --- /dev/null +++ b/modules/packages/helm/metadata.go @@ -0,0 +1,130 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package helm + +import ( + "archive/tar" + "compress/gzip" + "io" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" + "gopkg.in/yaml.v3" +) + +var ( + // ErrMissingChartFile indicates a missing Chart.yaml file + ErrMissingChartFile = util.NewInvalidArgumentErrorf("Chart.yaml file is missing") + // ErrInvalidName indicates an invalid package name + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + // ErrInvalidVersion indicates an invalid package version + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") + // ErrInvalidChart indicates an invalid chart + ErrInvalidChart = util.NewInvalidArgumentErrorf("chart is invalid") +) + +// Metadata for a Chart file. This models the structure of a Chart.yaml file. +type Metadata struct { + APIVersion string `json:"api_version" yaml:"apiVersion"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` + Name string `json:"name" yaml:"name"` + Version string `json:"version" yaml:"version"` + AppVersion string `json:"app_version,omitempty" yaml:"appVersion,omitempty"` + Home string `json:"home,omitempty" yaml:"home,omitempty"` + Sources []string `json:"sources,omitempty" yaml:"sources,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Keywords []string `json:"keywords,omitempty" yaml:"keywords,omitempty"` + Maintainers []*Maintainer `json:"maintainers,omitempty" yaml:"maintainers,omitempty"` + Icon string `json:"icon,omitempty" yaml:"icon,omitempty"` + Condition string `json:"condition,omitempty" yaml:"condition,omitempty"` + Tags string `json:"tags,omitempty" yaml:"tags,omitempty"` + Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"` + Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"` + KubeVersion string `json:"kube_version,omitempty" yaml:"kubeVersion,omitempty"` + Dependencies []*Dependency `json:"dependencies,omitempty" yaml:"dependencies,omitempty"` +} + +type Maintainer struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Email string `json:"email,omitempty" yaml:"email,omitempty"` + URL string `json:"url,omitempty" yaml:"url,omitempty"` +} + +type Dependency struct { + Name string `json:"name" yaml:"name"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` + Repository string `json:"repository" yaml:"repository"` + Condition string `json:"condition,omitempty" yaml:"condition,omitempty"` + Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + ImportValues []any `json:"import_values,omitempty" yaml:"import-values,omitempty"` + Alias string `json:"alias,omitempty" yaml:"alias,omitempty"` +} + +// ParseChartArchive parses the metadata of a Helm archive +func ParseChartArchive(r io.Reader) (*Metadata, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if hd.FileInfo().Name() == "Chart.yaml" { + if strings.Count(hd.Name, "/") != 1 { + continue + } + + return ParseChartFile(tr) + } + } + + return nil, ErrMissingChartFile +} + +// ParseChartFile parses a Chart.yaml file to retrieve the metadata of a Helm chart +func ParseChartFile(r io.Reader) (*Metadata, error) { + var metadata *Metadata + if err := yaml.NewDecoder(r).Decode(&metadata); err != nil { + return nil, err + } + + if metadata.APIVersion == "" { + return nil, ErrInvalidChart + } + + if metadata.Type != "" && metadata.Type != "application" && metadata.Type != "library" { + return nil, ErrInvalidChart + } + + if metadata.Name == "" { + return nil, ErrInvalidName + } + + if _, err := version.NewSemver(metadata.Version); err != nil { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(metadata.Home) { + metadata.Home = "" + } + + return metadata, nil +} diff --git a/modules/packages/maven/metadata.go b/modules/packages/maven/metadata.go new file mode 100644 index 0000000..42aa250 --- /dev/null +++ b/modules/packages/maven/metadata.go @@ -0,0 +1,93 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package maven + +import ( + "encoding/xml" + "io" + + "code.gitea.io/gitea/modules/validation" + + "golang.org/x/net/html/charset" +) + +// Metadata represents the metadata of a Maven package +type Metadata struct { + GroupID string `json:"group_id,omitempty"` + ArtifactID string `json:"artifact_id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Licenses []string `json:"licenses,omitempty"` + Dependencies []*Dependency `json:"dependencies,omitempty"` +} + +// Dependency represents a dependency of a Maven package +type Dependency struct { + GroupID string `json:"group_id,omitempty"` + ArtifactID string `json:"artifact_id,omitempty"` + Version string `json:"version,omitempty"` +} + +type pomStruct struct { + XMLName xml.Name `xml:"project"` + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Name string `xml:"name"` + Description string `xml:"description"` + URL string `xml:"url"` + Licenses []struct { + Name string `xml:"name"` + URL string `xml:"url"` + Distribution string `xml:"distribution"` + } `xml:"licenses>license"` + Dependencies []struct { + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Scope string `xml:"scope"` + } `xml:"dependencies>dependency"` +} + +// ParsePackageMetaData parses the metadata of a pom file +func ParsePackageMetaData(r io.Reader) (*Metadata, error) { + var pom pomStruct + + dec := xml.NewDecoder(r) + dec.CharsetReader = charset.NewReaderLabel + if err := dec.Decode(&pom); err != nil { + return nil, err + } + + if !validation.IsValidURL(pom.URL) { + pom.URL = "" + } + + licenses := make([]string, 0, len(pom.Licenses)) + for _, l := range pom.Licenses { + if l.Name != "" { + licenses = append(licenses, l.Name) + } + } + + dependencies := make([]*Dependency, 0, len(pom.Dependencies)) + for _, d := range pom.Dependencies { + dependencies = append(dependencies, &Dependency{ + GroupID: d.GroupID, + ArtifactID: d.ArtifactID, + Version: d.Version, + }) + } + + return &Metadata{ + GroupID: pom.GroupID, + ArtifactID: pom.ArtifactID, + Name: pom.Name, + Description: pom.Description, + ProjectURL: pom.URL, + Licenses: licenses, + Dependencies: dependencies, + }, nil +} diff --git a/modules/packages/maven/metadata_test.go b/modules/packages/maven/metadata_test.go new file mode 100644 index 0000000..d009301 --- /dev/null +++ b/modules/packages/maven/metadata_test.go @@ -0,0 +1,90 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package maven + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/encoding/charmap" +) + +const ( + groupID = "org.gitea" + artifactID = "my-project" + version = "1.0.1" + name = "My Gitea Project" + description = "Package Description" + projectURL = "https://gitea.io" + license = "MIT" + dependencyGroupID = "org.gitea.core" + dependencyArtifactID = "git" + dependencyVersion = "5.0.0" +) + +const pomContent = `<?xml version="1.0"?> +<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <groupId>` + groupID + `</groupId> + <artifactId>` + artifactID + `</artifactId> + <version>` + version + `</version> + <name>` + name + `</name> + <description>` + description + `</description> + <url>` + projectURL + `</url> + <licenses> + <license> + <name>` + license + `</name> + </license> + </licenses> + <dependencies> + <dependency> + <groupId>` + dependencyGroupID + `</groupId> + <artifactId>` + dependencyArtifactID + `</artifactId> + <version>` + dependencyVersion + `</version> + </dependency> + </dependencies> +</project>` + +func TestParsePackageMetaData(t *testing.T) { + t.Run("InvalidFile", func(t *testing.T) { + m, err := ParsePackageMetaData(strings.NewReader("")) + assert.Nil(t, m) + require.Error(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + m, err := ParsePackageMetaData(strings.NewReader(pomContent)) + require.NoError(t, err) + assert.NotNil(t, m) + + assert.Equal(t, groupID, m.GroupID) + assert.Equal(t, artifactID, m.ArtifactID) + assert.Equal(t, name, m.Name) + assert.Equal(t, description, m.Description) + assert.Equal(t, projectURL, m.ProjectURL) + assert.Len(t, m.Licenses, 1) + assert.Equal(t, license, m.Licenses[0]) + assert.Len(t, m.Dependencies, 1) + assert.Equal(t, dependencyGroupID, m.Dependencies[0].GroupID) + assert.Equal(t, dependencyArtifactID, m.Dependencies[0].ArtifactID) + assert.Equal(t, dependencyVersion, m.Dependencies[0].Version) + }) + + t.Run("Encoding", func(t *testing.T) { + // UTF-8 is default but the metadata could be encoded differently + pomContent8859_1, err := charmap.ISO8859_1.NewEncoder().String( + strings.ReplaceAll( + pomContent, + `<?xml version="1.0"?>`, + `<?xml version="1.0" encoding="ISO-8859-1"?>`, + ), + ) + require.NoError(t, err) + + m, err := ParsePackageMetaData(strings.NewReader(pomContent8859_1)) + require.NoError(t, err) + assert.NotNil(t, m) + }) +} diff --git a/modules/packages/multi_hasher.go b/modules/packages/multi_hasher.go new file mode 100644 index 0000000..83a4b5b --- /dev/null +++ b/modules/packages/multi_hasher.go @@ -0,0 +1,122 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package packages + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding" + "errors" + "hash" + "io" +) + +const ( + marshaledSizeMD5 = 92 + marshaledSizeSHA1 = 96 + marshaledSizeSHA256 = 108 + marshaledSizeSHA512 = 204 + + marshaledSize = marshaledSizeMD5 + marshaledSizeSHA1 + marshaledSizeSHA256 + marshaledSizeSHA512 +) + +// HashSummer provide a Sums method +type HashSummer interface { + Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) +} + +// MultiHasher calculates multiple checksums +type MultiHasher struct { + md5 hash.Hash + sha1 hash.Hash + sha256 hash.Hash + sha512 hash.Hash + + combinedWriter io.Writer +} + +// NewMultiHasher creates a multi hasher +func NewMultiHasher() *MultiHasher { + md5 := md5.New() + sha1 := sha1.New() + sha256 := sha256.New() + sha512 := sha512.New() + + combinedWriter := io.MultiWriter(md5, sha1, sha256, sha512) + + return &MultiHasher{ + md5, + sha1, + sha256, + sha512, + combinedWriter, + } +} + +// MarshalBinary implements encoding.BinaryMarshaler +func (h *MultiHasher) MarshalBinary() ([]byte, error) { + md5Bytes, err := h.md5.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha1Bytes, err := h.sha1.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha256Bytes, err := h.sha256.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha512Bytes, err := h.sha512.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + + b := make([]byte, 0, marshaledSize) + b = append(b, md5Bytes...) + b = append(b, sha1Bytes...) + b = append(b, sha256Bytes...) + b = append(b, sha512Bytes...) + return b, nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler +func (h *MultiHasher) UnmarshalBinary(b []byte) error { + if len(b) != marshaledSize { + return errors.New("invalid hash state size") + } + + if err := h.md5.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeMD5]); err != nil { + return err + } + + b = b[marshaledSizeMD5:] + if err := h.sha1.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA1]); err != nil { + return err + } + + b = b[marshaledSizeSHA1:] + if err := h.sha256.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA256]); err != nil { + return err + } + + b = b[marshaledSizeSHA256:] + return h.sha512.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[:marshaledSizeSHA512]) +} + +// Write implements io.Writer +func (h *MultiHasher) Write(p []byte) (int, error) { + return h.combinedWriter.Write(p) +} + +// Sums gets the MD5, SHA1, SHA256 and SHA512 checksums of the data +func (h *MultiHasher) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { + hashMD5 = h.md5.Sum(nil) + hashSHA1 = h.sha1.Sum(nil) + hashSHA256 = h.sha256.Sum(nil) + hashSHA512 = h.sha512.Sum(nil) + return hashMD5, hashSHA1, hashSHA256, hashSHA512 +} diff --git a/modules/packages/multi_hasher_test.go b/modules/packages/multi_hasher_test.go new file mode 100644 index 0000000..ca333cb --- /dev/null +++ b/modules/packages/multi_hasher_test.go @@ -0,0 +1,54 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package packages + +import ( + "encoding/hex" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + expectedMD5 = "e3bef03c5f3b7f6b3ab3e3053ed71e9c" + expectedSHA1 = "060b3b99f88e96085b4a68e095bc9e3d1d91e1bc" + expectedSHA256 = "6ccce4863b70f258d691f59609d31b4502e1ba5199942d3bc5d35d17a4ce771d" + expectedSHA512 = "7f70e439ba8c52025c1f06cdf6ae443c4b8ed2e90059cdb9bbbf8adf80846f185a24acca9245b128b226d61753b0d7ed46580a69c8999eeff3bc13a4d0bd816c" +) + +func TestMultiHasherSums(t *testing.T) { + t.Run("Sums", func(t *testing.T) { + h := NewMultiHasher() + h.Write([]byte("gitea")) + + hashMD5, hashSHA1, hashSHA256, hashSHA512 := h.Sums() + + assert.Equal(t, expectedMD5, hex.EncodeToString(hashMD5)) + assert.Equal(t, expectedSHA1, hex.EncodeToString(hashSHA1)) + assert.Equal(t, expectedSHA256, hex.EncodeToString(hashSHA256)) + assert.Equal(t, expectedSHA512, hex.EncodeToString(hashSHA512)) + }) + + t.Run("State", func(t *testing.T) { + h := NewMultiHasher() + h.Write([]byte("git")) + + state, err := h.MarshalBinary() + require.NoError(t, err) + + h2 := NewMultiHasher() + err = h2.UnmarshalBinary(state) + require.NoError(t, err) + + h2.Write([]byte("ea")) + + hashMD5, hashSHA1, hashSHA256, hashSHA512 := h2.Sums() + + assert.Equal(t, expectedMD5, hex.EncodeToString(hashMD5)) + assert.Equal(t, expectedSHA1, hex.EncodeToString(hashSHA1)) + assert.Equal(t, expectedSHA256, hex.EncodeToString(hashSHA256)) + assert.Equal(t, expectedSHA512, hex.EncodeToString(hashSHA512)) + }) +} diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go new file mode 100644 index 0000000..7d3d7cd --- /dev/null +++ b/modules/packages/npm/creator.go @@ -0,0 +1,289 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package npm + +import ( + "bytes" + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "fmt" + "io" + "regexp" + "strings" + "time" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +var ( + // ErrInvalidPackage indicates an invalid package + ErrInvalidPackage = util.NewInvalidArgumentErrorf("package is invalid") + // ErrInvalidPackageName indicates an invalid name + ErrInvalidPackageName = util.NewInvalidArgumentErrorf("package name is invalid") + // ErrInvalidPackageVersion indicates an invalid version + ErrInvalidPackageVersion = util.NewInvalidArgumentErrorf("package version is invalid") + // ErrInvalidAttachment indicates a invalid attachment + ErrInvalidAttachment = util.NewInvalidArgumentErrorf("package attachment is invalid") + // ErrInvalidIntegrity indicates an integrity validation error + ErrInvalidIntegrity = util.NewInvalidArgumentErrorf("failed to validate integrity") +) + +var nameMatch = regexp.MustCompile(`^(@[a-z0-9-][a-z0-9-._]*/)?[a-z0-9-][a-z0-9-._]*$`) + +// Package represents a npm package +type Package struct { + Name string + Version string + DistTags []string + Metadata Metadata + Filename string + Data []byte +} + +// PackageMetadata https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type PackageMetadata struct { + ID string `json:"_id"` + Name string `json:"name"` + Description string `json:"description"` + DistTags map[string]string `json:"dist-tags,omitempty"` + Versions map[string]*PackageMetadataVersion `json:"versions"` + Readme string `json:"readme,omitempty"` + Maintainers []User `json:"maintainers,omitempty"` + Time map[string]time.Time `json:"time,omitempty"` + Homepage string `json:"homepage,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Repository Repository `json:"repository,omitempty"` + Author User `json:"author"` + ReadmeFilename string `json:"readmeFilename,omitempty"` + Users map[string]bool `json:"users,omitempty"` + License string `json:"license,omitempty"` +} + +// PackageMetadataVersion documentation: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +// PackageMetadataVersion response: https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object +type PackageMetadataVersion struct { + ID string `json:"_id"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Author User `json:"author"` + Homepage string `json:"homepage,omitempty"` + License string `json:"license,omitempty"` + Repository Repository `json:"repository,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + BundleDependencies []string `json:"bundleDependencies,omitempty"` + DevDependencies map[string]string `json:"devDependencies,omitempty"` + PeerDependencies map[string]string `json:"peerDependencies,omitempty"` + Bin map[string]string `json:"bin,omitempty"` + OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` + Readme string `json:"readme,omitempty"` + Dist PackageDistribution `json:"dist"` + Maintainers []User `json:"maintainers,omitempty"` +} + +// PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type PackageDistribution struct { + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + Tarball string `json:"tarball"` + FileCount int `json:"fileCount,omitempty"` + UnpackedSize int `json:"unpackedSize,omitempty"` + NpmSignature string `json:"npm-signature,omitempty"` +} + +type PackageSearch struct { + Objects []*PackageSearchObject `json:"objects"` + Total int64 `json:"total"` +} + +type PackageSearchObject struct { + Package *PackageSearchPackage `json:"package"` +} + +type PackageSearchPackage struct { + Scope string `json:"scope"` + Name string `json:"name"` + Version string `json:"version"` + Date time.Time `json:"date"` + Description string `json:"description"` + Author User `json:"author"` + Publisher User `json:"publisher"` + Maintainers []User `json:"maintainers"` + Keywords []string `json:"keywords,omitempty"` + Links *PackageSearchPackageLinks `json:"links"` +} + +type PackageSearchPackageLinks struct { + Registry string `json:"npm"` + Homepage string `json:"homepage,omitempty"` + Repository string `json:"repository,omitempty"` +} + +// User https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type User struct { + Username string `json:"username,omitempty"` + Name string `json:"name"` + Email string `json:"email,omitempty"` + URL string `json:"url,omitempty"` +} + +// UnmarshalJSON is needed because User objects can be strings or objects +func (u *User) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + if err := json.Unmarshal(data, &u.Name); err != nil { + return err + } + case '{': + var tmp struct { + Username string `json:"username"` + Name string `json:"name"` + Email string `json:"email"` + URL string `json:"url"` + } + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + u.Username = tmp.Username + u.Name = tmp.Name + u.Email = tmp.Email + u.URL = tmp.URL + } + return nil +} + +// Repository https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version +type Repository struct { + Type string `json:"type"` + URL string `json:"url"` +} + +// PackageAttachment https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#package +type PackageAttachment struct { + ContentType string `json:"content_type"` + Data string `json:"data"` + Length int `json:"length"` +} + +type packageUpload struct { + PackageMetadata + Attachments map[string]*PackageAttachment `json:"_attachments"` +} + +// ParsePackage parses the content into a npm package +func ParsePackage(r io.Reader) (*Package, error) { + var upload packageUpload + if err := json.NewDecoder(r).Decode(&upload); err != nil { + return nil, err + } + + for _, meta := range upload.Versions { + if !validateName(meta.Name) { + return nil, ErrInvalidPackageName + } + + v, err := version.NewSemver(meta.Version) + if err != nil { + return nil, ErrInvalidPackageVersion + } + + scope := "" + name := meta.Name + nameParts := strings.SplitN(meta.Name, "/", 2) + if len(nameParts) == 2 { + scope = nameParts[0] + name = nameParts[1] + } + + if !validation.IsValidURL(meta.Homepage) { + meta.Homepage = "" + } + + p := &Package{ + Name: meta.Name, + Version: v.String(), + DistTags: make([]string, 0, 1), + Metadata: Metadata{ + Scope: scope, + Name: name, + Description: meta.Description, + Author: meta.Author.Name, + License: meta.License, + ProjectURL: meta.Homepage, + Keywords: meta.Keywords, + Dependencies: meta.Dependencies, + BundleDependencies: meta.BundleDependencies, + DevelopmentDependencies: meta.DevDependencies, + PeerDependencies: meta.PeerDependencies, + OptionalDependencies: meta.OptionalDependencies, + Bin: meta.Bin, + Readme: meta.Readme, + Repository: meta.Repository, + }, + } + + for tag := range upload.DistTags { + p.DistTags = append(p.DistTags, tag) + } + + p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version)) + + attachment := func() *PackageAttachment { + for _, a := range upload.Attachments { + return a + } + return nil + }() + if attachment == nil || len(attachment.Data) == 0 { + return nil, ErrInvalidAttachment + } + + data, err := base64.StdEncoding.DecodeString(attachment.Data) + if err != nil { + return nil, ErrInvalidAttachment + } + p.Data = data + + integrity := strings.SplitN(meta.Dist.Integrity, "-", 2) + if len(integrity) != 2 { + return nil, ErrInvalidIntegrity + } + integrityHash, err := base64.StdEncoding.DecodeString(integrity[1]) + if err != nil { + return nil, ErrInvalidIntegrity + } + var hash []byte + switch integrity[0] { + case "sha1": + tmp := sha1.Sum(data) + hash = tmp[:] + case "sha512": + tmp := sha512.Sum512(data) + hash = tmp[:] + } + if !bytes.Equal(integrityHash, hash) { + return nil, ErrInvalidIntegrity + } + + return p, nil + } + + return nil, ErrInvalidPackage +} + +func validateName(name string) bool { + if strings.TrimSpace(name) != name { + return false + } + if len(name) == 0 || len(name) > 214 { + return false + } + return nameMatch.MatchString(name) +} diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go new file mode 100644 index 0000000..b2cf1aa --- /dev/null +++ b/modules/packages/npm/creator_test.go @@ -0,0 +1,302 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package npm + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParsePackage(t *testing.T) { + packageScope := "@scope" + packageName := "test-package" + packageFullName := packageScope + "/" + packageName + packageVersion := "1.0.1-pre" + packageTag := "latest" + packageAuthor := "KN4CK3R" + packageBin := "gitea" + packageDescription := "Test Description" + data := "H4sIAAAAAAAA/ytITM5OTE/VL4DQelnF+XkMVAYGBgZmJiYK2MRBwNDcSIHB2NTMwNDQzMwAqA7IMDUxA9LUdgg2UFpcklgEdAql5kD8ogCnhwio5lJQUMpLzE1VslJQcihOzi9I1S9JLS7RhSYIJR2QgrLUouLM/DyQGkM9Az1D3YIiqExKanFyUWZBCVQ2BKhVwQVJDKwosbQkI78IJO/tZ+LsbRykxFXLNdA+HwWjYBSMgpENACgAbtAACAAA" + integrity := "sha512-yA4FJsVhetynGfOC1jFf79BuS+jrHbm0fhh+aHzCQkOaOBXKf9oBnC4a6DnLLnEsHQDRLYd00cwj8sCXpC+wIg==" + repository := Repository{ + Type: "gitea", + URL: "http://localhost:3000/gitea/test.git", + } + + t.Run("InvalidUpload", func(t *testing.T) { + p, err := ParsePackage(bytes.NewReader([]byte{0})) + assert.Nil(t, p) + require.Error(t, err) + }) + + t.Run("InvalidUploadNoData", func(t *testing.T) { + b, _ := json.Marshal(packageUpload{}) + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidPackage) + }) + + t.Run("InvalidPackageName", func(t *testing.T) { + test := func(t *testing.T, name string) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidPackageName) + } + + test(t, " test ") + test(t, " test") + test(t, "test ") + test(t, "te st") + test(t, "Test") + test(t, "_test") + test(t, ".test") + test(t, "^test") + test(t, "te^st") + test(t, "te|st") + test(t, "te)(st") + test(t, "te'st") + test(t, "te!st") + test(t, "te*st") + test(t, "te~st") + test(t, "invalid/scope") + test(t, "@invalid/_name") + test(t, "@invalid/.name") + }) + + t.Run("ValidPackageName", func(t *testing.T) { + test := func(t *testing.T, name string) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: name, + Name: name, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: name, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidPackageVersion) + } + + test(t, "test") + test(t, "@scope/name") + test(t, "@scope/q") + test(t, "q") + test(t, "@scope/package-name") + test(t, "@scope/package.name") + test(t, "@scope/package_name") + test(t, "123name") + test(t, "----") + test(t, packageFullName) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + version := "first-version" + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + version: { + Name: packageFullName, + Version: version, + }, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidPackageVersion) + }) + + t.Run("InvalidAttachment", func(t *testing.T) { + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + "dummy.tgz": {}, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidAttachment) + }) + + t.Run("InvalidData", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: "/", + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidAttachment) + }) + + t.Run("InvalidIntegrity", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Dist: PackageDistribution{ + Integrity: "sha512-test==", + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: data, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidIntegrity) + }) + + t.Run("InvalidIntegrity2", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Dist: PackageDistribution{ + Integrity: integrity, + }, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: base64.StdEncoding.EncodeToString([]byte("data")), + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrInvalidIntegrity) + }) + + t.Run("Valid", func(t *testing.T) { + filename := fmt.Sprintf("%s-%s.tgz", packageFullName, packageVersion) + b, _ := json.Marshal(packageUpload{ + PackageMetadata: PackageMetadata{ + ID: packageFullName, + Name: packageFullName, + DistTags: map[string]string{ + packageTag: packageVersion, + }, + Versions: map[string]*PackageMetadataVersion{ + packageVersion: { + Name: packageFullName, + Version: packageVersion, + Description: packageDescription, + Author: User{Name: packageAuthor}, + License: "MIT", + Homepage: "https://gitea.io/", + Readme: packageDescription, + Dependencies: map[string]string{ + "package": "1.2.0", + }, + Bin: map[string]string{ + "bin": packageBin, + }, + Dist: PackageDistribution{ + Integrity: integrity, + }, + Repository: repository, + }, + }, + }, + Attachments: map[string]*PackageAttachment{ + filename: { + Data: data, + }, + }, + }) + + p, err := ParsePackage(bytes.NewReader(b)) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, packageFullName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, []string{packageTag}, p.DistTags) + assert.Equal(t, fmt.Sprintf("%s-%s.tgz", strings.Split(packageFullName, "/")[1], packageVersion), p.Filename) + b, _ = base64.StdEncoding.DecodeString(data) + assert.Equal(t, b, p.Data) + assert.Equal(t, packageName, p.Metadata.Name) + assert.Equal(t, packageScope, p.Metadata.Scope) + assert.Equal(t, packageDescription, p.Metadata.Description) + assert.Equal(t, packageDescription, p.Metadata.Readme) + assert.Equal(t, packageAuthor, p.Metadata.Author) + assert.Equal(t, packageBin, p.Metadata.Bin["bin"]) + assert.Equal(t, "MIT", p.Metadata.License) + assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL) + assert.Contains(t, p.Metadata.Dependencies, "package") + assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"]) + assert.Equal(t, repository.Type, p.Metadata.Repository.Type) + assert.Equal(t, repository.URL, p.Metadata.Repository.URL) + }) +} diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go new file mode 100644 index 0000000..6bb77f3 --- /dev/null +++ b/modules/packages/npm/metadata.go @@ -0,0 +1,26 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package npm + +// TagProperty is the name of the property for tag management +const TagProperty = "npm.tag" + +// Metadata represents the metadata of a npm package +type Metadata struct { + Scope string `json:"scope,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Author string `json:"author,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + BundleDependencies []string `json:"bundleDependencies,omitempty"` + DevelopmentDependencies map[string]string `json:"development_dependencies,omitempty"` + PeerDependencies map[string]string `json:"peer_dependencies,omitempty"` + OptionalDependencies map[string]string `json:"optional_dependencies,omitempty"` + Bin map[string]string `json:"bin,omitempty"` + Readme string `json:"readme,omitempty"` + Repository Repository `json:"repository,omitempty"` +} diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go new file mode 100644 index 0000000..1e98ddf --- /dev/null +++ b/modules/packages/nuget/metadata.go @@ -0,0 +1,239 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "path/filepath" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +var ( + // ErrMissingNuspecFile indicates a missing Nuspec file + ErrMissingNuspecFile = util.NewInvalidArgumentErrorf("Nuspec file is missing") + // ErrNuspecFileTooLarge indicates a Nuspec file which is too large + ErrNuspecFileTooLarge = util.NewInvalidArgumentErrorf("Nuspec file is too large") + // ErrNuspecInvalidID indicates an invalid id in the Nuspec file + ErrNuspecInvalidID = util.NewInvalidArgumentErrorf("Nuspec file contains an invalid id") + // ErrNuspecInvalidVersion indicates an invalid version in the Nuspec file + ErrNuspecInvalidVersion = util.NewInvalidArgumentErrorf("Nuspec file contains an invalid version") +) + +// PackageType specifies the package type the metadata describes +type PackageType int + +const ( + // DependencyPackage represents a package (*.nupkg) + DependencyPackage PackageType = iota + 1 + // SymbolsPackage represents a symbol package (*.snupkg) + SymbolsPackage + + PropertySymbolID = "nuget.symbol.id" +) + +var idmatch = regexp.MustCompile(`\A\w+(?:[.-]\w+)*\z`) + +const maxNuspecFileSize = 3 * 1024 * 1024 + +// Package represents a Nuget package +type Package struct { + PackageType PackageType + ID string + Version string + Metadata *Metadata + NuspecContent *bytes.Buffer +} + +// Metadata represents the metadata of a Nuget package +type Metadata struct { + Description string `json:"description,omitempty"` + ReleaseNotes string `json:"release_notes,omitempty"` + Readme string `json:"readme,omitempty"` + Authors string `json:"authors,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + RequireLicenseAcceptance bool `json:"require_license_acceptance"` + Dependencies map[string][]Dependency `json:"dependencies,omitempty"` +} + +// Dependency represents a dependency of a Nuget package +type Dependency struct { + ID string `json:"id"` + Version string `json:"version"` +} + +// https://learn.microsoft.com/en-us/nuget/reference/nuspec +type nuspecPackage struct { + Metadata struct { + ID string `xml:"id"` + Version string `xml:"version"` + Authors string `xml:"authors"` + RequireLicenseAcceptance bool `xml:"requireLicenseAcceptance"` + ProjectURL string `xml:"projectUrl"` + Description string `xml:"description"` + ReleaseNotes string `xml:"releaseNotes"` + Readme string `xml:"readme"` + PackageTypes struct { + PackageType []struct { + Name string `xml:"name,attr"` + } `xml:"packageType"` + } `xml:"packageTypes"` + Repository struct { + URL string `xml:"url,attr"` + } `xml:"repository"` + Dependencies struct { + Dependency []struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + Exclude string `xml:"exclude,attr"` + } `xml:"dependency"` + Group []struct { + TargetFramework string `xml:"targetFramework,attr"` + Dependency []struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + Exclude string `xml:"exclude,attr"` + } `xml:"dependency"` + } `xml:"group"` + } `xml:"dependencies"` + } `xml:"metadata"` +} + +// ParsePackageMetaData parses the metadata of a Nuget package file +func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range archive.File { + if filepath.Dir(file.Name) != "." { + continue + } + if strings.HasSuffix(strings.ToLower(file.Name), ".nuspec") { + if file.UncompressedSize64 > maxNuspecFileSize { + return nil, ErrNuspecFileTooLarge + } + f, err := archive.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseNuspecMetaData(archive, f) + } + } + return nil, ErrMissingNuspecFile +} + +// ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package +func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { + var nuspecBuf bytes.Buffer + var p nuspecPackage + if err := xml.NewDecoder(io.TeeReader(r, &nuspecBuf)).Decode(&p); err != nil { + return nil, err + } + + if !idmatch.MatchString(p.Metadata.ID) { + return nil, ErrNuspecInvalidID + } + + v, err := version.NewSemver(p.Metadata.Version) + if err != nil { + return nil, ErrNuspecInvalidVersion + } + + if !validation.IsValidURL(p.Metadata.ProjectURL) { + p.Metadata.ProjectURL = "" + } + + packageType := DependencyPackage + for _, pt := range p.Metadata.PackageTypes.PackageType { + if pt.Name == "SymbolsPackage" { + packageType = SymbolsPackage + break + } + } + + m := &Metadata{ + Description: p.Metadata.Description, + ReleaseNotes: p.Metadata.ReleaseNotes, + Authors: p.Metadata.Authors, + ProjectURL: p.Metadata.ProjectURL, + RepositoryURL: p.Metadata.Repository.URL, + RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance, + Dependencies: make(map[string][]Dependency), + } + + if p.Metadata.Readme != "" { + f, err := archive.Open(p.Metadata.Readme) + if err == nil { + buf, _ := io.ReadAll(f) + m.Readme = string(buf) + _ = f.Close() + } + } + + if len(p.Metadata.Dependencies.Dependency) > 0 { + deps := make([]Dependency, 0, len(p.Metadata.Dependencies.Dependency)) + for _, dep := range p.Metadata.Dependencies.Dependency { + if dep.ID == "" || dep.Version == "" { + continue + } + deps = append(deps, Dependency{ + ID: dep.ID, + Version: dep.Version, + }) + } + m.Dependencies[""] = deps + } + for _, group := range p.Metadata.Dependencies.Group { + deps := make([]Dependency, 0, len(group.Dependency)) + for _, dep := range group.Dependency { + if dep.ID == "" || dep.Version == "" { + continue + } + deps = append(deps, Dependency{ + ID: dep.ID, + Version: dep.Version, + }) + } + if len(deps) > 0 { + m.Dependencies[group.TargetFramework] = deps + } + } + return &Package{ + PackageType: packageType, + ID: p.Metadata.ID, + Version: toNormalizedVersion(v), + Metadata: m, + NuspecContent: &nuspecBuf, + }, nil +} + +// https://learn.microsoft.com/en-us/nuget/concepts/package-versioning#normalized-version-numbers +// https://github.com/NuGet/NuGet.Client/blob/dccbd304b11103e08b97abf4cf4bcc1499d9235a/src/NuGet.Core/NuGet.Versioning/VersionFormatter.cs#L121 +func toNormalizedVersion(v *version.Version) string { + var buf bytes.Buffer + segments := v.Segments64() + fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2]) + if len(segments) > 3 && segments[3] > 0 { + fmt.Fprintf(&buf, ".%d", segments[3]) + } + pre := v.Prerelease() + if pre != "" { + fmt.Fprint(&buf, "-", pre) + } + return buf.String() +} diff --git a/modules/packages/nuget/metadata_test.go b/modules/packages/nuget/metadata_test.go new file mode 100644 index 0000000..ecce052 --- /dev/null +++ b/modules/packages/nuget/metadata_test.go @@ -0,0 +1,188 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package nuget + +import ( + "archive/zip" + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + id = "System.Gitea" + semver = "1.0.1" + authors = "Gitea Authors" + projectURL = "https://gitea.io" + description = "Package Description" + releaseNotes = "Package Release Notes" + readme = "Readme" + repositoryURL = "https://gitea.io/gitea/gitea" + targetFramework = ".NETStandard2.1" + dependencyID = "System.Text.Json" + dependencyVersion = "5.0.0" +) + +const nuspecContent = `<?xml version="1.0" encoding="utf-8"?> +<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata> + <id>` + id + `</id> + <version>` + semver + `</version> + <authors>` + authors + `</authors> + <requireLicenseAcceptance>true</requireLicenseAcceptance> + <projectUrl>` + projectURL + `</projectUrl> + <description>` + description + `</description> + <releaseNotes>` + releaseNotes + `</releaseNotes> + <repository url="` + repositoryURL + `" /> + <readme>README.md</readme> + <dependencies> + <group targetFramework="` + targetFramework + `"> + <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" /> + </group> + </dependencies> + </metadata> +</package>` + +const symbolsNuspecContent = `<?xml version="1.0" encoding="utf-8"?> +<package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata> + <id>` + id + `</id> + <version>` + semver + `</version> + <description>` + description + `</description> + <packageTypes> + <packageType name="SymbolsPackage" /> + </packageTypes> + <dependencies> + <group targetFramework="` + targetFramework + `" /> + </dependencies> + </metadata> +</package>` + +func TestParsePackageMetaData(t *testing.T) { + createArchive := func(files map[string]string) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + for name, content := range files { + w, _ := archive.Create(name) + w.Write([]byte(content)) + } + archive.Close() + return buf.Bytes() + } + + t.Run("MissingNuspecFile", func(t *testing.T) { + data := createArchive(map[string]string{"dummy.txt": ""}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.ErrorIs(t, err, ErrMissingNuspecFile) + }) + + t.Run("MissingNuspecFileInRoot", func(t *testing.T) { + data := createArchive(map[string]string{"sub/package.nuspec": ""}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.ErrorIs(t, err, ErrMissingNuspecFile) + }) + + t.Run("InvalidNuspecFile", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": ""}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.Error(t, err) + }) + + t.Run("InvalidPackageId", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": `<?xml version="1.0" encoding="utf-8"?> + <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata></metadata> + </package>`}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.ErrorIs(t, err, ErrNuspecInvalidID) + }) + + t.Run("InvalidPackageVersion", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": `<?xml version="1.0" encoding="utf-8"?> + <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata> + <id>` + id + `</id> + </metadata> + </package>`}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + assert.Nil(t, np) + require.ErrorIs(t, err, ErrNuspecInvalidVersion) + }) + + t.Run("MissingReadme", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": nuspecContent}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, np) + assert.Empty(t, np.Metadata.Readme) + }) + + t.Run("Dependency Package", func(t *testing.T) { + data := createArchive(map[string]string{ + "package.nuspec": nuspecContent, + "README.md": readme, + }) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, np) + assert.Equal(t, DependencyPackage, np.PackageType) + + assert.Equal(t, id, np.ID) + assert.Equal(t, semver, np.Version) + assert.Equal(t, authors, np.Metadata.Authors) + assert.Equal(t, projectURL, np.Metadata.ProjectURL) + assert.Equal(t, description, np.Metadata.Description) + assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes) + assert.Equal(t, readme, np.Metadata.Readme) + assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL) + assert.Len(t, np.Metadata.Dependencies, 1) + assert.Contains(t, np.Metadata.Dependencies, targetFramework) + deps := np.Metadata.Dependencies[targetFramework] + assert.Len(t, deps, 1) + assert.Equal(t, dependencyID, deps[0].ID) + assert.Equal(t, dependencyVersion, deps[0].Version) + + t.Run("NormalizedVersion", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": `<?xml version="1.0" encoding="utf-8"?> + <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"> + <metadata> + <id>test</id> + <version>1.04.5.2.5-rc.1+metadata</version> + </metadata> + </package>`}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, np) + assert.Equal(t, "1.4.5.2-rc.1", np.Version) + }) + }) + + t.Run("Symbols Package", func(t *testing.T) { + data := createArchive(map[string]string{"package.nuspec": symbolsNuspecContent}) + + np, err := ParsePackageMetaData(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.NotNil(t, np) + assert.Equal(t, SymbolsPackage, np.PackageType) + + assert.Equal(t, id, np.ID) + assert.Equal(t, semver, np.Version) + assert.Equal(t, description, np.Metadata.Description) + assert.Empty(t, np.Metadata.Dependencies) + }) +} diff --git a/modules/packages/nuget/symbol_extractor.go b/modules/packages/nuget/symbol_extractor.go new file mode 100644 index 0000000..81bf037 --- /dev/null +++ b/modules/packages/nuget/symbol_extractor.go @@ -0,0 +1,186 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/binary" + "fmt" + "io" + "path" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util" +) + +var ( + ErrMissingPdbFiles = util.NewInvalidArgumentErrorf("package does not contain PDB files") + ErrInvalidFiles = util.NewInvalidArgumentErrorf("package contains invalid files") + ErrInvalidPdbMagicNumber = util.NewInvalidArgumentErrorf("invalid Portable PDB magic number") + ErrMissingPdbStream = util.NewInvalidArgumentErrorf("missing PDB stream") +) + +type PortablePdb struct { + Name string + ID string + Content *packages.HashedBuffer +} + +type PortablePdbList []*PortablePdb + +func (l PortablePdbList) Close() { + for _, pdb := range l { + pdb.Content.Close() + } +} + +// ExtractPortablePdb extracts PDB files from a .snupkg file +func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) { + archive, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + var pdbs PortablePdbList + + err = func() error { + for _, file := range archive.File { + if strings.HasSuffix(file.Name, "/") { + continue + } + ext := strings.ToLower(filepath.Ext(file.Name)) + + switch ext { + case ".nuspec", ".xml", ".psmdcp", ".rels", ".p7s": + continue + case ".pdb": + f, err := archive.Open(file.Name) + if err != nil { + return err + } + + buf, err := packages.CreateHashedBufferFromReader(f) + + f.Close() + + if err != nil { + return err + } + + id, err := ParseDebugHeaderID(buf) + if err != nil { + buf.Close() + return fmt.Errorf("Invalid PDB file: %w", err) + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + buf.Close() + return err + } + + pdbs = append(pdbs, &PortablePdb{ + Name: path.Base(file.Name), + ID: id, + Content: buf, + }) + default: + return ErrInvalidFiles + } + } + return nil + }() + if err != nil { + pdbs.Close() + return nil, err + } + + if len(pdbs) == 0 { + return nil, ErrMissingPdbFiles + } + + return pdbs, nil +} + +// ParseDebugHeaderID TODO +func ParseDebugHeaderID(r io.ReadSeeker) (string, error) { + var magic uint32 + if err := binary.Read(r, binary.LittleEndian, &magic); err != nil { + return "", err + } + if magic != 0x424A5342 { + return "", ErrInvalidPdbMagicNumber + } + + if _, err := r.Seek(8, io.SeekCurrent); err != nil { + return "", err + } + + var versionStringSize int32 + if err := binary.Read(r, binary.LittleEndian, &versionStringSize); err != nil { + return "", err + } + if _, err := r.Seek(int64(versionStringSize), io.SeekCurrent); err != nil { + return "", err + } + if _, err := r.Seek(2, io.SeekCurrent); err != nil { + return "", err + } + + var streamCount int16 + if err := binary.Read(r, binary.LittleEndian, &streamCount); err != nil { + return "", err + } + + read4ByteAlignedString := func(r io.Reader) (string, error) { + b := make([]byte, 4) + var buf bytes.Buffer + for { + if _, err := r.Read(b); err != nil { + return "", err + } + if i := bytes.IndexByte(b, 0); i != -1 { + buf.Write(b[:i]) + return buf.String(), nil + } + buf.Write(b) + } + } + + for i := 0; i < int(streamCount); i++ { + var offset uint32 + if err := binary.Read(r, binary.LittleEndian, &offset); err != nil { + return "", err + } + if _, err := r.Seek(4, io.SeekCurrent); err != nil { + return "", err + } + name, err := read4ByteAlignedString(r) + if err != nil { + return "", err + } + + if name == "#Pdb" { + if _, err := r.Seek(int64(offset), io.SeekStart); err != nil { + return "", err + } + + b := make([]byte, 16) + if _, err := r.Read(b); err != nil { + return "", err + } + + data1 := binary.LittleEndian.Uint32(b[0:4]) + data2 := binary.LittleEndian.Uint16(b[4:6]) + data3 := binary.LittleEndian.Uint16(b[6:8]) + data4 := b[8:16] + + return fmt.Sprintf("%08x%04x%04x%04x%012x", data1, data2, data3, data4[:2], data4[2:]), nil + } + } + + return "", ErrMissingPdbStream +} diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go new file mode 100644 index 0000000..b767ed0 --- /dev/null +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package nuget + +import ( + "archive/zip" + "bytes" + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const pdbContent = `QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAABgB8AAAAWAAAACNQZGIAAAAA1AAAAAgBAAAj +fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB +AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` + +func TestExtractPortablePdb(t *testing.T) { + createArchive := func(name string, content []byte) []byte { + var buf bytes.Buffer + archive := zip.NewWriter(&buf) + w, _ := archive.Create(name) + w.Write(content) + archive.Close() + return buf.Bytes() + } + + t.Run("MissingPdbFiles", func(t *testing.T) { + var buf bytes.Buffer + zip.NewWriter(&buf).Close() + + pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + require.ErrorIs(t, err, ErrMissingPdbFiles) + assert.Empty(t, pdbs) + }) + + t.Run("InvalidFiles", func(t *testing.T) { + data := createArchive("sub/test.bin", []byte{}) + + pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data))) + require.ErrorIs(t, err, ErrInvalidFiles) + assert.Empty(t, pdbs) + }) + + t.Run("Valid", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(pdbContent) + data := createArchive("test.pdb", b) + + pdbs, err := ExtractPortablePdb(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + assert.Len(t, pdbs, 1) + assert.Equal(t, "test.pdb", pdbs[0].Name) + assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", pdbs[0].ID) + pdbs.Close() + }) +} + +func TestParseDebugHeaderID(t *testing.T) { + t.Run("InvalidPdbMagicNumber", func(t *testing.T) { + id, err := ParseDebugHeaderID(bytes.NewReader([]byte{0, 0, 0, 0})) + require.ErrorIs(t, err, ErrInvalidPdbMagicNumber) + assert.Empty(t, id) + }) + + t.Run("MissingPdbStream", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(`QlNKQgEAAQAAAAAADAAAAFBEQiB2MS4wAAAAAAAAAQB8AAAAWAAAACNVUwA=`) + + id, err := ParseDebugHeaderID(bytes.NewReader(b)) + require.ErrorIs(t, err, ErrMissingPdbStream) + assert.Empty(t, id) + }) + + t.Run("Valid", func(t *testing.T) { + b, _ := base64.StdEncoding.DecodeString(pdbContent) + + id, err := ParseDebugHeaderID(bytes.NewReader(b)) + require.NoError(t, err) + assert.Equal(t, "d910bb6948bd4c6cb40155bcf52c3c94", id) + }) +} diff --git a/modules/packages/pub/metadata.go b/modules/packages/pub/metadata.go new file mode 100644 index 0000000..afb464e --- /dev/null +++ b/modules/packages/pub/metadata.go @@ -0,0 +1,153 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package pub + +import ( + "archive/tar" + "compress/gzip" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" + "gopkg.in/yaml.v3" +) + +var ( + ErrMissingPubspecFile = util.NewInvalidArgumentErrorf("Pubspec file is missing") + ErrPubspecFileTooLarge = util.NewInvalidArgumentErrorf("Pubspec file is too large") + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") +) + +var namePattern = regexp.MustCompile(`\A[a-zA-Z_][a-zA-Z0-9_]*\z`) + +// https://github.com/dart-lang/pub-dev/blob/4d582302a8d10152a5cd6129f65bf4f4dbca239d/pkg/pub_package_reader/lib/pub_package_reader.dart#L143 +const maxPubspecFileSize = 128 * 1024 + +// Package represents a Pub package +type Package struct { + Name string + Version string + Metadata *Metadata +} + +// Metadata represents the metadata of a Pub package +type Metadata struct { + Description string `json:"description,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + Readme string `json:"readme,omitempty"` + Pubspec any `json:"pubspec"` +} + +type pubspecPackage struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Description string `yaml:"description"` + Homepage string `yaml:"homepage"` + Repository string `yaml:"repository"` + Documentation string `yaml:"documentation"` +} + +// ParsePackage parses the Pub package file +func ParsePackage(r io.Reader) (*Package, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + var p *Package + var readme string + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if hd.Name == "pubspec.yaml" { + if hd.Size > maxPubspecFileSize { + return nil, ErrPubspecFileTooLarge + } + p, err = ParsePubspecMetadata(tr) + if err != nil { + return nil, err + } + } else if strings.ToLower(hd.Name) == "readme.md" { + data, err := io.ReadAll(tr) + if err != nil { + return nil, err + } + readme = string(data) + } + } + + if p == nil { + return nil, ErrMissingPubspecFile + } + + p.Metadata.Readme = readme + + return p, nil +} + +// ParsePubspecMetadata parses a Pubspec file to retrieve the metadata of a Pub package +func ParsePubspecMetadata(r io.Reader) (*Package, error) { + buf, err := io.ReadAll(io.LimitReader(r, maxPubspecFileSize)) + if err != nil { + return nil, err + } + + var p pubspecPackage + if err := yaml.Unmarshal(buf, &p); err != nil { + return nil, err + } + + if !namePattern.MatchString(p.Name) { + return nil, ErrInvalidName + } + + v, err := version.NewSemver(p.Version) + if err != nil { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(p.Homepage) { + p.Homepage = "" + } + if !validation.IsValidURL(p.Repository) { + p.Repository = "" + } + + var pubspec any + if err := yaml.Unmarshal(buf, &pubspec); err != nil { + return nil, err + } + + return &Package{ + Name: p.Name, + Version: v.String(), + Metadata: &Metadata{ + Description: p.Description, + ProjectURL: p.Homepage, + RepositoryURL: p.Repository, + DocumentationURL: p.Documentation, + Pubspec: pubspec, + }, + }, nil +} diff --git a/modules/packages/pub/metadata_test.go b/modules/packages/pub/metadata_test.go new file mode 100644 index 0000000..5ed083b --- /dev/null +++ b/modules/packages/pub/metadata_test.go @@ -0,0 +1,136 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package pub + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + description = "Package Description" + projectURL = "https://gitea.com" + repositoryURL = "https://gitea.com/gitea/gitea" + documentationURL = "https://docs.gitea.com" +) + +const pubspecContent = `name: ` + packageName + ` +version: ` + packageVersion + ` +description: ` + description + ` +homepage: ` + projectURL + ` +repository: ` + repositoryURL + ` +documentation: ` + documentationURL + ` + +environment: + sdk: '>=2.16.0 <3.0.0' + +dependencies: + flutter: + sdk: flutter + path: '>=1.8.0 <3.0.0' + +dev_dependencies: + http: '>=0.13.0'` + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) io.Reader { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for filename, content := range files { + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + } + tw.Close() + zw.Close() + return &buf + } + + t.Run("MissingPubspecFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"dummy.txt": {}}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + require.ErrorIs(t, err, ErrMissingPubspecFile) + }) + + t.Run("PubspecFileTooLarge", func(t *testing.T) { + data := createArchive(map[string][]byte{"pubspec.yaml": make([]byte, 200*1024)}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + require.ErrorIs(t, err, ErrPubspecFileTooLarge) + }) + + t.Run("InvalidPubspecFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"pubspec.yaml": {}}) + + pp, err := ParsePackage(data) + assert.Nil(t, pp) + require.Error(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + data := createArchive(map[string][]byte{"pubspec.yaml": []byte(pubspecContent)}) + + pp, err := ParsePackage(data) + require.NoError(t, err) + assert.NotNil(t, pp) + assert.Empty(t, pp.Metadata.Readme) + }) + + t.Run("ValidWithReadme", func(t *testing.T) { + data := createArchive(map[string][]byte{"pubspec.yaml": []byte(pubspecContent), "README.md": []byte("readme")}) + + pp, err := ParsePackage(data) + require.NoError(t, err) + assert.NotNil(t, pp) + assert.Equal(t, "readme", pp.Metadata.Readme) + }) +} + +func TestParsePubspecMetadata(t *testing.T) { + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"123abc", "ab-cd"} { + pp, err := ParsePubspecMetadata(strings.NewReader(`name: ` + name)) + assert.Nil(t, pp) + require.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + pp, err := ParsePubspecMetadata(strings.NewReader(`name: dummy +version: invalid`)) + assert.Nil(t, pp) + require.ErrorIs(t, err, ErrInvalidVersion) + }) + + t.Run("Valid", func(t *testing.T) { + pp, err := ParsePubspecMetadata(strings.NewReader(pubspecContent)) + require.NoError(t, err) + assert.NotNil(t, pp) + + assert.Equal(t, packageName, pp.Name) + assert.Equal(t, packageVersion, pp.Version) + assert.Equal(t, description, pp.Metadata.Description) + assert.Equal(t, projectURL, pp.Metadata.ProjectURL) + assert.Equal(t, repositoryURL, pp.Metadata.RepositoryURL) + assert.Equal(t, documentationURL, pp.Metadata.DocumentationURL) + assert.NotNil(t, pp.Metadata.Pubspec) + }) +} diff --git a/modules/packages/pypi/metadata.go b/modules/packages/pypi/metadata.go new file mode 100644 index 0000000..125728c --- /dev/null +++ b/modules/packages/pypi/metadata.go @@ -0,0 +1,15 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package pypi + +// Metadata represents the metadata of a PyPI package +type Metadata struct { + Author string `json:"author,omitempty"` + Description string `json:"description,omitempty"` + LongDescription string `json:"long_description,omitempty"` + Summary string `json:"summary,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + License string `json:"license,omitempty"` + RequiresPython string `json:"requires_python,omitempty"` +} diff --git a/modules/packages/rpm/metadata.go b/modules/packages/rpm/metadata.go new file mode 100644 index 0000000..f4f78c2 --- /dev/null +++ b/modules/packages/rpm/metadata.go @@ -0,0 +1,298 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rpm + +import ( + "fmt" + "io" + "strings" + + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/validation" + + "github.com/sassoftware/go-rpmutils" +) + +const ( + PropertyMetadata = "rpm.metadata" + PropertyGroup = "rpm.group" + PropertyArchitecture = "rpm.architecture" + + SettingKeyPrivate = "rpm.key.private" + SettingKeyPublic = "rpm.key.public" + + RepositoryPackage = "_rpm" + RepositoryVersion = "_repository" +) + +const ( + // Can't use the syscall constants because they are not available for windows build. + sIFMT = 0xf000 + sIFDIR = 0x4000 + sIXUSR = 0x40 + sIXGRP = 0x8 + sIXOTH = 0x1 +) + +// https://rpm-software-management.github.io/rpm/manual/spec.html +// https://refspecs.linuxbase.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic/pkgformat.html + +type Package struct { + Name string + Version string + VersionMetadata *VersionMetadata + FileMetadata *FileMetadata +} + +type VersionMetadata struct { + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` +} + +type FileMetadata struct { + Architecture string `json:"architecture,omitempty"` + Epoch string `json:"epoch,omitempty"` + Version string `json:"version,omitempty"` + Release string `json:"release,omitempty"` + Vendor string `json:"vendor,omitempty"` + Group string `json:"group,omitempty"` + Packager string `json:"packager,omitempty"` + SourceRpm string `json:"source_rpm,omitempty"` + BuildHost string `json:"build_host,omitempty"` + BuildTime uint64 `json:"build_time,omitempty"` + FileTime uint64 `json:"file_time,omitempty"` + InstalledSize uint64 `json:"installed_size,omitempty"` + ArchiveSize uint64 `json:"archive_size,omitempty"` + + Provides []*Entry `json:"provide,omitempty"` + Requires []*Entry `json:"require,omitempty"` + Conflicts []*Entry `json:"conflict,omitempty"` + Obsoletes []*Entry `json:"obsolete,omitempty"` + + Files []*File `json:"files,omitempty"` + + Changelogs []*Changelog `json:"changelogs,omitempty"` +} + +type Entry struct { + Name string `json:"name" xml:"name,attr"` + Flags string `json:"flags,omitempty" xml:"flags,attr,omitempty"` + Version string `json:"version,omitempty" xml:"ver,attr,omitempty"` + Epoch string `json:"epoch,omitempty" xml:"epoch,attr,omitempty"` + Release string `json:"release,omitempty" xml:"rel,attr,omitempty"` +} + +type File struct { + Path string `json:"path" xml:",chardata"` + Type string `json:"type,omitempty" xml:"type,attr,omitempty"` + IsExecutable bool `json:"is_executable" xml:"-"` +} + +type Changelog struct { + Author string `json:"author,omitempty" xml:"author,attr"` + Date timeutil.TimeStamp `json:"date,omitempty" xml:"date,attr"` + Text string `json:"text,omitempty" xml:",chardata"` +} + +// ParsePackage parses the RPM package file +func ParsePackage(r io.Reader) (*Package, error) { + rpm, err := rpmutils.ReadRpm(r) + if err != nil { + return nil, err + } + + nevra, err := rpm.Header.GetNEVRA() + if err != nil { + return nil, err + } + + version := fmt.Sprintf("%s-%s", nevra.Version, nevra.Release) + if nevra.Epoch != "" && nevra.Epoch != "0" { + version = fmt.Sprintf("%s-%s", nevra.Epoch, version) + } + + p := &Package{ + Name: nevra.Name, + Version: version, + VersionMetadata: &VersionMetadata{ + Summary: getString(rpm.Header, rpmutils.SUMMARY), + Description: getString(rpm.Header, rpmutils.DESCRIPTION), + License: getString(rpm.Header, rpmutils.LICENSE), + ProjectURL: getString(rpm.Header, rpmutils.URL), + }, + FileMetadata: &FileMetadata{ + Architecture: nevra.Arch, + Epoch: nevra.Epoch, + Version: nevra.Version, + Release: nevra.Release, + Vendor: getString(rpm.Header, rpmutils.VENDOR), + Group: getString(rpm.Header, rpmutils.GROUP), + Packager: getString(rpm.Header, rpmutils.PACKAGER), + SourceRpm: getString(rpm.Header, rpmutils.SOURCERPM), + BuildHost: getString(rpm.Header, rpmutils.BUILDHOST), + BuildTime: getUInt64(rpm.Header, rpmutils.BUILDTIME), + FileTime: getUInt64(rpm.Header, rpmutils.FILEMTIMES), + InstalledSize: getUInt64(rpm.Header, rpmutils.SIZE), + ArchiveSize: getUInt64(rpm.Header, rpmutils.SIG_PAYLOADSIZE), + + Provides: getEntries(rpm.Header, rpmutils.PROVIDENAME, rpmutils.PROVIDEVERSION, rpmutils.PROVIDEFLAGS), + Requires: getEntries(rpm.Header, rpmutils.REQUIRENAME, rpmutils.REQUIREVERSION, rpmutils.REQUIREFLAGS), + Conflicts: getEntries(rpm.Header, rpmutils.CONFLICTNAME, rpmutils.CONFLICTVERSION, rpmutils.CONFLICTFLAGS), + Obsoletes: getEntries(rpm.Header, rpmutils.OBSOLETENAME, rpmutils.OBSOLETEVERSION, rpmutils.OBSOLETEFLAGS), + Files: getFiles(rpm.Header), + Changelogs: getChangelogs(rpm.Header), + }, + } + + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + p.VersionMetadata.ProjectURL = "" + } + + return p, nil +} + +func getString(h *rpmutils.RpmHeader, tag int) string { + values, err := h.GetStrings(tag) + if err != nil || len(values) < 1 { + return "" + } + return values[0] +} + +func getUInt64(h *rpmutils.RpmHeader, tag int) uint64 { + values, err := h.GetUint64s(tag) + if err != nil || len(values) < 1 { + return 0 + } + return values[0] +} + +func getEntries(h *rpmutils.RpmHeader, namesTag, versionsTag, flagsTag int) []*Entry { + names, err := h.GetStrings(namesTag) + if err != nil || len(names) == 0 { + return nil + } + flags, err := h.GetUint64s(flagsTag) + if err != nil || len(flags) == 0 { + return nil + } + versions, err := h.GetStrings(versionsTag) + if err != nil || len(versions) == 0 { + return nil + } + if len(names) != len(flags) || len(names) != len(versions) { + return nil + } + + entries := make([]*Entry, 0, len(names)) + for i := range names { + e := &Entry{ + Name: names[i], + } + + flags := flags[i] + if (flags&rpmutils.RPMSENSE_GREATER) != 0 && (flags&rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "GE" + } else if (flags&rpmutils.RPMSENSE_LESS) != 0 && (flags&rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "LE" + } else if (flags & rpmutils.RPMSENSE_GREATER) != 0 { + e.Flags = "GT" + } else if (flags & rpmutils.RPMSENSE_LESS) != 0 { + e.Flags = "LT" + } else if (flags & rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "EQ" + } + + version := versions[i] + if version != "" { + parts := strings.Split(version, "-") + + versionParts := strings.Split(parts[0], ":") + if len(versionParts) == 2 { + e.Version = versionParts[1] + e.Epoch = versionParts[0] + } else { + e.Version = versionParts[0] + e.Epoch = "0" + } + + if len(parts) > 1 { + e.Release = parts[1] + } + } + + entries = append(entries, e) + } + return entries +} + +func getFiles(h *rpmutils.RpmHeader) []*File { + baseNames, _ := h.GetStrings(rpmutils.BASENAMES) + dirNames, _ := h.GetStrings(rpmutils.DIRNAMES) + dirIndexes, _ := h.GetUint32s(rpmutils.DIRINDEXES) + fileFlags, _ := h.GetUint32s(rpmutils.FILEFLAGS) + fileModes, _ := h.GetUint32s(rpmutils.FILEMODES) + + files := make([]*File, 0, len(baseNames)) + for i := range baseNames { + if len(dirIndexes) <= i { + continue + } + dirIndex := dirIndexes[i] + if len(dirNames) <= int(dirIndex) { + continue + } + + var fileType string + var isExecutable bool + if i < len(fileFlags) && (fileFlags[i]&rpmutils.RPMFILE_GHOST) != 0 { + fileType = "ghost" + } else if i < len(fileModes) { + if (fileModes[i] & sIFMT) == sIFDIR { + fileType = "dir" + } else { + mode := fileModes[i] & ^uint32(sIFMT) + isExecutable = (mode&sIXUSR) != 0 || (mode&sIXGRP) != 0 || (mode&sIXOTH) != 0 + } + } + + files = append(files, &File{ + Path: dirNames[dirIndex] + baseNames[i], + Type: fileType, + IsExecutable: isExecutable, + }) + } + + return files +} + +func getChangelogs(h *rpmutils.RpmHeader) []*Changelog { + texts, err := h.GetStrings(rpmutils.CHANGELOGTEXT) + if err != nil || len(texts) == 0 { + return nil + } + authors, err := h.GetStrings(rpmutils.CHANGELOGNAME) + if err != nil || len(authors) == 0 { + return nil + } + times, err := h.GetUint32s(rpmutils.CHANGELOGTIME) + if err != nil || len(times) == 0 { + return nil + } + if len(texts) != len(authors) || len(texts) != len(times) { + return nil + } + + changelogs := make([]*Changelog, 0, len(texts)) + for i := range texts { + changelogs = append(changelogs, &Changelog{ + Author: authors[i], + Date: timeutil.TimeStamp(times[i]), + Text: texts[i], + }) + } + return changelogs +} diff --git a/modules/packages/rpm/metadata_test.go b/modules/packages/rpm/metadata_test.go new file mode 100644 index 0000000..dc9b480 --- /dev/null +++ b/modules/packages/rpm/metadata_test.go @@ -0,0 +1,164 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rpm + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParsePackage(t *testing.T) { + base64RpmPackageContent := `H4sICFayB2QCAGdpdGVhLXRlc3QtMS4wLjItMS14ODZfNjQucnBtAO2YV4gTQRjHJzl7wbNhhxVF +VNwk2zd2PdvZ9Sxnd3Z3NllNsmF3o6congVFsWFHRWwIImIXfRER0QcRfPBJEXvvBQvWSfZTT0VQ +8TF/MuU33zcz3+zOJGEe73lyuQBRBWKWRzDrEddjuVAkxLMc+lsFUOWfm5bvvReAalWECg/TsivU +dyKa0U61aVnl6wj0Uxe4nc8F92hZiaYE8CO/P0r7/Quegr0c7M/AvoCaGZEIWNGUqMHrhhGROIUT +Zc7gOAOraoQzCNZ0WdU0HpEI5jiB4zlek3gT85wqCBomhomxoGCs8wImWMImbxqKgXVNUKKaqShR +STKVKK9glFUNcf2g+/t27xs16v5x/eyOKftVGlIhyiuvvPLKK6+88sorr7zyyiuvvPKCO5HPnz+v +pGVhhXsTsFVeSstuWR9anwU+Bk3Vch5wTwL3JkHg+8C1gR8A169wj1KdpobAj4HbAT+Be5VewE+h +fz/g52AvBX4N9vHAb4AnA7+F8ePAH8BuA38ELgf+BLzQ50oIeBlw0OdAOXAlP57AGuCsbwGtbgCu +DrwRuAb4bwau6T/PwFbgWsDXgWuD/y3gOmC/B1wI/Bi4AcT3Arih3z9YCNzI9w9m/YKUG4Nd9N9z +pSZgHwrcFPgccFt//OADGE+F/q+Ao+D/FrijzwV1gbv4/QvaAHcFDgF3B5aB+wB3Be7rz1dQCtwP +eDxwMcw3GbgU7AasdwzYE8DjwT4L/CeAvRx4IvBCYA3iWQds+FzpDjABfghsAj8BTgA/A/b8+StX +A84A1wKe5s9fuRB4JpzHZv55rL8a/Dv49vpn/PErR4BvQX8Z+Db4l2W5CH2/f0W5+1fEoeFDBzFp +rE/FMcK4mWQSOzN+aDOIqztW2rPsFKIyqh7sQERR42RVMSKihnzVHlQ8Ag0YLBYNEIajkhmuR5Io +7nlpt2M4nJs0ZNkoYaUyZahMlSfJImr1n1WjFVNCPCaTZgYNGdGL8YN2mX8WHfA/C7ViHJK0pxHG +SrkeTiSI4T+7ubf85yrzRCQRQ5EVxVAjvIBVRY/KRFAVReIkhfARSddNSceayQkGliIKb0q8RAxJ +5QWNVxHIsW3Pz369bw+5jh5y0klE9Znqm0dF57b0HbGy2A5lVUBTZZrqZjdUjYoprFmpsBtHP5d0 ++ISltS2yk2mHuC4x+lgJMhgnidvuqy3b0suK0bm+tw3FMxI2zjm7/fA0MtQhplX2s7nYLZ2ZC0yg +CxJZDokhORTJlrlcCvG5OieGBERlVCs7CfuS6WzQ/T2j+9f92BWxTFEcp2IkYccYGp2LYySEfreq +irue4WRF5XkpKovw2wgpq2rZBI8bQZkzxEkiYaNwxnXCCVvHidzIiB3CM2yMYdNWmjDsaLovaE4c +x3a6mLaTxB7rEj3jWN4M2p7uwPaa1GfI8BHFfcZMKhkycnhR7y781/a+A4t7FpWWTupRUtKbegwZ +XMKwJinTSe70uhRcj55qNu3YHtE922Fdz7FTMTq9Q3TbMdiYrrPudMvT44S6u2miu138eC0tTN9D +2CFGHHtQsHHsGCRFDFbXuT9wx6mUTZfseydlkWZeJkW6xOgYjqXT+LA7I6XHaUx2xmUzqelWymA9 +rCXI9+D1BHbjsITssqhBNysw0tOWjcpmIh6+aViYPfftw8ZSGfRVPUqKiosZj5R5qGmk/8AjjRbZ +d8b3vvngdPHx3HvMeCarIk7VVSwbgoZVkceEVyOmyUmGxBGNYDVKSFSOGlIkGqWnUZFkiY/wsmhK +Mu0UFYgZ/bYnuvn/vz4wtCz8qMwsHUvP0PX3tbYFUctAPdrY6tiiDtcCddDECahx7SuVNP5dpmb5 +9tMDyaXb7OAlk5acuPn57ss9mw6Wym0m1Fq2cej7tUt2LL4/b8enXU2fndk+fvv57ndnt55/cQob +7tpp/pEjDS7cGPZ6BY430+7danDq6f42Nw49b9F7zp6BiKpJb9s5P0AYN2+L159cnrur636rx+v1 +7ae1K28QbMMcqI8CqwIrgwg9nTOp8Oj9q81plUY7ZuwXN8Vvs8wbAAA=` + rpmPackageContent, err := base64.StdEncoding.DecodeString(base64RpmPackageContent) + require.NoError(t, err) + + zr, err := gzip.NewReader(bytes.NewReader(rpmPackageContent)) + require.NoError(t, err) + + p, err := ParsePackage(zr) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.Equal(t, "gitea-test", p.Name) + assert.Equal(t, "1.0.2-1", p.Version) + assert.NotNil(t, p.VersionMetadata) + assert.NotNil(t, p.FileMetadata) + + assert.Equal(t, "MIT", p.VersionMetadata.License) + assert.Equal(t, "https://gitea.io", p.VersionMetadata.ProjectURL) + assert.Equal(t, "RPM package summary", p.VersionMetadata.Summary) + assert.Equal(t, "RPM package description", p.VersionMetadata.Description) + + assert.Equal(t, "x86_64", p.FileMetadata.Architecture) + assert.Equal(t, "0", p.FileMetadata.Epoch) + assert.Equal(t, "1.0.2", p.FileMetadata.Version) + assert.Equal(t, "1", p.FileMetadata.Release) + assert.Empty(t, p.FileMetadata.Vendor) + assert.Equal(t, "KN4CK3R", p.FileMetadata.Packager) + assert.Equal(t, "gitea-test-1.0.2-1.src.rpm", p.FileMetadata.SourceRpm) + assert.Equal(t, "e44b1687d04b", p.FileMetadata.BuildHost) + assert.EqualValues(t, 1678225964, p.FileMetadata.BuildTime) + assert.EqualValues(t, 1678225964, p.FileMetadata.FileTime) + assert.EqualValues(t, 13, p.FileMetadata.InstalledSize) + assert.EqualValues(t, 272, p.FileMetadata.ArchiveSize) + assert.Empty(t, p.FileMetadata.Conflicts) + assert.Empty(t, p.FileMetadata.Obsoletes) + + assert.ElementsMatch( + t, + []*Entry{ + { + Name: "gitea-test", + Flags: "EQ", + Version: "1.0.2", + Epoch: "0", + Release: "1", + }, + { + Name: "gitea-test(x86-64)", + Flags: "EQ", + Version: "1.0.2", + Epoch: "0", + Release: "1", + }, + }, + p.FileMetadata.Provides, + ) + assert.ElementsMatch( + t, + []*Entry{ + { + Name: "/bin/sh", + }, + { + Name: "/bin/sh", + }, + { + Name: "/bin/sh", + }, + { + Name: "rpmlib(CompressedFileNames)", + Flags: "LE", + Version: "3.0.4", + Epoch: "0", + Release: "1", + }, + { + Name: "rpmlib(FileDigests)", + Flags: "LE", + Version: "4.6.0", + Epoch: "0", + Release: "1", + }, + { + Name: "rpmlib(PayloadFilesHavePrefix)", + Flags: "LE", + Version: "4.0", + Epoch: "0", + Release: "1", + }, + { + Name: "rpmlib(PayloadIsXz)", + Flags: "LE", + Version: "5.2", + Epoch: "0", + Release: "1", + }, + }, + p.FileMetadata.Requires, + ) + assert.ElementsMatch( + t, + []*File{ + { + Path: "/usr/local/bin/hello", + IsExecutable: true, + }, + }, + p.FileMetadata.Files, + ) + assert.ElementsMatch( + t, + []*Changelog{ + { + Author: "KN4CK3R <dummy@gitea.io>", + Date: 1678276800, + Text: "- Changelog message.", + }, + }, + p.FileMetadata.Changelogs, + ) +} diff --git a/modules/packages/rubygems/marshal.go b/modules/packages/rubygems/marshal.go new file mode 100644 index 0000000..4e6a5fc --- /dev/null +++ b/modules/packages/rubygems/marshal.go @@ -0,0 +1,311 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "bufio" + "bytes" + "io" + "reflect" + + "code.gitea.io/gitea/modules/util" +) + +const ( + majorVersion = 4 + minorVersion = 8 + + typeNil = '0' + typeTrue = 'T' + typeFalse = 'F' + typeFixnum = 'i' + typeString = '"' + typeSymbol = ':' + typeSymbolLink = ';' + typeArray = '[' + typeIVar = 'I' + typeUserMarshal = 'U' + typeUserDef = 'u' + typeObject = 'o' +) + +var ( + // ErrUnsupportedType indicates an unsupported type + ErrUnsupportedType = util.NewInvalidArgumentErrorf("type is unsupported") + // ErrInvalidIntRange indicates an invalid number range + ErrInvalidIntRange = util.NewInvalidArgumentErrorf("number is not in valid range") +) + +// RubyUserMarshal is a Ruby object that has a marshal_load function. +type RubyUserMarshal struct { + Name string + Value any +} + +// RubyUserDef is a Ruby object that has a _load function. +type RubyUserDef struct { + Name string + Value any +} + +// RubyObject is a default Ruby object. +type RubyObject struct { + Name string + Member map[string]any +} + +// MarshalEncoder mimics Rubys Marshal class. +// Note: Only supports types used by the RubyGems package registry. +type MarshalEncoder struct { + w *bufio.Writer + symbols map[string]int +} + +// NewMarshalEncoder creates a new MarshalEncoder +func NewMarshalEncoder(w io.Writer) *MarshalEncoder { + return &MarshalEncoder{ + w: bufio.NewWriter(w), + symbols: map[string]int{}, + } +} + +// Encode encodes the given type +func (e *MarshalEncoder) Encode(v any) error { + if _, err := e.w.Write([]byte{majorVersion, minorVersion}); err != nil { + return err + } + + if err := e.marshal(v); err != nil { + return err + } + + return e.w.Flush() +} + +func (e *MarshalEncoder) marshal(v any) error { + if v == nil { + return e.marshalNil() + } + + val := reflect.ValueOf(v) + typ := reflect.TypeOf(v) + + if typ.Kind() == reflect.Ptr { + val = val.Elem() + typ = typ.Elem() + } + + switch typ.Kind() { + case reflect.Bool: + return e.marshalBool(val.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: + return e.marshalInt(val.Int()) + case reflect.String: + return e.marshalString(val.String()) + case reflect.Slice, reflect.Array: + return e.marshalArray(val) + } + + switch typ.Name() { + case "RubyUserMarshal": + return e.marshalUserMarshal(val.Interface().(RubyUserMarshal)) + case "RubyUserDef": + return e.marshalUserDef(val.Interface().(RubyUserDef)) + case "RubyObject": + return e.marshalObject(val.Interface().(RubyObject)) + } + + return ErrUnsupportedType +} + +func (e *MarshalEncoder) marshalNil() error { + return e.w.WriteByte(typeNil) +} + +func (e *MarshalEncoder) marshalBool(b bool) error { + if b { + return e.w.WriteByte(typeTrue) + } + return e.w.WriteByte(typeFalse) +} + +func (e *MarshalEncoder) marshalInt(i int64) error { + if err := e.w.WriteByte(typeFixnum); err != nil { + return err + } + + return e.marshalIntInternal(i) +} + +func (e *MarshalEncoder) marshalIntInternal(i int64) error { + if i == 0 { + return e.w.WriteByte(0) + } else if 0 < i && i < 123 { + return e.w.WriteByte(byte(i + 5)) + } else if -124 < i && i <= -1 { + return e.w.WriteByte(byte(i - 5)) + } + + var length int + if 122 < i && i <= 0xff { + length = 1 + } else if 0xff < i && i <= 0xffff { + length = 2 + } else if 0xffff < i && i <= 0xffffff { + length = 3 + } else if 0xffffff < i && i <= 0x3fffffff { + length = 4 + } else if -0x100 <= i && i < -123 { + length = -1 + } else if -0x10000 <= i && i < -0x100 { + length = -2 + } else if -0x1000000 <= i && i < -0x100000 { + length = -3 + } else if -0x40000000 <= i && i < -0x1000000 { + length = -4 + } else { + return ErrInvalidIntRange + } + + if err := e.w.WriteByte(byte(length)); err != nil { + return err + } + if length < 0 { + length = -length + } + + for c := 0; c < length; c++ { + if err := e.w.WriteByte(byte(i >> uint(8*c) & 0xff)); err != nil { + return err + } + } + + return nil +} + +func (e *MarshalEncoder) marshalString(str string) error { + if err := e.w.WriteByte(typeIVar); err != nil { + return err + } + + if err := e.marshalRawString(str); err != nil { + return err + } + + if err := e.marshalIntInternal(1); err != nil { + return err + } + + if err := e.marshalSymbol("E"); err != nil { + return err + } + + return e.marshalBool(true) +} + +func (e *MarshalEncoder) marshalRawString(str string) error { + if err := e.w.WriteByte(typeString); err != nil { + return err + } + + if err := e.marshalIntInternal(int64(len(str))); err != nil { + return err + } + + _, err := e.w.WriteString(str) + return err +} + +func (e *MarshalEncoder) marshalSymbol(str string) error { + if index, ok := e.symbols[str]; ok { + if err := e.w.WriteByte(typeSymbolLink); err != nil { + return err + } + return e.marshalIntInternal(int64(index)) + } + + e.symbols[str] = len(e.symbols) + + if err := e.w.WriteByte(typeSymbol); err != nil { + return err + } + + if err := e.marshalIntInternal(int64(len(str))); err != nil { + return err + } + + _, err := e.w.WriteString(str) + return err +} + +func (e *MarshalEncoder) marshalArray(arr reflect.Value) error { + if err := e.w.WriteByte(typeArray); err != nil { + return err + } + + length := arr.Len() + + if err := e.marshalIntInternal(int64(length)); err != nil { + return err + } + + for i := 0; i < length; i++ { + if err := e.marshal(arr.Index(i).Interface()); err != nil { + return err + } + } + return nil +} + +func (e *MarshalEncoder) marshalUserMarshal(userMarshal RubyUserMarshal) error { + if err := e.w.WriteByte(typeUserMarshal); err != nil { + return err + } + + if err := e.marshalSymbol(userMarshal.Name); err != nil { + return err + } + + return e.marshal(userMarshal.Value) +} + +func (e *MarshalEncoder) marshalUserDef(userDef RubyUserDef) error { + var buf bytes.Buffer + if err := NewMarshalEncoder(&buf).Encode(userDef.Value); err != nil { + return err + } + + if err := e.w.WriteByte(typeUserDef); err != nil { + return err + } + if err := e.marshalSymbol(userDef.Name); err != nil { + return err + } + if err := e.marshalIntInternal(int64(buf.Len())); err != nil { + return err + } + _, err := e.w.Write(buf.Bytes()) + return err +} + +func (e *MarshalEncoder) marshalObject(obj RubyObject) error { + if err := e.w.WriteByte(typeObject); err != nil { + return err + } + if err := e.marshalSymbol(obj.Name); err != nil { + return err + } + if err := e.marshalIntInternal(int64(len(obj.Member))); err != nil { + return err + } + for k, v := range obj.Member { + if err := e.marshalSymbol(k); err != nil { + return err + } + if err := e.marshal(v); err != nil { + return err + } + } + return nil +} diff --git a/modules/packages/rubygems/marshal_test.go b/modules/packages/rubygems/marshal_test.go new file mode 100644 index 0000000..8aa9160 --- /dev/null +++ b/modules/packages/rubygems/marshal_test.go @@ -0,0 +1,99 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMinimalEncoder(t *testing.T) { + cases := []struct { + Value any + Expected []byte + Error error + }{ + { + Value: nil, + Expected: []byte{4, 8, 0x30}, + }, + { + Value: true, + Expected: []byte{4, 8, 'T'}, + }, + { + Value: false, + Expected: []byte{4, 8, 'F'}, + }, + { + Value: 0, + Expected: []byte{4, 8, 'i', 0}, + }, + { + Value: 1, + Expected: []byte{4, 8, 'i', 6}, + }, + { + Value: -1, + Expected: []byte{4, 8, 'i', 0xfa}, + }, + { + Value: 0x1fffffff, + Expected: []byte{4, 8, 'i', 4, 0xff, 0xff, 0xff, 0x1f}, + }, + { + Value: 0x41000000, + Error: ErrInvalidIntRange, + }, + { + Value: "test", + Expected: []byte{4, 8, 'I', '"', 9, 't', 'e', 's', 't', 6, ':', 6, 'E', 'T'}, + }, + { + Value: []int{1, 2}, + Expected: []byte{4, 8, '[', 7, 'i', 6, 'i', 7}, + }, + { + Value: &RubyUserMarshal{ + Name: "Test", + Value: 4, + }, + Expected: []byte{4, 8, 'U', ':', 9, 'T', 'e', 's', 't', 'i', 9}, + }, + { + Value: &RubyUserDef{ + Name: "Test", + Value: 4, + }, + Expected: []byte{4, 8, 'u', ':', 9, 'T', 'e', 's', 't', 9, 4, 8, 'i', 9}, + }, + { + Value: &RubyObject{ + Name: "Test", + Member: map[string]any{ + "test": 4, + }, + }, + Expected: []byte{4, 8, 'o', ':', 9, 'T', 'e', 's', 't', 6, ':', 9, 't', 'e', 's', 't', 'i', 9}, + }, + { + Value: &struct { + Name string + }{ + "test", + }, + Error: ErrUnsupportedType, + }, + } + + for i, c := range cases { + var b bytes.Buffer + err := NewMarshalEncoder(&b).Encode(c.Value) + require.ErrorIs(t, err, c.Error) + assert.Equal(t, c.Expected, b.Bytes(), "case %d", i) + } +} diff --git a/modules/packages/rubygems/metadata.go b/modules/packages/rubygems/metadata.go new file mode 100644 index 0000000..8a97948 --- /dev/null +++ b/modules/packages/rubygems/metadata.go @@ -0,0 +1,220 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "archive/tar" + "compress/gzip" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "gopkg.in/yaml.v3" +) + +var ( + // ErrMissingMetadataFile indicates a missing metadata.gz file + ErrMissingMetadataFile = util.NewInvalidArgumentErrorf("metadata.gz file is missing") + // ErrInvalidName indicates an invalid id in the metadata.gz file + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + // ErrInvalidVersion indicates an invalid version in the metadata.gz file + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") +) + +var versionMatcher = regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`) + +// Package represents a RubyGems package +type Package struct { + Name string + Version string + Metadata *Metadata +} + +// Metadata represents the metadata of a RubyGems package +type Metadata struct { + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Summary string `json:"summary,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses []string `json:"licenses,omitempty"` + RequiredRubyVersion []VersionRequirement `json:"required_ruby_version,omitempty"` + RequiredRubygemsVersion []VersionRequirement `json:"required_rubygems_version,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RuntimeDependencies []Dependency `json:"runtime_dependencies,omitempty"` + DevelopmentDependencies []Dependency `json:"development_dependencies,omitempty"` +} + +// VersionRequirement represents a version restriction +type VersionRequirement struct { + Restriction string `json:"restriction"` + Version string `json:"version"` +} + +// Dependency represents a dependency of a RubyGems package +type Dependency struct { + Name string `json:"name"` + Version []VersionRequirement `json:"version"` +} + +type gemspec struct { + Name string `yaml:"name"` + Version struct { + Version string `yaml:"version"` + } `yaml:"version"` + Platform string `yaml:"platform"` + Authors []string `yaml:"authors"` + Autorequire any `yaml:"autorequire"` + Bindir string `yaml:"bindir"` + CertChain []any `yaml:"cert_chain"` + Date string `yaml:"date"` + Dependencies []struct { + Name string `yaml:"name"` + Requirement requirement `yaml:"requirement"` + Type string `yaml:"type"` + Prerelease bool `yaml:"prerelease"` + VersionRequirements requirement `yaml:"version_requirements"` + } `yaml:"dependencies"` + Description string `yaml:"description"` + Executables []string `yaml:"executables"` + Extensions []any `yaml:"extensions"` + ExtraRdocFiles []string `yaml:"extra_rdoc_files"` + Files []string `yaml:"files"` + Homepage string `yaml:"homepage"` + Licenses []string `yaml:"licenses"` + Metadata struct { + BugTrackerURI string `yaml:"bug_tracker_uri"` + ChangelogURI string `yaml:"changelog_uri"` + DocumentationURI string `yaml:"documentation_uri"` + SourceCodeURI string `yaml:"source_code_uri"` + } `yaml:"metadata"` + PostInstallMessage any `yaml:"post_install_message"` + RdocOptions []any `yaml:"rdoc_options"` + RequirePaths []string `yaml:"require_paths"` + RequiredRubyVersion requirement `yaml:"required_ruby_version"` + RequiredRubygemsVersion requirement `yaml:"required_rubygems_version"` + Requirements []any `yaml:"requirements"` + RubygemsVersion string `yaml:"rubygems_version"` + SigningKey any `yaml:"signing_key"` + SpecificationVersion int `yaml:"specification_version"` + Summary string `yaml:"summary"` + TestFiles []any `yaml:"test_files"` +} + +type requirement struct { + Requirements [][]any `yaml:"requirements"` +} + +// AsVersionRequirement converts into []VersionRequirement +func (r requirement) AsVersionRequirement() []VersionRequirement { + requirements := make([]VersionRequirement, 0, len(r.Requirements)) + for _, req := range r.Requirements { + if len(req) != 2 { + continue + } + restriction, ok := req[0].(string) + if !ok { + continue + } + vm, ok := req[1].(map[string]any) + if !ok { + continue + } + versionInt, ok := vm["version"] + if !ok { + continue + } + version, ok := versionInt.(string) + if !ok || version == "0" { + continue + } + + requirements = append(requirements, VersionRequirement{ + Restriction: restriction, + Version: version, + }) + } + return requirements +} + +// ParsePackageMetaData parses the metadata of a Gem package file +func ParsePackageMetaData(r io.Reader) (*Package, error) { + archive := tar.NewReader(r) + for { + hdr, err := archive.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Name == "metadata.gz" { + return parseMetadataFile(archive) + } + } + + return nil, ErrMissingMetadataFile +} + +func parseMetadataFile(r io.Reader) (*Package, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer zr.Close() + + var spec gemspec + if err := yaml.NewDecoder(zr).Decode(&spec); err != nil { + return nil, err + } + + if len(spec.Name) == 0 || strings.Contains(spec.Name, "/") { + return nil, ErrInvalidName + } + + if !versionMatcher.MatchString(spec.Version.Version) { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(spec.Homepage) { + spec.Homepage = "" + } + if !validation.IsValidURL(spec.Metadata.SourceCodeURI) { + spec.Metadata.SourceCodeURI = "" + } + + m := &Metadata{ + Platform: spec.Platform, + Description: spec.Description, + Summary: spec.Summary, + Authors: spec.Authors, + Licenses: spec.Licenses, + ProjectURL: spec.Homepage, + RequiredRubyVersion: spec.RequiredRubyVersion.AsVersionRequirement(), + RequiredRubygemsVersion: spec.RequiredRubygemsVersion.AsVersionRequirement(), + DevelopmentDependencies: make([]Dependency, 0, 5), + RuntimeDependencies: make([]Dependency, 0, 5), + } + + for _, gemdep := range spec.Dependencies { + dep := Dependency{ + Name: gemdep.Name, + Version: gemdep.Requirement.AsVersionRequirement(), + } + if gemdep.Type == ":runtime" { + m.RuntimeDependencies = append(m.RuntimeDependencies, dep) + } else { + m.DevelopmentDependencies = append(m.DevelopmentDependencies, dep) + } + } + + return &Package{ + Name: spec.Name, + Version: spec.Version.Version, + Metadata: m, + }, nil +} diff --git a/modules/packages/rubygems/metadata_test.go b/modules/packages/rubygems/metadata_test.go new file mode 100644 index 0000000..cd3a5bb --- /dev/null +++ b/modules/packages/rubygems/metadata_test.go @@ -0,0 +1,89 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "archive/tar" + "bytes" + "encoding/base64" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParsePackageMetaData(t *testing.T) { + createArchive := func(filename string, content []byte) io.Reader { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + tw.Close() + return &buf + } + + t.Run("MissingMetadataFile", func(t *testing.T) { + data := createArchive("dummy.txt", []byte{0}) + + rp, err := ParsePackageMetaData(data) + require.ErrorIs(t, err, ErrMissingMetadataFile) + assert.Nil(t, rp) + }) + + t.Run("Valid", func(t *testing.T) { + content, _ := base64.StdEncoding.DecodeString("H4sICHC/I2EEAG1ldGFkYXRhAAEeAOH/bmFtZTogZwp2ZXJzaW9uOgogIHZlcnNpb246IDEKWw35Tx4AAAA=") + data := createArchive("metadata.gz", content) + + rp, err := ParsePackageMetaData(data) + require.NoError(t, err) + assert.NotNil(t, rp) + }) +} + +func TestParseMetadataFile(t *testing.T) { + content, _ := base64.StdEncoding.DecodeString(`H4sIAMe7I2ECA9VVTW/UMBC9+1eYXvaUbJpSQBZUHJAqDlwK4kCFIseZzZrGH9iTqisEv52Js9nd +0KqggiqRXWnX45n3ZuZ5nCzL+JPQ15ulq7+AQnEORoj3HpReaSVRO8usNCB4qxEku4YQySbuCPo4 +bjHOd07HeZGfMt9JXLlgBB9imOxx7UIULOPnCZMMLsDXXgeiYbW2jQ6C0y9TELBSa6kJ6/IzaySS +R1mUx1nxIitPeFGI9M2L6eGfWAMebANWaUgktzN9M3lsKNmxutBb1AYyCibbNhsDFu+q9GK/Tc4z +d2IcLBl9js5eHaXFsLyvXeNz0LQyL/YoLx8EsiCMBZlx46k6sS2PDD5AgA5kJPNKdhH2elWzOv7n +uv9Q9Aau/6ngP84elvNpXh5oRVlB5/yW7BH0+qu0G4gqaI/JdEHBFBS5l+pKtsARIjIwUnfj8Le0 ++TrdJLl2DG5A9SjrjgZ1mG+4QbAD+G4ZZBUap6qVnnzGf6Rwp+vliBRqtnYGPBEKvkb0USyXE8mS +dVoR6hj07u0HZgAl3SRS8G/fmXcRK20jyq6rDMSYQFgidamqkXbbuspLXE/0k7GphtKqe67GuRC/ +yjAbmt9LsOMp8xMamFkSQ38fP5EFjdz8LA4do2C69VvqWXAJgrPbKZb58/xZXrKoW6ttW13Bhvzi +4ftn7/yUxd4YGcglvTmmY8aGY3ZwRn4CqcWcidUGAAA=`) + rp, err := parseMetadataFile(bytes.NewReader(content)) + require.NoError(t, err) + assert.NotNil(t, rp) + + assert.Equal(t, "gitea", rp.Name) + assert.Equal(t, "1.0.5", rp.Version) + assert.Equal(t, "ruby", rp.Metadata.Platform) + assert.Equal(t, "Gitea package", rp.Metadata.Summary) + assert.Equal(t, "RubyGems package test", rp.Metadata.Description) + assert.Equal(t, []string{"Gitea"}, rp.Metadata.Authors) + assert.Equal(t, "https://gitea.io/", rp.Metadata.ProjectURL) + assert.Equal(t, []string{"MIT"}, rp.Metadata.Licenses) + assert.Empty(t, rp.Metadata.RequiredRubygemsVersion) + assert.Len(t, rp.Metadata.RequiredRubyVersion, 1) + assert.Equal(t, ">=", rp.Metadata.RequiredRubyVersion[0].Restriction) + assert.Equal(t, "2.3.0", rp.Metadata.RequiredRubyVersion[0].Version) + assert.Len(t, rp.Metadata.RuntimeDependencies, 1) + assert.Equal(t, "runtime-dep", rp.Metadata.RuntimeDependencies[0].Name) + assert.Len(t, rp.Metadata.RuntimeDependencies[0].Version, 2) + assert.Equal(t, ">=", rp.Metadata.RuntimeDependencies[0].Version[0].Restriction) + assert.Equal(t, "1.2.0", rp.Metadata.RuntimeDependencies[0].Version[0].Version) + assert.Equal(t, "<", rp.Metadata.RuntimeDependencies[0].Version[1].Restriction) + assert.Equal(t, "2.0", rp.Metadata.RuntimeDependencies[0].Version[1].Version) + assert.Len(t, rp.Metadata.DevelopmentDependencies, 1) + assert.Equal(t, "dev-dep", rp.Metadata.DevelopmentDependencies[0].Name) + assert.Len(t, rp.Metadata.DevelopmentDependencies[0].Version, 1) + assert.Equal(t, "~>", rp.Metadata.DevelopmentDependencies[0].Version[0].Restriction) + assert.Equal(t, "5.2", rp.Metadata.DevelopmentDependencies[0].Version[0].Version) +} diff --git a/modules/packages/swift/metadata.go b/modules/packages/swift/metadata.go new file mode 100644 index 0000000..24c4262 --- /dev/null +++ b/modules/packages/swift/metadata.go @@ -0,0 +1,214 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package swift + +import ( + "archive/zip" + "fmt" + "io" + "path" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +var ( + ErrMissingManifestFile = util.NewInvalidArgumentErrorf("Package.swift file is missing") + ErrManifestFileTooLarge = util.NewInvalidArgumentErrorf("Package.swift file is too large") + ErrInvalidManifestVersion = util.NewInvalidArgumentErrorf("manifest version is invalid") + + manifestPattern = regexp.MustCompile(`\APackage(?:@swift-(\d+(?:\.\d+)?(?:\.\d+)?))?\.swift\z`) + toolsVersionPattern = regexp.MustCompile(`\A// swift-tools-version:(\d+(?:\.\d+)?(?:\.\d+)?)`) +) + +const ( + maxManifestFileSize = 128 * 1024 + + PropertyScope = "swift.scope" + PropertyName = "swift.name" + PropertyRepositoryURL = "swift.repository_url" +) + +// Package represents a Swift package +type Package struct { + RepositoryURLs []string + Metadata *Metadata +} + +// Metadata represents the metadata of a Swift package +type Metadata struct { + Description string `json:"description,omitempty"` + Keywords []string `json:"keywords,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + License string `json:"license,omitempty"` + Author Person `json:"author,omitempty"` + Manifests map[string]*Manifest `json:"manifests,omitempty"` +} + +// Manifest represents a Package.swift file +type Manifest struct { + Content string `json:"content"` + ToolsVersion string `json:"tools_version,omitempty"` +} + +// https://schema.org/SoftwareSourceCode +type SoftwareSourceCode struct { + Context []string `json:"@context"` + Type string `json:"@type"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description,omitempty"` + Keywords []string `json:"keywords,omitempty"` + CodeRepository string `json:"codeRepository,omitempty"` + License string `json:"license,omitempty"` + Author Person `json:"author"` + ProgrammingLanguage ProgrammingLanguage `json:"programmingLanguage"` + RepositoryURLs []string `json:"repositoryURLs,omitempty"` +} + +// https://schema.org/ProgrammingLanguage +type ProgrammingLanguage struct { + Type string `json:"@type"` + Name string `json:"name"` + URL string `json:"url"` +} + +// https://schema.org/Person +type Person struct { + Type string `json:"@type,omitempty"` + GivenName string `json:"givenName,omitempty"` + MiddleName string `json:"middleName,omitempty"` + FamilyName string `json:"familyName,omitempty"` +} + +func (p Person) String() string { + var sb strings.Builder + if p.GivenName != "" { + sb.WriteString(p.GivenName) + } + if p.MiddleName != "" { + if sb.Len() > 0 { + sb.WriteRune(' ') + } + sb.WriteString(p.MiddleName) + } + if p.FamilyName != "" { + if sb.Len() > 0 { + sb.WriteRune(' ') + } + sb.WriteString(p.FamilyName) + } + return sb.String() +} + +// ParsePackage parses the Swift package upload +func ParsePackage(sr io.ReaderAt, size int64, mr io.Reader) (*Package, error) { + zr, err := zip.NewReader(sr, size) + if err != nil { + return nil, err + } + + p := &Package{ + Metadata: &Metadata{ + Manifests: make(map[string]*Manifest), + }, + } + + for _, file := range zr.File { + manifestMatch := manifestPattern.FindStringSubmatch(path.Base(file.Name)) + if len(manifestMatch) == 0 { + continue + } + + if file.UncompressedSize64 > maxManifestFileSize { + return nil, ErrManifestFileTooLarge + } + + f, err := zr.Open(file.Name) + if err != nil { + return nil, err + } + + content, err := io.ReadAll(f) + + if err := f.Close(); err != nil { + return nil, err + } + + if err != nil { + return nil, err + } + + swiftVersion := "" + if len(manifestMatch) == 2 && manifestMatch[1] != "" { + v, err := version.NewSemver(manifestMatch[1]) + if err != nil { + return nil, ErrInvalidManifestVersion + } + swiftVersion = TrimmedVersionString(v) + } + + manifest := &Manifest{ + Content: string(content), + } + + toolsMatch := toolsVersionPattern.FindStringSubmatch(manifest.Content) + if len(toolsMatch) == 2 { + v, err := version.NewSemver(toolsMatch[1]) + if err != nil { + return nil, ErrInvalidManifestVersion + } + + manifest.ToolsVersion = TrimmedVersionString(v) + } + + p.Metadata.Manifests[swiftVersion] = manifest + } + + if _, found := p.Metadata.Manifests[""]; !found { + return nil, ErrMissingManifestFile + } + + if mr != nil { + var ssc *SoftwareSourceCode + if err := json.NewDecoder(mr).Decode(&ssc); err != nil { + return nil, err + } + + p.Metadata.Description = ssc.Description + p.Metadata.Keywords = ssc.Keywords + p.Metadata.License = ssc.License + p.Metadata.Author = Person{ + GivenName: ssc.Author.GivenName, + MiddleName: ssc.Author.MiddleName, + FamilyName: ssc.Author.FamilyName, + } + + p.Metadata.RepositoryURL = ssc.CodeRepository + if !validation.IsValidURL(p.Metadata.RepositoryURL) { + p.Metadata.RepositoryURL = "" + } + + p.RepositoryURLs = ssc.RepositoryURLs + } + + return p, nil +} + +// TrimmedVersionString returns the version string without the patch segment if it is zero +func TrimmedVersionString(v *version.Version) string { + segments := v.Segments64() + + var b strings.Builder + fmt.Fprintf(&b, "%d.%d", segments[0], segments[1]) + if segments[2] != 0 { + fmt.Fprintf(&b, ".%d", segments[2]) + } + return b.String() +} diff --git a/modules/packages/swift/metadata_test.go b/modules/packages/swift/metadata_test.go new file mode 100644 index 0000000..b223d8c --- /dev/null +++ b/modules/packages/swift/metadata_test.go @@ -0,0 +1,145 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package swift + +import ( + "archive/zip" + "bytes" + "strings" + "testing" + + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + packageDescription = "Package Description" + packageRepositoryURL = "https://gitea.io/gitea/gitea" + packageAuthor = "KN4CK3R" + packageLicense = "MIT" +) + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) *bytes.Reader { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for filename, content := range files { + w, _ := zw.Create(filename) + w.Write(content) + } + zw.Close() + return bytes.NewReader(buf.Bytes()) + } + + t.Run("MissingManifestFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"dummy.txt": {}}) + + p, err := ParsePackage(data, data.Size(), nil) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrMissingManifestFile) + }) + + t.Run("ManifestFileTooLarge", func(t *testing.T) { + data := createArchive(map[string][]byte{ + "Package.swift": make([]byte, maxManifestFileSize+1), + }) + + p, err := ParsePackage(data, data.Size(), nil) + assert.Nil(t, p) + require.ErrorIs(t, err, ErrManifestFileTooLarge) + }) + + t.Run("WithoutMetadata", func(t *testing.T) { + content1 := "// swift-tools-version:5.7\n//\n// Package.swift" + content2 := "// swift-tools-version:5.6\n//\n// Package@swift-5.6.swift" + + data := createArchive(map[string][]byte{ + "Package.swift": []byte(content1), + "Package@swift-5.5.swift": []byte(content2), + }) + + p, err := ParsePackage(data, data.Size(), nil) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.NotNil(t, p.Metadata) + assert.Empty(t, p.RepositoryURLs) + assert.Len(t, p.Metadata.Manifests, 2) + m := p.Metadata.Manifests[""] + assert.Equal(t, "5.7", m.ToolsVersion) + assert.Equal(t, content1, m.Content) + m = p.Metadata.Manifests["5.5"] + assert.Equal(t, "5.6", m.ToolsVersion) + assert.Equal(t, content2, m.Content) + }) + + t.Run("WithMetadata", func(t *testing.T) { + data := createArchive(map[string][]byte{ + "Package.swift": []byte("// swift-tools-version:5.7\n//\n// Package.swift"), + }) + + p, err := ParsePackage( + data, + data.Size(), + strings.NewReader(`{"name":"`+packageName+`","version":"`+packageVersion+`","description":"`+packageDescription+`","keywords":["swift","package"],"license":"`+packageLicense+`","codeRepository":"`+packageRepositoryURL+`","author":{"givenName":"`+packageAuthor+`"},"repositoryURLs":["`+packageRepositoryURL+`"]}`), + ) + assert.NotNil(t, p) + require.NoError(t, err) + + assert.NotNil(t, p.Metadata) + assert.Len(t, p.Metadata.Manifests, 1) + m := p.Metadata.Manifests[""] + assert.Equal(t, "5.7", m.ToolsVersion) + + assert.Equal(t, packageDescription, p.Metadata.Description) + assert.ElementsMatch(t, []string{"swift", "package"}, p.Metadata.Keywords) + assert.Equal(t, packageLicense, p.Metadata.License) + assert.Equal(t, packageAuthor, p.Metadata.Author.GivenName) + assert.Equal(t, packageRepositoryURL, p.Metadata.RepositoryURL) + assert.ElementsMatch(t, []string{packageRepositoryURL}, p.RepositoryURLs) + }) +} + +func TestTrimmedVersionString(t *testing.T) { + cases := []struct { + Version *version.Version + Expected string + }{ + { + Version: version.Must(version.NewVersion("1")), + Expected: "1.0", + }, + { + Version: version.Must(version.NewVersion("1.0")), + Expected: "1.0", + }, + { + Version: version.Must(version.NewVersion("1.0.0")), + Expected: "1.0", + }, + { + Version: version.Must(version.NewVersion("1.0.1")), + Expected: "1.0.1", + }, + { + Version: version.Must(version.NewVersion("1.0+meta")), + Expected: "1.0", + }, + { + Version: version.Must(version.NewVersion("1.0.0+meta")), + Expected: "1.0", + }, + { + Version: version.Must(version.NewVersion("1.0.1+meta")), + Expected: "1.0.1", + }, + } + + for _, c := range cases { + assert.Equal(t, c.Expected, TrimmedVersionString(c.Version)) + } +} diff --git a/modules/packages/vagrant/metadata.go b/modules/packages/vagrant/metadata.go new file mode 100644 index 0000000..6789533 --- /dev/null +++ b/modules/packages/vagrant/metadata.go @@ -0,0 +1,96 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package vagrant + +import ( + "archive/tar" + "compress/gzip" + "io" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" +) + +const ( + PropertyProvider = "vagrant.provider" +) + +// Metadata represents the metadata of a Vagrant package +type Metadata struct { + Author string `json:"author,omitempty"` + Description string `json:"description,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` +} + +// ParseMetadataFromBox parses the metadata of a box file +func ParseMetadataFromBox(r io.Reader) (*Metadata, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if hd.Name == "info.json" { + return ParseInfoFile(tr) + } + } + + return &Metadata{}, nil +} + +// ParseInfoFile parses a info.json file to retrieve the metadata of a Vagrant package +func ParseInfoFile(r io.Reader) (*Metadata, error) { + var values map[string]string + if err := json.NewDecoder(r).Decode(&values); err != nil { + return nil, err + } + + m := &Metadata{} + + // There is no defined format for this file, just try the common keys + for k, v := range values { + switch strings.ToLower(k) { + case "description": + fallthrough + case "short_description": + m.Description = v + case "website": + fallthrough + case "homepage": + fallthrough + case "url": + if validation.IsValidURL(v) { + m.ProjectURL = v + } + case "repository": + fallthrough + case "source": + if validation.IsValidURL(v) { + m.RepositoryURL = v + } + case "author": + fallthrough + case "authors": + m.Author = v + } + } + + return m, nil +} diff --git a/modules/packages/vagrant/metadata_test.go b/modules/packages/vagrant/metadata_test.go new file mode 100644 index 0000000..f467781 --- /dev/null +++ b/modules/packages/vagrant/metadata_test.go @@ -0,0 +1,111 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package vagrant + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + author = "gitea" + description = "Package Description" + projectURL = "https://gitea.io" + repositoryURL = "https://gitea.io/gitea/gitea" +) + +func TestParseMetadataFromBox(t *testing.T) { + createArchive := func(files map[string][]byte) io.Reader { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for filename, content := range files { + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + } + tw.Close() + zw.Close() + return &buf + } + + t.Run("MissingInfoFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"dummy.txt": {}}) + + metadata, err := ParseMetadataFromBox(data) + assert.NotNil(t, metadata) + require.NoError(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + content, err := json.Marshal(map[string]string{ + "description": description, + "author": author, + "website": projectURL, + "repository": repositoryURL, + }) + require.NoError(t, err) + + data := createArchive(map[string][]byte{"info.json": content}) + + metadata, err := ParseMetadataFromBox(data) + assert.NotNil(t, metadata) + require.NoError(t, err) + + assert.Equal(t, author, metadata.Author) + assert.Equal(t, description, metadata.Description) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) + }) +} + +func TestParseInfoFile(t *testing.T) { + t.Run("UnknownKeys", func(t *testing.T) { + content, err := json.Marshal(map[string]string{ + "package": "", + "dummy": "", + }) + require.NoError(t, err) + + metadata, err := ParseInfoFile(bytes.NewReader(content)) + assert.NotNil(t, metadata) + require.NoError(t, err) + + assert.Empty(t, metadata.Author) + assert.Empty(t, metadata.Description) + assert.Empty(t, metadata.ProjectURL) + assert.Empty(t, metadata.RepositoryURL) + }) + + t.Run("Valid", func(t *testing.T) { + content, err := json.Marshal(map[string]string{ + "description": description, + "author": author, + "website": projectURL, + "repository": repositoryURL, + }) + require.NoError(t, err) + + metadata, err := ParseInfoFile(bytes.NewReader(content)) + assert.NotNil(t, metadata) + require.NoError(t, err) + + assert.Equal(t, author, metadata.Author) + assert.Equal(t, description, metadata.Description) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) + }) +} |