From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- modules/packages/arch/metadata.go | 341 +++++++++++++++++++++++++ modules/packages/arch/metadata_test.go | 447 +++++++++++++++++++++++++++++++++ 2 files changed, 788 insertions(+) create mode 100644 modules/packages/arch/metadata.go create mode 100644 modules/packages/arch/metadata_test.go (limited to 'modules/packages/arch') diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go new file mode 100644 index 0000000..6cdde75 --- /dev/null +++ b/modules/packages/arch/metadata.go @@ -0,0 +1,341 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bufio" + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/mholt/archiver/v3" +) + +// Arch Linux Packages +// https://man.archlinux.org/man/PKGBUILD.5 + +const ( + PropertyDescription = "arch.description" + PropertyArch = "arch.architecture" + PropertyDistribution = "arch.distribution" + + SettingKeyPrivate = "arch.key.private" + SettingKeyPublic = "arch.key.public" + + RepositoryPackage = "_arch" + RepositoryVersion = "_repository" +) + +var ( + reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) + reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?(:.*)?$`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?$`) + + magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} + magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} + magicGZ = []byte{0x1F, 0x8B} +) + +type Package struct { + Name string `json:"name"` + Version string `json:"version"` // Includes version, release and epoch + CompressType string `json:"compress_type"` + VersionMetadata VersionMetadata + FileMetadata FileMetadata +} + +// Arch package metadata related to specific version. +// Version metadata the same across different architectures and distributions. +type VersionMetadata struct { + Base string `json:"base"` + Description string `json:"description"` + ProjectURL string `json:"project_url"` + Groups []string `json:"groups,omitempty"` + Provides []string `json:"provides,omitempty"` + License []string `json:"license,omitempty"` + Depends []string `json:"depends,omitempty"` + OptDepends []string `json:"opt_depends,omitempty"` + MakeDepends []string `json:"make_depends,omitempty"` + CheckDepends []string `json:"check_depends,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Replaces []string `json:"replaces,omitempty"` + Backup []string `json:"backup,omitempty"` + XData []string `json:"xdata,omitempty"` +} + +// FileMetadata Metadata related to specific package file. +// This metadata might vary for different architecture and distribution. +type FileMetadata struct { + CompressedSize int64 `json:"compressed_size"` + InstalledSize int64 `json:"installed_size"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + BuildDate int64 `json:"build_date"` + Packager string `json:"packager"` + Arch string `json:"arch"` + PgpSigned string `json:"pgp"` +} + +// ParsePackage Function that receives arch package archive data and returns it's metadata. +func ParsePackage(r *packages.HashedBuffer) (*Package, error) { + md5, _, sha256, _ := r.Sums() + _, err := r.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + header := make([]byte, 5) + _, err = r.Read(header) + if err != nil { + return nil, err + } + _, err = r.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + var tarball archiver.Reader + var tarballType string + if bytes.Equal(header[:len(magicZSTD)], magicZSTD) { + tarballType = "zst" + tarball = archiver.NewTarZstd() + } else if bytes.Equal(header[:len(magicXZ)], magicXZ) { + tarballType = "xz" + tarball = archiver.NewTarXz() + } else if bytes.Equal(header[:len(magicGZ)], magicGZ) { + tarballType = "gz" + tarball = archiver.NewTarGz() + } else { + return nil, errors.New("not supported compression") + } + err = tarball.Open(r, 0) + if err != nil { + return nil, err + } + defer tarball.Close() + + var pkg *Package + var mTree bool + + for { + f, err := tarball.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + switch f.Name() { + case ".PKGINFO": + pkg, err = ParsePackageInfo(tarballType, f) + if err != nil { + _ = f.Close() + return nil, err + } + case ".MTREE": + mTree = true + } + _ = f.Close() + } + + if pkg == nil { + return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found") + } + + if !mTree { + return nil, util.NewInvalidArgumentErrorf(".MTREE file not found") + } + + pkg.FileMetadata.CompressedSize = r.Size() + pkg.FileMetadata.MD5 = hex.EncodeToString(md5) + pkg.FileMetadata.SHA256 = hex.EncodeToString(sha256) + + return pkg, nil +} + +// ParsePackageInfo Function that accepts reader for .PKGINFO file from package archive, +// validates all field according to PKGBUILD spec and returns package. +func ParsePackageInfo(compressType string, r io.Reader) (*Package, error) { + p := &Package{ + CompressType: compressType, + } + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "#") { + continue + } + + key, value, find := strings.Cut(line, "=") + if !find { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch key { + case "pkgname": + p.Name = value + case "pkgbase": + p.VersionMetadata.Base = value + case "pkgver": + p.Version = value + case "pkgdesc": + p.VersionMetadata.Description = value + case "url": + p.VersionMetadata.ProjectURL = value + case "packager": + p.FileMetadata.Packager = value + case "arch": + p.FileMetadata.Arch = value + case "provides": + p.VersionMetadata.Provides = append(p.VersionMetadata.Provides, value) + case "license": + p.VersionMetadata.License = append(p.VersionMetadata.License, value) + case "depend": + p.VersionMetadata.Depends = append(p.VersionMetadata.Depends, value) + case "optdepend": + p.VersionMetadata.OptDepends = append(p.VersionMetadata.OptDepends, value) + case "makedepend": + p.VersionMetadata.MakeDepends = append(p.VersionMetadata.MakeDepends, value) + case "checkdepend": + p.VersionMetadata.CheckDepends = append(p.VersionMetadata.CheckDepends, value) + case "backup": + p.VersionMetadata.Backup = append(p.VersionMetadata.Backup, value) + case "group": + p.VersionMetadata.Groups = append(p.VersionMetadata.Groups, value) + case "conflict": + p.VersionMetadata.Conflicts = append(p.VersionMetadata.Conflicts, value) + case "replaces": + p.VersionMetadata.Replaces = append(p.VersionMetadata.Replaces, value) + case "xdata": + p.VersionMetadata.XData = append(p.VersionMetadata.XData, value) + case "builddate": + bd, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + p.FileMetadata.BuildDate = bd + case "size": + is, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + p.FileMetadata.InstalledSize = is + default: + return nil, util.NewInvalidArgumentErrorf("property is not supported %s", key) + } + } + + return p, errors.Join(scanner.Err(), ValidatePackageSpec(p)) +} + +// ValidatePackageSpec Arch package validation according to PKGBUILD specification. +func ValidatePackageSpec(p *Package) error { + if !reName.MatchString(p.Name) { + return util.NewInvalidArgumentErrorf("invalid package name") + } + if !reName.MatchString(p.VersionMetadata.Base) { + return util.NewInvalidArgumentErrorf("invalid package base") + } + if !reVer.MatchString(p.Version) { + return util.NewInvalidArgumentErrorf("invalid package version") + } + if p.FileMetadata.Arch == "" { + return util.NewInvalidArgumentErrorf("architecture should be specified") + } + if p.VersionMetadata.ProjectURL != "" { + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + return util.NewInvalidArgumentErrorf("invalid project URL") + } + } + for _, checkDepend := range p.VersionMetadata.CheckDepends { + if !rePkgVer.MatchString(checkDepend) { + return util.NewInvalidArgumentErrorf("invalid check dependency: %s", checkDepend) + } + } + for _, depend := range p.VersionMetadata.Depends { + if !rePkgVer.MatchString(depend) { + return util.NewInvalidArgumentErrorf("invalid dependency: %s", depend) + } + } + for _, makeDepend := range p.VersionMetadata.MakeDepends { + if !rePkgVer.MatchString(makeDepend) { + return util.NewInvalidArgumentErrorf("invalid make dependency: %s", makeDepend) + } + } + for _, provide := range p.VersionMetadata.Provides { + if !rePkgVer.MatchString(provide) { + return util.NewInvalidArgumentErrorf("invalid provides: %s", provide) + } + } + for _, conflict := range p.VersionMetadata.Conflicts { + if !rePkgVer.MatchString(conflict) { + return util.NewInvalidArgumentErrorf("invalid conflicts: %s", conflict) + } + } + for _, replace := range p.VersionMetadata.Replaces { + if !rePkgVer.MatchString(replace) { + return util.NewInvalidArgumentErrorf("invalid replaces: %s", replace) + } + } + for _, optDepend := range p.VersionMetadata.OptDepends { + if !reOptDep.MatchString(optDepend) { + return util.NewInvalidArgumentErrorf("invalid optional dependency: %s", optDepend) + } + } + for _, b := range p.VersionMetadata.Backup { + if strings.HasPrefix(b, "/") { + return util.NewInvalidArgumentErrorf("backup file contains leading forward slash") + } + } + return nil +} + +// Desc Create pacman package description file. +func (p *Package) Desc() string { + entries := []string{ + "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.%s", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType), + "NAME", p.Name, + "BASE", p.VersionMetadata.Base, + "VERSION", p.Version, + "DESC", p.VersionMetadata.Description, + "GROUPS", strings.Join(p.VersionMetadata.Groups, "\n"), + "CSIZE", fmt.Sprintf("%d", p.FileMetadata.CompressedSize), + "ISIZE", fmt.Sprintf("%d", p.FileMetadata.InstalledSize), + "MD5SUM", p.FileMetadata.MD5, + "SHA256SUM", p.FileMetadata.SHA256, + "PGPSIG", p.FileMetadata.PgpSigned, + "URL", p.VersionMetadata.ProjectURL, + "LICENSE", strings.Join(p.VersionMetadata.License, "\n"), + "ARCH", p.FileMetadata.Arch, + "BUILDDATE", fmt.Sprintf("%d", p.FileMetadata.BuildDate), + "PACKAGER", p.FileMetadata.Packager, + "REPLACES", strings.Join(p.VersionMetadata.Replaces, "\n"), + "CONFLICTS", strings.Join(p.VersionMetadata.Conflicts, "\n"), + "PROVIDES", strings.Join(p.VersionMetadata.Provides, "\n"), + "DEPENDS", strings.Join(p.VersionMetadata.Depends, "\n"), + "OPTDEPENDS", strings.Join(p.VersionMetadata.OptDepends, "\n"), + "MAKEDEPENDS", strings.Join(p.VersionMetadata.MakeDepends, "\n"), + "CHECKDEPENDS", strings.Join(p.VersionMetadata.CheckDepends, "\n"), + } + + var buf bytes.Buffer + for i := 0; i < len(entries); i += 2 { + if entries[i+1] != "" { + _, _ = fmt.Fprintf(&buf, "%%%s%%\n%s\n\n", entries[i], entries[i+1]) + } + } + return buf.String() +} diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go new file mode 100644 index 0000000..ddb35ca --- /dev/null +++ b/modules/packages/arch/metadata_test.go @@ -0,0 +1,447 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bytes" + "errors" + "os" + "strings" + "testing" + "testing/fstest" + "time" + + "code.gitea.io/gitea/modules/packages" + + "github.com/mholt/archiver/v3" + "github.com/stretchr/testify/require" +) + +func TestParsePackage(t *testing.T) { + // Minimal PKGINFO contents and test FS + const PKGINFO = `pkgname = a +pkgbase = b +pkgver = 1-2 +arch = x86_64 +` + fs := fstest.MapFS{ + "pkginfo": &fstest.MapFile{ + Data: []byte(PKGINFO), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + "mtree": &fstest.MapFile{ + Data: []byte("data"), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + } + + // Test .PKGINFO file + pinf, err := fs.Stat("pkginfo") + require.NoError(t, err) + + pfile, err := fs.Open("pkginfo") + require.NoError(t, err) + + parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO") + require.NoError(t, err) + + // Test .MTREE file + minf, err := fs.Stat("mtree") + require.NoError(t, err) + + mfile, err := fs.Open("mtree") + require.NoError(t, err) + + marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE") + require.NoError(t, err) + + t.Run("normal archive", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + require.NoError(t, errors.Join(pfile.Close(), err)) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: minf, + CustomName: marcname, + }, + ReadCloser: mfile, + }) + require.NoError(t, errors.Join(mfile.Close(), archive.Close(), err)) + + reader, err := packages.CreateHashedBufferFromReader(&buf) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + _, err = ParsePackage(reader) + + require.NoError(t, err) + }) + + t.Run("missing .PKGINFO", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + require.NoError(t, archive.Close()) + + reader, err := packages.CreateHashedBufferFromReader(&buf) + require.NoError(t, err) + + defer reader.Close() + _, err = ParsePackage(reader) + + require.Error(t, err) + require.Contains(t, err.Error(), ".PKGINFO file not found") + }) + + t.Run("missing .MTREE", func(t *testing.T) { + var buf bytes.Buffer + + pfile, err := fs.Open("pkginfo") + require.NoError(t, err) + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + require.NoError(t, errors.Join(pfile.Close(), archive.Close(), err)) + reader, err := packages.CreateHashedBufferFromReader(&buf) + require.NoError(t, err) + + defer reader.Close() + _, err = ParsePackage(reader) + + require.Error(t, err) + require.Contains(t, err.Error(), ".MTREE file not found") + }) +} + +func TestParsePackageInfo(t *testing.T) { + const PKGINFO = `# Generated by makepkg 6.0.2 +# using fakeroot version 1.31 +pkgname = a +pkgbase = b +pkgver = 1-2 +pkgdesc = comment +url = https://example.com/ +group = group +builddate = 3 +packager = Name Surname +size = 5 +arch = x86_64 +license = BSD +provides = pvd +depend = smth +optdepend = hex +checkdepend = ola +makedepend = cmake +backup = usr/bin/paket1 +` + p, err := ParsePackageInfo("zst", strings.NewReader(PKGINFO)) + require.NoError(t, err) + require.Equal(t, Package{ + CompressType: "zst", + Name: "a", + Version: "1-2", + VersionMetadata: VersionMetadata{ + Base: "b", + Description: "comment", + ProjectURL: "https://example.com/", + Groups: []string{"group"}, + Provides: []string{"pvd"}, + License: []string{"BSD"}, + Depends: []string{"smth"}, + OptDepends: []string{"hex"}, + MakeDepends: []string{"cmake"}, + CheckDepends: []string{"ola"}, + Backup: []string{"usr/bin/paket1"}, + }, + FileMetadata: FileMetadata{ + InstalledSize: 5, + BuildDate: 3, + Packager: "Name Surname ", + Arch: "x86_64", + }, + }, *p) +} + +func TestValidatePackageSpec(t *testing.T) { + newpkg := func() Package { + return Package{ + Name: "abc", + Version: "1-1", + VersionMetadata: VersionMetadata{ + Base: "ghx", + Description: "whoami", + ProjectURL: "https://example.com/", + Groups: []string{"gnome"}, + Provides: []string{"abc", "def"}, + License: []string{"GPL"}, + Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"}, + OptDepends: []string{"git", "libgcc=1.0", "gzip>1.0", "gz>=1.0", "lz<1.0", "gzip<=1.0", "zstd>1.0:foo bar"}, + MakeDepends: []string{"chrom"}, + CheckDepends: []string{"bariy"}, + Backup: []string{"etc/pacman.d/filo"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 1, + InstalledSize: 2, + SHA256: "def", + BuildDate: 3, + Packager: "smon", + Arch: "x86_64", + }, + } + } + + t.Run("valid package", func(t *testing.T) { + p := newpkg() + + err := ValidatePackageSpec(&p) + + require.NoError(t, err) + }) + + t.Run("invalid package name", func(t *testing.T) { + p := newpkg() + p.Name = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package name") + }) + + t.Run("invalid package base", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "una-luna?" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.Version = "una-luna" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package version") + }) + + t.Run("missing architecture", func(t *testing.T) { + p := newpkg() + p.FileMetadata.Arch = "" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "architecture should be specified") + }) + + t.Run("invalid URL", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.ProjectURL = "http%%$#" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid project URL") + }) + + t.Run("invalid check dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.CheckDepends = []string{"Err^_^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid check dependency") + }) + + t.Run("invalid dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Depends = []string{"^^abc"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid dependency") + }) + + t.Run("invalid make dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.MakeDepends = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid make dependency") + }) + + t.Run("invalid provides", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Provides = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid provides") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.OptDepends = []string{"^m^:MM"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid optional dependency") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Backup = []string{"/ola/cola"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "backup file contains leading forward slash") + }) +} + +func TestDescString(t *testing.T) { + const pkgdesc = `%FILENAME% +zstd-1.5.5-1-x86_64.pkg.tar.zst + +%NAME% +zstd + +%BASE% +zstd + +%VERSION% +1.5.5-1 + +%DESC% +Zstandard - Fast real-time compression algorithm + +%GROUPS% +dummy1 +dummy2 + +%CSIZE% +401 + +%ISIZE% +1500453 + +%MD5SUM% +5016660ef3d9aa148a7b72a08d3df1b2 + +%SHA256SUM% +9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd + +%URL% +https://facebook.github.io/zstd/ + +%LICENSE% +BSD +GPL2 + +%ARCH% +x86_64 + +%BUILDDATE% +1681646714 + +%PACKAGER% +Jelle van der Waa + +%PROVIDES% +libzstd.so=1-64 + +%DEPENDS% +glibc +gcc-libs +zlib +xz +lz4 + +%OPTDEPENDS% +dummy3 +dummy4 + +%MAKEDEPENDS% +cmake +gtest +ninja + +%CHECKDEPENDS% +dummy5 +dummy6 + +` + + md := &Package{ + CompressType: "zst", + Name: "zstd", + Version: "1.5.5-1", + VersionMetadata: VersionMetadata{ + Base: "zstd", + Description: "Zstandard - Fast real-time compression algorithm", + ProjectURL: "https://facebook.github.io/zstd/", + Groups: []string{"dummy1", "dummy2"}, + Provides: []string{"libzstd.so=1-64"}, + License: []string{"BSD", "GPL2"}, + Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}, + OptDepends: []string{"dummy3", "dummy4"}, + MakeDepends: []string{"cmake", "gtest", "ninja"}, + CheckDepends: []string{"dummy5", "dummy6"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 401, + InstalledSize: 1500453, + MD5: "5016660ef3d9aa148a7b72a08d3df1b2", + SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd", + BuildDate: 1681646714, + Packager: "Jelle van der Waa ", + Arch: "x86_64", + }, + } + require.Equal(t, pkgdesc, md.Desc()) +} -- cgit v1.2.3