From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- modules/packages/rubygems/marshal.go | 311 +++++++++++++++++++++++++++++ modules/packages/rubygems/marshal_test.go | 99 +++++++++ modules/packages/rubygems/metadata.go | 220 ++++++++++++++++++++ modules/packages/rubygems/metadata_test.go | 89 +++++++++ 4 files changed, 719 insertions(+) create mode 100644 modules/packages/rubygems/marshal.go create mode 100644 modules/packages/rubygems/marshal_test.go create mode 100644 modules/packages/rubygems/metadata.go create mode 100644 modules/packages/rubygems/metadata_test.go (limited to 'modules/packages/rubygems') diff --git a/modules/packages/rubygems/marshal.go b/modules/packages/rubygems/marshal.go new file mode 100644 index 0000000..4e6a5fc --- /dev/null +++ b/modules/packages/rubygems/marshal.go @@ -0,0 +1,311 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "bufio" + "bytes" + "io" + "reflect" + + "code.gitea.io/gitea/modules/util" +) + +const ( + majorVersion = 4 + minorVersion = 8 + + typeNil = '0' + typeTrue = 'T' + typeFalse = 'F' + typeFixnum = 'i' + typeString = '"' + typeSymbol = ':' + typeSymbolLink = ';' + typeArray = '[' + typeIVar = 'I' + typeUserMarshal = 'U' + typeUserDef = 'u' + typeObject = 'o' +) + +var ( + // ErrUnsupportedType indicates an unsupported type + ErrUnsupportedType = util.NewInvalidArgumentErrorf("type is unsupported") + // ErrInvalidIntRange indicates an invalid number range + ErrInvalidIntRange = util.NewInvalidArgumentErrorf("number is not in valid range") +) + +// RubyUserMarshal is a Ruby object that has a marshal_load function. +type RubyUserMarshal struct { + Name string + Value any +} + +// RubyUserDef is a Ruby object that has a _load function. +type RubyUserDef struct { + Name string + Value any +} + +// RubyObject is a default Ruby object. +type RubyObject struct { + Name string + Member map[string]any +} + +// MarshalEncoder mimics Rubys Marshal class. +// Note: Only supports types used by the RubyGems package registry. +type MarshalEncoder struct { + w *bufio.Writer + symbols map[string]int +} + +// NewMarshalEncoder creates a new MarshalEncoder +func NewMarshalEncoder(w io.Writer) *MarshalEncoder { + return &MarshalEncoder{ + w: bufio.NewWriter(w), + symbols: map[string]int{}, + } +} + +// Encode encodes the given type +func (e *MarshalEncoder) Encode(v any) error { + if _, err := e.w.Write([]byte{majorVersion, minorVersion}); err != nil { + return err + } + + if err := e.marshal(v); err != nil { + return err + } + + return e.w.Flush() +} + +func (e *MarshalEncoder) marshal(v any) error { + if v == nil { + return e.marshalNil() + } + + val := reflect.ValueOf(v) + typ := reflect.TypeOf(v) + + if typ.Kind() == reflect.Ptr { + val = val.Elem() + typ = typ.Elem() + } + + switch typ.Kind() { + case reflect.Bool: + return e.marshalBool(val.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: + return e.marshalInt(val.Int()) + case reflect.String: + return e.marshalString(val.String()) + case reflect.Slice, reflect.Array: + return e.marshalArray(val) + } + + switch typ.Name() { + case "RubyUserMarshal": + return e.marshalUserMarshal(val.Interface().(RubyUserMarshal)) + case "RubyUserDef": + return e.marshalUserDef(val.Interface().(RubyUserDef)) + case "RubyObject": + return e.marshalObject(val.Interface().(RubyObject)) + } + + return ErrUnsupportedType +} + +func (e *MarshalEncoder) marshalNil() error { + return e.w.WriteByte(typeNil) +} + +func (e *MarshalEncoder) marshalBool(b bool) error { + if b { + return e.w.WriteByte(typeTrue) + } + return e.w.WriteByte(typeFalse) +} + +func (e *MarshalEncoder) marshalInt(i int64) error { + if err := e.w.WriteByte(typeFixnum); err != nil { + return err + } + + return e.marshalIntInternal(i) +} + +func (e *MarshalEncoder) marshalIntInternal(i int64) error { + if i == 0 { + return e.w.WriteByte(0) + } else if 0 < i && i < 123 { + return e.w.WriteByte(byte(i + 5)) + } else if -124 < i && i <= -1 { + return e.w.WriteByte(byte(i - 5)) + } + + var length int + if 122 < i && i <= 0xff { + length = 1 + } else if 0xff < i && i <= 0xffff { + length = 2 + } else if 0xffff < i && i <= 0xffffff { + length = 3 + } else if 0xffffff < i && i <= 0x3fffffff { + length = 4 + } else if -0x100 <= i && i < -123 { + length = -1 + } else if -0x10000 <= i && i < -0x100 { + length = -2 + } else if -0x1000000 <= i && i < -0x100000 { + length = -3 + } else if -0x40000000 <= i && i < -0x1000000 { + length = -4 + } else { + return ErrInvalidIntRange + } + + if err := e.w.WriteByte(byte(length)); err != nil { + return err + } + if length < 0 { + length = -length + } + + for c := 0; c < length; c++ { + if err := e.w.WriteByte(byte(i >> uint(8*c) & 0xff)); err != nil { + return err + } + } + + return nil +} + +func (e *MarshalEncoder) marshalString(str string) error { + if err := e.w.WriteByte(typeIVar); err != nil { + return err + } + + if err := e.marshalRawString(str); err != nil { + return err + } + + if err := e.marshalIntInternal(1); err != nil { + return err + } + + if err := e.marshalSymbol("E"); err != nil { + return err + } + + return e.marshalBool(true) +} + +func (e *MarshalEncoder) marshalRawString(str string) error { + if err := e.w.WriteByte(typeString); err != nil { + return err + } + + if err := e.marshalIntInternal(int64(len(str))); err != nil { + return err + } + + _, err := e.w.WriteString(str) + return err +} + +func (e *MarshalEncoder) marshalSymbol(str string) error { + if index, ok := e.symbols[str]; ok { + if err := e.w.WriteByte(typeSymbolLink); err != nil { + return err + } + return e.marshalIntInternal(int64(index)) + } + + e.symbols[str] = len(e.symbols) + + if err := e.w.WriteByte(typeSymbol); err != nil { + return err + } + + if err := e.marshalIntInternal(int64(len(str))); err != nil { + return err + } + + _, err := e.w.WriteString(str) + return err +} + +func (e *MarshalEncoder) marshalArray(arr reflect.Value) error { + if err := e.w.WriteByte(typeArray); err != nil { + return err + } + + length := arr.Len() + + if err := e.marshalIntInternal(int64(length)); err != nil { + return err + } + + for i := 0; i < length; i++ { + if err := e.marshal(arr.Index(i).Interface()); err != nil { + return err + } + } + return nil +} + +func (e *MarshalEncoder) marshalUserMarshal(userMarshal RubyUserMarshal) error { + if err := e.w.WriteByte(typeUserMarshal); err != nil { + return err + } + + if err := e.marshalSymbol(userMarshal.Name); err != nil { + return err + } + + return e.marshal(userMarshal.Value) +} + +func (e *MarshalEncoder) marshalUserDef(userDef RubyUserDef) error { + var buf bytes.Buffer + if err := NewMarshalEncoder(&buf).Encode(userDef.Value); err != nil { + return err + } + + if err := e.w.WriteByte(typeUserDef); err != nil { + return err + } + if err := e.marshalSymbol(userDef.Name); err != nil { + return err + } + if err := e.marshalIntInternal(int64(buf.Len())); err != nil { + return err + } + _, err := e.w.Write(buf.Bytes()) + return err +} + +func (e *MarshalEncoder) marshalObject(obj RubyObject) error { + if err := e.w.WriteByte(typeObject); err != nil { + return err + } + if err := e.marshalSymbol(obj.Name); err != nil { + return err + } + if err := e.marshalIntInternal(int64(len(obj.Member))); err != nil { + return err + } + for k, v := range obj.Member { + if err := e.marshalSymbol(k); err != nil { + return err + } + if err := e.marshal(v); err != nil { + return err + } + } + return nil +} diff --git a/modules/packages/rubygems/marshal_test.go b/modules/packages/rubygems/marshal_test.go new file mode 100644 index 0000000..8aa9160 --- /dev/null +++ b/modules/packages/rubygems/marshal_test.go @@ -0,0 +1,99 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMinimalEncoder(t *testing.T) { + cases := []struct { + Value any + Expected []byte + Error error + }{ + { + Value: nil, + Expected: []byte{4, 8, 0x30}, + }, + { + Value: true, + Expected: []byte{4, 8, 'T'}, + }, + { + Value: false, + Expected: []byte{4, 8, 'F'}, + }, + { + Value: 0, + Expected: []byte{4, 8, 'i', 0}, + }, + { + Value: 1, + Expected: []byte{4, 8, 'i', 6}, + }, + { + Value: -1, + Expected: []byte{4, 8, 'i', 0xfa}, + }, + { + Value: 0x1fffffff, + Expected: []byte{4, 8, 'i', 4, 0xff, 0xff, 0xff, 0x1f}, + }, + { + Value: 0x41000000, + Error: ErrInvalidIntRange, + }, + { + Value: "test", + Expected: []byte{4, 8, 'I', '"', 9, 't', 'e', 's', 't', 6, ':', 6, 'E', 'T'}, + }, + { + Value: []int{1, 2}, + Expected: []byte{4, 8, '[', 7, 'i', 6, 'i', 7}, + }, + { + Value: &RubyUserMarshal{ + Name: "Test", + Value: 4, + }, + Expected: []byte{4, 8, 'U', ':', 9, 'T', 'e', 's', 't', 'i', 9}, + }, + { + Value: &RubyUserDef{ + Name: "Test", + Value: 4, + }, + Expected: []byte{4, 8, 'u', ':', 9, 'T', 'e', 's', 't', 9, 4, 8, 'i', 9}, + }, + { + Value: &RubyObject{ + Name: "Test", + Member: map[string]any{ + "test": 4, + }, + }, + Expected: []byte{4, 8, 'o', ':', 9, 'T', 'e', 's', 't', 6, ':', 9, 't', 'e', 's', 't', 'i', 9}, + }, + { + Value: &struct { + Name string + }{ + "test", + }, + Error: ErrUnsupportedType, + }, + } + + for i, c := range cases { + var b bytes.Buffer + err := NewMarshalEncoder(&b).Encode(c.Value) + require.ErrorIs(t, err, c.Error) + assert.Equal(t, c.Expected, b.Bytes(), "case %d", i) + } +} diff --git a/modules/packages/rubygems/metadata.go b/modules/packages/rubygems/metadata.go new file mode 100644 index 0000000..8a97948 --- /dev/null +++ b/modules/packages/rubygems/metadata.go @@ -0,0 +1,220 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "archive/tar" + "compress/gzip" + "io" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "gopkg.in/yaml.v3" +) + +var ( + // ErrMissingMetadataFile indicates a missing metadata.gz file + ErrMissingMetadataFile = util.NewInvalidArgumentErrorf("metadata.gz file is missing") + // ErrInvalidName indicates an invalid id in the metadata.gz file + ErrInvalidName = util.NewInvalidArgumentErrorf("package name is invalid") + // ErrInvalidVersion indicates an invalid version in the metadata.gz file + ErrInvalidVersion = util.NewInvalidArgumentErrorf("package version is invalid") +) + +var versionMatcher = regexp.MustCompile(`\A[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\z`) + +// Package represents a RubyGems package +type Package struct { + Name string + Version string + Metadata *Metadata +} + +// Metadata represents the metadata of a RubyGems package +type Metadata struct { + Platform string `json:"platform,omitempty"` + Description string `json:"description,omitempty"` + Summary string `json:"summary,omitempty"` + Authors []string `json:"authors,omitempty"` + Licenses []string `json:"licenses,omitempty"` + RequiredRubyVersion []VersionRequirement `json:"required_ruby_version,omitempty"` + RequiredRubygemsVersion []VersionRequirement `json:"required_rubygems_version,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RuntimeDependencies []Dependency `json:"runtime_dependencies,omitempty"` + DevelopmentDependencies []Dependency `json:"development_dependencies,omitempty"` +} + +// VersionRequirement represents a version restriction +type VersionRequirement struct { + Restriction string `json:"restriction"` + Version string `json:"version"` +} + +// Dependency represents a dependency of a RubyGems package +type Dependency struct { + Name string `json:"name"` + Version []VersionRequirement `json:"version"` +} + +type gemspec struct { + Name string `yaml:"name"` + Version struct { + Version string `yaml:"version"` + } `yaml:"version"` + Platform string `yaml:"platform"` + Authors []string `yaml:"authors"` + Autorequire any `yaml:"autorequire"` + Bindir string `yaml:"bindir"` + CertChain []any `yaml:"cert_chain"` + Date string `yaml:"date"` + Dependencies []struct { + Name string `yaml:"name"` + Requirement requirement `yaml:"requirement"` + Type string `yaml:"type"` + Prerelease bool `yaml:"prerelease"` + VersionRequirements requirement `yaml:"version_requirements"` + } `yaml:"dependencies"` + Description string `yaml:"description"` + Executables []string `yaml:"executables"` + Extensions []any `yaml:"extensions"` + ExtraRdocFiles []string `yaml:"extra_rdoc_files"` + Files []string `yaml:"files"` + Homepage string `yaml:"homepage"` + Licenses []string `yaml:"licenses"` + Metadata struct { + BugTrackerURI string `yaml:"bug_tracker_uri"` + ChangelogURI string `yaml:"changelog_uri"` + DocumentationURI string `yaml:"documentation_uri"` + SourceCodeURI string `yaml:"source_code_uri"` + } `yaml:"metadata"` + PostInstallMessage any `yaml:"post_install_message"` + RdocOptions []any `yaml:"rdoc_options"` + RequirePaths []string `yaml:"require_paths"` + RequiredRubyVersion requirement `yaml:"required_ruby_version"` + RequiredRubygemsVersion requirement `yaml:"required_rubygems_version"` + Requirements []any `yaml:"requirements"` + RubygemsVersion string `yaml:"rubygems_version"` + SigningKey any `yaml:"signing_key"` + SpecificationVersion int `yaml:"specification_version"` + Summary string `yaml:"summary"` + TestFiles []any `yaml:"test_files"` +} + +type requirement struct { + Requirements [][]any `yaml:"requirements"` +} + +// AsVersionRequirement converts into []VersionRequirement +func (r requirement) AsVersionRequirement() []VersionRequirement { + requirements := make([]VersionRequirement, 0, len(r.Requirements)) + for _, req := range r.Requirements { + if len(req) != 2 { + continue + } + restriction, ok := req[0].(string) + if !ok { + continue + } + vm, ok := req[1].(map[string]any) + if !ok { + continue + } + versionInt, ok := vm["version"] + if !ok { + continue + } + version, ok := versionInt.(string) + if !ok || version == "0" { + continue + } + + requirements = append(requirements, VersionRequirement{ + Restriction: restriction, + Version: version, + }) + } + return requirements +} + +// ParsePackageMetaData parses the metadata of a Gem package file +func ParsePackageMetaData(r io.Reader) (*Package, error) { + archive := tar.NewReader(r) + for { + hdr, err := archive.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Name == "metadata.gz" { + return parseMetadataFile(archive) + } + } + + return nil, ErrMissingMetadataFile +} + +func parseMetadataFile(r io.Reader) (*Package, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer zr.Close() + + var spec gemspec + if err := yaml.NewDecoder(zr).Decode(&spec); err != nil { + return nil, err + } + + if len(spec.Name) == 0 || strings.Contains(spec.Name, "/") { + return nil, ErrInvalidName + } + + if !versionMatcher.MatchString(spec.Version.Version) { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(spec.Homepage) { + spec.Homepage = "" + } + if !validation.IsValidURL(spec.Metadata.SourceCodeURI) { + spec.Metadata.SourceCodeURI = "" + } + + m := &Metadata{ + Platform: spec.Platform, + Description: spec.Description, + Summary: spec.Summary, + Authors: spec.Authors, + Licenses: spec.Licenses, + ProjectURL: spec.Homepage, + RequiredRubyVersion: spec.RequiredRubyVersion.AsVersionRequirement(), + RequiredRubygemsVersion: spec.RequiredRubygemsVersion.AsVersionRequirement(), + DevelopmentDependencies: make([]Dependency, 0, 5), + RuntimeDependencies: make([]Dependency, 0, 5), + } + + for _, gemdep := range spec.Dependencies { + dep := Dependency{ + Name: gemdep.Name, + Version: gemdep.Requirement.AsVersionRequirement(), + } + if gemdep.Type == ":runtime" { + m.RuntimeDependencies = append(m.RuntimeDependencies, dep) + } else { + m.DevelopmentDependencies = append(m.DevelopmentDependencies, dep) + } + } + + return &Package{ + Name: spec.Name, + Version: spec.Version.Version, + Metadata: m, + }, nil +} diff --git a/modules/packages/rubygems/metadata_test.go b/modules/packages/rubygems/metadata_test.go new file mode 100644 index 0000000..cd3a5bb --- /dev/null +++ b/modules/packages/rubygems/metadata_test.go @@ -0,0 +1,89 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rubygems + +import ( + "archive/tar" + "bytes" + "encoding/base64" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParsePackageMetaData(t *testing.T) { + createArchive := func(filename string, content []byte) io.Reader { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + tw.Close() + return &buf + } + + t.Run("MissingMetadataFile", func(t *testing.T) { + data := createArchive("dummy.txt", []byte{0}) + + rp, err := ParsePackageMetaData(data) + require.ErrorIs(t, err, ErrMissingMetadataFile) + assert.Nil(t, rp) + }) + + t.Run("Valid", func(t *testing.T) { + content, _ := base64.StdEncoding.DecodeString("H4sICHC/I2EEAG1ldGFkYXRhAAEeAOH/bmFtZTogZwp2ZXJzaW9uOgogIHZlcnNpb246IDEKWw35Tx4AAAA=") + data := createArchive("metadata.gz", content) + + rp, err := ParsePackageMetaData(data) + require.NoError(t, err) + assert.NotNil(t, rp) + }) +} + +func TestParseMetadataFile(t *testing.T) { + content, _ := base64.StdEncoding.DecodeString(`H4sIAMe7I2ECA9VVTW/UMBC9+1eYXvaUbJpSQBZUHJAqDlwK4kCFIseZzZrGH9iTqisEv52Js9nd +0KqggiqRXWnX45n3ZuZ5nCzL+JPQ15ulq7+AQnEORoj3HpReaSVRO8usNCB4qxEku4YQySbuCPo4 +bjHOd07HeZGfMt9JXLlgBB9imOxx7UIULOPnCZMMLsDXXgeiYbW2jQ6C0y9TELBSa6kJ6/IzaySS +R1mUx1nxIitPeFGI9M2L6eGfWAMebANWaUgktzN9M3lsKNmxutBb1AYyCibbNhsDFu+q9GK/Tc4z +d2IcLBl9js5eHaXFsLyvXeNz0LQyL/YoLx8EsiCMBZlx46k6sS2PDD5AgA5kJPNKdhH2elWzOv7n +uv9Q9Aau/6ngP84elvNpXh5oRVlB5/yW7BH0+qu0G4gqaI/JdEHBFBS5l+pKtsARIjIwUnfj8Le0 ++TrdJLl2DG5A9SjrjgZ1mG+4QbAD+G4ZZBUap6qVnnzGf6Rwp+vliBRqtnYGPBEKvkb0USyXE8mS +dVoR6hj07u0HZgAl3SRS8G/fmXcRK20jyq6rDMSYQFgidamqkXbbuspLXE/0k7GphtKqe67GuRC/ +yjAbmt9LsOMp8xMamFkSQ38fP5EFjdz8LA4do2C69VvqWXAJgrPbKZb58/xZXrKoW6ttW13Bhvzi +4ftn7/yUxd4YGcglvTmmY8aGY3ZwRn4CqcWcidUGAAA=`) + rp, err := parseMetadataFile(bytes.NewReader(content)) + require.NoError(t, err) + assert.NotNil(t, rp) + + assert.Equal(t, "gitea", rp.Name) + assert.Equal(t, "1.0.5", rp.Version) + assert.Equal(t, "ruby", rp.Metadata.Platform) + assert.Equal(t, "Gitea package", rp.Metadata.Summary) + assert.Equal(t, "RubyGems package test", rp.Metadata.Description) + assert.Equal(t, []string{"Gitea"}, rp.Metadata.Authors) + assert.Equal(t, "https://gitea.io/", rp.Metadata.ProjectURL) + assert.Equal(t, []string{"MIT"}, rp.Metadata.Licenses) + assert.Empty(t, rp.Metadata.RequiredRubygemsVersion) + assert.Len(t, rp.Metadata.RequiredRubyVersion, 1) + assert.Equal(t, ">=", rp.Metadata.RequiredRubyVersion[0].Restriction) + assert.Equal(t, "2.3.0", rp.Metadata.RequiredRubyVersion[0].Version) + assert.Len(t, rp.Metadata.RuntimeDependencies, 1) + assert.Equal(t, "runtime-dep", rp.Metadata.RuntimeDependencies[0].Name) + assert.Len(t, rp.Metadata.RuntimeDependencies[0].Version, 2) + assert.Equal(t, ">=", rp.Metadata.RuntimeDependencies[0].Version[0].Restriction) + assert.Equal(t, "1.2.0", rp.Metadata.RuntimeDependencies[0].Version[0].Version) + assert.Equal(t, "<", rp.Metadata.RuntimeDependencies[0].Version[1].Restriction) + assert.Equal(t, "2.0", rp.Metadata.RuntimeDependencies[0].Version[1].Version) + assert.Len(t, rp.Metadata.DevelopmentDependencies, 1) + assert.Equal(t, "dev-dep", rp.Metadata.DevelopmentDependencies[0].Name) + assert.Len(t, rp.Metadata.DevelopmentDependencies[0].Version, 1) + assert.Equal(t, "~>", rp.Metadata.DevelopmentDependencies[0].Version[0].Restriction) + assert.Equal(t, "5.2", rp.Metadata.DevelopmentDependencies[0].Version[0].Version) +} -- cgit v1.2.3