summaryrefslogtreecommitdiffstats
path: root/routers/api/packages/container/manifest.go
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--routers/api/packages/container/manifest.go483
1 files changed, 483 insertions, 0 deletions
diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go
new file mode 100644
index 0000000..4a79a58
--- /dev/null
+++ b/routers/api/packages/container/manifest.go
@@ -0,0 +1,483 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+
+ "code.gitea.io/gitea/models/db"
+ packages_model "code.gitea.io/gitea/models/packages"
+ container_model "code.gitea.io/gitea/models/packages/container"
+ user_model "code.gitea.io/gitea/models/user"
+ "code.gitea.io/gitea/modules/json"
+ "code.gitea.io/gitea/modules/log"
+ packages_module "code.gitea.io/gitea/modules/packages"
+ container_module "code.gitea.io/gitea/modules/packages/container"
+ "code.gitea.io/gitea/modules/util"
+ notify_service "code.gitea.io/gitea/services/notify"
+ packages_service "code.gitea.io/gitea/services/packages"
+
+ digest "github.com/opencontainers/go-digest"
+ oci "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func isValidMediaType(mt string) bool {
+ return strings.HasPrefix(mt, "application/vnd.docker.") || strings.HasPrefix(mt, "application/vnd.oci.")
+}
+
+func isImageManifestMediaType(mt string) bool {
+ return strings.EqualFold(mt, oci.MediaTypeImageManifest) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.v2+json")
+}
+
+func isImageIndexMediaType(mt string) bool {
+ return strings.EqualFold(mt, oci.MediaTypeImageIndex) || strings.EqualFold(mt, "application/vnd.docker.distribution.manifest.list.v2+json")
+}
+
+// manifestCreationInfo describes a manifest to create
+type manifestCreationInfo struct {
+ MediaType string
+ Owner *user_model.User
+ Creator *user_model.User
+ Image string
+ Reference string
+ IsTagged bool
+ Properties map[string]string
+}
+
+func processManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
+ var index oci.Index
+ if err := json.NewDecoder(buf).Decode(&index); err != nil {
+ return "", err
+ }
+
+ if index.SchemaVersion != 2 {
+ return "", errUnsupported.WithMessage("Schema version is not supported")
+ }
+
+ if _, err := buf.Seek(0, io.SeekStart); err != nil {
+ return "", err
+ }
+
+ if !isValidMediaType(mci.MediaType) {
+ mci.MediaType = index.MediaType
+ if !isValidMediaType(mci.MediaType) {
+ return "", errManifestInvalid.WithMessage("MediaType not recognized")
+ }
+ }
+
+ if isImageManifestMediaType(mci.MediaType) {
+ return processImageManifest(ctx, mci, buf)
+ } else if isImageIndexMediaType(mci.MediaType) {
+ return processImageManifestIndex(ctx, mci, buf)
+ }
+ return "", errManifestInvalid
+}
+
+func processImageManifest(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
+ manifestDigest := ""
+
+ err := func() error {
+ var manifest oci.Manifest
+ if err := json.NewDecoder(buf).Decode(&manifest); err != nil {
+ return err
+ }
+
+ if _, err := buf.Seek(0, io.SeekStart); err != nil {
+ return err
+ }
+
+ ctx, committer, err := db.TxContext(ctx)
+ if err != nil {
+ return err
+ }
+ defer committer.Close()
+
+ configDescriptor, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
+ OwnerID: mci.Owner.ID,
+ Image: mci.Image,
+ Digest: string(manifest.Config.Digest),
+ })
+ if err != nil {
+ return err
+ }
+
+ configReader, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(configDescriptor.Blob.HashSHA256))
+ if err != nil {
+ return err
+ }
+ defer configReader.Close()
+
+ metadata, err := container_module.ParseImageConfig(manifest.Config.MediaType, configReader)
+ if err != nil {
+ return err
+ }
+
+ blobReferences := make([]*blobReference, 0, 1+len(manifest.Layers))
+
+ blobReferences = append(blobReferences, &blobReference{
+ Digest: manifest.Config.Digest,
+ MediaType: manifest.Config.MediaType,
+ File: configDescriptor,
+ ExpectedSize: manifest.Config.Size,
+ })
+
+ for _, layer := range manifest.Layers {
+ pfd, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
+ OwnerID: mci.Owner.ID,
+ Image: mci.Image,
+ Digest: string(layer.Digest),
+ })
+ if err != nil {
+ return err
+ }
+
+ blobReferences = append(blobReferences, &blobReference{
+ Digest: layer.Digest,
+ MediaType: layer.MediaType,
+ File: pfd,
+ ExpectedSize: layer.Size,
+ })
+ }
+
+ pv, err := createPackageAndVersion(ctx, mci, metadata)
+ if err != nil {
+ return err
+ }
+
+ uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_model.UploadVersion)
+ if err != nil && err != packages_model.ErrPackageNotExist {
+ return err
+ }
+
+ for _, ref := range blobReferences {
+ if err := createFileFromBlobReference(ctx, pv, uploadVersion, ref); err != nil {
+ return err
+ }
+ }
+
+ pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf)
+ removeBlob := false
+ defer func() {
+ if removeBlob {
+ contentStore := packages_module.NewContentStore()
+ if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
+ log.Error("Error deleting package blob from content store: %v", err)
+ }
+ }
+ }()
+ if err != nil {
+ removeBlob = created
+ return err
+ }
+
+ if err := committer.Commit(); err != nil {
+ removeBlob = created
+ return err
+ }
+
+ if err := notifyPackageCreate(ctx, mci.Creator, pv); err != nil {
+ return err
+ }
+
+ manifestDigest = digest
+
+ return nil
+ }()
+ if err != nil {
+ return "", err
+ }
+
+ return manifestDigest, nil
+}
+
+func processImageManifestIndex(ctx context.Context, mci *manifestCreationInfo, buf *packages_module.HashedBuffer) (string, error) {
+ manifestDigest := ""
+
+ err := func() error {
+ var index oci.Index
+ if err := json.NewDecoder(buf).Decode(&index); err != nil {
+ return err
+ }
+
+ if _, err := buf.Seek(0, io.SeekStart); err != nil {
+ return err
+ }
+
+ ctx, committer, err := db.TxContext(ctx)
+ if err != nil {
+ return err
+ }
+ defer committer.Close()
+
+ metadata := &container_module.Metadata{
+ Type: container_module.TypeOCI,
+ Manifests: make([]*container_module.Manifest, 0, len(index.Manifests)),
+ }
+
+ for _, manifest := range index.Manifests {
+ if !isImageManifestMediaType(manifest.MediaType) {
+ return errManifestInvalid
+ }
+
+ platform := container_module.DefaultPlatform
+ if manifest.Platform != nil {
+ platform = fmt.Sprintf("%s/%s", manifest.Platform.OS, manifest.Platform.Architecture)
+ if manifest.Platform.Variant != "" {
+ platform = fmt.Sprintf("%s/%s", platform, manifest.Platform.Variant)
+ }
+ }
+
+ pfd, err := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{
+ OwnerID: mci.Owner.ID,
+ Image: mci.Image,
+ Digest: string(manifest.Digest),
+ IsManifest: true,
+ })
+ if err != nil {
+ if err == container_model.ErrContainerBlobNotExist {
+ return errManifestBlobUnknown
+ }
+ return err
+ }
+
+ size, err := packages_model.CalculateFileSize(ctx, &packages_model.PackageFileSearchOptions{
+ VersionID: pfd.File.VersionID,
+ })
+ if err != nil {
+ return err
+ }
+
+ metadata.Manifests = append(metadata.Manifests, &container_module.Manifest{
+ Platform: platform,
+ Digest: string(manifest.Digest),
+ Size: size,
+ })
+ }
+
+ pv, err := createPackageAndVersion(ctx, mci, metadata)
+ if err != nil {
+ return err
+ }
+
+ pb, created, digest, err := createManifestBlob(ctx, mci, pv, buf)
+ removeBlob := false
+ defer func() {
+ if removeBlob {
+ contentStore := packages_module.NewContentStore()
+ if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
+ log.Error("Error deleting package blob from content store: %v", err)
+ }
+ }
+ }()
+ if err != nil {
+ removeBlob = created
+ return err
+ }
+
+ if err := committer.Commit(); err != nil {
+ removeBlob = created
+ return err
+ }
+
+ if err := notifyPackageCreate(ctx, mci.Creator, pv); err != nil {
+ return err
+ }
+
+ manifestDigest = digest
+
+ return nil
+ }()
+ if err != nil {
+ return "", err
+ }
+
+ return manifestDigest, nil
+}
+
+func notifyPackageCreate(ctx context.Context, doer *user_model.User, pv *packages_model.PackageVersion) error {
+ pd, err := packages_model.GetPackageDescriptor(ctx, pv)
+ if err != nil {
+ return err
+ }
+
+ notify_service.PackageCreate(ctx, doer, pd)
+
+ return nil
+}
+
+func createPackageAndVersion(ctx context.Context, mci *manifestCreationInfo, metadata *container_module.Metadata) (*packages_model.PackageVersion, error) {
+ created := true
+ p := &packages_model.Package{
+ OwnerID: mci.Owner.ID,
+ Type: packages_model.TypeContainer,
+ Name: strings.ToLower(mci.Image),
+ LowerName: strings.ToLower(mci.Image),
+ }
+ var err error
+ if p, err = packages_model.TryInsertPackage(ctx, p); err != nil {
+ if err == packages_model.ErrDuplicatePackage {
+ created = false
+ } else {
+ log.Error("Error inserting package: %v", err)
+ return nil, err
+ }
+ }
+
+ if created {
+ if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypePackage, p.ID, container_module.PropertyRepository, strings.ToLower(mci.Owner.LowerName+"/"+mci.Image)); err != nil {
+ log.Error("Error setting package property: %v", err)
+ return nil, err
+ }
+ }
+
+ metadata.IsTagged = mci.IsTagged
+
+ metadataJSON, err := json.Marshal(metadata)
+ if err != nil {
+ return nil, err
+ }
+
+ _pv := &packages_model.PackageVersion{
+ PackageID: p.ID,
+ CreatorID: mci.Creator.ID,
+ Version: strings.ToLower(mci.Reference),
+ LowerVersion: strings.ToLower(mci.Reference),
+ MetadataJSON: string(metadataJSON),
+ }
+ var pv *packages_model.PackageVersion
+ if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
+ if err == packages_model.ErrDuplicatePackageVersion {
+ if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
+ return nil, err
+ }
+
+ // keep download count on overwrite
+ _pv.DownloadCount = pv.DownloadCount
+
+ if pv, err = packages_model.GetOrInsertVersion(ctx, _pv); err != nil {
+ log.Error("Error inserting package: %v", err)
+ return nil, err
+ }
+ } else {
+ log.Error("Error inserting package: %v", err)
+ return nil, err
+ }
+ }
+
+ if err := packages_service.CheckCountQuotaExceeded(ctx, mci.Creator, mci.Owner); err != nil {
+ return nil, err
+ }
+
+ if mci.IsTagged {
+ if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestTagged, ""); err != nil {
+ log.Error("Error setting package version property: %v", err)
+ return nil, err
+ }
+ }
+ for _, manifest := range metadata.Manifests {
+ if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, container_module.PropertyManifestReference, manifest.Digest); err != nil {
+ log.Error("Error setting package version property: %v", err)
+ return nil, err
+ }
+ }
+
+ return pv, nil
+}
+
+type blobReference struct {
+ Digest digest.Digest
+ MediaType string
+ Name string
+ File *packages_model.PackageFileDescriptor
+ ExpectedSize int64
+ IsLead bool
+}
+
+func createFileFromBlobReference(ctx context.Context, pv, uploadVersion *packages_model.PackageVersion, ref *blobReference) error {
+ if ref.File.Blob.Size != ref.ExpectedSize {
+ return errSizeInvalid
+ }
+
+ if ref.Name == "" {
+ ref.Name = strings.ToLower(fmt.Sprintf("sha256_%s", ref.File.Blob.HashSHA256))
+ }
+
+ pf := &packages_model.PackageFile{
+ VersionID: pv.ID,
+ BlobID: ref.File.Blob.ID,
+ Name: ref.Name,
+ LowerName: ref.Name,
+ IsLead: ref.IsLead,
+ }
+ var err error
+ if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
+ if err == packages_model.ErrDuplicatePackageFile {
+ // Skip this blob because the manifest contains the same filesystem layer multiple times.
+ return nil
+ }
+ log.Error("Error inserting package file: %v", err)
+ return err
+ }
+
+ props := map[string]string{
+ container_module.PropertyMediaType: ref.MediaType,
+ container_module.PropertyDigest: string(ref.Digest),
+ }
+ for name, value := range props {
+ if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, name, value); err != nil {
+ log.Error("Error setting package file property: %v", err)
+ return err
+ }
+ }
+
+ // Remove the file from the blob upload version
+ if uploadVersion != nil && ref.File.File != nil && uploadVersion.ID == ref.File.File.VersionID {
+ if err := packages_service.DeletePackageFile(ctx, ref.File.File); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *packages_model.PackageVersion, buf *packages_module.HashedBuffer) (*packages_model.PackageBlob, bool, string, error) {
+ pb, exists, err := packages_model.GetOrInsertBlob(ctx, packages_service.NewPackageBlob(buf))
+ if err != nil {
+ log.Error("Error inserting package blob: %v", err)
+ return nil, false, "", err
+ }
+ // FIXME: Workaround to be removed in v1.20
+ // https://github.com/go-gitea/gitea/issues/19586
+ if exists {
+ err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256))
+ if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
+ log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
+ exists = false
+ }
+ }
+ if !exists {
+ contentStore := packages_module.NewContentStore()
+ if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil {
+ log.Error("Error saving package blob in content store: %v", err)
+ return nil, false, "", err
+ }
+ }
+
+ manifestDigest := digestFromHashSummer(buf)
+ err = createFileFromBlobReference(ctx, pv, nil, &blobReference{
+ Digest: digest.Digest(manifestDigest),
+ MediaType: mci.MediaType,
+ Name: container_model.ManifestFilename,
+ File: &packages_model.PackageFileDescriptor{Blob: pb},
+ ExpectedSize: pb.Size,
+ IsLead: true,
+ })
+
+ return pb, !exists, manifestDigest, err
+}