From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- services/packages/cleanup/cleanup.go | 198 +++++++++++++++++++++++ services/packages/cleanup/cleanup_sha256_test.go | 116 +++++++++++++ services/packages/cleanup/main_test.go | 14 ++ 3 files changed, 328 insertions(+) create mode 100644 services/packages/cleanup/cleanup.go create mode 100644 services/packages/cleanup/cleanup_sha256_test.go create mode 100644 services/packages/cleanup/main_test.go (limited to 'services/packages/cleanup') diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go new file mode 100644 index 0000000..ab419a9 --- /dev/null +++ b/services/packages/cleanup/cleanup.go @@ -0,0 +1,198 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package container + +import ( + "context" + "fmt" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" + packages_module "code.gitea.io/gitea/modules/packages" + packages_service "code.gitea.io/gitea/services/packages" + alpine_service "code.gitea.io/gitea/services/packages/alpine" + arch_service "code.gitea.io/gitea/services/packages/arch" + cargo_service "code.gitea.io/gitea/services/packages/cargo" + container_service "code.gitea.io/gitea/services/packages/container" + debian_service "code.gitea.io/gitea/services/packages/debian" + rpm_service "code.gitea.io/gitea/services/packages/rpm" +) + +// Task method to execute cleanup rules and cleanup expired package data +func CleanupTask(ctx context.Context, olderThan time.Duration) error { + if err := ExecuteCleanupRules(ctx); err != nil { + return err + } + + return CleanupExpiredData(ctx, olderThan) +} + +func ExecuteCleanupRules(outerCtx context.Context) error { + ctx, committer, err := db.TxContext(outerCtx) + if err != nil { + return err + } + defer committer.Close() + + err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + select { + case <-outerCtx.Done(): + return db.ErrCancelledf("While processing package cleanup rules") + default: + } + + if err := pcr.CompiledPattern(); err != nil { + return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + } + + anyVersionDeleted := false + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: optional.Some(false), + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) + } + versionDeleted := false + for _, pv := range pvs { + if pcr.Type == packages_model.TypeContainer { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) + continue + } + + log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) + + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) + } + + versionDeleted = true + anyVersionDeleted = true + } + + if versionDeleted { + if pcr.Type == packages_model.TypeCargo { + owner, err := user_model.GetUserByID(ctx, pcr.OwnerID) + if err != nil { + return fmt.Errorf("GetUserByID failed: %w", err) + } + if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil { + return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err) + } + } + } + } + + if anyVersionDeleted { + if pcr.Type == packages_model.TypeDebian { + if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err) + } + } else if pcr.Type == packages_model.TypeAlpine { + if err := alpine_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: alpine.BuildAllRepositoryFiles failed: %w", pcr.ID, err) + } + } else if pcr.Type == packages_model.TypeRpm { + if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err) + } + } else if pcr.Type == packages_model.TypeArch { + if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err) + } + } + } + return nil + }) + if err != nil { + return err + } + + return committer.Commit() +} + +func CleanupExpiredData(outerCtx context.Context, olderThan time.Duration) error { + ctx, committer, err := db.TxContext(outerCtx) + if err != nil { + return err + } + defer committer.Close() + + if err := container_service.Cleanup(ctx, olderThan); err != nil { + return err + } + + pIDs, err := packages_model.FindUnreferencedPackages(ctx) + if err != nil { + return err + } + for _, pID := range pIDs { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, pID); err != nil { + return err + } + if err := packages_model.DeletePackageByID(ctx, pID); err != nil { + return err + } + } + + pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan) + if err != nil { + return err + } + + for _, pb := range pbs { + if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { + return err + } + } + + if err := committer.Commit(); err != nil { + return err + } + + contentStore := packages_module.NewContentStore() + for _, pb := range pbs { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob [%v]: %v", pb.ID, err) + } + } + + return nil +} diff --git a/services/packages/cleanup/cleanup_sha256_test.go b/services/packages/cleanup/cleanup_sha256_test.go new file mode 100644 index 0000000..6d7cc47 --- /dev/null +++ b/services/packages/cleanup/cleanup_sha256_test.go @@ -0,0 +1,116 @@ +// Copyright 2024 The Forgejo Authors. +// SPDX-License-Identifier: GPL-3.0-or-later + +package container + +import ( + "testing" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/timeutil" + container_service "code.gitea.io/gitea/services/packages/container" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCleanupSHA256(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + defer test.MockVariableValue(&container_service.SHA256BatchSize, 1)() + + ctx := db.DefaultContext + + createContainer := func(t *testing.T, name, version, digest string, created timeutil.TimeStamp) { + t.Helper() + + ownerID := int64(2001) + + p := packages.Package{ + OwnerID: ownerID, + LowerName: name, + Type: packages.TypeContainer, + } + _, err := db.GetEngine(ctx).Insert(&p) + // package_version").Where("version = ?", multiTag).Update(&packages_model.PackageVersion{MetadataJSON: `corrupted "manifests":[{ bad`}) + require.NoError(t, err) + + var metadata string + if digest != "" { + m := container_module.Metadata{ + Manifests: []*container_module.Manifest{ + { + Digest: digest, + }, + }, + } + mt, err := json.Marshal(m) + require.NoError(t, err) + metadata = string(mt) + } + v := packages.PackageVersion{ + PackageID: p.ID, + LowerVersion: version, + MetadataJSON: metadata, + CreatedUnix: created, + } + _, err = db.GetEngine(ctx).NoAutoTime().Insert(&v) + require.NoError(t, err) + } + + cleanupAndCheckLogs := func(t *testing.T, olderThan time.Duration, expected ...string) { + t.Helper() + logChecker, cleanup := test.NewLogChecker(log.DEFAULT, log.TRACE) + logChecker.Filter(expected...) + logChecker.StopMark(container_service.SHA256LogFinish) + defer cleanup() + + require.NoError(t, CleanupExpiredData(ctx, olderThan)) + + logFiltered, logStopped := logChecker.Check(5 * time.Second) + assert.True(t, logStopped) + filtered := make([]bool, 0, len(expected)) + for range expected { + filtered = append(filtered, true) + } + assert.EqualValues(t, filtered, logFiltered, expected) + } + + ancient := 1 * time.Hour + + t.Run("no packages, cleanup nothing", func(t *testing.T) { + cleanupAndCheckLogs(t, ancient, "Nothing to cleanup") + }) + + orphan := "orphan" + createdLongAgo := timeutil.TimeStamp(time.Now().Add(-(ancient * 2)).Unix()) + createdRecently := timeutil.TimeStamp(time.Now().Add(-(ancient / 2)).Unix()) + + t.Run("an orphaned package created a long time ago is removed", func(t *testing.T) { + createContainer(t, orphan, "sha256:"+orphan, "", createdLongAgo) + cleanupAndCheckLogs(t, ancient, "Removing 1 entries from `package_version`") + cleanupAndCheckLogs(t, ancient, "Nothing to cleanup") + }) + + t.Run("a newly created orphaned package is not cleaned up", func(t *testing.T) { + createContainer(t, orphan, "sha256:"+orphan, "", createdRecently) + cleanupAndCheckLogs(t, ancient, "1 out of 1 container image(s) are not deleted because they were created less than") + cleanupAndCheckLogs(t, 0, "Removing 1 entries from `package_version`") + cleanupAndCheckLogs(t, 0, "Nothing to cleanup") + }) + + t.Run("a referenced package is not removed", func(t *testing.T) { + referenced := "referenced" + digest := "sha256:" + referenced + createContainer(t, referenced, digest, "", createdRecently) + index := "index" + createContainer(t, index, index, digest, createdRecently) + cleanupAndCheckLogs(t, ancient, "Nothing to cleanup") + }) +} diff --git a/services/packages/cleanup/main_test.go b/services/packages/cleanup/main_test.go new file mode 100644 index 0000000..ded3d76 --- /dev/null +++ b/services/packages/cleanup/main_test.go @@ -0,0 +1,14 @@ +// Copyright 2024 The Forgejo Authors. +// SPDX-License-Identifier: GPL-3.0-or-later + +package container + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m) +} -- cgit v1.2.3