Adding upstream version 9.0.3.

Signed-off-by: Daniel Baumann <daniel@debian.org>

3 files changed, 328 insertions, 0 deletions

diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go
new file mode 100644
index 0000000..ab419a9
--- /dev/null
+++ b/services/packages/cleanup/cleanup.go
@@ -0,0 +1,198 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package container
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/models/db"
+	packages_model "code.gitea.io/gitea/models/packages"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/optional"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	packages_service "code.gitea.io/gitea/services/packages"
+	alpine_service "code.gitea.io/gitea/services/packages/alpine"
+	arch_service "code.gitea.io/gitea/services/packages/arch"
+	cargo_service "code.gitea.io/gitea/services/packages/cargo"
+	container_service "code.gitea.io/gitea/services/packages/container"
+	debian_service "code.gitea.io/gitea/services/packages/debian"
+	rpm_service "code.gitea.io/gitea/services/packages/rpm"
+)
+
+// Task method to execute cleanup rules and cleanup expired package data
+func CleanupTask(ctx context.Context, olderThan time.Duration) error {
+	if err := ExecuteCleanupRules(ctx); err != nil {
+		return err
+	}
+
+	return CleanupExpiredData(ctx, olderThan)
+}
+
+func ExecuteCleanupRules(outerCtx context.Context) error {
+	ctx, committer, err := db.TxContext(outerCtx)
+	if err != nil {
+		return err
+	}
+	defer committer.Close()
+
+	err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
+		select {
+		case <-outerCtx.Done():
+			return db.ErrCancelledf("While processing package cleanup rules")
+		default:
+		}
+
+		if err := pcr.CompiledPattern(); err != nil {
+			return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
+		}
+
+		olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
+
+		packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
+		if err != nil {
+			return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
+		}
+
+		anyVersionDeleted := false
+		for _, p := range packages {
+			pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
+				PackageID:  p.ID,
+				IsInternal: optional.Some(false),
+				Sort:       packages_model.SortCreatedDesc,
+				Paginator:  db.NewAbsoluteListOptions(pcr.KeepCount, 200),
+			})
+			if err != nil {
+				return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
+			}
+			versionDeleted := false
+			for _, pv := range pvs {
+				if pcr.Type == packages_model.TypeContainer {
+					if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
+						return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
+					} else if skip {
+						log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
+						continue
+					}
+				}
+
+				toMatch := pv.LowerVersion
+				if pcr.MatchFullName {
+					toMatch = p.LowerName + "/" + pv.LowerVersion
+				}
+
+				if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
+					log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+				if pv.CreatedUnix.AsLocalTime().After(olderThan) {
+					log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+				if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
+					log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
+					continue
+				}
+
+				log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
+
+				if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
+				}
+
+				versionDeleted = true
+				anyVersionDeleted = true
+			}
+
+			if versionDeleted {
+				if pcr.Type == packages_model.TypeCargo {
+					owner, err := user_model.GetUserByID(ctx, pcr.OwnerID)
+					if err != nil {
+						return fmt.Errorf("GetUserByID failed: %w", err)
+					}
+					if err := cargo_service.UpdatePackageIndexIfExists(ctx, owner, owner, p.ID); err != nil {
+						return fmt.Errorf("CleanupRule [%d]: cargo.UpdatePackageIndexIfExists failed: %w", pcr.ID, err)
+					}
+				}
+			}
+		}
+
+		if anyVersionDeleted {
+			if pcr.Type == packages_model.TypeDebian {
+				if err := debian_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: debian.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+				}
+			} else if pcr.Type == packages_model.TypeAlpine {
+				if err := alpine_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: alpine.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+				}
+			} else if pcr.Type == packages_model.TypeRpm {
+				if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+				}
+			} else if pcr.Type == packages_model.TypeArch {
+				if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil {
+					return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return committer.Commit()
+}
+
+func CleanupExpiredData(outerCtx context.Context, olderThan time.Duration) error {
+	ctx, committer, err := db.TxContext(outerCtx)
+	if err != nil {
+		return err
+	}
+	defer committer.Close()
+
+	if err := container_service.Cleanup(ctx, olderThan); err != nil {
+		return err
+	}
+
+	pIDs, err := packages_model.FindUnreferencedPackages(ctx)
+	if err != nil {
+		return err
+	}
+	for _, pID := range pIDs {
+		if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, pID); err != nil {
+			return err
+		}
+		if err := packages_model.DeletePackageByID(ctx, pID); err != nil {
+			return err
+		}
+	}
+
+	pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan)
+	if err != nil {
+		return err
+	}
+
+	for _, pb := range pbs {
+		if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil {
+			return err
+		}
+	}
+
+	if err := committer.Commit(); err != nil {
+		return err
+	}
+
+	contentStore := packages_module.NewContentStore()
+	for _, pb := range pbs {
+		if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
+			log.Error("Error deleting package blob [%v]: %v", pb.ID, err)
+		}
+	}
+
+	return nil
+}
diff --git a/services/packages/cleanup/cleanup_sha256_test.go b/services/packages/cleanup/cleanup_sha256_test.go
new file mode 100644
index 0000000..6d7cc47
--- /dev/null
+++ b/services/packages/cleanup/cleanup_sha256_test.go
@@ -0,0 +1,116 @@
+// Copyright 2024 The Forgejo Authors.
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package container
+
+import (
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/log"
+	container_module "code.gitea.io/gitea/modules/packages/container"
+	"code.gitea.io/gitea/modules/test"
+	"code.gitea.io/gitea/modules/timeutil"
+	container_service "code.gitea.io/gitea/services/packages/container"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCleanupSHA256(t *testing.T) {
+	require.NoError(t, unittest.PrepareTestDatabase())
+	defer test.MockVariableValue(&container_service.SHA256BatchSize, 1)()
+
+	ctx := db.DefaultContext
+
+	createContainer := func(t *testing.T, name, version, digest string, created timeutil.TimeStamp) {
+		t.Helper()
+
+		ownerID := int64(2001)
+
+		p := packages.Package{
+			OwnerID:   ownerID,
+			LowerName: name,
+			Type:      packages.TypeContainer,
+		}
+		_, err := db.GetEngine(ctx).Insert(&p)
+		// package_version").Where("version = ?", multiTag).Update(&packages_model.PackageVersion{MetadataJSON: `corrupted "manifests":[{ bad`})
+		require.NoError(t, err)
+
+		var metadata string
+		if digest != "" {
+			m := container_module.Metadata{
+				Manifests: []*container_module.Manifest{
+					{
+						Digest: digest,
+					},
+				},
+			}
+			mt, err := json.Marshal(m)
+			require.NoError(t, err)
+			metadata = string(mt)
+		}
+		v := packages.PackageVersion{
+			PackageID:    p.ID,
+			LowerVersion: version,
+			MetadataJSON: metadata,
+			CreatedUnix:  created,
+		}
+		_, err = db.GetEngine(ctx).NoAutoTime().Insert(&v)
+		require.NoError(t, err)
+	}
+
+	cleanupAndCheckLogs := func(t *testing.T, olderThan time.Duration, expected ...string) {
+		t.Helper()
+		logChecker, cleanup := test.NewLogChecker(log.DEFAULT, log.TRACE)
+		logChecker.Filter(expected...)
+		logChecker.StopMark(container_service.SHA256LogFinish)
+		defer cleanup()
+
+		require.NoError(t, CleanupExpiredData(ctx, olderThan))
+
+		logFiltered, logStopped := logChecker.Check(5 * time.Second)
+		assert.True(t, logStopped)
+		filtered := make([]bool, 0, len(expected))
+		for range expected {
+			filtered = append(filtered, true)
+		}
+		assert.EqualValues(t, filtered, logFiltered, expected)
+	}
+
+	ancient := 1 * time.Hour
+
+	t.Run("no packages, cleanup nothing", func(t *testing.T) {
+		cleanupAndCheckLogs(t, ancient, "Nothing to cleanup")
+	})
+
+	orphan := "orphan"
+	createdLongAgo := timeutil.TimeStamp(time.Now().Add(-(ancient * 2)).Unix())
+	createdRecently := timeutil.TimeStamp(time.Now().Add(-(ancient / 2)).Unix())
+
+	t.Run("an orphaned package created a long time ago is removed", func(t *testing.T) {
+		createContainer(t, orphan, "sha256:"+orphan, "", createdLongAgo)
+		cleanupAndCheckLogs(t, ancient, "Removing 1 entries from `package_version`")
+		cleanupAndCheckLogs(t, ancient, "Nothing to cleanup")
+	})
+
+	t.Run("a newly created orphaned package is not cleaned up", func(t *testing.T) {
+		createContainer(t, orphan, "sha256:"+orphan, "", createdRecently)
+		cleanupAndCheckLogs(t, ancient, "1 out of 1 container image(s) are not deleted because they were created less than")
+		cleanupAndCheckLogs(t, 0, "Removing 1 entries from `package_version`")
+		cleanupAndCheckLogs(t, 0, "Nothing to cleanup")
+	})
+
+	t.Run("a referenced package is not removed", func(t *testing.T) {
+		referenced := "referenced"
+		digest := "sha256:" + referenced
+		createContainer(t, referenced, digest, "", createdRecently)
+		index := "index"
+		createContainer(t, index, index, digest, createdRecently)
+		cleanupAndCheckLogs(t, ancient, "Nothing to cleanup")
+	})
+}
diff --git a/services/packages/cleanup/main_test.go b/services/packages/cleanup/main_test.go
new file mode 100644
index 0000000..ded3d76
--- /dev/null
+++ b/services/packages/cleanup/main_test.go
@@ -0,0 +1,14 @@
+// Copyright 2024 The Forgejo Authors.
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package container
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+)
+
+func TestMain(m *testing.M) {
+	unittest.MainTest(m)
+}