Adding upstream version 9.0.0.

Signed-off-by: Daniel Baumann <daniel@debian.org>

1 files changed, 492 insertions, 0 deletions

diff --git a/cmd/dump.go b/cmd/dump.go
new file mode 100644
index 0000000..5c64199
--- /dev/null
+++ b/cmd/dump.go
@@ -0,0 +1,492 @@
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Copyright 2016 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/util"
+
+	"code.forgejo.org/go-chi/session"
+	"github.com/mholt/archiver/v3"
+	"github.com/urfave/cli/v2"
+)
+
+func addReader(w archiver.Writer, r io.ReadCloser, info os.FileInfo, customName string, verbose bool) error {
+	if verbose {
+		log.Info("Adding file %s", customName)
+	}
+
+	return w.Write(archiver.File{
+		FileInfo: archiver.FileInfo{
+			FileInfo:   info,
+			CustomName: customName,
+		},
+		ReadCloser: r,
+	})
+}
+
+func addFile(w archiver.Writer, filePath, absPath string, verbose bool) error {
+	file, err := os.Open(absPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	return addReader(w, file, fileInfo, filePath, verbose)
+}
+
+func isSubdir(upper, lower string) (bool, error) {
+	if relPath, err := filepath.Rel(upper, lower); err != nil {
+		return false, err
+	} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
+		return true, nil
+	}
+	return false, nil
+}
+
+type outputType struct {
+	Enum     []string
+	Default  string
+	selected string
+}
+
+func (o outputType) Join() string {
+	return strings.Join(o.Enum, ", ")
+}
+
+func (o *outputType) Set(value string) error {
+	for _, enum := range o.Enum {
+		if enum == value {
+			o.selected = value
+			return nil
+		}
+	}
+
+	return fmt.Errorf("allowed values are %s", o.Join())
+}
+
+func (o outputType) String() string {
+	if o.selected == "" {
+		return o.Default
+	}
+	return o.selected
+}
+
+var outputTypeEnum = &outputType{
+	Enum:    []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"},
+	Default: "zip",
+}
+
+// CmdDump represents the available dump sub-command.
+var CmdDump = &cli.Command{
+	Name:  "dump",
+	Usage: "Dump Forgejo files and database",
+	Description: `Dump compresses all related files and database into zip file.
+It can be used for backup and capture Forgejo server image to send to maintainer`,
+	Action: runDump,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "file",
+			Aliases: []string{"f"},
+			Value:   fmt.Sprintf("forgejo-dump-%d.zip", time.Now().Unix()),
+			Usage:   "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
+		},
+		&cli.BoolFlag{
+			Name:    "verbose",
+			Aliases: []string{"V"},
+			Usage:   "Show process details",
+		},
+		&cli.BoolFlag{
+			Name:    "quiet",
+			Aliases: []string{"q"},
+			Usage:   "Only display warnings and errors",
+		},
+		&cli.StringFlag{
+			Name:    "tempdir",
+			Aliases: []string{"t"},
+			Value:   os.TempDir(),
+			Usage:   "Temporary dir path",
+		},
+		&cli.StringFlag{
+			Name:    "database",
+			Aliases: []string{"d"},
+			Usage:   "Specify the database SQL syntax: sqlite3, mysql, postgres",
+		},
+		&cli.BoolFlag{
+			Name:    "skip-repository",
+			Aliases: []string{"R"},
+			Usage:   "Skip the repository dumping",
+		},
+		&cli.BoolFlag{
+			Name:    "skip-log",
+			Aliases: []string{"L"},
+			Usage:   "Skip the log dumping",
+		},
+		&cli.BoolFlag{
+			Name:  "skip-custom-dir",
+			Usage: "Skip custom directory",
+		},
+		&cli.BoolFlag{
+			Name:  "skip-lfs-data",
+			Usage: "Skip LFS data",
+		},
+		&cli.BoolFlag{
+			Name:  "skip-attachment-data",
+			Usage: "Skip attachment data",
+		},
+		&cli.BoolFlag{
+			Name:  "skip-package-data",
+			Usage: "Skip package data",
+		},
+		&cli.BoolFlag{
+			Name:  "skip-index",
+			Usage: "Skip bleve index data",
+		},
+		&cli.GenericFlag{
+			Name:  "type",
+			Value: outputTypeEnum,
+			Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
+		},
+	},
+}
+
+func fatal(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+	log.Fatal(format, args...)
+}
+
+func runDump(ctx *cli.Context) error {
+	var file *os.File
+	fileName := ctx.String("file")
+	outType := ctx.String("type")
+	if fileName == "-" {
+		file = os.Stdout
+		setupConsoleLogger(log.FATAL, log.CanColorStderr, os.Stderr)
+	} else {
+		for _, suffix := range outputTypeEnum.Enum {
+			if strings.HasSuffix(fileName, "."+suffix) {
+				fileName = strings.TrimSuffix(fileName, "."+suffix)
+				break
+			}
+		}
+		fileName += "." + outType
+	}
+	setting.MustInstalled()
+
+	// make sure we are logging to the console no matter what the configuration tells us do to
+	// FIXME: don't use CfgProvider directly
+	if _, err := setting.CfgProvider.Section("log").NewKey("MODE", "console"); err != nil {
+		fatal("Setting logging mode to console failed: %v", err)
+	}
+	if _, err := setting.CfgProvider.Section("log.console").NewKey("STDERR", "true"); err != nil {
+		fatal("Setting console logger to stderr failed: %v", err)
+	}
+
+	// Set loglevel to Warn if quiet-mode is requested
+	if ctx.Bool("quiet") {
+		if _, err := setting.CfgProvider.Section("log.console").NewKey("LEVEL", "Warn"); err != nil {
+			fatal("Setting console log-level failed: %v", err)
+		}
+	}
+
+	if !setting.InstallLock {
+		log.Error("Is '%s' really the right config path?\n", setting.CustomConf)
+		return fmt.Errorf("forgejo is not initialized")
+	}
+	setting.LoadSettings() // cannot access session settings otherwise
+
+	verbose := ctx.Bool("verbose")
+	if verbose && ctx.Bool("quiet") {
+		return fmt.Errorf("--quiet and --verbose cannot both be set")
+	}
+
+	stdCtx, cancel := installSignals()
+	defer cancel()
+
+	err := db.InitEngine(stdCtx)
+	if err != nil {
+		return err
+	}
+
+	if err := storage.Init(); err != nil {
+		return err
+	}
+
+	if file == nil {
+		file, err = os.Create(fileName)
+		if err != nil {
+			fatal("Unable to open %s: %v", fileName, err)
+		}
+	}
+	defer file.Close()
+
+	absFileName, err := filepath.Abs(fileName)
+	if err != nil {
+		return err
+	}
+
+	var iface any
+	if fileName == "-" {
+		iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
+	} else {
+		iface, err = archiver.ByExtension(fileName)
+	}
+	if err != nil {
+		fatal("Unable to get archiver for extension: %v", err)
+	}
+
+	w, _ := iface.(archiver.Writer)
+	if err := w.Create(file); err != nil {
+		fatal("Creating archiver.Writer failed: %v", err)
+	}
+	defer w.Close()
+
+	if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
+		log.Info("Skip dumping local repositories")
+	} else {
+		log.Info("Dumping local repositories... %s", setting.RepoRootPath)
+		if err := addRecursiveExclude(w, "repos", setting.RepoRootPath, []string{absFileName}, verbose); err != nil {
+			fatal("Failed to include repositories: %v", err)
+		}
+
+		if ctx.IsSet("skip-lfs-data") && ctx.Bool("skip-lfs-data") {
+			log.Info("Skip dumping LFS data")
+		} else if !setting.LFS.StartServer {
+			log.Info("LFS isn't enabled. Skip dumping LFS data")
+		} else if err := storage.LFS.IterateObjects("", func(objPath string, object storage.Object) error {
+			info, err := object.Stat()
+			if err != nil {
+				return err
+			}
+
+			return addReader(w, object, info, path.Join("data", "lfs", objPath), verbose)
+		}); err != nil {
+			fatal("Failed to dump LFS objects: %v", err)
+		}
+	}
+
+	tmpDir := ctx.String("tempdir")
+	if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
+		fatal("Path does not exist: %s", tmpDir)
+	}
+
+	dbDump, err := os.CreateTemp(tmpDir, "forgejo-db.sql")
+	if err != nil {
+		fatal("Failed to create tmp file: %v", err)
+	}
+	defer func() {
+		_ = dbDump.Close()
+		if err := util.Remove(dbDump.Name()); err != nil {
+			log.Warn("Unable to remove temporary file: %s: Error: %v", dbDump.Name(), err)
+		}
+	}()
+
+	targetDBType := ctx.String("database")
+	if len(targetDBType) > 0 && targetDBType != setting.Database.Type.String() {
+		log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType)
+	} else {
+		log.Info("Dumping database...")
+	}
+
+	if err := db.DumpDatabase(dbDump.Name(), targetDBType); err != nil {
+		fatal("Failed to dump database: %v", err)
+	}
+
+	if err := addFile(w, "forgejo-db.sql", dbDump.Name(), verbose); err != nil {
+		fatal("Failed to include forgejo-db.sql: %v", err)
+	}
+
+	if len(setting.CustomConf) > 0 {
+		log.Info("Adding custom configuration file from %s", setting.CustomConf)
+		if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
+			fatal("Failed to include specified app.ini: %v", err)
+		}
+	}
+
+	if ctx.IsSet("skip-custom-dir") && ctx.Bool("skip-custom-dir") {
+		log.Info("Skipping custom directory")
+	} else {
+		customDir, err := os.Stat(setting.CustomPath)
+		if err == nil && customDir.IsDir() {
+			if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
+				if err := addRecursiveExclude(w, "custom", setting.CustomPath, []string{absFileName}, verbose); err != nil {
+					fatal("Failed to include custom: %v", err)
+				}
+			} else {
+				log.Info("Custom dir %s is inside data dir %s, skipped", setting.CustomPath, setting.AppDataPath)
+			}
+		} else {
+			log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath)
+		}
+	}
+
+	isExist, err := util.IsExist(setting.AppDataPath)
+	if err != nil {
+		log.Error("Unable to check if %s exists. Error: %v", setting.AppDataPath, err)
+	}
+	if isExist {
+		log.Info("Packing data directory...%s", setting.AppDataPath)
+
+		var excludes []string
+		if setting.SessionConfig.OriginalProvider == "file" {
+			var opts session.Options
+			if err = json.Unmarshal([]byte(setting.SessionConfig.ProviderConfig), &opts); err != nil {
+				return err
+			}
+			excludes = append(excludes, opts.ProviderConfig)
+		}
+
+		if ctx.IsSet("skip-index") && ctx.Bool("skip-index") {
+			excludes = append(excludes, setting.Indexer.RepoPath)
+			excludes = append(excludes, setting.Indexer.IssuePath)
+		}
+
+		excludes = append(excludes, setting.RepoRootPath)
+		excludes = append(excludes, setting.LFS.Storage.Path)
+		excludes = append(excludes, setting.Attachment.Storage.Path)
+		excludes = append(excludes, setting.Packages.Storage.Path)
+		excludes = append(excludes, setting.Log.RootPath)
+		excludes = append(excludes, absFileName)
+		if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
+			fatal("Failed to include data directory: %v", err)
+		}
+	}
+
+	if ctx.IsSet("skip-attachment-data") && ctx.Bool("skip-attachment-data") {
+		log.Info("Skip dumping attachment data")
+	} else if err := storage.Attachments.IterateObjects("", func(objPath string, object storage.Object) error {
+		info, err := object.Stat()
+		if err != nil {
+			return err
+		}
+
+		return addReader(w, object, info, path.Join("data", "attachments", objPath), verbose)
+	}); err != nil {
+		fatal("Failed to dump attachments: %v", err)
+	}
+
+	if ctx.IsSet("skip-package-data") && ctx.Bool("skip-package-data") {
+		log.Info("Skip dumping package data")
+	} else if !setting.Packages.Enabled {
+		log.Info("Packages isn't enabled. Skip dumping package data")
+	} else if err := storage.Packages.IterateObjects("", func(objPath string, object storage.Object) error {
+		info, err := object.Stat()
+		if err != nil {
+			return err
+		}
+
+		return addReader(w, object, info, path.Join("data", "packages", objPath), verbose)
+	}); err != nil {
+		fatal("Failed to dump packages: %v", err)
+	}
+
+	// Doesn't check if LogRootPath exists before processing --skip-log intentionally,
+	// ensuring that it's clear the dump is skipped whether the directory's initialized
+	// yet or not.
+	if ctx.IsSet("skip-log") && ctx.Bool("skip-log") {
+		log.Info("Skip dumping log files")
+	} else {
+		isExist, err := util.IsExist(setting.Log.RootPath)
+		if err != nil {
+			log.Error("Unable to check if %s exists. Error: %v", setting.Log.RootPath, err)
+		}
+		if isExist {
+			if err := addRecursiveExclude(w, "log", setting.Log.RootPath, []string{absFileName}, verbose); err != nil {
+				fatal("Failed to include log: %v", err)
+			}
+		}
+	}
+
+	if fileName != "-" {
+		if err = w.Close(); err != nil {
+			_ = util.Remove(fileName)
+			fatal("Failed to save %s: %v", fileName, err)
+		}
+
+		if err := os.Chmod(fileName, 0o600); err != nil {
+			log.Info("Can't change file access permissions mask to 0600: %v", err)
+		}
+	}
+
+	if fileName != "-" {
+		log.Info("Finish dumping in file %s", fileName)
+	} else {
+		log.Info("Finish dumping to stdout")
+	}
+
+	return nil
+}
+
+// addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
+func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
+	absPath, err := filepath.Abs(absPath)
+	if err != nil {
+		return err
+	}
+	dir, err := os.Open(absPath)
+	if err != nil {
+		return err
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdir(0)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		currentAbsPath := filepath.Join(absPath, file.Name())
+		currentInsidePath := path.Join(insidePath, file.Name())
+
+		if util.SliceContainsString(excludeAbsPath, currentAbsPath) {
+			log.Debug("Skipping %q because matched an excluded path.", currentAbsPath)
+			continue
+		}
+
+		if file.IsDir() {
+			if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
+				return err
+			}
+			if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
+				return err
+			}
+		} else {
+			// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
+			shouldAdd := file.Mode().IsRegular()
+			if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
+				target, err := filepath.EvalSymlinks(currentAbsPath)
+				if err != nil {
+					return err
+				}
+				targetStat, err := os.Stat(target)
+				if err != nil {
+					return err
+				}
+				shouldAdd = targetStat.Mode().IsRegular()
+			}
+			if shouldAdd {
+				if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}