2 files changed, 366 insertions, 0 deletions
diff --git a/modules/assetfs/layered.go b/modules/assetfs/layered.go
new file mode 100644
index 0000000..9678d23
--- /dev/null
+++ b/modules/assetfs/layered.go
@@ -0,0 +1,256 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package assetfs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"code.gitea.io/gitea/modules/container"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/fsnotify/fsnotify"
+)
+
+// Layer represents a layer in a layered asset file-system. It has a name and works like http.FileSystem
+type Layer struct {
+	name      string
+	fs        http.FileSystem
+	localPath string
+}
+
+func (l *Layer) Name() string {
+	return l.name
+}
+
+// Open opens the named file. The caller is responsible for closing the file.
+func (l *Layer) Open(name string) (http.File, error) {
+	return l.fs.Open(name)
+}
+
+// Local returns a new Layer with the given name, it serves files from the given local path.
+func Local(name, base string, sub ...string) *Layer {
+	// TODO: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before
+	// Ideally, the caller should guarantee the base is absolute, guessing a relative path based on the current working directory is unreliable.
+	base, err := filepath.Abs(base)
+	if err != nil {
+		// This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths.
+		panic(fmt.Sprintf("Unable to get absolute path for %q: %v", base, err))
+	}
+	root := util.FilePathJoinAbs(base, sub...)
+	return &Layer{name: name, fs: http.Dir(root), localPath: root}
+}
+
+// Bindata returns a new Layer with the given name, it serves files from the given bindata asset.
+func Bindata(name string, fs http.FileSystem) *Layer {
+	return &Layer{name: name, fs: fs}
+}
+
+// LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers.
+// The first layer is the top layer, and it will be used first.
+// If the file is not found in the top layer, it will be searched in the next layer.
+type LayeredFS struct {
+	layers []*Layer
+}
+
+// Layered returns a new LayeredFS with the given layers. The first layer is the top layer.
+func Layered(layers ...*Layer) *LayeredFS {
+	return &LayeredFS{layers: layers}
+}
+
+// Open opens the named file. The caller is responsible for closing the file.
+func (l *LayeredFS) Open(name string) (http.File, error) {
+	for _, layer := range l.layers {
+		f, err := layer.Open(name)
+		if err == nil || !os.IsNotExist(err) {
+			return f, err
+		}
+	}
+	return nil, fs.ErrNotExist
+}
+
+// ReadFile reads the named file.
+func (l *LayeredFS) ReadFile(elems ...string) ([]byte, error) {
+	bs, _, err := l.ReadLayeredFile(elems...)
+	return bs, err
+}
+
+// ReadLayeredFile reads the named file, and returns the layer name.
+func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) {
+	name := util.PathJoinRel(elems...)
+	for _, layer := range l.layers {
+		f, err := layer.Open(name)
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return nil, layer.name, err
+		}
+		bs, err := io.ReadAll(f)
+		_ = f.Close()
+		return bs, layer.name, err
+	}
+	return nil, "", fs.ErrNotExist
+}
+
+func shouldInclude(info fs.FileInfo, fileMode ...bool) bool {
+	if util.CommonSkip(info.Name()) {
+		return false
+	}
+	if len(fileMode) == 0 {
+		return true
+	} else if len(fileMode) == 1 {
+		return fileMode[0] == !info.Mode().IsDir()
+	}
+	panic("too many arguments for fileMode in shouldInclude")
+}
+
+func readDir(layer *Layer, name string) ([]fs.FileInfo, error) {
+	f, err := layer.Open(name)
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return f.Readdir(-1)
+}
+
+// ListFiles lists files/directories in the given directory. The fileMode controls the returned files.
+// * omitted: all files and directories will be returned.
+// * true: only files will be returned.
+// * false: only directories will be returned.
+// The returned files are sorted by name.
+func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) {
+	fileSet := make(container.Set[string])
+	for _, layer := range l.layers {
+		infos, err := readDir(layer, name)
+		if err != nil {
+			return nil, err
+		}
+		for _, info := range infos {
+			if shouldInclude(info, fileMode...) {
+				fileSet.Add(info.Name())
+			}
+		}
+	}
+	files := fileSet.Values()
+	sort.Strings(files)
+	return files, nil
+}
+
+// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively.
+// The fileMode controls the returned files:
+// * omitted: all files and directories will be returned.
+// * true: only files will be returned.
+// * false: only directories will be returned.
+// The returned files are sorted by name.
+func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) {
+	return listAllFiles(l.layers, name, fileMode...)
+}
+
+func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) {
+	fileSet := make(container.Set[string])
+	var list func(dir string) error
+	list = func(dir string) error {
+		for _, layer := range layers {
+			infos, err := readDir(layer, dir)
+			if err != nil {
+				return err
+			}
+			for _, info := range infos {
+				path := util.PathJoinRelX(dir, info.Name())
+				if shouldInclude(info, fileMode...) {
+					fileSet.Add(path)
+				}
+				if info.IsDir() {
+					if err = list(path); err != nil {
+						return err
+					}
+				}
+			}
+		}
+		return nil
+	}
+	if err := list(name); err != nil {
+		return nil, err
+	}
+	files := fileSet.Values()
+	sort.Strings(files)
+	return files, nil
+}
+
+// WatchLocalChanges watches local changes in the file-system. It's used to help to reload assets when the local file-system changes.
+func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) {
+	ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Asset Local FileSystem Watcher", process.SystemProcessType, true)
+	defer finished()
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Error("Unable to create watcher for asset local file-system: %v", err)
+		return
+	}
+	defer watcher.Close()
+
+	for _, layer := range l.layers {
+		if layer.localPath == "" {
+			continue
+		}
+		layerDirs, err := listAllFiles([]*Layer{layer}, ".", false)
+		if err != nil {
+			log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err)
+			continue
+		}
+		layerDirs = append(layerDirs, ".")
+		for _, dir := range layerDirs {
+			if err = watcher.Add(util.FilePathJoinAbs(layer.localPath, dir)); err != nil && !os.IsNotExist(err) {
+				log.Error("Unable to watch directory %s: %v", dir, err)
+			}
+		}
+	}
+
+	debounce := util.Debounce(100 * time.Millisecond)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case event, ok := <-watcher.Events:
+			if !ok {
+				return
+			}
+			log.Trace("Watched asset local file-system had event: %v", event)
+			debounce(callback)
+		case err, ok := <-watcher.Errors:
+			if !ok {
+				return
+			}
+			log.Error("Watched asset local file-system had error: %v", err)
+		}
+	}
+}
+
+// GetFileLayerName returns the name of the first-seen layer that contains the given file.
+func (l *LayeredFS) GetFileLayerName(elems ...string) string {
+	name := util.PathJoinRel(elems...)
+	for _, layer := range l.layers {
+		f, err := layer.Open(name)
+		if os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return ""
+		}
+		_ = f.Close()
+		return layer.name
+	}
+	return ""
+}
diff --git a/modules/assetfs/layered_test.go b/modules/assetfs/layered_test.go
new file mode 100644
index 0000000..58876d9
--- /dev/null
+++ b/modules/assetfs/layered_test.go
@@ -0,0 +1,110 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package assetfs
+
+import (
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLayered(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "assetfs-layers")
+	dir1 := filepath.Join(dir, "l1")
+	dir2 := filepath.Join(dir, "l2")
+
+	mkdir := func(elems ...string) {
+		require.NoError(t, os.MkdirAll(filepath.Join(elems...), 0o755))
+	}
+	write := func(content string, elems ...string) {
+		require.NoError(t, os.WriteFile(filepath.Join(elems...), []byte(content), 0o644))
+	}
+
+	// d1 & f1: only in "l1"; d2 & f2: only in "l2"
+	// da & fa: in both "l1" and "l2"
+	mkdir(dir1, "d1")
+	mkdir(dir1, "da")
+	mkdir(dir1, "da/sub1")
+
+	mkdir(dir2, "d2")
+	mkdir(dir2, "da")
+	mkdir(dir2, "da/sub2")
+
+	write("dummy", dir1, ".DS_Store")
+	write("f1", dir1, "f1")
+	write("fa-1", dir1, "fa")
+	write("d1-f", dir1, "d1/f")
+	write("da-f-1", dir1, "da/f")
+
+	write("f2", dir2, "f2")
+	write("fa-2", dir2, "fa")
+	write("d2-f", dir2, "d2/f")
+	write("da-f-2", dir2, "da/f")
+
+	assets := Layered(Local("l1", dir1), Local("l2", dir2))
+
+	f, err := assets.Open("f1")
+	require.NoError(t, err)
+	bs, err := io.ReadAll(f)
+	require.NoError(t, err)
+	assert.EqualValues(t, "f1", string(bs))
+	_ = f.Close()
+
+	assertRead := func(expected string, expectedErr error, elems ...string) {
+		bs, err := assets.ReadFile(elems...)
+		if err != nil {
+			require.ErrorIs(t, err, expectedErr)
+		} else {
+			require.NoError(t, err)
+			assert.Equal(t, expected, string(bs))
+		}
+	}
+	assertRead("f1", nil, "f1")
+	assertRead("f2", nil, "f2")
+	assertRead("fa-1", nil, "fa")
+
+	assertRead("d1-f", nil, "d1/f")
+	assertRead("d2-f", nil, "d2/f")
+	assertRead("da-f-1", nil, "da/f")
+
+	assertRead("", fs.ErrNotExist, "no-such")
+
+	files, err := assets.ListFiles(".", true)
+	require.NoError(t, err)
+	assert.EqualValues(t, []string{"f1", "f2", "fa"}, files)
+
+	files, err = assets.ListFiles(".", false)
+	require.NoError(t, err)
+	assert.EqualValues(t, []string{"d1", "d2", "da"}, files)
+
+	files, err = assets.ListFiles(".")
+	require.NoError(t, err)
+	assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files)
+
+	files, err = assets.ListAllFiles(".", true)
+	require.NoError(t, err)
+	assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files)
+
+	files, err = assets.ListAllFiles(".", false)
+	require.NoError(t, err)
+	assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files)
+
+	files, err = assets.ListAllFiles(".")
+	require.NoError(t, err)
+	assert.EqualValues(t, []string{
+		"d1", "d1/f",
+		"d2", "d2/f",
+		"da", "da/f", "da/sub1", "da/sub2",
+		"f1", "f2", "fa",
+	}, files)
+
+	assert.Empty(t, assets.GetFileLayerName("no-such"))
+	assert.EqualValues(t, "l1", assets.GetFileLayerName("f1"))
+	assert.EqualValues(t, "l2", assets.GetFileLayerName("f2"))
+}