mirror of
https://github.com/go-gitea/gitea
synced 2025-07-22 18:28:37 +00:00
Use a general approach to access custom/static/builtin assets (#24022)
The idea is to use a Layered Asset File-system (modules/assetfs/layered.go) For example: when there are 2 layers: "custom", "builtin", when access to asset "my/page.tmpl", the Layered Asset File-system will first try to use "custom" assets, if not found, then use "builtin" assets. This approach will hugely simplify a lot of code, make them testable. Other changes: * Simplify the AssetsHandlerFunc code * Simplify the `gitea embedded` sub-command code --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
wxiaoguang
GitHub
260
modules/assetfs/layered.go
Normal file
260
modules/assetfs/layered.go
Normal file
@@ -0,0 +1,260 @@
|
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package assetfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/process"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
)
|
||||
|
||||
// Layer represents a layer in a layered asset file-system. It has a name and works like http.FileSystem
|
||||
type Layer struct {
|
||||
name string
|
||||
fs http.FileSystem
|
||||
localPath string
|
||||
}
|
||||
|
||||
func (l *Layer) Name() string {
|
||||
return l.name
|
||||
}
|
||||
|
||||
// Open opens the named file. The caller is responsible for closing the file.
|
||||
func (l *Layer) Open(name string) (http.File, error) {
|
||||
return l.fs.Open(name)
|
||||
}
|
||||
|
||||
// Local returns a new Layer with the given name, it serves files from the given local path.
|
||||
func Local(name, base string, sub ...string) *Layer {
|
||||
// TODO: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before
|
||||
// Ideally, the caller should guarantee the base is absolute, guessing a relative path based on the current working directory is unreliable.
|
||||
base, err := filepath.Abs(base)
|
||||
if err != nil {
|
||||
// This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths.
|
||||
panic(fmt.Sprintf("Unable to get absolute path for %q: %v", base, err))
|
||||
}
|
||||
root := util.FilePathJoinAbs(base, sub...)
|
||||
return &Layer{name: name, fs: http.Dir(root), localPath: root}
|
||||
}
|
||||
|
||||
// Bindata returns a new Layer with the given name, it serves files from the given bindata asset.
|
||||
func Bindata(name string, fs http.FileSystem) *Layer {
|
||||
return &Layer{name: name, fs: fs}
|
||||
}
|
||||
|
||||
// LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers.
|
||||
// The first layer is the top layer, and it will be used first.
|
||||
// If the file is not found in the top layer, it will be searched in the next layer.
|
||||
type LayeredFS struct {
|
||||
layers []*Layer
|
||||
}
|
||||
|
||||
// Layered returns a new LayeredFS with the given layers. The first layer is the top layer.
|
||||
func Layered(layers ...*Layer) *LayeredFS {
|
||||
return &LayeredFS{layers: layers}
|
||||
}
|
||||
|
||||
// Open opens the named file. The caller is responsible for closing the file.
|
||||
func (l *LayeredFS) Open(name string) (http.File, error) {
|
||||
for _, layer := range l.layers {
|
||||
f, err := layer.Open(name)
|
||||
if err == nil || !os.IsNotExist(err) {
|
||||
return f, err
|
||||
}
|
||||
}
|
||||
return nil, fs.ErrNotExist
|
||||
}
|
||||
|
||||
// ReadFile reads the named file.
|
||||
func (l *LayeredFS) ReadFile(elems ...string) ([]byte, error) {
|
||||
bs, _, err := l.ReadLayeredFile(elems...)
|
||||
return bs, err
|
||||
}
|
||||
|
||||
// ReadLayeredFile reads the named file, and returns the layer name.
|
||||
func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) {
|
||||
name := util.PathJoinRel(elems...)
|
||||
for _, layer := range l.layers {
|
||||
f, err := layer.Open(name)
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, layer.name, err
|
||||
}
|
||||
bs, err := io.ReadAll(f)
|
||||
_ = f.Close()
|
||||
return bs, layer.name, err
|
||||
}
|
||||
return nil, "", fs.ErrNotExist
|
||||
}
|
||||
|
||||
func shouldInclude(info fs.FileInfo, fileMode ...bool) bool {
|
||||
if util.CommonSkip(info.Name()) {
|
||||
return false
|
||||
}
|
||||
if len(fileMode) == 0 {
|
||||
return true
|
||||
} else if len(fileMode) == 1 {
|
||||
return fileMode[0] == !info.Mode().IsDir()
|
||||
}
|
||||
panic("too many arguments for fileMode in shouldInclude")
|
||||
}
|
||||
|
||||
func readDir(layer *Layer, name string) ([]fs.FileInfo, error) {
|
||||
f, err := layer.Open(name)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return f.Readdir(-1)
|
||||
}
|
||||
|
||||
// ListFiles lists files/directories in the given directory. The fileMode controls the returned files.
|
||||
// * omitted: all files and directories will be returned.
|
||||
// * true: only files will be returned.
|
||||
// * false: only directories will be returned.
|
||||
// The returned files are sorted by name.
|
||||
func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) {
|
||||
fileMap := map[string]bool{}
|
||||
for _, layer := range l.layers {
|
||||
infos, err := readDir(layer, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, info := range infos {
|
||||
if shouldInclude(info, fileMode...) {
|
||||
fileMap[info.Name()] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
files := make([]string, 0, len(fileMap))
|
||||
for file := range fileMap {
|
||||
files = append(files, file)
|
||||
}
|
||||
sort.Strings(files)
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively.
|
||||
// The fileMode controls the returned files:
|
||||
// * omitted: all files and directories will be returned.
|
||||
// * true: only files will be returned.
|
||||
// * false: only directories will be returned.
|
||||
// The returned files are sorted by name.
|
||||
func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) {
|
||||
return listAllFiles(l.layers, name, fileMode...)
|
||||
}
|
||||
|
||||
func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) {
|
||||
fileMap := map[string]bool{}
|
||||
var list func(dir string) error
|
||||
list = func(dir string) error {
|
||||
for _, layer := range layers {
|
||||
infos, err := readDir(layer, dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, info := range infos {
|
||||
path := util.PathJoinRelX(dir, info.Name())
|
||||
if shouldInclude(info, fileMode...) {
|
||||
fileMap[path] = true
|
||||
}
|
||||
if info.IsDir() {
|
||||
if err = list(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := list(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var files []string
|
||||
for file := range fileMap {
|
||||
files = append(files, file)
|
||||
}
|
||||
sort.Strings(files)
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// WatchLocalChanges watches local changes in the file-system. It's used to help to reload assets when the local file-system changes.
|
||||
func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) {
|
||||
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Asset Local FileSystem Watcher", process.SystemProcessType, true)
|
||||
defer finished()
|
||||
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
log.Error("Unable to create watcher for asset local file-system: %v", err)
|
||||
return
|
||||
}
|
||||
defer watcher.Close()
|
||||
|
||||
for _, layer := range l.layers {
|
||||
if layer.localPath == "" {
|
||||
continue
|
||||
}
|
||||
layerDirs, err := listAllFiles([]*Layer{layer}, ".", false)
|
||||
if err != nil {
|
||||
log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err)
|
||||
continue
|
||||
}
|
||||
for _, dir := range layerDirs {
|
||||
if err = watcher.Add(util.FilePathJoinAbs(layer.localPath, dir)); err != nil {
|
||||
log.Error("Unable to watch directory %s: %v", dir, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debounce := util.Debounce(100 * time.Millisecond)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case event, ok := <-watcher.Events:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
log.Trace("Watched asset local file-system had event: %v", event)
|
||||
debounce(callback)
|
||||
case err, ok := <-watcher.Errors:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
log.Error("Watched asset local file-system had error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetFileLayerName returns the name of the first-seen layer that contains the given file.
|
||||
func (l *LayeredFS) GetFileLayerName(elems ...string) string {
|
||||
name := util.PathJoinRel(elems...)
|
||||
for _, layer := range l.layers {
|
||||
f, err := layer.Open(name)
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return ""
|
||||
}
|
||||
_ = f.Close()
|
||||
return layer.name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
109
modules/assetfs/layered_test.go
Normal file
109
modules/assetfs/layered_test.go
Normal file
@@ -0,0 +1,109 @@
|
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package assetfs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestLayered(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "assetfs-layers")
|
||||
dir1 := filepath.Join(dir, "l1")
|
||||
dir2 := filepath.Join(dir, "l2")
|
||||
|
||||
mkdir := func(elems ...string) {
|
||||
assert.NoError(t, os.MkdirAll(filepath.Join(elems...), 0o755))
|
||||
}
|
||||
write := func(content string, elems ...string) {
|
||||
assert.NoError(t, os.WriteFile(filepath.Join(elems...), []byte(content), 0o644))
|
||||
}
|
||||
|
||||
// d1 & f1: only in "l1"; d2 & f2: only in "l2"
|
||||
// da & fa: in both "l1" and "l2"
|
||||
mkdir(dir1, "d1")
|
||||
mkdir(dir1, "da")
|
||||
mkdir(dir1, "da/sub1")
|
||||
|
||||
mkdir(dir2, "d2")
|
||||
mkdir(dir2, "da")
|
||||
mkdir(dir2, "da/sub2")
|
||||
|
||||
write("dummy", dir1, ".DS_Store")
|
||||
write("f1", dir1, "f1")
|
||||
write("fa-1", dir1, "fa")
|
||||
write("d1-f", dir1, "d1/f")
|
||||
write("da-f-1", dir1, "da/f")
|
||||
|
||||
write("f2", dir2, "f2")
|
||||
write("fa-2", dir2, "fa")
|
||||
write("d2-f", dir2, "d2/f")
|
||||
write("da-f-2", dir2, "da/f")
|
||||
|
||||
assets := Layered(Local("l1", dir1), Local("l2", dir2))
|
||||
|
||||
f, err := assets.Open("f1")
|
||||
assert.NoError(t, err)
|
||||
bs, err := io.ReadAll(f)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, "f1", string(bs))
|
||||
_ = f.Close()
|
||||
|
||||
assertRead := func(expected string, expectedErr error, elems ...string) {
|
||||
bs, err := assets.ReadFile(elems...)
|
||||
if err != nil {
|
||||
assert.ErrorAs(t, err, &expectedErr)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, expected, string(bs))
|
||||
}
|
||||
}
|
||||
assertRead("f1", nil, "f1")
|
||||
assertRead("f2", nil, "f2")
|
||||
assertRead("fa-1", nil, "fa")
|
||||
|
||||
assertRead("d1-f", nil, "d1/f")
|
||||
assertRead("d2-f", nil, "d2/f")
|
||||
assertRead("da-f-1", nil, "da/f")
|
||||
|
||||
assertRead("", fs.ErrNotExist, "no-such")
|
||||
|
||||
files, err := assets.ListFiles(".", true)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, []string{"f1", "f2", "fa"}, files)
|
||||
|
||||
files, err = assets.ListFiles(".", false)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, []string{"d1", "d2", "da"}, files)
|
||||
|
||||
files, err = assets.ListFiles(".")
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files)
|
||||
|
||||
files, err = assets.ListAllFiles(".", true)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files)
|
||||
|
||||
files, err = assets.ListAllFiles(".", false)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files)
|
||||
|
||||
files, err = assets.ListAllFiles(".")
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, []string{
|
||||
"d1", "d1/f",
|
||||
"d2", "d2/f",
|
||||
"da", "da/f", "da/sub1", "da/sub2",
|
||||
"f1", "f2", "fa",
|
||||
}, files)
|
||||
|
||||
assert.Empty(t, assets.GetFileLayerName("no-such"))
|
||||
assert.EqualValues(t, "l1", assets.GetFileLayerName("f1"))
|
||||
assert.EqualValues(t, "l2", assets.GetFileLayerName("f2"))
|
||||
}
|
||||
Reference in New Issue
Block a user