tdsds.com/gitea
1
1
Fork 0
mirror of https://github.com/go-gitea/gitea synced 2025-09-10 10:48:28 +00:00
Code Releases Activity

Use github.com/mholt/archives replace github.com/mholt/archiver (#35390)

Browse Source 
Fix #32620

---------

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: junoberryferry <user@example.tld>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
junoberryferry
2025-09-01 15:40:12 -04:00
committed by GitHub
parent 1717af72c4
commit ec75bdbe68
6 changed files with 451 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
modules/dump/dumper.go
View File

@@ -4,8 +4,11 @@
package dump
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
@@ -16,7 +19,7 @@ import (
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"github.com/mholt/archiver/v3"
"github.com/mholt/archives"
)
var SupportedOutputTypes = []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"}
@@ -60,37 +63,122 @@ func IsSubdir(upper, lower string) (bool, error) {
}
type Dumper struct {
Writer archiver.Writer
Verbose bool
jobs chan archives.ArchiveAsyncJob
errArchiveAsync chan error
errArchiveJob chan error
globalExcludeAbsPaths []string
}
func (dumper *Dumper) AddReader(r io.ReadCloser, info os.FileInfo, customName string) error {
if dumper.Verbose {
log.Info("Adding file %s", customName)
func NewDumper(ctx context.Context, format string, output io.Writer) (*Dumper, error) {
d := &Dumper{
jobs: make(chan archives.ArchiveAsyncJob, 1),
errArchiveAsync: make(chan error, 1),
errArchiveJob: make(chan error, 1),
}
return dumper.Writer.Write(archiver.File{
FileInfo: archiver.FileInfo{
FileInfo: info,
CustomName: customName,
},
ReadCloser: r,
// TODO: in the future, we could completely drop the "mholt/archives" dependency.
// Then we only need to support "zip" and ".tar.gz" natively, and let users provide custom command line tools
// like "zstd" or "xz" with compression-level arguments.
var comp archives.ArchiverAsync
switch format {
case "zip":
comp = archives.Zip{}
case "tar":
comp = archives.Tar{}
case "tar.sz":
comp = archives.CompressedArchive{Compression: archives.Sz{}, Archival: archives.Tar{}}
case "tar.gz":
comp = archives.CompressedArchive{Compression: archives.Gz{}, Archival: archives.Tar{}}
case "tar.xz":
comp = archives.CompressedArchive{Compression: archives.Xz{}, Archival: archives.Tar{}}
case "tar.bz2":
comp = archives.CompressedArchive{Compression: archives.Bz2{}, Archival: archives.Tar{}}
case "tar.br":
comp = archives.CompressedArchive{Compression: archives.Brotli{}, Archival: archives.Tar{}}
case "tar.lz4":
comp = archives.CompressedArchive{Compression: archives.Lz4{}, Archival: archives.Tar{}}
case "tar.zst":
comp = archives.CompressedArchive{Compression: archives.Zstd{}, Archival: archives.Tar{}}
default:
return nil, fmt.Errorf("unsupported format: %s", format)
}
go func() {
d.errArchiveAsync <- comp.ArchiveAsync(ctx, output, d.jobs)
close(d.errArchiveAsync)
}()
return d, nil
}
func (dumper *Dumper) runArchiveJob(job archives.ArchiveAsyncJob) error {
dumper.jobs <- job
select {
case err := <-dumper.errArchiveAsync:
if err == nil {
return errors.New("archiver has been closed")
}
return err
case err := <-dumper.errArchiveJob:
return err
}
}
// AddFileByPath adds a file by its filesystem path
func (dumper *Dumper) AddFileByPath(filePath, absPath string) error {
if dumper.Verbose {
log.Info("Adding local file %s", filePath)
}
fileInfo, err := os.Stat(absPath)
if err != nil {
return err
}
archiveFileInfo := archives.FileInfo{
FileInfo: fileInfo,
NameInArchive: filePath,
Open: func() (fs.File, error) { return os.Open(absPath) },
}
return dumper.runArchiveJob(archives.ArchiveAsyncJob{
File: archiveFileInfo,
Result: dumper.errArchiveJob,
})
}
func (dumper *Dumper) AddFile(filePath, absPath string) error {
file, err := os.Open(absPath)
if err != nil {
return err
type readerFile struct {
r io.Reader
info os.FileInfo
}
var _ fs.File = (*readerFile)(nil)
func (f *readerFile) Stat() (fs.FileInfo, error) { return f.info, nil }
func (f *readerFile) Read(bytes []byte) (int, error) { return f.r.Read(bytes) }
func (f *readerFile) Close() error { return nil }
// AddFileByReader adds a file's contents from a Reader
func (dumper *Dumper) AddFileByReader(r io.Reader, info os.FileInfo, customName string) error {
if dumper.Verbose {
log.Info("Adding storage file %s", customName)
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
fileInfo := archives.FileInfo{
FileInfo: info,
NameInArchive: customName,
Open: func() (fs.File, error) { return &readerFile{r, info}, nil },
}
return dumper.AddReader(file, fileInfo, filePath)
return dumper.runArchiveJob(archives.ArchiveAsyncJob{
File: fileInfo,
Result: dumper.errArchiveJob,
})
}
func (dumper *Dumper) Close() error {
close(dumper.jobs)
return <-dumper.errArchiveAsync
}
func (dumper *Dumper) normalizeFilePath(absPath string) string {
@@ -143,7 +231,7 @@ func (dumper *Dumper) addFileOrDir(insidePath, absPath string, excludes []string
currentInsidePath := path.Join(insidePath, file.Name())
if file.IsDir() {
if err := dumper.AddFile(currentInsidePath, currentAbsPath); err != nil {
if err := dumper.AddFileByPath(currentInsidePath, currentAbsPath); err != nil {
return err
}
if err = dumper.addFileOrDir(currentInsidePath, currentAbsPath, excludes); err != nil {
@@ -164,7 +252,7 @@ func (dumper *Dumper) addFileOrDir(insidePath, absPath string, excludes []string
shouldAdd = targetStat.Mode().IsRegular()
}
if shouldAdd {
if err = dumper.AddFile(currentInsidePath, currentAbsPath); err != nil {
if err = dumper.AddFileByPath(currentInsidePath, currentAbsPath); err != nil {
return err
}
}

94
modules/dump/dumper_test.go
View File

@@ -4,6 +4,8 @@
package dump
import (
"archive/tar"
"bytes"
"fmt"
"io"
"os"
@@ -14,8 +16,8 @@ import (
"code.gitea.io/gitea/modules/timeutil"
"github.com/mholt/archiver/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPrepareFileNameAndType(t *testing.T) {
@@ -67,28 +69,26 @@ func TestIsSubDir(t *testing.T) {
assert.False(t, isSub)
}
type testWriter struct {
added []string
}
func TestDumperIntegration(t *testing.T) {
var buf bytes.Buffer
dumper, err := NewDumper(t.Context(), "zip", &buf)
require.NoError(t, err)
func (t *testWriter) Create(out io.Writer) error {
return nil
}
tmpDir := t.TempDir()
_ = os.WriteFile(filepath.Join(tmpDir, "test.txt"), nil, 0o644)
f, _ := os.Open(filepath.Join(tmpDir, "test.txt"))
func (t *testWriter) Write(f archiver.File) error {
t.added = append(t.added, f.Name())
return nil
}
fi, _ := f.Stat()
err = dumper.AddFileByReader(f, fi, "test.txt")
require.NoError(t, err)
func (t *testWriter) Close() error {
return nil
err = dumper.Close()
require.NoError(t, err)
assert.Positive(t, buf.Len())
}
func TestDumper(t *testing.T) {
sortStrings := func(s []string) []string {
sort.Strings(s)
return s
}
tmpDir := t.TempDir()
_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude1"), 0o755)
_ = os.MkdirAll(filepath.Join(tmpDir, "include/exclude2"), 0o755)
@@ -98,16 +98,54 @@ func TestDumper(t *testing.T) {
_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude1/a-1"), nil, 0o644)
_ = os.WriteFile(filepath.Join(tmpDir, "include/exclude2/a-2"), nil, 0o644)
tw := &testWriter{}
d := &Dumper{Writer: tw}
d.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1"))
err := d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")})
assert.NoError(t, err)
assert.Equal(t, sortStrings([]string{"include/a", "include/sub", "include/sub/b"}), sortStrings(tw.added))
sortStrings := func(s []string) []string {
sort.Strings(s)
return s
}
tw = &testWriter{}
d = &Dumper{Writer: tw}
err = d.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil)
assert.NoError(t, err)
assert.Equal(t, sortStrings([]string{"include/exclude2", "include/exclude2/a-2", "include/a", "include/sub", "include/sub/b", "include/exclude1", "include/exclude1/a-1"}), sortStrings(tw.added))
t.Run("IncludesWithExcludes", func(t *testing.T) {
var buf bytes.Buffer
dumper, err := NewDumper(t.Context(), "tar", &buf)
require.NoError(t, err)
dumper.GlobalExcludeAbsPath(filepath.Join(tmpDir, "include/exclude1"))
err = dumper.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), []string{filepath.Join(tmpDir, "include/exclude2")})
require.NoError(t, err)
err = dumper.Close()
require.NoError(t, err)
files := extractTarFileNames(t, &buf)
expected := []string{"include/a", "include/sub", "include/sub/b"}
assert.Equal(t, sortStrings(expected), sortStrings(files))
})
t.Run("IncludesAll", func(t *testing.T) {
var buf bytes.Buffer
dumper, err := NewDumper(t.Context(), "tar", &buf)
require.NoError(t, err)
err = dumper.AddRecursiveExclude("include", filepath.Join(tmpDir, "include"), nil)
require.NoError(t, err)
err = dumper.Close()
require.NoError(t, err)
files := extractTarFileNames(t, &buf)
expected := []string{
"include/exclude2", "include/exclude2/a-2",
"include/a", "include/sub", "include/sub/b",
"include/exclude1", "include/exclude1/a-1",
}
assert.Equal(t, sortStrings(expected), sortStrings(files))
})
}
func extractTarFileNames(t *testing.T, buf *bytes.Buffer) (fileNames []string) {
tr := tar.NewReader(buf)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
require.NoError(t, err, "Error reading tar archive")
fileNames = append(fileNames, hdr.Name)
}
return fileNames
}