1
1
mirror of https://github.com/go-gitea/gitea synced 2025-01-05 07:24:25 +00:00

Inherit submodules from template repository content (#16237)

Fix #10316

---------

Signed-off-by: Steffen Schröter <steffen@vexar.de>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Steffen Schröter 2025-01-01 03:55:13 +01:00 committed by GitHub
parent 92a2900a2d
commit 57eb9d0b64
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 291 additions and 137 deletions

View File

@ -242,7 +242,7 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte {
return out
}
// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream
// This carefully avoids allocations - except where fnameBuf is too small.
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
//
@ -250,7 +250,7 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte {
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <binary HASH>
//
// We don't attempt to convert the raw HASH to save a lot of time
func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
var readBytes []byte
// Read the Mode & fname
@ -260,7 +260,7 @@ func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBu
}
idx := bytes.IndexByte(readBytes, ' ')
if idx < 0 {
log.Debug("missing space in readBytes ParseTreeLine: %s", readBytes)
log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes)
return mode, fname, sha, n, &ErrNotExist{}
}

78
modules/git/parse.go Normal file
View File

@ -0,0 +1,78 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"bytes"
"fmt"
"strconv"
"strings"
"code.gitea.io/gitea/modules/optional"
)
var sepSpace = []byte{' '}
type LsTreeEntry struct {
ID ObjectID
EntryMode EntryMode
Name string
Size optional.Option[int64]
}
func parseLsTreeLine(line []byte) (*LsTreeEntry, error) {
// expect line to be of the form:
// <mode> <type> <sha> <space-padded-size>\t<filename>
// <mode> <type> <sha>\t<filename>
var err error
posTab := bytes.IndexByte(line, '\t')
if posTab == -1 {
return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
}
entry := new(LsTreeEntry)
entryAttrs := line[:posTab]
entryName := line[posTab+1:]
entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
if len(entryAttrs) > 0 {
entrySize := entryAttrs // the last field is the space-padded-size
size, _ := strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
entry.Size = optional.Some(size)
}
switch string(entryMode) {
case "100644":
entry.EntryMode = EntryModeBlob
case "100755":
entry.EntryMode = EntryModeExec
case "120000":
entry.EntryMode = EntryModeSymlink
case "160000":
entry.EntryMode = EntryModeCommit
case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
entry.EntryMode = EntryModeTree
default:
return nil, fmt.Errorf("unknown type: %v", string(entryMode))
}
entry.ID, err = NewIDFromString(string(entryObjectID))
if err != nil {
return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
}
if len(entryName) > 0 && entryName[0] == '"' {
entry.Name, err = strconv.Unquote(string(entryName))
if err != nil {
return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
}
} else {
entry.Name = string(entryName)
}
return entry, nil
}

View File

@ -10,8 +10,6 @@ import (
"bytes"
"fmt"
"io"
"strconv"
"strings"
"code.gitea.io/gitea/modules/log"
)
@ -21,71 +19,30 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
return parseTreeEntries(data, nil)
}
var sepSpace = []byte{' '}
// parseTreeEntries FIXME this function's design is not right, it should make the caller read all data into memory
func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
var err error
entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1)
for pos := 0; pos < len(data); {
// expect line to be of the form:
// <mode> <type> <sha> <space-padded-size>\t<filename>
// <mode> <type> <sha>\t<filename>
posEnd := bytes.IndexByte(data[pos:], '\n')
if posEnd == -1 {
posEnd = len(data)
} else {
posEnd += pos
}
line := data[pos:posEnd]
posTab := bytes.IndexByte(line, '\t')
if posTab == -1 {
return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
}
entry := new(TreeEntry)
entry.ptree = ptree
entryAttrs := line[:posTab]
entryName := line[posTab+1:]
entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
if len(entryAttrs) > 0 {
entrySize := entryAttrs // the last field is the space-padded-size
entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
entry.sized = true
}
switch string(entryMode) {
case "100644":
entry.entryMode = EntryModeBlob
case "100755":
entry.entryMode = EntryModeExec
case "120000":
entry.entryMode = EntryModeSymlink
case "160000":
entry.entryMode = EntryModeCommit
case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
entry.entryMode = EntryModeTree
default:
return nil, fmt.Errorf("unknown type: %v", string(entryMode))
}
entry.ID, err = NewIDFromString(string(entryObjectID))
lsTreeLine, err := parseLsTreeLine(line)
if err != nil {
return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
return nil, err
}
if len(entryName) > 0 && entryName[0] == '"' {
entry.name, err = strconv.Unquote(string(entryName))
if err != nil {
return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
}
} else {
entry.name = string(entryName)
entry := &TreeEntry{
ptree: ptree,
ID: lsTreeLine.ID,
entryMode: lsTreeLine.EntryMode,
name: lsTreeLine.Name,
size: lsTreeLine.Size.Value(),
sized: lsTreeLine.Size.Has(),
}
pos = posEnd + 1
entries = append(entries, entry)
}
@ -100,7 +57,7 @@ func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.
loop:
for sz > 0 {
mode, fname, sha, count, err := ParseTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
mode, fname, sha, count, err := ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
if err != nil {
if err == io.EOF {
break loop

View File

@ -114,7 +114,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
case "tree":
var n int64
for n < size {
mode, fname, binObjectID, count, err := git.ParseTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
mode, fname, binObjectID, count, err := git.ParseCatFileTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
if err != nil {
return nil, err
}

66
modules/git/submodule.go Normal file
View File

@ -0,0 +1,66 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"bufio"
"context"
"fmt"
"os"
"code.gitea.io/gitea/modules/log"
)
type TemplateSubmoduleCommit struct {
Path string
Commit string
}
// GetTemplateSubmoduleCommits returns a list of submodules paths and their commits from a repository
// This function is only for generating new repos based on existing template, the template couldn't be too large.
func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []TemplateSubmoduleCommit, _ error) {
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
return nil, err
}
opts := &RunOpts{
Dir: repoPath,
Stdout: stdoutWriter,
PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error {
_ = stdoutWriter.Close()
defer stdoutReader.Close()
scanner := bufio.NewScanner(stdoutReader)
for scanner.Scan() {
entry, err := parseLsTreeLine(scanner.Bytes())
if err != nil {
cancel()
return err
}
if entry.EntryMode == EntryModeCommit {
submoduleCommits = append(submoduleCommits, TemplateSubmoduleCommit{Path: entry.Name, Commit: entry.ID.String()})
}
}
return scanner.Err()
},
}
err = NewCommand(ctx, "ls-tree", "-r", "--", "HEAD").Run(opts)
if err != nil {
return nil, fmt.Errorf("GetTemplateSubmoduleCommits: error running git ls-tree: %v", err)
}
return submoduleCommits, nil
}
// AddTemplateSubmoduleIndexes Adds the given submodules to the git index.
// It is only for generating new repos based on existing template, requires the .gitmodules file to be already present in the work dir.
func AddTemplateSubmoduleIndexes(ctx context.Context, repoPath string, submodules []TemplateSubmoduleCommit) error {
for _, submodule := range submodules {
cmd := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path)
if stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}); err != nil {
log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Path, repoPath, stdout, err)
return err
}
}
return nil
}

View File

@ -0,0 +1,48 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetTemplateSubmoduleCommits(t *testing.T) {
testRepoPath := filepath.Join(testReposDir, "repo4_submodules")
submodules, err := GetTemplateSubmoduleCommits(DefaultContext, testRepoPath)
require.NoError(t, err)
assert.Len(t, submodules, 2)
assert.EqualValues(t, "<°)))><", submodules[0].Path)
assert.EqualValues(t, "d2932de67963f23d43e1c7ecf20173e92ee6c43c", submodules[0].Commit)
assert.EqualValues(t, "libtest", submodules[1].Path)
assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[1].Commit)
}
func TestAddTemplateSubmoduleIndexes(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
var err error
_, _, err = NewCommand(ctx, "init").RunStdString(&RunOpts{Dir: tmpDir})
require.NoError(t, err)
_ = os.Mkdir(filepath.Join(tmpDir, "new-dir"), 0o755)
err = AddTemplateSubmoduleIndexes(ctx, tmpDir, []TemplateSubmoduleCommit{{Path: "new-dir", Commit: "1234567890123456789012345678901234567890"}})
require.NoError(t, err)
_, _, err = NewCommand(ctx, "add", "--all").RunStdString(&RunOpts{Dir: tmpDir})
require.NoError(t, err)
_, _, err = NewCommand(ctx, "-c", "user.name=a", "-c", "user.email=b", "commit", "-m=test").RunStdString(&RunOpts{Dir: tmpDir})
require.NoError(t, err)
submodules, err := GetTemplateSubmoduleCommits(DefaultContext, tmpDir)
require.NoError(t, err)
assert.Len(t, submodules, 1)
assert.EqualValues(t, "new-dir", submodules[0].Path)
assert.EqualValues(t, "1234567890123456789012345678901234567890", submodules[0].Commit)
}

View File

@ -0,0 +1 @@
ref: refs/heads/master

View File

@ -0,0 +1,4 @@
[core]
repositoryformatversion = 0
filemode = true
bare = true

View File

@ -0,0 +1,2 @@
x<01><>[
Β0EύΞ*ζ_<CEB6>ι$MΡ5tifBk IΕ•Ή7ζk~ήΓ9ά<39>—εά ό¦π.jΦΘ Ε δΙ"zΒ`ί#I<>irF…µΝΉΐΨ$%ΉΒης|4)°―?tΌΙ=”Λ:K¦ο­#[$DΏ―ϋΏ^<5E><>…΅®Σy½HU/<2F>f?G

View File

@ -0,0 +1 @@
e1e59caba97193d48862d6809912043871f37437

View File

@ -17,7 +17,7 @@ func NewTree(repo *Repository, id ObjectID) *Tree {
}
}
// SubTree get a sub tree by the sub dir path
// SubTree get a subtree by the sub dir path
func (t *Tree) SubTree(rpath string) (*Tree, error) {
if len(rpath) == 0 {
return t, nil
@ -63,7 +63,7 @@ func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error
return filelist, err
}
// GetTreePathLatestCommitID returns the latest commit of a tree path
// GetTreePathLatestCommit returns the latest commit of a tree path
func (repo *Repository) GetTreePathLatestCommit(refName, treePath string) (*Commit, error) {
stdout, _, err := NewCommand(repo.Ctx, "rev-list", "-1").
AddDynamicArguments(refName).AddDashesAndList(treePath).

View File

@ -17,7 +17,6 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
ptree: t,
ID: t.ID,
name: "",
fullName: "",
entryMode: EntryModeTree,
}, nil
}

View File

@ -9,23 +9,17 @@ import "code.gitea.io/gitea/modules/log"
// TreeEntry the leaf in the git tree
type TreeEntry struct {
ID ObjectID
ID ObjectID
ptree *Tree
entryMode EntryMode
name string
size int64
sized bool
fullName string
size int64
sized bool
}
// Name returns the name of the entry
func (te *TreeEntry) Name() string {
if te.fullName != "" {
return te.fullName
}
return te.name
}

View File

@ -9,7 +9,6 @@ import (
"context"
"fmt"
"os"
"path"
"path/filepath"
"regexp"
"strconv"
@ -123,7 +122,7 @@ func (gt *GiteaTemplate) Globs() []glob.Glob {
return gt.globs
}
func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) {
func readGiteaTemplateFile(tmpDir string) (*GiteaTemplate, error) {
gtPath := filepath.Join(tmpDir, ".gitea", "template")
if _, err := os.Stat(gtPath); os.IsNotExist(err) {
return nil, nil
@ -136,12 +135,55 @@ func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) {
return nil, err
}
gt := &GiteaTemplate{
Path: gtPath,
Content: content,
}
return &GiteaTemplate{Path: gtPath, Content: content}, nil
}
return gt, nil
func processGiteaTemplateFile(tmpDir string, templateRepo, generateRepo *repo_model.Repository, giteaTemplateFile *GiteaTemplate) error {
if err := util.Remove(giteaTemplateFile.Path); err != nil {
return fmt.Errorf("remove .giteatemplate: %w", err)
}
if len(giteaTemplateFile.Globs()) == 0 {
return nil // Avoid walking tree if there are no globs
}
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
return filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash)
for _, g := range giteaTemplateFile.Globs() {
if g.Match(base) {
content, err := os.ReadFile(path)
if err != nil {
return err
}
generatedContent := []byte(generateExpansion(string(content), templateRepo, generateRepo, false))
if err := os.WriteFile(path, generatedContent, 0o644); err != nil {
return err
}
substPath := filepath.FromSlash(filepath.Join(tmpDirSlash, generateExpansion(base, templateRepo, generateRepo, true)))
// Create parent subdirectories if needed or continue silently if it exists
if err = os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil {
return err
}
// Substitute filename variables
if err = os.Rename(path, substPath); err != nil {
return err
}
break
}
}
return nil
}) // end: WalkDir
}
func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository, tmpDir string) error {
@ -167,81 +209,43 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r
return fmt.Errorf("git clone: %w", err)
}
if err := util.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
// Get active submodules from the template
submodules, err := git.GetTemplateSubmoduleCommits(ctx, tmpDir)
if err != nil {
return fmt.Errorf("GetTemplateSubmoduleCommits: %w", err)
}
if err = util.RemoveAll(filepath.Join(tmpDir, ".git")); err != nil {
return fmt.Errorf("remove git dir: %w", err)
}
// Variable expansion
gt, err := checkGiteaTemplate(tmpDir)
giteaTemplateFile, err := readGiteaTemplateFile(tmpDir)
if err != nil {
return fmt.Errorf("checkGiteaTemplate: %w", err)
return fmt.Errorf("readGiteaTemplateFile: %w", err)
}
if gt != nil {
if err := util.Remove(gt.Path); err != nil {
return fmt.Errorf("remove .giteatemplate: %w", err)
}
// Avoid walking tree if there are no globs
if len(gt.Globs()) > 0 {
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash)
for _, g := range gt.Globs() {
if g.Match(base) {
content, err := os.ReadFile(path)
if err != nil {
return err
}
if err := os.WriteFile(path,
[]byte(generateExpansion(string(content), templateRepo, generateRepo, false)),
0o644); err != nil {
return err
}
substPath := filepath.FromSlash(filepath.Join(tmpDirSlash,
generateExpansion(base, templateRepo, generateRepo, true)))
// Create parent subdirectories if needed or continue silently if it exists
if err := os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil {
return err
}
// Substitute filename variables
if err := os.Rename(path, substPath); err != nil {
return err
}
break
}
}
return nil
}); err != nil {
return err
}
if giteaTemplateFile != nil {
err = processGiteaTemplateFile(tmpDir, templateRepo, generateRepo, giteaTemplateFile)
if err != nil {
return err
}
}
if err := git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil {
if err = git.InitRepository(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil {
return err
}
repoPath := repo.RepoPath()
if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin").AddDynamicArguments(repoPath).
if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin").AddDynamicArguments(repo.RepoPath()).
RunStdString(&git.RunOpts{Dir: tmpDir, Env: env}); err != nil {
log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err)
return fmt.Errorf("git remote add: %w", err)
}
if err = git.AddTemplateSubmoduleIndexes(ctx, tmpDir, submodules); err != nil {
return fmt.Errorf("failed to add submodules: %v", err)
}
// set default branch based on whether it's specified in the newly generated repo or not
defaultBranch := repo.DefaultBranch
if strings.TrimSpace(defaultBranch) == "" {

View File

@ -19,9 +19,9 @@
{{svg "octicon-file-submodule"}}
{{$refURL := $subModuleFile.RefURL AppUrl $.Repository.FullName $.SSHDomain}} {{/* FIXME: the usage of AppUrl seems incorrect, it would be fixed in the future, use AppSubUrl instead */}}
{{if $refURL}}
<a class="muted" href="{{$refURL}}">{{$entry.Name}}</a><span class="at">@</span><a href="{{$refURL}}/commit/{{PathEscape $subModuleFile.RefID}}">{{ShortSha $subModuleFile.RefID}}</a>
<a class="muted" href="{{$refURL}}">{{$entry.Name}}</a> <span class="at">@</span> <a href="{{$refURL}}/commit/{{PathEscape $subModuleFile.RefID}}">{{ShortSha $subModuleFile.RefID}}</a>
{{else}}
{{$entry.Name}}<span class="at">@</span>{{ShortSha $subModuleFile.RefID}}
{{$entry.Name}} <span class="at">@</span> {{ShortSha $subModuleFile.RefID}}
{{end}}
{{else}}
{{if $entry.IsDir}}