tdsds.com/gitea
1
1
Fork 0
mirror of https://github.com/go-gitea/gitea synced 2025-12-07 13:28:25 +00:00
Code Releases Activity

Merge branch 'main' into allow-force-push-protected-branches

Browse Source
This commit is contained in:
Henry Goodman
2024-01-31 12:06:44 +11:00
committed by GitHub
parent 21cb9bc590 3e8414179c
commit d102809251
375 changed files with 5214 additions and 2316 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/charset/charset.go
+22 -21
View File
@@ -22,17 +22,21 @@ import (
// UTF8BOM is the utf-8 byte-order marker
var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
type ConvertOpts struct {
KeepBOM bool
}
// ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible
func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader {
buf := make([]byte, 2048)
n, err := util.ReadAtMost(rd, buf)
if err != nil {
return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
}
charsetLabel, err := DetectEncoding(buf[:n])
if err != nil || charsetLabel == "UTF-8" {
return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd)
}
encoding, _ := charset.Lookup(charsetLabel)
@@ -42,20 +46,20 @@ func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
return transform.NewReader(
io.MultiReader(
bytes.NewReader(RemoveBOMIfPresent(buf[:n])),
bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)),
rd,
),
encoding.NewDecoder(),
)
}
// ToUTF8WithErr converts content to UTF8 encoding
func ToUTF8WithErr(content []byte) (string, error) {
// ToUTF8 converts content to UTF8 encoding
func ToUTF8(content []byte, opts ConvertOpts) (string, error) {
charsetLabel, err := DetectEncoding(content)
if err != nil {
return "", err
} else if charsetLabel == "UTF-8" {
return string(RemoveBOMIfPresent(content)), nil
return string(MaybeRemoveBOM(content, opts)), nil
}
encoding, _ := charset.Lookup(charsetLabel)
@@ -70,28 +74,22 @@ func ToUTF8WithErr(content []byte) (string, error) {
result = append(result, content[n:]...)
}
result = RemoveBOMIfPresent(result)
result = MaybeRemoveBOM(result, opts)
return string(result), err
}
// ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible
func ToUTF8WithFallback(content []byte) []byte {
bs, _ := io.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content)))
func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte {
bs, _ := io.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content), opts))
return bs
}
// ToUTF8 converts content to UTF8 encoding and ignore error
func ToUTF8(content string) string {
res, _ := ToUTF8WithErr([]byte(content))
return res
}
// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible
func ToUTF8DropErrors(content []byte) []byte {
func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte {
charsetLabel, err := DetectEncoding(content)
if err != nil || charsetLabel == "UTF-8" {
return RemoveBOMIfPresent(content)
return MaybeRemoveBOM(content, opts)
}
encoding, _ := charset.Lookup(charsetLabel)
@@ -117,11 +115,14 @@ func ToUTF8DropErrors(content []byte) []byte {
}
}
return RemoveBOMIfPresent(decoded)
return MaybeRemoveBOM(decoded, opts)
}
// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte
func RemoveBOMIfPresent(content []byte) []byte {
// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false
func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte {
if opts.KeepBOM {
return content
}
if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) {
return content[3:]
}
modules/charset/charset_test.go
+34 -99
View File
@@ -30,15 +30,15 @@ func resetDefaultCharsetsOrder() {
}
}
func TestRemoveBOMIfPresent(t *testing.T) {
res := RemoveBOMIfPresent([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
func TestMaybeRemoveBOM(t *testing.T) {
res := MaybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
res = RemoveBOMIfPresent([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
res = MaybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
}
func TestToUTF8WithErr(t *testing.T) {
func TestToUTF8(t *testing.T) {
resetDefaultCharsetsOrder()
var res string
var err error
@@ -47,53 +47,53 @@ func TestToUTF8WithErr(t *testing.T) {
// locale, so some conversions might behave differently. For that reason, we don't
// depend on particular conversions but in expected behaviors.
res, err = ToUTF8WithErr([]byte{0x41, 0x42, 0x43})
res, err = ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, "ABC", res)
// "áéíóú"
res, err = ToUTF8WithErr([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
res, err = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
// "áéíóú"
res, err = ToUTF8WithErr([]byte{
res, err = ToUTF8([]byte{
0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
0xc3, 0xba,
})
}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
res, err = ToUTF8WithErr([]byte{
res, err = ToUTF8([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
})
}, ConvertOpts{})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
res, err = ToUTF8WithErr([]byte{
res, err = ToUTF8([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
})
}, ConvertOpts{})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
res, err = ToUTF8WithErr([]byte{
res, err = ToUTF8([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e,
})
}, ConvertOpts{})
assert.NoError(t, err)
stringMustStartWith(t, "Hola,", res)
stringMustEndWith(t, "AAA.", res)
// Japanese (Shift-JIS)
// 日属秘ぞしちゅ。
res, err = ToUTF8WithErr([]byte{
res, err = ToUTF8([]byte{
0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
0xBF, 0x82, 0xE3, 0x81, 0x42,
})
}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, []byte{
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
@@ -101,7 +101,7 @@ func TestToUTF8WithErr(t *testing.T) {
},
[]byte(res))
res, err = ToUTF8WithErr([]byte{0x00, 0x00, 0x00, 0x00})
res, err = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
assert.NoError(t, err)
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
}
@@ -109,22 +109,22 @@ func TestToUTF8WithErr(t *testing.T) {
func TestToUTF8WithFallback(t *testing.T) {
resetDefaultCharsetsOrder()
// "ABC"
res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43})
res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
// "áéíóú"
res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
// UTF8 BOM + "áéíóú"
res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
// "Hola, así cómo ños"
res = ToUTF8WithFallback([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73,
})
}, ConvertOpts{})
assert.Equal(t, []byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63,
0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73,
@@ -133,126 +133,65 @@ func TestToUTF8WithFallback(t *testing.T) {
// "Hola, así cómo "
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Equal(t, minmatch, res[0:len(minmatch)])
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Equal(t, minmatch, res[0:len(minmatch)])
// Japanese (Shift-JIS)
// "日属秘ぞしちゅ。"
res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
assert.Equal(t, []byte{
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
}, res)
res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00})
res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
}
func TestToUTF8(t *testing.T) {
resetDefaultCharsetsOrder()
// Note: golang compiler seems so behave differently depending on the current
// locale, so some conversions might behave differently. For that reason, we don't
// depend on particular conversions but in expected behaviors.
res := ToUTF8(string([]byte{0x41, 0x42, 0x43}))
assert.Equal(t, "ABC", res)
// "áéíóú"
res = ToUTF8(string([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}))
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
// BOM + "áéíóú"
res = ToUTF8(string([]byte{
0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3,
0xc3, 0xba,
}))
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res))
// Latin1
// Hola, así cómo ños
res = ToUTF8(string([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73,
}))
assert.Equal(t, []byte{
0x48, 0x6f, 0x6c, 0x61, 0x2c, 0x20, 0x61, 0x73, 0xc3, 0xad, 0x20, 0x63,
0xc3, 0xb3, 0x6d, 0x6f, 0x20, 0xc3, 0xb1, 0x6f, 0x73,
}, []byte(res))
// Latin1
// Hola, así cómo \x07ños
res = ToUTF8(string([]byte{
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63,
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73,
}))
// Hola,
bytesMustStartWith(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C}, []byte(res))
// This test FAILS
// res = ToUTF8("Hola, así cómo \x81ños")
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
// assert.Regexp(t, "^Hola, así cómo", res)
// Japanese (Shift-JIS)
// 日属秘ぞしちゅ。
res = ToUTF8(string([]byte{
0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82,
0xBF, 0x82, 0xE3, 0x81, 0x42,
}))
assert.Equal(t, []byte{
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
},
[]byte(res))
res = ToUTF8("\x00\x00\x00\x00")
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res))
}
func TestToUTF8DropErrors(t *testing.T) {
resetDefaultCharsetsOrder()
// "ABC"
res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43})
res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}, ConvertOpts{})
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res)
// "áéíóú"
res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
// UTF8 BOM + "áéíóú"
res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba})
res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{})
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res)
// "Hola, así cómo ños"
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73})
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}, ConvertOpts{})
assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8])
assert.Equal(t, []byte{0x73}, res[len(res)-1:])
// "Hola, así cómo "
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20}
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73})
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{})
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Equal(t, minmatch, res[0:len(minmatch)])
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73})
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{})
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those
assert.Equal(t, minmatch, res[0:len(minmatch)])
// Japanese (Shift-JIS)
// "日属秘ぞしちゅ。"
res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42})
res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{})
assert.Equal(t, []byte{
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3,
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82,
}, res)
res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{})
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
}
@@ -302,10 +241,6 @@ func stringMustEndWith(t *testing.T, expected, value string) {
assert.Equal(t, expected, value[len(value)-len(expected):])
}
func bytesMustStartWith(t *testing.T, expected, value []byte) {
assert.Equal(t, expected, value[:len(expected)])
}
func TestToUTF8WithFallbackReader(t *testing.T) {
resetDefaultCharsetsOrder()
@@ -317,7 +252,7 @@ func TestToUTF8WithFallbackReader(t *testing.T) {
}
input = input[:testLen]
input += "// Выключаем"
rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)))
rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{})
r, _ := io.ReadAll(rd)
assert.EqualValuesf(t, input, string(r), "testing string len=%d", testLen)
}
modules/context/api.go
+4 -5
View File
@@ -11,11 +11,11 @@ import (
"net/url"
"strings"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
mc "code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@@ -224,7 +224,7 @@ func APIContexter() func(http.Handler) http.Handler {
defer baseCleanUp()
ctx.Base.AppendContextValue(apiContextKey, ctx)
ctx.Base.AppendContextValueFunc(git.RepositoryContextKey, func() any { return ctx.Repo.GitRepo })
ctx.Base.AppendContextValueFunc(gitrepo.RepositoryContextKey, func() any { return ctx.Repo.GitRepo })
// If request sends files, parse them here otherwise the Query() can't be parsed and the CsrfToken will be invalid.
if ctx.Req.Method == "POST" && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") {
@@ -278,10 +278,9 @@ func ReferencesGitRepo(allowEmpty ...bool) func(ctx *APIContext) (cancel context
// For API calls.
if ctx.Repo.GitRepo == nil {
repoPath := repo_model.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name)
gitRepo, err := git.OpenRepository(ctx, repoPath)
gitRepo, err := gitrepo.OpenRepository(ctx, ctx.Repo.Repository)
if err != nil {
ctx.Error(http.StatusInternalServerError, "RepoRef Invalid repo "+repoPath, err)
ctx.Error(http.StatusInternalServerError, fmt.Sprintf("Open Repository %v failed", ctx.Repo.Repository.FullName()), err)
return cancel
}
ctx.Repo.GitRepo = gitRepo
modules/context/context.go
+2 -2
View File
@@ -17,7 +17,7 @@ import (
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
mc "code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/templates"
@@ -163,7 +163,7 @@ func Contexter() func(next http.Handler) http.Handler {
ctx.Data["PageData"] = ctx.PageData
ctx.Base.AppendContextValue(WebContextKey, ctx)
ctx.Base.AppendContextValueFunc(git.RepositoryContextKey, func() any { return ctx.Repo.GitRepo })
ctx.Base.AppendContextValueFunc(gitrepo.RepositoryContextKey, func() any { return ctx.Repo.GitRepo })
ctx.Csrf = PrepareCSRFProtector(csrfOpts, ctx)
modules/context/package.go
+1 -1
View File
@@ -93,7 +93,7 @@ func packageAssignment(ctx *packageAssignmentCtx, errCb func(int, string, any))
}
func determineAccessMode(ctx *Base, pkg *Package, doer *user_model.User) (perm.AccessMode, error) {
if setting.Service.RequireSignInView && doer == nil {
if setting.Service.RequireSignInView && (doer == nil || doer.IsGhost()) {
return perm.AccessModeNone, nil
}
modules/context/repo.go
+5 -5
View File
@@ -23,6 +23,7 @@ import (
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
code_indexer "code.gitea.io/gitea/modules/indexer/code"
"code.gitea.io/gitea/modules/log"
repo_module "code.gitea.io/gitea/modules/repository"
@@ -633,7 +634,7 @@ func RepoAssignment(ctx *Context) context.CancelFunc {
return nil
}
gitRepo, err := git.OpenRepository(ctx, repo_model.RepoPath(userName, repoName))
gitRepo, err := gitrepo.OpenRepository(ctx, repo)
if err != nil {
if strings.Contains(err.Error(), "repository does not exist") || strings.Contains(err.Error(), "no such file or directory") {
log.Error("Repository %-v has a broken repository on the file system: %s Error: %v", ctx.Repo.Repository, ctx.Repo.Repository.RepoPath(), err)
@@ -645,7 +646,7 @@ func RepoAssignment(ctx *Context) context.CancelFunc {
}
return nil
}
ctx.ServerError("RepoAssignment Invalid repo "+repo_model.RepoPath(userName, repoName), err)
ctx.ServerError("RepoAssignment Invalid repo "+repo.FullName(), err)
return nil
}
if ctx.Repo.GitRepo != nil {
@@ -920,10 +921,9 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
)
if ctx.Repo.GitRepo == nil {
repoPath := repo_model.RepoPath(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name)
ctx.Repo.GitRepo, err = git.OpenRepository(ctx, repoPath)
ctx.Repo.GitRepo, err = gitrepo.OpenRepository(ctx, ctx.Repo.Repository)
if err != nil {
ctx.ServerError("RepoRef Invalid repo "+repoPath, err)
ctx.ServerError(fmt.Sprintf("Open Repository %v failed", ctx.Repo.Repository.FullName()), err)
return nil
}
// We opened it, we should close it
modules/contexttest/context_tests.go
+3 -3
View File
@@ -18,7 +18,7 @@ import (
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/templates"
"code.gitea.io/gitea/modules/translation"
"code.gitea.io/gitea/modules/web/middleware"
@@ -107,7 +107,7 @@ func LoadRepoCommit(t *testing.T, ctx gocontext.Context) {
assert.FailNow(t, "context is not *context.Context or *context.APIContext")
}
gitRepo, err := git.OpenRepository(ctx, repo.Repository.RepoPath())
gitRepo, err := gitrepo.OpenRepository(ctx, repo.Repository)
assert.NoError(t, err)
defer gitRepo.Close()
branch, err := gitRepo.GetHEADBranch()
@@ -137,7 +137,7 @@ func LoadUser(t *testing.T, ctx gocontext.Context, userID int64) {
func LoadGitRepo(t *testing.T, ctx *context.Context) {
assert.NoError(t, ctx.Repo.Repository.LoadOwner(ctx))
var err error
ctx.Repo.GitRepo, err = git.OpenRepository(ctx, ctx.Repo.Repository.RepoPath())
ctx.Repo.GitRepo, err = gitrepo.OpenRepository(ctx, ctx.Repo.Repository)
assert.NoError(t, err)
}
modules/doctor/authorizedkeys.go
-96
View File
@@ -1,96 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"bufio"
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"strings"
asymkey_model "code.gitea.io/gitea/models/asymkey"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
const tplCommentPrefix = `# gitea public key`
func checkAuthorizedKeys(ctx context.Context, logger log.Logger, autofix bool) error {
if setting.SSH.StartBuiltinServer || !setting.SSH.CreateAuthorizedKeysFile {
return nil
}
fPath := filepath.Join(setting.SSH.RootPath, "authorized_keys")
f, err := os.Open(fPath)
if err != nil {
if !autofix {
logger.Critical("Unable to open authorized_keys file. ERROR: %v", err)
return fmt.Errorf("Unable to open authorized_keys file. ERROR: %w", err)
}
logger.Warn("Unable to open authorized_keys. (ERROR: %v). Attempting to rewrite...", err)
if err = asymkey_model.RewriteAllPublicKeys(ctx); err != nil {
logger.Critical("Unable to rewrite authorized_keys file. ERROR: %v", err)
return fmt.Errorf("Unable to rewrite authorized_keys file. ERROR: %w", err)
}
}
defer f.Close()
linesInAuthorizedKeys := make(container.Set[string])
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, tplCommentPrefix) {
continue
}
linesInAuthorizedKeys.Add(line)
}
f.Close()
// now we regenerate and check if there are any lines missing
regenerated := &bytes.Buffer{}
if err := asymkey_model.RegeneratePublicKeys(ctx, regenerated); err != nil {
logger.Critical("Unable to regenerate authorized_keys file. ERROR: %v", err)
return fmt.Errorf("Unable to regenerate authorized_keys file. ERROR: %w", err)
}
scanner = bufio.NewScanner(regenerated)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, tplCommentPrefix) {
continue
}
if linesInAuthorizedKeys.Contains(line) {
continue
}
if !autofix {
logger.Critical(
"authorized_keys file %q is out of date.\nRegenerate it with:\n\t\"%s\"\nor\n\t\"%s\"",
fPath,
"gitea admin regenerate keys",
"gitea doctor --run authorized-keys --fix")
return fmt.Errorf(`authorized_keys is out of date and should be regenerated with "gitea admin regenerate keys" or "gitea doctor --run authorized-keys --fix"`)
}
logger.Warn("authorized_keys is out of date. Attempting rewrite...")
err = asymkey_model.RewriteAllPublicKeys(ctx)
if err != nil {
logger.Critical("Unable to rewrite authorized_keys file. ERROR: %v", err)
return fmt.Errorf("Unable to rewrite authorized_keys file. ERROR: %w", err)
}
}
return nil
}
func init() {
Register(&Check{
Title: "Check if OpenSSH authorized_keys file is up-to-date",
Name: "authorized-keys",
IsDefault: true,
Run: checkAuthorizedKeys,
Priority: 4,
})
}
modules/doctor/breaking.go
-97
View File
@@ -1,97 +0,0 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"fmt"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
"xorm.io/builder"
)
func iterateUserAccounts(ctx context.Context, each func(*user.User) error) error {
err := db.Iterate(
ctx,
builder.Gt{"id": 0},
func(ctx context.Context, bean *user.User) error {
return each(bean)
},
)
return err
}
// Since 1.16.4 new restrictions has been set on email addresses. However users with invalid email
// addresses would be currently facing a error due to their invalid email address.
// Ref: https://github.com/go-gitea/gitea/pull/19085 & https://github.com/go-gitea/gitea/pull/17688
func checkUserEmail(ctx context.Context, logger log.Logger, _ bool) error {
// We could use quirky SQL to get all users that start without a [a-zA-Z0-9], but that would mean
// DB provider-specific SQL and only works _now_. So instead we iterate through all user accounts
// and use the user.ValidateEmail function to be future-proof.
var invalidUserCount int64
if err := iterateUserAccounts(ctx, func(u *user.User) error {
// Only check for users, skip
if u.Type != user.UserTypeIndividual {
return nil
}
if err := user.ValidateEmail(u.Email); err != nil {
invalidUserCount++
logger.Warn("User[id=%d name=%q] have not a valid e-mail: %v", u.ID, u.Name, err)
}
return nil
}); err != nil {
return fmt.Errorf("iterateUserAccounts: %w", err)
}
if invalidUserCount == 0 {
logger.Info("All users have a valid e-mail.")
} else {
logger.Warn("%d user(s) have a non-valid e-mail.", invalidUserCount)
}
return nil
}
// From time to time Gitea makes changes to the reserved usernames and which symbols
// are allowed for various reasons. This check helps with detecting users that, according
// to our reserved names, don't have a valid username.
func checkUserName(ctx context.Context, logger log.Logger, _ bool) error {
var invalidUserCount int64
if err := iterateUserAccounts(ctx, func(u *user.User) error {
if err := user.IsUsableUsername(u.Name); err != nil {
invalidUserCount++
logger.Warn("User[id=%d] does not have a valid username: %v", u.ID, err)
}
return nil
}); err != nil {
return fmt.Errorf("iterateUserAccounts: %w", err)
}
if invalidUserCount == 0 {
logger.Info("All users have a valid username.")
} else {
logger.Warn("%d user(s) have a non-valid username.", invalidUserCount)
}
return nil
}
func init() {
Register(&Check{
Title: "Check if users has an valid email address",
Name: "check-user-email",
IsDefault: false,
Run: checkUserEmail,
Priority: 9,
})
Register(&Check{
Title: "Check if users have a valid username",
Name: "check-user-names",
IsDefault: false,
Run: checkUserName,
Priority: 9,
})
}
modules/doctor/checkOldArchives.go
-59
View File
@@ -1,59 +0,0 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"os"
"path/filepath"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
)
func checkOldArchives(ctx context.Context, logger log.Logger, autofix bool) error {
numRepos := 0
numReposUpdated := 0
err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
if repo.IsEmpty {
return nil
}
p := filepath.Join(repo.RepoPath(), "archives")
isDir, err := util.IsDir(p)
if err != nil {
log.Warn("check if %s is directory failed: %v", p, err)
}
if isDir {
numRepos++
if autofix {
if err := os.RemoveAll(p); err == nil {
numReposUpdated++
} else {
log.Warn("remove %s failed: %v", p, err)
}
}
}
return nil
})
if autofix {
logger.Info("%d / %d old archives in repository deleted", numReposUpdated, numRepos)
} else {
logger.Info("%d old archives in repository need to be deleted", numRepos)
}
return err
}
func init() {
Register(&Check{
Title: "Check old archives",
Name: "check-old-archives",
IsDefault: false,
Run: checkOldArchives,
Priority: 7,
})
}
modules/doctor/dbconsistency.go
-245
View File
@@ -1,245 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
actions_model "code.gitea.io/gitea/models/actions"
activities_model "code.gitea.io/gitea/models/activities"
"code.gitea.io/gitea/models/db"
issues_model "code.gitea.io/gitea/models/issues"
"code.gitea.io/gitea/models/migrations"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
type consistencyCheck struct {
Name string
Counter func(context.Context) (int64, error)
Fixer func(context.Context) (int64, error)
FixedMessage string
}
func (c *consistencyCheck) Run(ctx context.Context, logger log.Logger, autofix bool) error {
count, err := c.Counter(ctx)
if err != nil {
logger.Critical("Error: %v whilst counting %s", err, c.Name)
return err
}
if count > 0 {
if autofix {
var fixed int64
if fixed, err = c.Fixer(ctx); err != nil {
logger.Critical("Error: %v whilst fixing %s", err, c.Name)
return err
}
prompt := "Deleted"
if c.FixedMessage != "" {
prompt = c.FixedMessage
}
if fixed < 0 {
logger.Info(prompt+" %d %s", count, c.Name)
} else {
logger.Info(prompt+" %d/%d %s", fixed, count, c.Name)
}
} else {
logger.Warn("Found %d %s", count, c.Name)
}
}
return nil
}
func asFixer(fn func(ctx context.Context) error) func(ctx context.Context) (int64, error) {
return func(ctx context.Context) (int64, error) {
err := fn(ctx)
return -1, err
}
}
func genericOrphanCheck(name, subject, refobject, joincond string) consistencyCheck {
return consistencyCheck{
Name: name,
Counter: func(ctx context.Context) (int64, error) {
return db.CountOrphanedObjects(ctx, subject, refobject, joincond)
},
Fixer: func(ctx context.Context) (int64, error) {
err := db.DeleteOrphanedObjects(ctx, subject, refobject, joincond)
return -1, err
},
}
}
func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) error {
// make sure DB version is uptodate
if err := db.InitEngineWithMigration(ctx, migrations.EnsureUpToDate); err != nil {
logger.Critical("Model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded")
return err
}
consistencyChecks := []consistencyCheck{
{
// find labels without existing repo or org
Name: "Orphaned Labels without existing repository or organisation",
Counter: issues_model.CountOrphanedLabels,
Fixer: asFixer(issues_model.DeleteOrphanedLabels),
},
{
// find IssueLabels without existing label
Name: "Orphaned Issue Labels without existing label",
Counter: issues_model.CountOrphanedIssueLabels,
Fixer: asFixer(issues_model.DeleteOrphanedIssueLabels),
},
{
// find issues without existing repository
Name: "Orphaned Issues without existing repository",
Counter: issues_model.CountOrphanedIssues,
Fixer: asFixer(issues_model.DeleteOrphanedIssues),
},
// find releases without existing repository
genericOrphanCheck("Orphaned Releases without existing repository",
"release", "repository", "`release`.repo_id=repository.id"),
// find pulls without existing issues
genericOrphanCheck("Orphaned PullRequests without existing issue",
"pull_request", "issue", "pull_request.issue_id=issue.id"),
// find pull requests without base repository
genericOrphanCheck("Pull request entries without existing base repository",
"pull_request", "repository", "pull_request.base_repo_id=repository.id"),
// find tracked times without existing issues/pulls
genericOrphanCheck("Orphaned TrackedTimes without existing issue",
"tracked_time", "issue", "tracked_time.issue_id=issue.id"),
// find attachments without existing issues or releases
{
Name: "Orphaned Attachments without existing issues or releases",
Counter: repo_model.CountOrphanedAttachments,
Fixer: asFixer(repo_model.DeleteOrphanedAttachments),
},
// find null archived repositories
{
Name: "Repositories with is_archived IS NULL",
Counter: repo_model.CountNullArchivedRepository,
Fixer: repo_model.FixNullArchivedRepository,
FixedMessage: "Fixed",
},
// find label comments with empty labels
{
Name: "Label comments with empty labels",
Counter: issues_model.CountCommentTypeLabelWithEmptyLabel,
Fixer: issues_model.FixCommentTypeLabelWithEmptyLabel,
FixedMessage: "Fixed",
},
// find label comments with labels from outside the repository
{
Name: "Label comments with labels from outside the repository",
Counter: issues_model.CountCommentTypeLabelWithOutsideLabels,
Fixer: issues_model.FixCommentTypeLabelWithOutsideLabels,
FixedMessage: "Removed",
},
// find issue_label with labels from outside the repository
{
Name: "IssueLabels with Labels from outside the repository",
Counter: issues_model.CountIssueLabelWithOutsideLabels,
Fixer: issues_model.FixIssueLabelWithOutsideLabels,
FixedMessage: "Removed",
},
{
Name: "Action with created_unix set as an empty string",
Counter: activities_model.CountActionCreatedUnixString,
Fixer: activities_model.FixActionCreatedUnixString,
FixedMessage: "Set to zero",
},
{
Name: "Action Runners without existing owner",
Counter: actions_model.CountRunnersWithoutBelongingOwner,
Fixer: actions_model.FixRunnersWithoutBelongingOwner,
FixedMessage: "Removed",
},
{
Name: "Topics with empty repository count",
Counter: repo_model.CountOrphanedTopics,
Fixer: repo_model.DeleteOrphanedTopics,
FixedMessage: "Removed",
},
}
// TODO: function to recalc all counters
if setting.Database.Type.IsPostgreSQL() {
consistencyChecks = append(consistencyChecks, consistencyCheck{
Name: "Sequence values",
Counter: db.CountBadSequences,
Fixer: asFixer(db.FixBadSequences),
FixedMessage: "Updated",
})
}
consistencyChecks = append(consistencyChecks,
// find protected branches without existing repository
genericOrphanCheck("Protected Branches without existing repository",
"protected_branch", "repository", "protected_branch.repo_id=repository.id"),
// find branches without existing repository
genericOrphanCheck("Branches without existing repository",
"branch", "repository", "branch.repo_id=repository.id"),
// find LFS locks without existing repository
genericOrphanCheck("LFS locks without existing repository",
"lfs_lock", "repository", "lfs_lock.repo_id=repository.id"),
// find collaborations without users
genericOrphanCheck("Collaborations without existing user",
"collaboration", "user", "collaboration.user_id=`user`.id"),
// find collaborations without repository
genericOrphanCheck("Collaborations without existing repository",
"collaboration", "repository", "collaboration.repo_id=repository.id"),
// find access without users
genericOrphanCheck("Access entries without existing user",
"access", "user", "access.user_id=`user`.id"),
// find access without repository
genericOrphanCheck("Access entries without existing repository",
"access", "repository", "access.repo_id=repository.id"),
// find action without repository
genericOrphanCheck("Action entries without existing repository",
"action", "repository", "action.repo_id=repository.id"),
// find action without user
genericOrphanCheck("Action entries without existing user",
"action", "user", "action.act_user_id=`user`.id"),
// find OAuth2Grant without existing user
genericOrphanCheck("Orphaned OAuth2Grant without existing User",
"oauth2_grant", "user", "oauth2_grant.user_id=`user`.id"),
// find OAuth2Application without existing user
genericOrphanCheck("Orphaned OAuth2Application without existing User",
"oauth2_application", "user", "oauth2_application.uid=`user`.id"),
// find OAuth2AuthorizationCode without existing OAuth2Grant
genericOrphanCheck("Orphaned OAuth2AuthorizationCode without existing OAuth2Grant",
"oauth2_authorization_code", "oauth2_grant", "oauth2_authorization_code.grant_id=oauth2_grant.id"),
// find stopwatches without existing user
genericOrphanCheck("Orphaned Stopwatches without existing User",
"stopwatch", "user", "stopwatch.user_id=`user`.id"),
// find stopwatches without existing issue
genericOrphanCheck("Orphaned Stopwatches without existing Issue",
"stopwatch", "issue", "stopwatch.issue_id=`issue`.id"),
// find redirects without existing user.
genericOrphanCheck("Orphaned Redirects without existing redirect user",
"user_redirect", "user", "user_redirect.redirect_user_id=`user`.id"),
)
for _, c := range consistencyChecks {
if err := c.Run(ctx, logger, autofix); err != nil {
return err
}
}
return nil
}
func init() {
Register(&Check{
Title: "Check consistency of database",
Name: "check-db-consistency",
IsDefault: false,
Run: checkDBConsistency,
Priority: 3,
})
}
modules/doctor/dbversion.go
-42
View File
@@ -1,42 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/log"
)
func checkDBVersion(ctx context.Context, logger log.Logger, autofix bool) error {
logger.Info("Expected database version: %d", migrations.ExpectedVersion())
if err := db.InitEngineWithMigration(ctx, migrations.EnsureUpToDate); err != nil {
if !autofix {
logger.Critical("Error: %v during ensure up to date", err)
return err
}
logger.Warn("Got Error: %v during ensure up to date", err)
logger.Warn("Attempting to migrate to the latest DB version to fix this.")
err = db.InitEngineWithMigration(ctx, migrations.Migrate)
if err != nil {
logger.Critical("Error: %v during migration", err)
}
return err
}
return nil
}
func init() {
Register(&Check{
Title: "Check Database Version",
Name: "check-db-version",
IsDefault: true,
Run: checkDBVersion,
AbortIfFailed: false,
Priority: 2,
})
}
modules/doctor/doctor.go
-127
View File
@@ -1,127 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"fmt"
"os"
"sort"
"strings"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
// Check represents a Doctor check
type Check struct {
Title string
Name string
IsDefault bool
Run func(ctx context.Context, logger log.Logger, autofix bool) error
AbortIfFailed bool
SkipDatabaseInitialization bool
Priority int
}
func initDBSkipLogger(ctx context.Context) error {
setting.MustInstalled()
setting.LoadDBSetting()
if err := db.InitEngine(ctx); err != nil {
return fmt.Errorf("db.InitEngine: %w", err)
}
// some doctor sub-commands need to use git command
if err := git.InitFull(ctx); err != nil {
return fmt.Errorf("git.InitFull: %w", err)
}
return nil
}
type doctorCheckLogger struct {
colorize bool
}
var _ log.BaseLogger = (*doctorCheckLogger)(nil)
func (d *doctorCheckLogger) Log(skip int, level log.Level, format string, v ...any) {
_, _ = fmt.Fprintf(os.Stdout, format+"\n", v...)
}
func (d *doctorCheckLogger) GetLevel() log.Level {
return log.TRACE
}
type doctorCheckStepLogger struct {
colorize bool
}
var _ log.BaseLogger = (*doctorCheckStepLogger)(nil)
func (d *doctorCheckStepLogger) Log(skip int, level log.Level, format string, v ...any) {
levelChar := fmt.Sprintf("[%s]", strings.ToUpper(level.String()[0:1]))
var levelArg any = levelChar
if d.colorize {
levelArg = log.NewColoredValue(levelChar, level.ColorAttributes()...)
}
args := append([]any{levelArg}, v...)
_, _ = fmt.Fprintf(os.Stdout, " - %s "+format+"\n", args...)
}
func (d *doctorCheckStepLogger) GetLevel() log.Level {
return log.TRACE
}
// Checks is the list of available commands
var Checks []*Check
// RunChecks runs the doctor checks for the provided list
func RunChecks(ctx context.Context, colorize, autofix bool, checks []*Check) error {
SortChecks(checks)
// the checks output logs by a special logger, they do not use the default logger
logger := log.BaseLoggerToGeneralLogger(&doctorCheckLogger{colorize: colorize})
loggerStep := log.BaseLoggerToGeneralLogger(&doctorCheckStepLogger{colorize: colorize})
dbIsInit := false
for i, check := range checks {
if !dbIsInit && !check.SkipDatabaseInitialization {
// Only open database after the most basic configuration check
if err := initDBSkipLogger(ctx); err != nil {
logger.Error("Error whilst initializing the database: %v", err)
logger.Error("Check if you are using the right config file. You can use a --config directive to specify one.")
return nil
}
dbIsInit = true
}
logger.Info("\n[%d] %s", i+1, check.Title)
if err := check.Run(ctx, loggerStep, autofix); err != nil {
if check.AbortIfFailed {
logger.Critical("FAIL")
return err
}
logger.Error("ERROR")
} else {
logger.Info("OK")
}
}
logger.Info("\nAll done (checks: %d).", len(checks))
return nil
}
// Register registers a command with the list
func Register(command *Check) {
Checks = append(Checks, command)
}
func SortChecks(checks []*Check) {
sort.SliceStable(checks, func(i, j int) bool {
if checks[i].Priority == checks[j].Priority {
return checks[i].Name < checks[j].Name
}
if checks[i].Priority == 0 {
return false
}
return checks[i].Priority < checks[j].Priority
})
}
modules/doctor/fix16961.go
-322
View File
@@ -1,322 +0,0 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"bytes"
"context"
"errors"
"fmt"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unit"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/builder"
)
// #16831 revealed that the dump command that was broken in 1.14.3-1.14.6 and 1.15.0 (#15885).
// This led to repo_unit and login_source cfg not being converted to JSON in the dump
// Unfortunately although it was hoped that there were only a few users affected it
// appears that many users are affected.
// We therefore need to provide a doctor command to fix this repeated issue #16961
func parseBool16961(bs []byte) (bool, error) {
if bytes.EqualFold(bs, []byte("%!s(bool=false)")) {
return false, nil
}
if bytes.EqualFold(bs, []byte("%!s(bool=true)")) {
return true, nil
}
return false, fmt.Errorf("unexpected bool format: %s", string(bs))
}
func fixUnitConfig16961(bs []byte, cfg *repo_model.UnitConfig) (fixed bool, err error) {
err = json.UnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return false, nil
}
// Handle #16961
if string(bs) != "&{}" && len(bs) != 0 {
return false, err
}
return true, nil
}
func fixExternalWikiConfig16961(bs []byte, cfg *repo_model.ExternalWikiConfig) (fixed bool, err error) {
err = json.UnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return false, nil
}
if len(bs) < 3 {
return false, err
}
if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
return false, err
}
cfg.ExternalWikiURL = string(bs[2 : len(bs)-1])
return true, nil
}
func fixExternalTrackerConfig16961(bs []byte, cfg *repo_model.ExternalTrackerConfig) (fixed bool, err error) {
err = json.UnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return false, nil
}
// Handle #16961
if len(bs) < 3 {
return false, err
}
if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
return false, err
}
parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
if len(parts) != 3 {
return false, err
}
cfg.ExternalTrackerURL = string(bytes.Join(parts[:len(parts)-2], []byte{' '}))
cfg.ExternalTrackerFormat = string(parts[len(parts)-2])
cfg.ExternalTrackerStyle = string(parts[len(parts)-1])
return true, nil
}
func fixPullRequestsConfig16961(bs []byte, cfg *repo_model.PullRequestsConfig) (fixed bool, err error) {
err = json.UnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return false, nil
}
// Handle #16961
if len(bs) < 3 {
return false, err
}
if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
return false, err
}
// PullRequestsConfig was the following in 1.14
// type PullRequestsConfig struct {
// IgnoreWhitespaceConflicts bool
// AllowMerge bool
// AllowRebase bool
// AllowRebaseMerge bool
// AllowSquash bool
// AllowManualMerge bool
// AutodetectManualMerge bool
// }
//
// 1.15 added in addition:
// DefaultDeleteBranchAfterMerge bool
// DefaultMergeStyle MergeStyle
parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
if len(parts) < 7 {
return false, err
}
var parseErr error
cfg.IgnoreWhitespaceConflicts, parseErr = parseBool16961(parts[0])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.AllowMerge, parseErr = parseBool16961(parts[1])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.AllowRebase, parseErr = parseBool16961(parts[2])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.AllowRebaseMerge, parseErr = parseBool16961(parts[3])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.AllowSquash, parseErr = parseBool16961(parts[4])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.AllowManualMerge, parseErr = parseBool16961(parts[5])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.AutodetectManualMerge, parseErr = parseBool16961(parts[6])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
// 1.14 unit
if len(parts) == 7 {
return true, nil
}
if len(parts) < 9 {
return false, err
}
cfg.DefaultDeleteBranchAfterMerge, parseErr = parseBool16961(parts[7])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.DefaultMergeStyle = repo_model.MergeStyle(string(bytes.Join(parts[8:], []byte{' '})))
return true, nil
}
func fixIssuesConfig16961(bs []byte, cfg *repo_model.IssuesConfig) (fixed bool, err error) {
err = json.UnmarshalHandleDoubleEncode(bs, &cfg)
if err == nil {
return false, nil
}
// Handle #16961
if len(bs) < 3 {
return false, err
}
if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
return false, err
}
parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
if len(parts) != 3 {
return false, err
}
var parseErr error
cfg.EnableTimetracker, parseErr = parseBool16961(parts[0])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.AllowOnlyContributorsToTrackTime, parseErr = parseBool16961(parts[1])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
cfg.EnableDependencies, parseErr = parseBool16961(parts[2])
if parseErr != nil {
return false, errors.Join(err, parseErr)
}
return true, nil
}
func fixBrokenRepoUnit16961(repoUnit *repo_model.RepoUnit, bs []byte) (fixed bool, err error) {
// Shortcut empty or null values
if len(bs) == 0 {
return false, nil
}
switch repoUnit.Type {
case unit.TypeCode, unit.TypeReleases, unit.TypeWiki, unit.TypeProjects:
cfg := &repo_model.UnitConfig{}
repoUnit.Config = cfg
if fixed, err := fixUnitConfig16961(bs, cfg); !fixed {
return false, err
}
case unit.TypeExternalWiki:
cfg := &repo_model.ExternalWikiConfig{}
repoUnit.Config = cfg
if fixed, err := fixExternalWikiConfig16961(bs, cfg); !fixed {
return false, err
}
case unit.TypeExternalTracker:
cfg := &repo_model.ExternalTrackerConfig{}
repoUnit.Config = cfg
if fixed, err := fixExternalTrackerConfig16961(bs, cfg); !fixed {
return false, err
}
case unit.TypePullRequests:
cfg := &repo_model.PullRequestsConfig{}
repoUnit.Config = cfg
if fixed, err := fixPullRequestsConfig16961(bs, cfg); !fixed {
return false, err
}
case unit.TypeIssues:
cfg := &repo_model.IssuesConfig{}
repoUnit.Config = cfg
if fixed, err := fixIssuesConfig16961(bs, cfg); !fixed {
return false, err
}
default:
panic(fmt.Sprintf("unrecognized repo unit type: %v", repoUnit.Type))
}
return true, nil
}
func fixBrokenRepoUnits16961(ctx context.Context, logger log.Logger, autofix bool) error {
// RepoUnit describes all units of a repository
type RepoUnit struct {
ID int64
RepoID int64
Type unit.Type
Config []byte
CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
}
count := 0
err := db.Iterate(
ctx,
builder.Gt{
"id": 0,
},
func(ctx context.Context, unit *RepoUnit) error {
bs := unit.Config
repoUnit := &repo_model.RepoUnit{
ID: unit.ID,
RepoID: unit.RepoID,
Type: unit.Type,
CreatedUnix: unit.CreatedUnix,
}
if fixed, err := fixBrokenRepoUnit16961(repoUnit, bs); !fixed {
return err
}
count++
if !autofix {
return nil
}
return repo_model.UpdateRepoUnit(ctx, repoUnit)
},
)
if err != nil {
logger.Critical("Unable to iterate across repounits to fix the broken units: Error %v", err)
return err
}
if !autofix {
if count == 0 {
logger.Info("Found no broken repo_units")
} else {
logger.Warn("Found %d broken repo_units", count)
}
return nil
}
logger.Info("Fixed %d broken repo_units", count)
return nil
}
func init() {
Register(&Check{
Title: "Check for incorrectly dumped repo_units (See #16961)",
Name: "fix-broken-repo-units",
IsDefault: false,
Run: fixBrokenRepoUnits16961,
Priority: 7,
})
}
modules/doctor/fix16961_test.go
-271
View File
@@ -1,271 +0,0 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"testing"
repo_model "code.gitea.io/gitea/models/repo"
"github.com/stretchr/testify/assert"
)
func Test_fixUnitConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
wantFixed bool
wantErr bool
}{
{
name: "empty",
bs: "",
wantFixed: true,
wantErr: false,
},
{
name: "normal: {}",
bs: "{}",
wantFixed: false,
wantErr: false,
},
{
name: "broken but fixable: &{}",
bs: "&{}",
wantFixed: true,
wantErr: false,
},
{
name: "broken but unfixable: &{asdasd}",
bs: "&{asdasd}",
wantFixed: false,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotFixed, err := fixUnitConfig16961([]byte(tt.bs), &repo_model.UnitConfig{})
if (err != nil) != tt.wantErr {
t.Errorf("fixUnitConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixUnitConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
})
}
}
func Test_fixExternalWikiConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
expected string
wantFixed bool
wantErr bool
}{
{
name: "normal: {\"ExternalWikiURL\":\"http://someurl\"}",
bs: "{\"ExternalWikiURL\":\"http://someurl\"}",
expected: "http://someurl",
wantFixed: false,
wantErr: false,
},
{
name: "broken: &{http://someurl}",
bs: "&{http://someurl}",
expected: "http://someurl",
wantFixed: true,
wantErr: false,
},
{
name: "broken but unfixable: http://someurl",
bs: "http://someurl",
wantFixed: false,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &repo_model.ExternalWikiConfig{}
gotFixed, err := fixExternalWikiConfig16961([]byte(tt.bs), cfg)
if (err != nil) != tt.wantErr {
t.Errorf("fixExternalWikiConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixExternalWikiConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
if cfg.ExternalWikiURL != tt.expected {
t.Errorf("fixExternalWikiConfig_16961().ExternalWikiURL = %v, want %v", cfg.ExternalWikiURL, tt.expected)
}
})
}
}
func Test_fixExternalTrackerConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
expected repo_model.ExternalTrackerConfig
wantFixed bool
wantErr bool
}{
{
name: "normal",
bs: `{"ExternalTrackerURL":"a","ExternalTrackerFormat":"b","ExternalTrackerStyle":"c"}`,
expected: repo_model.ExternalTrackerConfig{
ExternalTrackerURL: "a",
ExternalTrackerFormat: "b",
ExternalTrackerStyle: "c",
},
wantFixed: false,
wantErr: false,
},
{
name: "broken",
bs: "&{a b c}",
expected: repo_model.ExternalTrackerConfig{
ExternalTrackerURL: "a",
ExternalTrackerFormat: "b",
ExternalTrackerStyle: "c",
},
wantFixed: true,
wantErr: false,
},
{
name: "broken - too many fields",
bs: "&{a b c d}",
wantFixed: false,
wantErr: true,
},
{
name: "broken - wrong format",
bs: "a b c d}",
wantFixed: false,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &repo_model.ExternalTrackerConfig{}
gotFixed, err := fixExternalTrackerConfig16961([]byte(tt.bs), cfg)
if (err != nil) != tt.wantErr {
t.Errorf("fixExternalTrackerConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixExternalTrackerConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
if cfg.ExternalTrackerFormat != tt.expected.ExternalTrackerFormat {
t.Errorf("fixExternalTrackerConfig_16961().ExternalTrackerFormat = %v, want %v", tt.expected.ExternalTrackerFormat, cfg.ExternalTrackerFormat)
}
if cfg.ExternalTrackerStyle != tt.expected.ExternalTrackerStyle {
t.Errorf("fixExternalTrackerConfig_16961().ExternalTrackerStyle = %v, want %v", tt.expected.ExternalTrackerStyle, cfg.ExternalTrackerStyle)
}
if cfg.ExternalTrackerURL != tt.expected.ExternalTrackerURL {
t.Errorf("fixExternalTrackerConfig_16961().ExternalTrackerURL = %v, want %v", tt.expected.ExternalTrackerURL, cfg.ExternalTrackerURL)
}
})
}
}
func Test_fixPullRequestsConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
expected repo_model.PullRequestsConfig
wantFixed bool
wantErr bool
}{
{
name: "normal",
bs: `{"IgnoreWhitespaceConflicts":false,"AllowMerge":false,"AllowRebase":false,"AllowRebaseMerge":false,"AllowSquash":false,"AllowManualMerge":false,"AutodetectManualMerge":false,"DefaultDeleteBranchAfterMerge":false,"DefaultMergeStyle":""}`,
},
{
name: "broken - 1.14",
bs: `&{%!s(bool=false) %!s(bool=true) %!s(bool=true) %!s(bool=true) %!s(bool=true) %!s(bool=false) %!s(bool=false)}`,
expected: repo_model.PullRequestsConfig{
IgnoreWhitespaceConflicts: false,
AllowMerge: true,
AllowRebase: true,
AllowRebaseMerge: true,
AllowSquash: true,
AllowManualMerge: false,
AutodetectManualMerge: false,
},
wantFixed: true,
},
{
name: "broken - 1.15",
bs: `&{%!s(bool=false) %!s(bool=true) %!s(bool=true) %!s(bool=true) %!s(bool=true) %!s(bool=false) %!s(bool=false) %!s(bool=false) merge}`,
expected: repo_model.PullRequestsConfig{
AllowMerge: true,
AllowRebase: true,
AllowRebaseMerge: true,
AllowSquash: true,
DefaultMergeStyle: repo_model.MergeStyleMerge,
},
wantFixed: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &repo_model.PullRequestsConfig{}
gotFixed, err := fixPullRequestsConfig16961([]byte(tt.bs), cfg)
if (err != nil) != tt.wantErr {
t.Errorf("fixPullRequestsConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixPullRequestsConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
assert.EqualValues(t, &tt.expected, cfg)
})
}
}
func Test_fixIssuesConfig_16961(t *testing.T) {
tests := []struct {
name string
bs string
expected repo_model.IssuesConfig
wantFixed bool
wantErr bool
}{
{
name: "normal",
bs: `{"EnableTimetracker":true,"AllowOnlyContributorsToTrackTime":true,"EnableDependencies":true}`,
expected: repo_model.IssuesConfig{
EnableTimetracker: true,
AllowOnlyContributorsToTrackTime: true,
EnableDependencies: true,
},
},
{
name: "broken",
bs: `&{%!s(bool=true) %!s(bool=true) %!s(bool=true)}`,
expected: repo_model.IssuesConfig{
EnableTimetracker: true,
AllowOnlyContributorsToTrackTime: true,
EnableDependencies: true,
},
wantFixed: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &repo_model.IssuesConfig{}
gotFixed, err := fixIssuesConfig16961([]byte(tt.bs), cfg)
if (err != nil) != tt.wantErr {
t.Errorf("fixIssuesConfig_16961() error = %v, wantErr %v", err, tt.wantErr)
return
}
if gotFixed != tt.wantFixed {
t.Errorf("fixIssuesConfig_16961() = %v, want %v", gotFixed, tt.wantFixed)
}
assert.EqualValues(t, &tt.expected, cfg)
})
}
}
modules/doctor/fix8312.go
-61
View File
@@ -1,61 +0,0 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/db"
org_model "code.gitea.io/gitea/models/organization"
"code.gitea.io/gitea/models/perm"
"code.gitea.io/gitea/modules/log"
"xorm.io/builder"
)
func fixOwnerTeamCreateOrgRepo(ctx context.Context, logger log.Logger, autofix bool) error {
count := 0
err := db.Iterate(
ctx,
builder.Eq{"authorize": perm.AccessModeOwner, "can_create_org_repo": false},
func(ctx context.Context, team *org_model.Team) error {
team.CanCreateOrgRepo = true
count++
if !autofix {
return nil
}
return models.UpdateTeam(ctx, team, false, false)
},
)
if err != nil {
logger.Critical("Unable to iterate across repounits to fix incorrect can_create_org_repo: Error %v", err)
return err
}
if !autofix {
if count == 0 {
logger.Info("Found no team with incorrect can_create_org_repo")
} else {
logger.Warn("Found %d teams with incorrect can_create_org_repo", count)
}
return nil
}
logger.Info("Fixed %d teams with incorrect can_create_org_repo", count)
return nil
}
func init() {
Register(&Check{
Title: "Check for incorrect can_create_org_repo for org owner teams",
Name: "fix-owner-team-create-org-repo",
IsDefault: false,
Run: fixOwnerTeamCreateOrgRepo,
Priority: 7,
})
}
modules/doctor/heads.go
-88
View File
@@ -1,88 +0,0 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
)
func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool) error {
numRepos := 0
numHeadsBroken := 0
numDefaultBranchesBroken := 0
numReposUpdated := 0
err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
numRepos++
_, _, defaultBranchErr := git.NewCommand(ctx, "rev-parse").AddDashesAndList(repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
head, _, headErr := git.NewCommand(ctx, "symbolic-ref", "--short", "HEAD").RunStdString(&git.RunOpts{Dir: repo.RepoPath()})
// what we expect: default branch is valid, and HEAD points to it
if headErr == nil && defaultBranchErr == nil && head == repo.DefaultBranch {
return nil
}
if headErr != nil {
numHeadsBroken++
}
if defaultBranchErr != nil {
numDefaultBranchesBroken++
}
// if default branch is broken, let the user fix that in the UI
if defaultBranchErr != nil {
logger.Warn("Default branch for %s/%s doesn't point to a valid commit", repo.OwnerName, repo.Name)
return nil
}
// if we're not autofixing, that's all we can do
if !autofix {
return nil
}
// otherwise, let's try fixing HEAD
err := git.NewCommand(ctx, "symbolic-ref").AddDashesAndList("HEAD", git.BranchPrefix+repo.DefaultBranch).Run(&git.RunOpts{Dir: repo.RepoPath()})
if err != nil {
logger.Warn("Failed to fix HEAD for %s/%s: %v", repo.OwnerName, repo.Name, err)
return nil
}
numReposUpdated++
return nil
})
if err != nil {
logger.Critical("Error when fixing repo HEADs: %v", err)
}
if autofix {
logger.Info("Out of %d repos, HEADs for %d are now fixed and HEADS for %d are still broken", numRepos, numReposUpdated, numDefaultBranchesBroken+numHeadsBroken-numReposUpdated)
} else {
if numHeadsBroken == 0 && numDefaultBranchesBroken == 0 {
logger.Info("All %d repos have their HEADs in the correct state", numRepos)
} else {
if numHeadsBroken == 0 && numDefaultBranchesBroken != 0 {
logger.Critical("Default branches are broken for %d/%d repos", numDefaultBranchesBroken, numRepos)
} else if numHeadsBroken != 0 && numDefaultBranchesBroken == 0 {
logger.Warn("HEADs are broken for %d/%d repos", numHeadsBroken, numRepos)
} else {
logger.Critical("Out of %d repos, HEADS are broken for %d and default branches are broken for %d", numRepos, numHeadsBroken, numDefaultBranchesBroken)
}
}
}
return err
}
func init() {
Register(&Check{
Title: "Synchronize repo HEADs",
Name: "synchronize-repo-heads",
IsDefault: true,
Run: synchronizeRepoHeads,
Priority: 7,
})
}
modules/doctor/lfs.go
-51
View File
@@ -1,51 +0,0 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"fmt"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/repository"
)
func init() {
Register(&Check{
Title: "Garbage collect LFS",
Name: "gc-lfs",
IsDefault: false,
Run: garbageCollectLFSCheck,
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
}
func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool) error {
if !setting.LFS.StartServer {
return fmt.Errorf("LFS support is disabled")
}
if err := repository.GarbageCollectLFSMetaObjects(ctx, repository.GarbageCollectLFSMetaObjectsOptions{
LogDetail: logger.Info,
AutoFix: autofix,
// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
// objects.
//
// It is likely that a week is potentially excessive but it should definitely be enough that any
// unassociated LFS object is genuinely unassociated.
OlderThan: time.Now().Add(-24 * time.Hour * 7),
// We don't set the UpdatedLessRecentlyThan because we want to do a full GC
}); err != nil {
return err
}
return checkStorage(&checkStorageOptions{LFS: true})(ctx, logger, autofix)
}
modules/doctor/mergebase.go
-114
View File
@@ -1,114 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"fmt"
"strings"
"code.gitea.io/gitea/models/db"
issues_model "code.gitea.io/gitea/models/issues"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"xorm.io/builder"
)
func iteratePRs(ctx context.Context, repo *repo_model.Repository, each func(*repo_model.Repository, *issues_model.PullRequest) error) error {
return db.Iterate(
ctx,
builder.Eq{"base_repo_id": repo.ID},
func(ctx context.Context, bean *issues_model.PullRequest) error {
return each(repo, bean)
},
)
}
func checkPRMergeBase(ctx context.Context, logger log.Logger, autofix bool) error {
numRepos := 0
numPRs := 0
numPRsUpdated := 0
err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
numRepos++
return iteratePRs(ctx, repo, func(repo *repo_model.Repository, pr *issues_model.PullRequest) error {
numPRs++
pr.BaseRepo = repo
repoPath := repo.RepoPath()
oldMergeBase := pr.MergeBase
if !pr.HasMerged {
var err error
pr.MergeBase, _, err = git.NewCommand(ctx, "merge-base").AddDashesAndList(pr.BaseBranch, pr.GetGitRefName()).RunStdString(&git.RunOpts{Dir: repoPath})
if err != nil {
var err2 error
pr.MergeBase, _, err2 = git.NewCommand(ctx, "rev-parse").AddDynamicArguments(git.BranchPrefix + pr.BaseBranch).RunStdString(&git.RunOpts{Dir: repoPath})
if err2 != nil {
logger.Warn("Unable to get merge base for PR ID %d, #%d onto %s in %s/%s. Error: %v & %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err, err2)
return nil
}
}
} else {
parentsString, _, err := git.NewCommand(ctx, "rev-list", "--parents", "-n", "1").AddDynamicArguments(pr.MergedCommitID).RunStdString(&git.RunOpts{Dir: repoPath})
if err != nil {
logger.Warn("Unable to get parents for merged PR ID %d, #%d onto %s in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err)
return nil
}
parents := strings.Split(strings.TrimSpace(parentsString), " ")
if len(parents) < 2 {
return nil
}
refs := append([]string{}, parents[1:]...)
refs = append(refs, pr.GetGitRefName())
cmd := git.NewCommand(ctx, "merge-base").AddDashesAndList(refs...)
pr.MergeBase, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath})
if err != nil {
logger.Warn("Unable to get merge base for merged PR ID %d, #%d onto %s in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err)
return nil
}
}
pr.MergeBase = strings.TrimSpace(pr.MergeBase)
if pr.MergeBase != oldMergeBase {
if autofix {
if err := pr.UpdateCols(ctx, "merge_base"); err != nil {
logger.Critical("Failed to update merge_base. ERROR: %v", err)
return fmt.Errorf("Failed to update merge_base. ERROR: %w", err)
}
} else {
logger.Info("#%d onto %s in %s/%s: MergeBase should be %s but is %s", pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, oldMergeBase, pr.MergeBase)
}
numPRsUpdated++
}
return nil
})
})
if autofix {
logger.Info("%d PR mergebases updated of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
} else {
if numPRsUpdated == 0 {
logger.Info("All %d PRs in %d repos have a correct mergebase", numPRs, numRepos)
} else if err == nil {
logger.Critical("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
return fmt.Errorf("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
} else {
logger.Warn("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)
}
}
return err
}
func init() {
Register(&Check{
Title: "Recalculate merge bases",
Name: "recalculate-merge-bases",
IsDefault: false,
Run: checkPRMergeBase,
Priority: 7,
})
}
modules/doctor/misc.go
-298
View File
@@ -1,298 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"fmt"
"os"
"os/exec"
"path"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
lru "github.com/hashicorp/golang-lru/v2"
"xorm.io/builder"
)
func iterateRepositories(ctx context.Context, each func(*repo_model.Repository) error) error {
err := db.Iterate(
ctx,
builder.Gt{"id": 0},
func(ctx context.Context, bean *repo_model.Repository) error {
return each(bean)
},
)
return err
}
func checkScriptType(ctx context.Context, logger log.Logger, autofix bool) error {
path, err := exec.LookPath(setting.ScriptType)
if err != nil {
logger.Critical("ScriptType \"%q\" is not on the current PATH. Error: %v", setting.ScriptType, err)
return fmt.Errorf("ScriptType \"%q\" is not on the current PATH. Error: %w", setting.ScriptType, err)
}
logger.Info("ScriptType %s is on the current PATH at %s", setting.ScriptType, path)
return nil
}
func checkHooks(ctx context.Context, logger log.Logger, autofix bool) error {
if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
results, err := repository.CheckDelegateHooks(repo.RepoPath())
if err != nil {
logger.Critical("Unable to check delegate hooks for repo %-v. ERROR: %v", repo, err)
return fmt.Errorf("Unable to check delegate hooks for repo %-v. ERROR: %w", repo, err)
}
if len(results) > 0 && autofix {
logger.Warn("Regenerated hooks for %s", repo.FullName())
if err := repository.CreateDelegateHooks(repo.RepoPath()); err != nil {
logger.Critical("Unable to recreate delegate hooks for %-v. ERROR: %v", repo, err)
return fmt.Errorf("Unable to recreate delegate hooks for %-v. ERROR: %w", repo, err)
}
}
for _, result := range results {
logger.Warn(result)
}
return nil
}); err != nil {
logger.Critical("Errors noted whilst checking delegate hooks.")
return err
}
return nil
}
func checkUserStarNum(ctx context.Context, logger log.Logger, autofix bool) error {
if autofix {
if err := models.DoctorUserStarNum(ctx); err != nil {
logger.Critical("Unable update User Stars numbers")
return err
}
logger.Info("Updated User Stars numbers.")
} else {
logger.Info("No check available for User Stars numbers (skipped)")
}
return nil
}
func checkEnablePushOptions(ctx context.Context, logger log.Logger, autofix bool) error {
numRepos := 0
numNeedUpdate := 0
if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
numRepos++
r, err := git.OpenRepository(ctx, repo.RepoPath())
if err != nil {
return err
}
defer r.Close()
if autofix {
_, _, err := git.NewCommand(ctx, "config", "receive.advertisePushOptions", "true").RunStdString(&git.RunOpts{Dir: r.Path})
return err
}
value, _, err := git.NewCommand(ctx, "config", "receive.advertisePushOptions").RunStdString(&git.RunOpts{Dir: r.Path})
if err != nil {
return err
}
result, valid := git.ParseBool(strings.TrimSpace(value))
if !result || !valid {
numNeedUpdate++
logger.Info("%s: does not have receive.advertisePushOptions set correctly: %q", repo.FullName(), value)
}
return nil
}); err != nil {
logger.Critical("Unable to EnablePushOptions: %v", err)
return err
}
if autofix {
logger.Info("Enabled push options for %d repositories.", numRepos)
} else {
logger.Info("Checked %d repositories, %d need updates.", numRepos, numNeedUpdate)
}
return nil
}
func checkDaemonExport(ctx context.Context, logger log.Logger, autofix bool) error {
numRepos := 0
numNeedUpdate := 0
cache, err := lru.New[int64, any](512)
if err != nil {
logger.Critical("Unable to create cache: %v", err)
return err
}
if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
numRepos++
if owner, has := cache.Get(repo.OwnerID); has {
repo.Owner = owner.(*user_model.User)
} else {
if err := repo.LoadOwner(ctx); err != nil {
return err
}
cache.Add(repo.OwnerID, repo.Owner)
}
// Create/Remove git-daemon-export-ok for git-daemon...
daemonExportFile := path.Join(repo.RepoPath(), `git-daemon-export-ok`)
isExist, err := util.IsExist(daemonExportFile)
if err != nil {
log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err)
return err
}
isPublic := !repo.IsPrivate && repo.Owner.Visibility == structs.VisibleTypePublic
if isPublic != isExist {
numNeedUpdate++
if autofix {
if !isPublic && isExist {
if err = util.Remove(daemonExportFile); err != nil {
log.Error("Failed to remove %s: %v", daemonExportFile, err)
}
} else if isPublic && !isExist {
if f, err := os.Create(daemonExportFile); err != nil {
log.Error("Failed to create %s: %v", daemonExportFile, err)
} else {
f.Close()
}
}
}
}
return nil
}); err != nil {
logger.Critical("Unable to checkDaemonExport: %v", err)
return err
}
if autofix {
logger.Info("Updated git-daemon-export-ok files for %d of %d repositories.", numNeedUpdate, numRepos)
} else {
logger.Info("Checked %d repositories, %d need updates.", numRepos, numNeedUpdate)
}
return nil
}
func checkCommitGraph(ctx context.Context, logger log.Logger, autofix bool) error {
numRepos := 0
numNeedUpdate := 0
numWritten := 0
if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
numRepos++
commitGraphExists := func() (bool, error) {
// Check commit-graph exists
commitGraphFile := path.Join(repo.RepoPath(), `objects/info/commit-graph`)
isExist, err := util.IsExist(commitGraphFile)
if err != nil {
logger.Error("Unable to check if %s exists. Error: %v", commitGraphFile, err)
return false, err
}
if !isExist {
commitGraphsDir := path.Join(repo.RepoPath(), `objects/info/commit-graphs`)
isExist, err = util.IsExist(commitGraphsDir)
if err != nil {
logger.Error("Unable to check if %s exists. Error: %v", commitGraphsDir, err)
return false, err
}
}
return isExist, nil
}
isExist, err := commitGraphExists()
if err != nil {
return err
}
if !isExist {
numNeedUpdate++
if autofix {
if err := git.WriteCommitGraph(ctx, repo.RepoPath()); err != nil {
logger.Error("Unable to write commit-graph in %s. Error: %v", repo.FullName(), err)
return err
}
isExist, err := commitGraphExists()
if err != nil {
return err
}
if isExist {
numWritten++
logger.Info("Commit-graph written: %s", repo.FullName())
} else {
logger.Warn("No commit-graph written: %s", repo.FullName())
}
}
}
return nil
}); err != nil {
logger.Critical("Unable to checkCommitGraph: %v", err)
return err
}
if autofix {
logger.Info("Wrote commit-graph files for %d of %d repositories.", numWritten, numRepos)
} else {
logger.Info("Checked %d repositories, %d without commit-graphs.", numRepos, numNeedUpdate)
}
return nil
}
func init() {
Register(&Check{
Title: "Check if SCRIPT_TYPE is available",
Name: "script-type",
IsDefault: false,
Run: checkScriptType,
Priority: 5,
})
Register(&Check{
Title: "Check if hook files are up-to-date and executable",
Name: "hooks",
IsDefault: false,
Run: checkHooks,
Priority: 6,
})
Register(&Check{
Title: "Recalculate Stars number for all user",
Name: "recalculate-stars-number",
IsDefault: false,
Run: checkUserStarNum,
Priority: 6,
})
Register(&Check{
Title: "Check that all git repositories have receive.advertisePushOptions set to true",
Name: "enable-push-options",
IsDefault: false,
Run: checkEnablePushOptions,
Priority: 7,
})
Register(&Check{
Title: "Check git-daemon-export-ok files",
Name: "check-git-daemon-export-ok",
IsDefault: false,
Run: checkDaemonExport,
Priority: 8,
})
Register(&Check{
Title: "Check commit-graphs",
Name: "check-commit-graphs",
IsDefault: false,
Run: checkCommitGraph,
Priority: 9,
})
}
modules/doctor/paths.go
-124
View File
@@ -1,124 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"fmt"
"os"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
type configurationFile struct {
Name string
Path string
IsDirectory bool
Required bool
Writable bool
}
func checkConfigurationFile(logger log.Logger, autofix bool, fileOpts configurationFile) error {
logger.Info(`%-26s %q`, log.NewColoredValue(fileOpts.Name+":", log.Reset), fileOpts.Path)
fi, err := os.Stat(fileOpts.Path)
if err != nil {
if os.IsNotExist(err) && autofix && fileOpts.IsDirectory {
if err := os.MkdirAll(fileOpts.Path, 0o777); err != nil {
logger.Error(" Directory does not exist and could not be created. ERROR: %v", err)
return fmt.Errorf("Configuration directory: \"%q\" does not exist and could not be created. ERROR: %w", fileOpts.Path, err)
}
fi, err = os.Stat(fileOpts.Path)
}
}
if err != nil {
if fileOpts.Required {
logger.Error(" Is REQUIRED but is not accessible. ERROR: %v", err)
return fmt.Errorf("Configuration file \"%q\" is not accessible but is required. Error: %w", fileOpts.Path, err)
}
logger.Warn(" NOTICE: is not accessible (Error: %v)", err)
// this is a non-critical error
return nil
}
if fileOpts.IsDirectory && !fi.IsDir() {
logger.Error(" ERROR: not a directory")
return fmt.Errorf("Configuration directory \"%q\" is not a directory. Error: %w", fileOpts.Path, err)
} else if !fileOpts.IsDirectory && !fi.Mode().IsRegular() {
logger.Error(" ERROR: not a regular file")
return fmt.Errorf("Configuration file \"%q\" is not a regular file. Error: %w", fileOpts.Path, err)
} else if fileOpts.Writable {
if err := isWritableDir(fileOpts.Path); err != nil {
logger.Error(" ERROR: is required to be writable but is not writable: %v", err)
return fmt.Errorf("Configuration file \"%q\" is required to be writable but is not. Error: %w", fileOpts.Path, err)
}
}
return nil
}
func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix bool) error {
if fi, err := os.Stat(setting.CustomConf); err != nil || !fi.Mode().IsRegular() {
logger.Error("Failed to find configuration file at '%s'.", setting.CustomConf)
logger.Error("If you've never ran Gitea yet, this is normal and '%s' will be created for you on first run.", setting.CustomConf)
logger.Error("Otherwise check that you are running this command from the correct path and/or provide a `--config` parameter.")
logger.Critical("Cannot proceed without a configuration file")
return err
}
setting.MustInstalled()
configurationFiles := []configurationFile{
{"Configuration File Path", setting.CustomConf, false, true, false},
{"Repository Root Path", setting.RepoRootPath, true, true, true},
{"Data Root Path", setting.AppDataPath, true, true, true},
{"Custom File Root Path", setting.CustomPath, true, false, false},
{"Work directory", setting.AppWorkPath, true, true, false},
{"Log Root Path", setting.Log.RootPath, true, true, true},
}
if !setting.HasBuiltinBindata {
configurationFiles = append(configurationFiles, configurationFile{"Static File Root Path", setting.StaticRootPath, true, true, false})
}
numberOfErrors := 0
for _, configurationFile := range configurationFiles {
if err := checkConfigurationFile(logger, autofix, configurationFile); err != nil {
numberOfErrors++
}
}
if numberOfErrors > 0 {
logger.Critical("Please check your configuration files and try again.")
return fmt.Errorf("%d configuration files with errors", numberOfErrors)
}
return nil
}
func isWritableDir(path string) error {
// There's no platform-independent way of checking if a directory is writable
// https://stackoverflow.com/questions/20026320/how-to-tell-if-folder-exists-and-is-writable
tmpFile, err := os.CreateTemp(path, "doctors-order")
if err != nil {
return err
}
if err := os.Remove(tmpFile.Name()); err != nil {
fmt.Printf("Warning: can't remove temporary file: '%s'\n", tmpFile.Name()) //nolint:forbidigo
}
tmpFile.Close()
return nil
}
func init() {
Register(&Check{
Title: "Check paths and basic configuration",
Name: "paths",
IsDefault: true,
Run: checkConfigurationFiles,
AbortIfFailed: true,
SkipDatabaseInitialization: true,
Priority: 1,
})
}
modules/doctor/repository.go
-80
View File
@@ -1,80 +0,0 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage"
repo_service "code.gitea.io/gitea/services/repository"
"xorm.io/builder"
)
func handleDeleteOrphanedRepos(ctx context.Context, logger log.Logger, autofix bool) error {
test := &consistencyCheck{
Name: "Repos with no existing owner",
Counter: countOrphanedRepos,
Fixer: deleteOrphanedRepos,
FixedMessage: "Deleted all content related to orphaned repos",
}
return test.Run(ctx, logger, autofix)
}
// countOrphanedRepos count repository where user of owner_id do not exist
func countOrphanedRepos(ctx context.Context) (int64, error) {
return db.CountOrphanedObjects(ctx, "repository", "user", "repository.owner_id=`user`.id")
}
// deleteOrphanedRepos delete repository where user of owner_id do not exist
func deleteOrphanedRepos(ctx context.Context) (int64, error) {
if err := storage.Init(); err != nil {
return 0, err
}
batchSize := db.MaxBatchInsertSize("repository")
e := db.GetEngine(ctx)
var deleted int64
adminUser := &user_model.User{IsAdmin: true}
for {
select {
case <-ctx.Done():
return deleted, ctx.Err()
default:
var ids []int64
if err := e.Table("`repository`").
Join("LEFT", "`user`", "repository.owner_id=`user`.id").
Where(builder.IsNull{"`user`.id"}).
Select("`repository`.id").Limit(batchSize).Find(&ids); err != nil {
return deleted, err
}
// if we don't get ids we have deleted them all
if len(ids) == 0 {
return deleted, nil
}
for _, id := range ids {
if err := repo_service.DeleteRepositoryDirectly(ctx, adminUser, id, true); err != nil {
return deleted, err
}
deleted++
}
}
}
}
func init() {
Register(&Check{
Title: "Deleted all content related to orphaned repos",
Name: "delete-orphaned-repos",
IsDefault: false,
Run: handleDeleteOrphanedRepos,
Priority: 4,
})
}
modules/doctor/storage.go
-270
View File
@@ -1,270 +0,0 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"errors"
"io/fs"
"strings"
"code.gitea.io/gitea/models/git"
"code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/log"
packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
)
type commonStorageCheckOptions struct {
storer storage.ObjectStorage
isOrphaned func(path string, obj storage.Object, stat fs.FileInfo) (bool, error)
name string
}
func commonCheckStorage(ctx context.Context, logger log.Logger, autofix bool, opts *commonStorageCheckOptions) error {
totalCount, orphanedCount := 0, 0
totalSize, orphanedSize := int64(0), int64(0)
var pathsToDelete []string
if err := opts.storer.IterateObjects("", func(p string, obj storage.Object) error {
defer obj.Close()
totalCount++
stat, err := obj.Stat()
if err != nil {
return err
}
totalSize += stat.Size()
orphaned, err := opts.isOrphaned(p, obj, stat)
if err != nil {
return err
}
if orphaned {
orphanedCount++
orphanedSize += stat.Size()
if autofix {
pathsToDelete = append(pathsToDelete, p)
}
}
return nil
}); err != nil {
logger.Error("Error whilst iterating %s storage: %v", opts.name, err)
return err
}
if orphanedCount > 0 {
if autofix {
var deletedNum int
for _, p := range pathsToDelete {
if err := opts.storer.Delete(p); err != nil {
log.Error("Error whilst deleting %s from %s storage: %v", p, opts.name, err)
} else {
deletedNum++
}
}
logger.Info("Deleted %d/%d orphaned %s(s)", deletedNum, orphanedCount, opts.name)
} else {
logger.Warn("Found %d/%d (%s/%s) orphaned %s(s)", orphanedCount, totalCount, base.FileSize(orphanedSize), base.FileSize(totalSize), opts.name)
}
} else {
logger.Info("Found %d (%s) %s(s)", totalCount, base.FileSize(totalSize), opts.name)
}
return nil
}
type checkStorageOptions struct {
All bool
Attachments bool
LFS bool
Avatars bool
RepoAvatars bool
RepoArchives bool
Packages bool
}
// checkStorage will return a doctor check function to check the requested storage types for "orphaned" stored object/files and optionally delete them
func checkStorage(opts *checkStorageOptions) func(ctx context.Context, logger log.Logger, autofix bool) error {
return func(ctx context.Context, logger log.Logger, autofix bool) error {
if err := storage.Init(); err != nil {
logger.Error("storage.Init failed: %v", err)
return err
}
if opts.Attachments || opts.All {
if err := commonCheckStorage(ctx, logger, autofix,
&commonStorageCheckOptions{
storer: storage.Attachments,
isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
exists, err := repo.ExistAttachmentsByUUID(ctx, stat.Name())
return !exists, err
},
name: "attachment",
}); err != nil {
return err
}
}
if opts.LFS || opts.All {
if !setting.LFS.StartServer {
logger.Info("LFS isn't enabled (skipped)")
return nil
}
if err := commonCheckStorage(ctx, logger, autofix,
&commonStorageCheckOptions{
storer: storage.LFS,
isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
// The oid of an LFS stored object is the name but with all the path.Separators removed
oid := strings.ReplaceAll(path, "/", "")
exists, err := git.ExistsLFSObject(ctx, oid)
return !exists, err
},
name: "LFS file",
}); err != nil {
return err
}
}
if opts.Avatars || opts.All {
if err := commonCheckStorage(ctx, logger, autofix,
&commonStorageCheckOptions{
storer: storage.Avatars,
isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
exists, err := user.ExistsWithAvatarAtStoragePath(ctx, path)
return !exists, err
},
name: "avatar",
}); err != nil {
return err
}
}
if opts.RepoAvatars || opts.All {
if err := commonCheckStorage(ctx, logger, autofix,
&commonStorageCheckOptions{
storer: storage.RepoAvatars,
isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
exists, err := repo.ExistsWithAvatarAtStoragePath(ctx, path)
return !exists, err
},
name: "repo avatar",
}); err != nil {
return err
}
}
if opts.RepoArchives || opts.All {
if err := commonCheckStorage(ctx, logger, autofix,
&commonStorageCheckOptions{
storer: storage.RepoAvatars,
isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
exists, err := repo.ExistsRepoArchiverWithStoragePath(ctx, path)
if err == nil || errors.Is(err, util.ErrInvalidArgument) {
// invalid arguments mean that the object is not a valid repo archiver and it should be removed
return !exists, nil
}
return !exists, err
},
name: "repo archive",
}); err != nil {
return err
}
}
if opts.Packages || opts.All {
if !setting.Packages.Enabled {
logger.Info("Packages isn't enabled (skipped)")
return nil
}
if err := commonCheckStorage(ctx, logger, autofix,
&commonStorageCheckOptions{
storer: storage.Packages,
isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
key, err := packages_module.RelativePathToKey(path)
if err != nil {
// If there is an error here then the relative path does not match a valid package
// Therefore it is orphaned by default
return true, nil
}
exists, err := packages.ExistPackageBlobWithSHA(ctx, string(key))
return !exists, err
},
name: "package blob",
}); err != nil {
return err
}
}
return nil
}
}
func init() {
Register(&Check{
Title: "Check if there are orphaned storage files",
Name: "storages",
IsDefault: false,
Run: checkStorage(&checkStorageOptions{All: true}),
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
Register(&Check{
Title: "Check if there are orphaned attachments in storage",
Name: "storage-attachments",
IsDefault: false,
Run: checkStorage(&checkStorageOptions{Attachments: true}),
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
Register(&Check{
Title: "Check if there are orphaned lfs files in storage",
Name: "storage-lfs",
IsDefault: false,
Run: checkStorage(&checkStorageOptions{LFS: true}),
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
Register(&Check{
Title: "Check if there are orphaned avatars in storage",
Name: "storage-avatars",
IsDefault: false,
Run: checkStorage(&checkStorageOptions{Avatars: true, RepoAvatars: true}),
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
Register(&Check{
Title: "Check if there are orphaned archives in storage",
Name: "storage-archives",
IsDefault: false,
Run: checkStorage(&checkStorageOptions{RepoArchives: true}),
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
Register(&Check{
Title: "Check if there are orphaned package blobs in storage",
Name: "storage-packages",
IsDefault: false,
Run: checkStorage(&checkStorageOptions{Packages: true}),
AbortIfFailed: false,
SkipDatabaseInitialization: false,
Priority: 1,
})
}
modules/doctor/usertype.go
-41
View File
@@ -1,41 +0,0 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
)
func checkUserType(ctx context.Context, logger log.Logger, autofix bool) error {
count, err := user_model.CountWrongUserType(ctx)
if err != nil {
logger.Critical("Error: %v whilst counting wrong user types")
return err
}
if count > 0 {
if autofix {
if count, err = user_model.FixWrongUserType(ctx); err != nil {
logger.Critical("Error: %v whilst fixing wrong user types")
return err
}
logger.Info("%d users with wrong type fixed", count)
} else {
logger.Warn("%d users with wrong type exist", count)
}
}
return nil
}
func init() {
Register(&Check{
Title: "Check if user with wrong type exist",
Name: "check-user-type",
IsDefault: true,
Run: checkUserType,
Priority: 3,
})
}
modules/git/blame_sha256_test.go
+144
View File
@@ -0,0 +1,144 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
)
func TestReadingBlameOutputSha256(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
t.Run("Without .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo5_pulls_sha256")
assert.NoError(t, err)
defer repo.Close()
commit, err := repo.GetCommit("0b69b7bb649b5d46e14cabb6468685e5dd721290acc7ffe604d37cde57927345")
assert.NoError(t, err)
parts := []*BlamePart{
{
Sha: "1e35a51dc00fd7de730344c07061acfe80e8117e075ac979b6a29a3a045190ca",
Lines: []string{
"# test_repo",
"Test repository for testing migration from github to gitea",
},
},
{
Sha: "0b69b7bb649b5d46e14cabb6468685e5dd721290acc7ffe604d37cde57927345",
Lines: []string{"", "Do not make any changes to this repo it is used for unit testing"},
PreviousSha: "1e35a51dc00fd7de730344c07061acfe80e8117e075ac979b6a29a3a045190ca",
PreviousPath: "README.md",
},
}
for _, bypass := range []bool{false, true} {
blameReader, err := CreateBlameReader(ctx, Sha256ObjectFormat, "./tests/repos/repo5_pulls_sha256", commit, "README.md", bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()
assert.False(t, blameReader.UsesIgnoreRevs())
for _, part := range parts {
actualPart, err := blameReader.NextPart()
assert.NoError(t, err)
assert.Equal(t, part, actualPart)
}
// make sure all parts have been read
actualPart, err := blameReader.NextPart()
assert.Nil(t, actualPart)
assert.NoError(t, err)
}
})
t.Run("With .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo6_blame_sha256")
assert.NoError(t, err)
defer repo.Close()
full := []*BlamePart{
{
Sha: "ab2b57a4fa476fb2edb74dafa577caf918561abbaa8fba0c8dc63c412e17a7cc",
Lines: []string{"line", "line"},
},
{
Sha: "9347b0198cd1f25017579b79d0938fa89dba34ad2514f0dd92f6bc975ed1a2fe",
Lines: []string{"changed line"},
PreviousSha: "ab2b57a4fa476fb2edb74dafa577caf918561abbaa8fba0c8dc63c412e17a7cc",
PreviousPath: "blame.txt",
},
{
Sha: "ab2b57a4fa476fb2edb74dafa577caf918561abbaa8fba0c8dc63c412e17a7cc",
Lines: []string{"line", "line", ""},
},
}
cases := []struct {
CommitID string
UsesIgnoreRevs bool
Bypass bool
Parts []*BlamePart
}{
{
CommitID: "e2f5660e15159082902960af0ed74fc144921d2b0c80e069361853b3ece29ba3",
UsesIgnoreRevs: true,
Bypass: false,
Parts: []*BlamePart{
{
Sha: "ab2b57a4fa476fb2edb74dafa577caf918561abbaa8fba0c8dc63c412e17a7cc",
Lines: []string{"line", "line", "changed line", "line", "line", ""},
},
},
},
{
CommitID: "e2f5660e15159082902960af0ed74fc144921d2b0c80e069361853b3ece29ba3",
UsesIgnoreRevs: false,
Bypass: true,
Parts: full,
},
{
CommitID: "9347b0198cd1f25017579b79d0938fa89dba34ad2514f0dd92f6bc975ed1a2fe",
UsesIgnoreRevs: false,
Bypass: false,
Parts: full,
},
{
CommitID: "9347b0198cd1f25017579b79d0938fa89dba34ad2514f0dd92f6bc975ed1a2fe",
UsesIgnoreRevs: false,
Bypass: false,
Parts: full,
},
}
for _, c := range cases {
commit, err := repo.GetCommit(c.CommitID)
assert.NoError(t, err)
blameReader, err := CreateBlameReader(ctx, repo.objectFormat, "./tests/repos/repo6_blame_sha256", commit, "blame.txt", c.Bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()
assert.Equal(t, c.UsesIgnoreRevs, blameReader.UsesIgnoreRevs())
for _, part := range c.Parts {
actualPart, err := blameReader.NextPart()
assert.NoError(t, err)
assert.Equal(t, part, actualPart)
}
// make sure all parts have been read
actualPart, err := blameReader.NextPart()
assert.Nil(t, actualPart)
assert.NoError(t, err)
}
})
}
modules/git/commit_reader.go
+2
View File
@@ -85,6 +85,8 @@ readLoop:
commit.Committer.Decode(data)
_, _ = payloadSB.Write(line)
case "gpgsig":
fallthrough
case "gpgsig-sha256": // FIXME: no intertop, so only 1 exists at present.
_, _ = signatureSB.Write(data)
_ = signatureSB.WriteByte('\n')
pgpsig = true
modules/git/commit_sha256_test.go
+195
View File
@@ -0,0 +1,195 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !gogit
package git
import (
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCommitsCountSha256(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare_sha256")
commitsCount, err := CommitsCount(DefaultContext,
CommitsCountOptions{
RepoPath: bareRepo1Path,
Revision: []string{"f004f41359117d319dedd0eaab8c5259ee2263da839dcba33637997458627fdc"},
})
assert.NoError(t, err)
assert.Equal(t, int64(3), commitsCount)
}
func TestCommitsCountWithoutBaseSha256(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare_sha256")
commitsCount, err := CommitsCount(DefaultContext,
CommitsCountOptions{
RepoPath: bareRepo1Path,
Not: "main",
Revision: []string{"branch1"},
})
assert.NoError(t, err)
assert.Equal(t, int64(2), commitsCount)
}
func TestGetFullCommitIDSha256(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare_sha256")
id, err := GetFullCommitID(DefaultContext, bareRepo1Path, "f004f4")
assert.NoError(t, err)
assert.Equal(t, "f004f41359117d319dedd0eaab8c5259ee2263da839dcba33637997458627fdc", id)
}
func TestGetFullCommitIDErrorSha256(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare_sha256")
id, err := GetFullCommitID(DefaultContext, bareRepo1Path, "unknown")
assert.Empty(t, id)
if assert.Error(t, err) {
assert.EqualError(t, err, "object does not exist [id: unknown, rel_path: ]")
}
}
func TestCommitFromReaderSha256(t *testing.T) {
commitString := `9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0 commit 1114
tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e
parent 26e9ccc29fad747e9c5d9f4c9ddeb7eff61cc45ef6a8dc258cbeb181afc055e8
author Adam Majer <amajer@suse.de> 1698676906 +0100
committer Adam Majer <amajer@suse.de> 1698676906 +0100
gpgsig-sha256 -----BEGIN PGP SIGNATURE-----
` + " " + `
iQIrBAABCgAtFiEES+fB08xlgTrzSdQvhkUIsBsmec8FAmU/wKoPHGFtYWplckBz
dXNlLmRlAAoJEIZFCLAbJnnP4s4PQIJATa++WPzR6/H4etT7bsOGoMyguEJYyWOd
aTybplzT7QAL7h2to0QszGabtzMJPIA39xSFZNYNN30voK5YyyYibXluPKgjemfK
WNXwF+gkwgZI38gSvKf+vlqI+EYyIFe19wOhiju0m8SIlB5NEPiWHa17q2mqmqqx
1FWa2JdqLPYjAtSLFXeSZegrY5V1FxdemyMUONkg8YO9OSIMZiE0GsnnOXQ3xcT4
JTCnmlUxIKw689UiEY80JopUIq+Wl7+qq9507IYYSUCyB6JazL42AKMzVCbD+qBP
oOzh/hafYgk9H9qCQXaLbmvs17zXRpicig1bAzqgAy1FDelvpERyRTydEajSLIG6
U1cRCkgXCZ0NfsYNPPmBa8b3+rnstypXYTbyMwTln7FfUAaGo6o9JYiPMkzxlmsy
zfp/tcaY8+LlBL9aOJjtv+a0p+HrpCGd6CCa4ARfphTLq8QRSSh8uzlB9N+6HnRI
VAEUo6ecdDxSpyt2naeg9pKus/BRi7P6g4B1hkk/zZstUX/QP4IQuAJbXjkvsC+X
HKRr3NlRM/DygzTyj0gN74uoa0goCIbyAQhiT42nm0cuhM7uN/W0ayrlZjGF1cbR
8NCJUL2Nwj0ywKIavC99Ipkb8AsFwpVT6U6effs6
=xybZ
-----END PGP SIGNATURE-----
signed commit`
sha := &Sha256Hash{
0x94, 0x33, 0xb2, 0xa6, 0x2b, 0x96, 0x4c, 0x17, 0xa4, 0x48, 0x5a, 0xe1, 0x80, 0xf4, 0x5f, 0x59,
0x5d, 0x3e, 0x69, 0xd3, 0x1b, 0x78, 0x60, 0x87, 0x77, 0x5e, 0x28, 0xc6, 0xb6, 0x39, 0x9d, 0xf0,
}
gitRepo, err := openRepositoryWithDefaultContext(filepath.Join(testReposDir, "repo1_bare_sha256"))
assert.NoError(t, err)
assert.NotNil(t, gitRepo)
defer gitRepo.Close()
commitFromReader, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString))
assert.NoError(t, err)
if !assert.NotNil(t, commitFromReader) {
return
}
assert.EqualValues(t, sha, commitFromReader.ID)
assert.EqualValues(t, `-----BEGIN PGP SIGNATURE-----
iQIrBAABCgAtFiEES+fB08xlgTrzSdQvhkUIsBsmec8FAmU/wKoPHGFtYWplckBz
dXNlLmRlAAoJEIZFCLAbJnnP4s4PQIJATa++WPzR6/H4etT7bsOGoMyguEJYyWOd
aTybplzT7QAL7h2to0QszGabtzMJPIA39xSFZNYNN30voK5YyyYibXluPKgjemfK
WNXwF+gkwgZI38gSvKf+vlqI+EYyIFe19wOhiju0m8SIlB5NEPiWHa17q2mqmqqx
1FWa2JdqLPYjAtSLFXeSZegrY5V1FxdemyMUONkg8YO9OSIMZiE0GsnnOXQ3xcT4
JTCnmlUxIKw689UiEY80JopUIq+Wl7+qq9507IYYSUCyB6JazL42AKMzVCbD+qBP
oOzh/hafYgk9H9qCQXaLbmvs17zXRpicig1bAzqgAy1FDelvpERyRTydEajSLIG6
U1cRCkgXCZ0NfsYNPPmBa8b3+rnstypXYTbyMwTln7FfUAaGo6o9JYiPMkzxlmsy
zfp/tcaY8+LlBL9aOJjtv+a0p+HrpCGd6CCa4ARfphTLq8QRSSh8uzlB9N+6HnRI
VAEUo6ecdDxSpyt2naeg9pKus/BRi7P6g4B1hkk/zZstUX/QP4IQuAJbXjkvsC+X
HKRr3NlRM/DygzTyj0gN74uoa0goCIbyAQhiT42nm0cuhM7uN/W0ayrlZjGF1cbR
8NCJUL2Nwj0ywKIavC99Ipkb8AsFwpVT6U6effs6
=xybZ
-----END PGP SIGNATURE-----
`, commitFromReader.Signature.Signature)
assert.EqualValues(t, `tree e7f9e96dd79c09b078cac8b303a7d3b9d65ff9b734e86060a4d20409fd379f9e
parent 26e9ccc29fad747e9c5d9f4c9ddeb7eff61cc45ef6a8dc258cbeb181afc055e8
author Adam Majer <amajer@suse.de> 1698676906 +0100
committer Adam Majer <amajer@suse.de> 1698676906 +0100
signed commit`, commitFromReader.Signature.Payload)
assert.EqualValues(t, "Adam Majer <amajer@suse.de>", commitFromReader.Author.String())
commitFromReader2, err := CommitFromReader(gitRepo, sha, strings.NewReader(commitString+"\n\n"))
assert.NoError(t, err)
commitFromReader.CommitMessage += "\n\n"
commitFromReader.Signature.Payload += "\n\n"
assert.EqualValues(t, commitFromReader, commitFromReader2)
}
func TestHasPreviousCommitSha256(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare_sha256")
repo, err := openRepositoryWithDefaultContext(bareRepo1Path)
assert.NoError(t, err)
defer repo.Close()
commit, err := repo.GetCommit("f004f41359117d319dedd0eaab8c5259ee2263da839dcba33637997458627fdc")
assert.NoError(t, err)
parentSHA := MustIDFromString("b0ec7af4547047f12d5093e37ef8f1b3b5415ed8ee17894d43a34d7d34212e9c")
notParentSHA := MustIDFromString("42e334efd04cd36eea6da0599913333c26116e1a537ca76e5b6e4af4dda00236")
assert.Equal(t, repo.objectFormat, parentSHA.Type())
assert.Equal(t, repo.objectFormat.Name(), "sha256")
haz, err := commit.HasPreviousCommit(parentSHA)
assert.NoError(t, err)
assert.True(t, haz)
hazNot, err := commit.HasPreviousCommit(notParentSHA)
assert.NoError(t, err)
assert.False(t, hazNot)
selfNot, err := commit.HasPreviousCommit(commit.ID)
assert.NoError(t, err)
assert.False(t, selfNot)
}
func TestGetCommitFileStatusMergesSha256(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo6_merge_sha256")
commitFileStatus, err := GetCommitFileStatus(DefaultContext, bareRepo1Path, "d2e5609f630dd8db500f5298d05d16def282412e3e66ed68cc7d0833b29129a1")
assert.NoError(t, err)
expected := CommitFileStatus{
[]string{
"add_file.txt",
},
[]string{},
[]string{
"to_modify.txt",
},
}
assert.Equal(t, expected.Added, commitFileStatus.Added)
assert.Equal(t, expected.Removed, commitFileStatus.Removed)
assert.Equal(t, expected.Modified, commitFileStatus.Modified)
expected = CommitFileStatus{
[]string{},
[]string{
"to_remove.txt",
},
[]string{},
}
commitFileStatus, err = GetCommitFileStatus(DefaultContext, bareRepo1Path, "da1ded40dc8e5b7c564171f4bf2fc8370487decfb1cb6a99ef28f3ed73d09172")
assert.NoError(t, err)
assert.Equal(t, expected.Added, commitFileStatus.Added)
assert.Equal(t, expected.Removed, commitFileStatus.Removed)
assert.Equal(t, expected.Modified, commitFileStatus.Modified)
}
modules/git/foreachref/format_test.go
+2 -2
View File
@@ -49,9 +49,9 @@ func TestFormat_Flag(t *testing.T) {
{
name: "multiple fields",
givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"),
givenFormat: foreachref.NewFormat("refname:lstrip=2", "objecttype", "objectname"),
wantFlag: "refname:short %(refname:short)%00objecttype %(objecttype)%00objectname %(objectname)%00%00",
wantFlag: "refname:lstrip=2 %(refname:lstrip=2)%00objecttype %(objecttype)%00objectname %(objectname)%00%00",
},
}
modules/git/git.go
+7 -5
View File
@@ -166,10 +166,6 @@ func InitSimple(ctx context.Context) error {
// InitFull initializes git module with version check and change global variables, sync gitconfig.
// It should only be called once at the beginning of the program initialization (TestMain/GlobalInitInstalled) as this code makes unsynchronized changes to variables.
func InitFull(ctx context.Context) (err error) {
if err = checkInit(); err != nil {
return err
}
if err = InitSimple(ctx); err != nil {
return err
}
@@ -189,7 +185,13 @@ func InitFull(ctx context.Context) (err error) {
globalCommandArgs = append(globalCommandArgs, "-c", "credential.helper=")
}
SupportProcReceive = CheckGitVersionAtLeast("2.29") == nil
SupportHashSha256 = CheckGitVersionAtLeast("2.42") == nil
SupportHashSha256 = CheckGitVersionAtLeast("2.42") == nil && !isGogit
if SupportHashSha256 {
SupportedObjectFormats = append(SupportedObjectFormats, Sha256ObjectFormat)
} else {
log.Warn("sha256 hash support is disabled - requires Git >= 2.42. Gogit is currently unsupported")
}
if setting.LFS.StartServer {
if CheckGitVersionAtLeast("2.1.2") != nil {
return errors.New("LFS server support requires Git >= 2.1.2")
modules/git/object_format.go
+59 -6
View File
@@ -5,6 +5,7 @@ package git
import (
"crypto/sha1"
"crypto/sha256"
"regexp"
"strconv"
)
@@ -12,6 +13,9 @@ import (
// sha1Pattern can be used to determine if a string is an valid sha
var sha1Pattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`)
// sha256Pattern can be used to determine if a string is an valid sha
var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{4,64}$`)
type ObjectFormat interface {
// Name returns the name of the object format
Name() string
@@ -29,11 +33,12 @@ type ObjectFormat interface {
ComputeHash(t ObjectType, content []byte) ObjectID
}
/* SHA1 Type */
type Sha1ObjectFormatImpl struct{}
var (
emptyObjectID = &Sha1Hash{}
emptyTree = &Sha1Hash{
emptySha1ObjectID = &Sha1Hash{}
emptySha1Tree = &Sha1Hash{
0x4b, 0x82, 0x5d, 0xc6, 0x42, 0xcb, 0x6e, 0xb9, 0xa0, 0x60,
0xe5, 0x4b, 0xf8, 0xd6, 0x92, 0x88, 0xfb, 0xee, 0x49, 0x04,
}
@@ -41,11 +46,11 @@ var (
func (Sha1ObjectFormatImpl) Name() string { return "sha1" }
func (Sha1ObjectFormatImpl) EmptyObjectID() ObjectID {
return emptyObjectID
return emptySha1ObjectID
}
func (Sha1ObjectFormatImpl) EmptyTree() ObjectID {
return emptyTree
return emptySha1Tree
}
func (Sha1ObjectFormatImpl) FullLength() int { return 40 }
func (Sha1ObjectFormatImpl) IsValid(input string) bool {
@@ -72,11 +77,59 @@ func (h Sha1ObjectFormatImpl) ComputeHash(t ObjectType, content []byte) ObjectID
return &sha1
}
var Sha1ObjectFormat ObjectFormat = Sha1ObjectFormatImpl{}
/* SHA256 Type */
type Sha256ObjectFormatImpl struct{}
var (
emptySha256ObjectID = &Sha256Hash{}
emptySha256Tree = &Sha256Hash{
0x6e, 0xf1, 0x9b, 0x41, 0x22, 0x5c, 0x53, 0x69, 0xf1, 0xc1,
0x04, 0xd4, 0x5d, 0x8d, 0x85, 0xef, 0xa9, 0xb0, 0x57, 0xb5,
0x3b, 0x14, 0xb4, 0xb9, 0xb9, 0x39, 0xdd, 0x74, 0xde, 0xcc,
0x53, 0x21,
}
)
func (Sha256ObjectFormatImpl) Name() string { return "sha256" }
func (Sha256ObjectFormatImpl) EmptyObjectID() ObjectID {
return emptySha256ObjectID
}
func (Sha256ObjectFormatImpl) EmptyTree() ObjectID {
return emptySha256Tree
}
func (Sha256ObjectFormatImpl) FullLength() int { return 64 }
func (Sha256ObjectFormatImpl) IsValid(input string) bool {
return sha256Pattern.MatchString(input)
}
func (Sha256ObjectFormatImpl) MustID(b []byte) ObjectID {
var id Sha256Hash
copy(id[0:32], b)
return &id
}
// ComputeHash compute the hash for a given ObjectType and content
func (h Sha256ObjectFormatImpl) ComputeHash(t ObjectType, content []byte) ObjectID {
hasher := sha256.New()
_, _ = hasher.Write(t.Bytes())
_, _ = hasher.Write([]byte(" "))
_, _ = hasher.Write([]byte(strconv.FormatInt(int64(len(content)), 10)))
_, _ = hasher.Write([]byte{0})
// HashSum generates a SHA256 for the provided hash
var sha256 Sha1Hash
copy(sha256[:], hasher.Sum(nil))
return &sha256
}
var (
Sha1ObjectFormat ObjectFormat = Sha1ObjectFormatImpl{}
Sha256ObjectFormat ObjectFormat = Sha256ObjectFormatImpl{}
)
var SupportedObjectFormats = []ObjectFormat{
Sha1ObjectFormat,
// TODO: add sha256
}
func ObjectFormatFromName(name string) ObjectFormat {
modules/git/object_id.go
+16
View File
@@ -16,6 +16,7 @@ type ObjectID interface {
Type() ObjectFormat
}
/* SHA1 */
type Sha1Hash [20]byte
func (h *Sha1Hash) String() string {
@@ -39,6 +40,21 @@ func MustIDFromString(hexHash string) ObjectID {
return id
}
/* SHA256 */
type Sha256Hash [32]byte
func (h *Sha256Hash) String() string {
return hex.EncodeToString(h[:])
}
func (h *Sha256Hash) IsZero() bool {
empty := Sha256Hash{}
return bytes.Equal(empty[:], h[:])
}
func (h *Sha256Hash) RawValue() []byte { return h[:] }
func (*Sha256Hash) Type() ObjectFormat { return Sha256ObjectFormat }
/* utility */
func NewIDFromString(hexHash string) (ObjectID, error) {
var theObjectFormat ObjectFormat
for _, objectFormat := range SupportedObjectFormats {
modules/git/object_id_gogit.go
+2
View File
@@ -13,6 +13,8 @@ func ParseGogitHash(h plumbing.Hash) ObjectID {
switch hash.Size {
case 20:
return Sha1ObjectFormat.MustID(h[:])
case 32:
return Sha256ObjectFormat.MustID(h[:])
}
return nil
modules/git/repo.go
+5 -6
View File
@@ -97,15 +97,14 @@ func InitRepository(ctx context.Context, repoPath string, bare bool, objectForma
}
cmd := NewCommand(ctx, "init")
if !IsValidObjectFormat(objectFormatName) {
return fmt.Errorf("invalid object format: %s", objectFormatName)
}
if SupportHashSha256 {
if objectFormatName == "" {
objectFormatName = Sha1ObjectFormat.Name()
}
if !IsValidObjectFormat(objectFormatName) {
return fmt.Errorf("invalid object format: %s", objectFormatName)
}
cmd.AddOptionValues("--object-format", objectFormatName)
}
if bare {
cmd.AddArguments("--bare")
}
modules/git/repo_archive.go
+5
View File
@@ -8,6 +8,7 @@ import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
@@ -62,11 +63,15 @@ func (repo *Repository) CreateArchive(ctx context.Context, format ArchiveType, t
cmd.AddOptionFormat("--format=%s", format.String())
cmd.AddDynamicArguments(commitID)
// Avoid LFS hooks getting installed because of /etc/gitconfig, which can break pull requests.
env := append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1")
var stderr strings.Builder
err := cmd.Run(&RunOpts{
Dir: repo.Path,
Stdout: target,
Stderr: &stderr,
Env: env,
})
if err != nil {
return ConcatenateError(err, stderr.String())
modules/git/repo_base.go
+1 -43
View File
@@ -3,46 +3,4 @@
package git
import (
"context"
"io"
)
// contextKey is a value for use with context.WithValue.
type contextKey struct {
name string
}
// RepositoryContextKey is a context key. It is used with context.Value() to get the current Repository for the context
var RepositoryContextKey = &contextKey{"repository"}
// RepositoryFromContext attempts to get the repository from the context
func RepositoryFromContext(ctx context.Context, path string) *Repository {
value := ctx.Value(RepositoryContextKey)
if value == nil {
return nil
}
if repo, ok := value.(*Repository); ok && repo != nil {
if repo.Path == path {
return repo
}
}
return nil
}
type nopCloser func()
func (nopCloser) Close() error { return nil }
// RepositoryFromContextOrOpen attempts to get the repository from the context or just opens it
func RepositoryFromContextOrOpen(ctx context.Context, path string) (*Repository, io.Closer, error) {
gitRepo := RepositoryFromContext(ctx, path)
if gitRepo != nil {
return gitRepo, nopCloser(nil), nil
}
gitRepo, err := OpenRepository(ctx, path)
return gitRepo, gitRepo, err
}
var isGogit bool
modules/git/repo_base_gogit.go
+14 -1
View File
@@ -14,6 +14,7 @@ import (
gitealog "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/osfs"
gogit "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
@@ -21,6 +22,10 @@ import (
"github.com/go-git/go-git/v5/storage/filesystem"
)
func init() {
isGogit = true
}
// Repository represents a Git repository.
type Repository struct {
Path string
@@ -58,7 +63,15 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) {
return nil, err
}
}
storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true, LargeObjectThreshold: setting.Git.LargeObjectThreshold})
// the "clone --shared" repo doesn't work well with go-git AlternativeFS, https://github.com/go-git/go-git/issues/1006
// so use "/" for AlternatesFS, I guess it is the same behavior as current nogogit (no limitation or check for the "objects/info/alternates" paths), trust the "clone" command executed by the server.
var altFs billy.Filesystem
if setting.IsWindows {
altFs = osfs.New(filepath.VolumeName(setting.RepoRootPath) + "\\") // TODO: does it really work for Windows? Need some time to check.
} else {
altFs = osfs.New("/")
}
storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true, LargeObjectThreshold: setting.Git.LargeObjectThreshold, AlternatesFS: altFs})
gogitRepo, err := gogit.Open(storage, fs)
if err != nil {
return nil, err
modules/git/repo_base_nogogit.go
+4
View File
@@ -15,6 +15,10 @@ import (
"code.gitea.io/gitea/modules/log"
)
func init() {
isGogit = false
}
// Repository represents a Git repository.
type Repository struct {
Path string
modules/git/repo_branch.go
-23
View File
@@ -86,29 +86,6 @@ func (repo *Repository) GetBranch(branch string) (*Branch, error) {
}, nil
}
// GetBranchesByPath returns a branch by it's path
// if limit = 0 it will not limit
func GetBranchesByPath(ctx context.Context, path string, skip, limit int) ([]*Branch, int, error) {
gitRepo, err := OpenRepository(ctx, path)
if err != nil {
return nil, 0, err
}
defer gitRepo.Close()
return gitRepo.GetBranches(skip, limit)
}
// GetBranchCommitID returns a branch commit ID by its name
func GetBranchCommitID(ctx context.Context, path, branch string) (string, error) {
gitRepo, err := OpenRepository(ctx, path)
if err != nil {
return "", err
}
defer gitRepo.Close()
return gitRepo.GetBranchCommitID(branch)
}
// GetBranches returns a slice of *git.Branch
func (repo *Repository) GetBranches(skip, limit int) ([]*Branch, int, error) {
brs, countAll, err := repo.GetBranchNames(skip, limit)
modules/git/repo_branch_gogit.go
-29
View File
@@ -7,7 +7,6 @@
package git
import (
"context"
"sort"
"strings"
@@ -95,34 +94,6 @@ func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) {
return branchNames, len(branchData), nil
}
// WalkReferences walks all the references from the repository
// refType should be empty, ObjectTag or ObjectBranch. All other values are equivalent to empty.
func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refname string) error) (int, error) {
repo := RepositoryFromContext(ctx, repoPath)
if repo == nil {
var err error
repo, err = OpenRepository(ctx, repoPath)
if err != nil {
return 0, err
}
defer repo.Close()
}
i := 0
iter, err := repo.gogitRepo.References()
if err != nil {
return i, err
}
defer iter.Close()
err = iter.ForEach(func(ref *plumbing.Reference) error {
err := walkfn(ref.Hash().String(), string(ref.Name()))
i++
return err
})
return i, err
}
// WalkReferences walks all the references from the repository
func (repo *Repository) WalkReferences(arg ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) {
i := 0
modules/git/repo_branch_nogogit.go
+4 -9
View File
@@ -65,11 +65,6 @@ func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) {
return callShowRef(repo.Ctx, repo.Path, BranchPrefix, TrustedCmdArgs{BranchPrefix, "--sort=-committerdate"}, skip, limit)
}
// WalkReferences walks all the references from the repository
func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refname string) error) (int, error) {
return walkShowRef(ctx, repoPath, nil, 0, 0, walkfn)
}
// WalkReferences walks all the references from the repository
// refType should be empty, ObjectTag or ObjectBranch. All other values are equivalent to empty.
func (repo *Repository) WalkReferences(refType ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) {
@@ -81,12 +76,12 @@ func (repo *Repository) WalkReferences(refType ObjectType, skip, limit int, walk
args = TrustedCmdArgs{BranchPrefix, "--sort=-committerdate"}
}
return walkShowRef(repo.Ctx, repo.Path, args, skip, limit, walkfn)
return WalkShowRef(repo.Ctx, repo.Path, args, skip, limit, walkfn)
}
// callShowRef return refs, if limit = 0 it will not limit
func callShowRef(ctx context.Context, repoPath, trimPrefix string, extraArgs TrustedCmdArgs, skip, limit int) (branchNames []string, countAll int, err error) {
countAll, err = walkShowRef(ctx, repoPath, extraArgs, skip, limit, func(_, branchName string) error {
countAll, err = WalkShowRef(ctx, repoPath, extraArgs, skip, limit, func(_, branchName string) error {
branchName = strings.TrimPrefix(branchName, trimPrefix)
branchNames = append(branchNames, branchName)
@@ -95,7 +90,7 @@ func callShowRef(ctx context.Context, repoPath, trimPrefix string, extraArgs Tru
return branchNames, countAll, err
}
func walkShowRef(ctx context.Context, repoPath string, extraArgs TrustedCmdArgs, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) {
func WalkShowRef(ctx context.Context, repoPath string, extraArgs TrustedCmdArgs, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) {
stdoutReader, stdoutWriter := io.Pipe()
defer func() {
_ = stdoutReader.Close()
@@ -189,7 +184,7 @@ func walkShowRef(ctx context.Context, repoPath string, extraArgs TrustedCmdArgs,
// GetRefsBySha returns all references filtered with prefix that belong to a sha commit hash
func (repo *Repository) GetRefsBySha(sha, prefix string) ([]string, error) {
var revList []string
_, err := walkShowRef(repo.Ctx, repo.Path, nil, 0, 0, func(walkSha, refname string) error {
_, err := WalkShowRef(repo.Ctx, repo.Path, nil, 0, 0, func(walkSha, refname string) error {
if walkSha == sha && strings.HasPrefix(refname, prefix) {
revList = append(revList, refname)
}
modules/git/repo_commitgraph_gogit.go
+2 -2
View File
@@ -12,7 +12,7 @@ import (
gitealog "code.gitea.io/gitea/modules/log"
"github.com/go-git/go-git/v5/plumbing/format/commitgraph"
commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
@@ -22,7 +22,7 @@ func (r *Repository) CommitNodeIndex() (cgobject.CommitNodeIndex, *os.File) {
file, err := os.Open(indexPath)
if err == nil {
var index commitgraph.Index // TODO: in newer go-git, it might need to use "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2" package to compile
var index commitgraph.Index
index, err = commitgraph.OpenFileIndex(file)
if err == nil {
return cgobject.NewGraphCommitNodeIndex(index, r.gogitRepo.Storer), file
modules/git/repo_tag.go
+4 -2
View File
@@ -112,7 +112,9 @@ func (repo *Repository) GetTagWithID(idStr, name string) (*Tag, error) {
// GetTagInfos returns all tag infos of the repository.
func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) {
forEachRefFmt := foreachref.NewFormat("objecttype", "refname:short", "object", "objectname", "creator", "contents", "contents:signature")
// Generally, refname:short should be equal to refname:lstrip=2 except core.warnAmbiguousRefs is used to select the strict abbreviation mode.
// https://git-scm.com/docs/git-for-each-ref#Documentation/git-for-each-ref.txt-refname
forEachRefFmt := foreachref.NewFormat("objecttype", "refname:lstrip=2", "object", "objectname", "creator", "contents", "contents:signature")
stdoutReader, stdoutWriter := io.Pipe()
defer stdoutReader.Close()
@@ -162,7 +164,7 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) {
func parseTagRef(objectFormat ObjectFormat, ref map[string]string) (tag *Tag, err error) {
tag = &Tag{
Type: ref["objecttype"],
Name: ref["refname:short"],
Name: ref["refname:lstrip=2"],
}
tag.ID, err = NewIDFromString(ref["objectname"])
modules/git/repo_tag_test.go
+9 -9
View File
@@ -208,8 +208,8 @@ func TestRepository_parseTagRef(t *testing.T) {
name: "lightweight tag",
givenRef: map[string]string{
"objecttype": "commit",
"refname:short": "v1.9.1",
"objecttype": "commit",
"refname:lstrip=2": "v1.9.1",
// object will be empty for lightweight tags
"object": "",
"objectname": "ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889",
@@ -237,8 +237,8 @@ func TestRepository_parseTagRef(t *testing.T) {
name: "annotated tag",
givenRef: map[string]string{
"objecttype": "tag",
"refname:short": "v0.0.1",
"objecttype": "tag",
"refname:lstrip=2": "v0.0.1",
// object will refer to commit hash for annotated tag
"object": "3325fd8a973321fd59455492976c042dde3fd1ca",
"objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9",
@@ -266,11 +266,11 @@ func TestRepository_parseTagRef(t *testing.T) {
name: "annotated tag with signature",
givenRef: map[string]string{
"objecttype": "tag",
"refname:short": "v0.0.1",
"object": "3325fd8a973321fd59455492976c042dde3fd1ca",
"objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9",
"creator": "Foo Bar <foo@bar.com> 1565789218 +0300",
"objecttype": "tag",
"refname:lstrip=2": "v0.0.1",
"object": "3325fd8a973321fd59455492976c042dde3fd1ca",
"objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9",
"creator": "Foo Bar <foo@bar.com> 1565789218 +0300",
"contents": `Add changelog of v1.9.1 (#7859)
* add changelog of v1.9.1
modules/git/tests/repos/repo1_bare_sha256/HEAD
+1
View File
@@ -0,0 +1 @@
ref: refs/heads/main
modules/git/tests/repos/repo1_bare_sha256/config
+6
View File
@@ -0,0 +1,6 @@
[core]
repositoryformatversion = 1
filemode = true
bare = true
[extensions]
objectformat = sha256
modules/git/tests/repos/repo1_bare_sha256/description
+1
View File
@@ -0,0 +1 @@
Unnamed repository; edit this file 'description' to name the repository.
modules/git/tests/repos/repo1_bare_sha256/info/exclude
+6
View File
@@ -0,0 +1,6 @@
# git ls-files --others --exclude-from=.git/info/exclude
# Lines that start with '#' are comments.
# For a project mostly in C, the following would be a good set of
# exclude patterns (uncomment them if you want to use them):
# *.[oa]
# *~
modules/git/tests/repos/repo1_bare_sha256/info/refs
+7
View File
@@ -0,0 +1,7 @@
42e334efd04cd36eea6da0599913333c26116e1a537ca76e5b6e4af4dda00236 refs/heads/branch1
5bc2249e32e0ba40a08879fba2bd4e97a13cb345831549f4bc5649525da8f6cc refs/heads/branch2
9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0 refs/heads/main
29a82d4fc02e19190fb489cc90d5730ed91970b49f4e39acda2798b3dd4f814e refs/tags/signed-tag
9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0 refs/tags/signed-tag^{}
171822a62559f3aa28a00aa3785dbe915d6a8eb02712682740db44fc8bd2187a refs/tags/test
6aae864a3d1d0d6a5be0cc64028c1e7021e2632b031fd8eb82afc5a283d1c3d1 refs/tags/test^{}
modules/git/tests/repos/repo1_bare_sha256/objects/info/commit-graph
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo1_bare_sha256/objects/info/packs
+2
View File
@@ -0,0 +1,2 @@
P pack-c01aa121b9c5e345fe0da2f9be78665970b0c38c6b495d5fc034bc7a7b95334b.pack
modules/git/tests/repos/repo1_bare_sha256/objects/pack/pack-c01aa121b9c5e345fe0da2f9be78665970b0c38c6b495d5fc034bc7a7b95334b.bitmap
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo1_bare_sha256/objects/pack/pack-c01aa121b9c5e345fe0da2f9be78665970b0c38c6b495d5fc034bc7a7b95334b.idx
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo1_bare_sha256/objects/pack/pack-c01aa121b9c5e345fe0da2f9be78665970b0c38c6b495d5fc034bc7a7b95334b.pack
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo1_bare_sha256/objects/pack/pack-c01aa121b9c5e345fe0da2f9be78665970b0c38c6b495d5fc034bc7a7b95334b.rev
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo1_bare_sha256/packed-refs
+8
View File
@@ -0,0 +1,8 @@
# pack-refs with: peeled fully-peeled sorted
42e334efd04cd36eea6da0599913333c26116e1a537ca76e5b6e4af4dda00236 refs/heads/branch1
5bc2249e32e0ba40a08879fba2bd4e97a13cb345831549f4bc5649525da8f6cc refs/heads/branch2
9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0 refs/heads/main
29a82d4fc02e19190fb489cc90d5730ed91970b49f4e39acda2798b3dd4f814e refs/tags/signed-tag
^9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0
171822a62559f3aa28a00aa3785dbe915d6a8eb02712682740db44fc8bd2187a refs/tags/test
^6aae864a3d1d0d6a5be0cc64028c1e7021e2632b031fd8eb82afc5a283d1c3d1
modules/git/tests/repos/repo1_bare_sha256/refs/heads/main
+1
View File
@@ -0,0 +1 @@
9433b2a62b964c17a4485ae180f45f595d3e69d31b786087775e28c6b6399df0
modules/git/tests/repos/repo5_pulls_sha256/HEAD
+1
View File
@@ -0,0 +1 @@
ref: refs/heads/main
modules/git/tests/repos/repo5_pulls_sha256/config
+6
View File
@@ -0,0 +1,6 @@
[core]
repositoryformatversion = 1
filemode = true
bare = true
[extensions]
objectformat = sha256
modules/git/tests/repos/repo5_pulls_sha256/description
+1
View File
@@ -0,0 +1 @@
Unnamed repository; edit this file 'description' to name the repository.
modules/git/tests/repos/repo5_pulls_sha256/info/refs
+4
View File
@@ -0,0 +1,4 @@
35ecd0f946c8baeb76fa5a3876f46bf35218655e2304d8505026fa4bfb496a4b refs/heads/main
35ecd0f946c8baeb76fa5a3876f46bf35218655e2304d8505026fa4bfb496a4b refs/heads/main-clone
7f50a4906503378b0bbb7d61bd2ca8d8d8ff4f7a2474980f99402d742ccc9665 refs/heads/test-patch-1
1e35a51dc00fd7de730344c07061acfe80e8117e075ac979b6a29a3a045190ca refs/tags/v0.9.99
modules/git/tests/repos/repo5_pulls_sha256/objects/info/commit-graph
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo5_pulls_sha256/objects/info/packs
+2
View File
@@ -0,0 +1,2 @@
P pack-bfe8f09d42ef5dd1610bf42641fe145d4a02b788eb26c31022a362312660a29d.pack
modules/git/tests/repos/repo5_pulls_sha256/objects/pack/pack-bfe8f09d42ef5dd1610bf42641fe145d4a02b788eb26c31022a362312660a29d.bitmap
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo5_pulls_sha256/objects/pack/pack-bfe8f09d42ef5dd1610bf42641fe145d4a02b788eb26c31022a362312660a29d.idx
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo5_pulls_sha256/objects/pack/pack-bfe8f09d42ef5dd1610bf42641fe145d4a02b788eb26c31022a362312660a29d.pack
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo5_pulls_sha256/objects/pack/pack-bfe8f09d42ef5dd1610bf42641fe145d4a02b788eb26c31022a362312660a29d.rev
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo5_pulls_sha256/packed-refs
+5
View File
@@ -0,0 +1,5 @@
# pack-refs with: peeled fully-peeled sorted
35ecd0f946c8baeb76fa5a3876f46bf35218655e2304d8505026fa4bfb496a4b refs/heads/main
35ecd0f946c8baeb76fa5a3876f46bf35218655e2304d8505026fa4bfb496a4b refs/heads/main-clone
7f50a4906503378b0bbb7d61bd2ca8d8d8ff4f7a2474980f99402d742ccc9665 refs/heads/test-patch-1
1e35a51dc00fd7de730344c07061acfe80e8117e075ac979b6a29a3a045190ca refs/tags/v0.9.99
modules/git/tests/repos/repo5_pulls_sha256/refs/heads/main
+1
View File
@@ -0,0 +1 @@
35ecd0f946c8baeb76fa5a3876f46bf35218655e2304d8505026fa4bfb496a4b
modules/git/tests/repos/repo6_blame_sha256/HEAD
+1
View File
@@ -0,0 +1 @@
ref: refs/heads/main
modules/git/tests/repos/repo6_blame_sha256/config
+6
View File
@@ -0,0 +1,6 @@
[core]
repositoryformatversion = 1
filemode = true
bare = true
[extensions]
objectformat = sha256
modules/git/tests/repos/repo6_blame_sha256/description
+1
View File
@@ -0,0 +1 @@
Unnamed repository; edit this file 'description' to name the repository.
modules/git/tests/repos/repo6_blame_sha256/info/exclude
+6
View File
@@ -0,0 +1,6 @@
# git ls-files --others --exclude-from=.git/info/exclude
# Lines that start with '#' are comments.
# For a project mostly in C, the following would be a good set of
# exclude patterns (uncomment them if you want to use them):
# *.[oa]
# *~
modules/git/tests/repos/repo6_blame_sha256/info/refs
+1
View File
@@ -0,0 +1 @@
e2f5660e15159082902960af0ed74fc144921d2b0c80e069361853b3ece29ba3 refs/heads/main
modules/git/tests/repos/repo6_blame_sha256/objects/info/commit-graph
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_blame_sha256/objects/info/packs
+2
View File
@@ -0,0 +1,2 @@
P pack-fcb8a221b76025fd8415d3c562b611ac24312a5ffc3d3703d7c5cc906bdaee8e.pack
modules/git/tests/repos/repo6_blame_sha256/objects/pack/pack-fcb8a221b76025fd8415d3c562b611ac24312a5ffc3d3703d7c5cc906bdaee8e.bitmap
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_blame_sha256/objects/pack/pack-fcb8a221b76025fd8415d3c562b611ac24312a5ffc3d3703d7c5cc906bdaee8e.idx
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_blame_sha256/objects/pack/pack-fcb8a221b76025fd8415d3c562b611ac24312a5ffc3d3703d7c5cc906bdaee8e.pack
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_blame_sha256/objects/pack/pack-fcb8a221b76025fd8415d3c562b611ac24312a5ffc3d3703d7c5cc906bdaee8e.rev
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_blame_sha256/packed-refs
+2
View File
@@ -0,0 +1,2 @@
# pack-refs with: peeled fully-peeled sorted
e2f5660e15159082902960af0ed74fc144921d2b0c80e069361853b3ece29ba3 refs/heads/main
modules/git/tests/repos/repo6_blame_sha256/refs/refs/main
+1
View File
@@ -0,0 +1 @@
e2f5660e15159082902960af0ed74fc144921d2b0c80e069361853b3ece29ba3
modules/git/tests/repos/repo6_merge_sha256/HEAD
+1
View File
@@ -0,0 +1 @@
ref: refs/heads/main
modules/git/tests/repos/repo6_merge_sha256/config
+6
View File
@@ -0,0 +1,6 @@
[core]
repositoryformatversion = 1
filemode = true
bare = true
[extensions]
objectformat = sha256
modules/git/tests/repos/repo6_merge_sha256/description
+1
View File
@@ -0,0 +1 @@
Unnamed repository; edit this file 'description' to name the repository.
modules/git/tests/repos/repo6_merge_sha256/info/exclude
+6
View File
@@ -0,0 +1,6 @@
# git ls-files --others --exclude-from=.git/info/exclude
# Lines that start with '#' are comments.
# For a project mostly in C, the following would be a good set of
# exclude patterns (uncomment them if you want to use them):
# *.[oa]
# *~
modules/git/tests/repos/repo6_merge_sha256/info/refs
+4
View File
@@ -0,0 +1,4 @@
d2e5609f630dd8db500f5298d05d16def282412e3e66ed68cc7d0833b29129a1 refs/heads/main
b45258e9823233edea2d40d183742f29630e1e69300479fb4a55eabfe9b1d8bf refs/heads/merge/add_file
ff2b996e2fa366146300e4c9e51ccb6818147b360e46fa1437334f4a690955ce refs/heads/merge/modify_file
da1ded40dc8e5b7c564171f4bf2fc8370487decfb1cb6a99ef28f3ed73d09172 refs/heads/merge/remove_file
modules/git/tests/repos/repo6_merge_sha256/objects/info/commit-graph
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/objects/info/packs
+3
View File
@@ -0,0 +1,3 @@
P pack-2fff0848f8d8eab8f7902ac91ab6a096c7530f577d5c0a79c63d9ac2b44f7510.pack
P pack-65162b86afdbac3c566696d487e67bb2a4a5501ca1fa3528fad8a9474fba7e50.pack
modules/git/tests/repos/repo6_merge_sha256/objects/pack/pack-2fff0848f8d8eab8f7902ac91ab6a096c7530f577d5c0a79c63d9ac2b44f7510.bitmap
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/objects/pack/pack-2fff0848f8d8eab8f7902ac91ab6a096c7530f577d5c0a79c63d9ac2b44f7510.idx
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/objects/pack/pack-2fff0848f8d8eab8f7902ac91ab6a096c7530f577d5c0a79c63d9ac2b44f7510.pack
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/objects/pack/pack-2fff0848f8d8eab8f7902ac91ab6a096c7530f577d5c0a79c63d9ac2b44f7510.rev
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/objects/pack/pack-65162b86afdbac3c566696d487e67bb2a4a5501ca1fa3528fad8a9474fba7e50.idx
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/objects/pack/pack-65162b86afdbac3c566696d487e67bb2a4a5501ca1fa3528fad8a9474fba7e50.mtimes
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/objects/pack/pack-65162b86afdbac3c566696d487e67bb2a4a5501ca1fa3528fad8a9474fba7e50.pack
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/objects/pack/pack-65162b86afdbac3c566696d487e67bb2a4a5501ca1fa3528fad8a9474fba7e50.rev
BIN
View File
Binary file not shown.
modules/git/tests/repos/repo6_merge_sha256/packed-refs
+5
View File
@@ -0,0 +1,5 @@
# pack-refs with: peeled fully-peeled sorted
d2e5609f630dd8db500f5298d05d16def282412e3e66ed68cc7d0833b29129a1 refs/heads/main
b45258e9823233edea2d40d183742f29630e1e69300479fb4a55eabfe9b1d8bf refs/heads/merge/add_file
ff2b996e2fa366146300e4c9e51ccb6818147b360e46fa1437334f4a690955ce refs/heads/merge/modify_file
da1ded40dc8e5b7c564171f4bf2fc8370487decfb1cb6a99ef28f3ed73d09172 refs/heads/merge/remove_file
modules/git/tests/repos/repo6_merge_sha256/refs/heads/main
+1
View File
@@ -0,0 +1 @@
d2e5609f630dd8db500f5298d05d16def282412e3e66ed68cc7d0833b29129a1
modules/gitrepo/branch.go
+32
View File
@@ -0,0 +1,32 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package gitrepo
import (
"context"
"code.gitea.io/gitea/modules/git"
)
// GetBranchesByPath returns a branch by its path
// if limit = 0 it will not limit
func GetBranchesByPath(ctx context.Context, repo Repository, skip, limit int) ([]*git.Branch, int, error) {
gitRepo, err := OpenRepository(ctx, repo)
if err != nil {
return nil, 0, err
}
defer gitRepo.Close()
return gitRepo.GetBranches(skip, limit)
}
func GetBranchCommitID(ctx context.Context, repo Repository, branch string) (string, error) {
gitRepo, err := OpenRepository(ctx, repo)
if err != nil {
return "", err
}
defer gitRepo.Close()
return gitRepo.GetBranchCommitID(branch)
}
modules/gitrepo/gitrepo.go
+103
View File
@@ -0,0 +1,103 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package gitrepo
import (
"context"
"io"
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
)
type Repository interface {
GetName() string
GetOwnerName() string
}
func repoPath(repo Repository) string {
return filepath.Join(setting.RepoRootPath, strings.ToLower(repo.GetOwnerName()), strings.ToLower(repo.GetName())+".git")
}
func wikiPath(repo Repository) string {
return filepath.Join(setting.RepoRootPath, strings.ToLower(repo.GetOwnerName()), strings.ToLower(repo.GetName())+".wiki.git")
}
// OpenRepository opens the repository at the given relative path with the provided context.
func OpenRepository(ctx context.Context, repo Repository) (*git.Repository, error) {
return git.OpenRepository(ctx, repoPath(repo))
}
func OpenWikiRepository(ctx context.Context, repo Repository) (*git.Repository, error) {
return git.OpenRepository(ctx, wikiPath(repo))
}
// contextKey is a value for use with context.WithValue.
type contextKey struct {
name string
}
// RepositoryContextKey is a context key. It is used with context.Value() to get the current Repository for the context
var RepositoryContextKey = &contextKey{"repository"}
// RepositoryFromContext attempts to get the repository from the context
func repositoryFromContext(ctx context.Context, repo Repository) *git.Repository {
value := ctx.Value(RepositoryContextKey)
if value == nil {
return nil
}
if gitRepo, ok := value.(*git.Repository); ok && gitRepo != nil {
if gitRepo.Path == repoPath(repo) {
return gitRepo
}
}
return nil
}
type nopCloser func()
func (nopCloser) Close() error { return nil }
// RepositoryFromContextOrOpen attempts to get the repository from the context or just opens it
func RepositoryFromContextOrOpen(ctx context.Context, repo Repository) (*git.Repository, io.Closer, error) {
gitRepo := repositoryFromContext(ctx, repo)
if gitRepo != nil {
return gitRepo, nopCloser(nil), nil
}
gitRepo, err := OpenRepository(ctx, repo)
return gitRepo, gitRepo, err
}
// repositoryFromContextPath attempts to get the repository from the context
func repositoryFromContextPath(ctx context.Context, path string) *git.Repository {
value := ctx.Value(RepositoryContextKey)
if value == nil {
return nil
}
if repo, ok := value.(*git.Repository); ok && repo != nil {
if repo.Path == path {
return repo
}
}
return nil
}
// RepositoryFromContextOrOpenPath attempts to get the repository from the context or just opens it
// Deprecated: Use RepositoryFromContextOrOpen instead
func RepositoryFromContextOrOpenPath(ctx context.Context, path string) (*git.Repository, io.Closer, error) {
gitRepo := repositoryFromContextPath(ctx, path)
if gitRepo != nil {
return gitRepo, nopCloser(nil), nil
}
gitRepo, err := git.OpenRepository(ctx, path)
return gitRepo, gitRepo, err
}

Some files were not shown because too many files have changed in this diff Show More