1
1
mirror of https://github.com/go-gitea/gitea synced 2025-07-22 18:28:37 +00:00

Add a storage layer for attachments (#11387)

* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>
This commit is contained in:
Lunny Xiao
2020-08-18 12:23:45 +08:00
committed by GitHub
parent 02fbe1e5dc
commit 62e6c9bc6c
330 changed files with 62099 additions and 331 deletions

20
vendor/gopkg.in/ini.v1/.travis.yml generated vendored
View File

@@ -1,20 +0,0 @@
sudo: false
language: go
go:
- 1.6.x
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- 1.11.x
- 1.12.x
- 1.13.x
install: skip
script:
- go get golang.org/x/tools/cmd/cover
- go get github.com/smartystreets/goconvey
- mkdir -p $HOME/gopath/src/gopkg.in
- ln -s $HOME/gopath/src/github.com/go-ini/ini $HOME/gopath/src/gopkg.in/ini.v1
- cd $HOME/gopath/src/gopkg.in/ini.v1
- go test -v -cover -race

2
vendor/gopkg.in/ini.v1/Makefile generated vendored
View File

@@ -6,7 +6,7 @@ test:
go test -v -cover -race
bench:
go test -v -cover -race -test.bench=. -test.benchmem
go test -v -cover -test.bench=. -test.benchmem
vet:
go vet

8
vendor/gopkg.in/ini.v1/README.md generated vendored
View File

@@ -1,6 +1,9 @@
# INI
[![Build Status](https://img.shields.io/travis/go-ini/ini/master.svg?style=for-the-badge&logo=travis)](https://travis-ci.org/go-ini/ini) [![Sourcegraph](https://img.shields.io/badge/view%20on-Sourcegraph-brightgreen.svg?style=for-the-badge&logo=sourcegraph)](https://sourcegraph.com/github.com/go-ini/ini)
[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/go-ini/ini/Go?logo=github&style=for-the-badge)](https://github.com/go-ini/ini/actions?query=workflow%3AGo)
[![codecov](https://img.shields.io/codecov/c/github/go-ini/ini/master?logo=codecov&style=for-the-badge)](https://codecov.io/gh/go-ini/ini)
[![GoDoc](https://img.shields.io/badge/GoDoc-Reference-blue?style=for-the-badge&logo=go)](https://pkg.go.dev/github.com/go-ini/ini?tab=doc)
[![Sourcegraph](https://img.shields.io/badge/view%20on-Sourcegraph-brightgreen.svg?style=for-the-badge&logo=sourcegraph)](https://sourcegraph.com/github.com/go-ini/ini)
![](https://avatars0.githubusercontent.com/u/10216035?v=3&s=200)
@@ -8,7 +11,7 @@ Package ini provides INI file read and write functionality in Go.
## Features
- Load from multiple data sources(`[]byte`, file and `io.ReadCloser`) with overwrites.
- Load from multiple data sources(file, `[]byte`, `io.Reader` and `io.ReadCloser`) with overwrites.
- Read with recursion values.
- Read with parent-child sections.
- Read with auto-increment key names.
@@ -33,6 +36,7 @@ Please add `-u` flag to update in the future.
- [Getting Started](https://ini.unknwon.io/docs/intro/getting_started)
- [API Documentation](https://gowalker.org/gopkg.in/ini.v1)
- 中国大陆镜像https://ini.unknwon.cn
## License

9
vendor/gopkg.in/ini.v1/codecov.yml generated vendored Normal file
View File

@@ -0,0 +1,9 @@
coverage:
range: "60...95"
status:
project:
default:
threshold: 1%
comment:
layout: 'diff, files'

View File

@@ -68,6 +68,8 @@ func parseDataSource(source interface{}) (dataSource, error) {
return &sourceData{s}, nil
case io.ReadCloser:
return &sourceReadCloser{s}, nil
case io.Reader:
return &sourceReadCloser{ioutil.NopCloser(s)}, nil
default:
return nil, fmt.Errorf("error parsing data source: unknown type %q", s)
}

159
vendor/gopkg.in/ini.v1/file.go generated vendored
View File

@@ -25,7 +25,7 @@ import (
"sync"
)
// File represents a combination of a or more INI file(s) in memory.
// File represents a combination of one or more INI files in memory.
type File struct {
options LoadOptions
dataSources []dataSource
@@ -36,8 +36,12 @@ type File struct {
// To keep data in order.
sectionList []string
// To keep track of the index of a section with same name.
// This meta list is only used with non-unique section names are allowed.
sectionIndexes []int
// Actual data is stored here.
sections map[string]*Section
sections map[string][]*Section
NameMapper
ValueMapper
@@ -48,27 +52,37 @@ func newFile(dataSources []dataSource, opts LoadOptions) *File {
if len(opts.KeyValueDelimiters) == 0 {
opts.KeyValueDelimiters = "=:"
}
if len(opts.KeyValueDelimiterOnWrite) == 0 {
opts.KeyValueDelimiterOnWrite = "="
}
return &File{
BlockMode: true,
dataSources: dataSources,
sections: make(map[string]*Section),
sectionList: make([]string, 0, 10),
sections: make(map[string][]*Section),
options: opts,
}
}
// Empty returns an empty file object.
func Empty() *File {
// Ignore error here, we sure our data is good.
f, _ := Load([]byte(""))
func Empty(opts ...LoadOptions) *File {
var opt LoadOptions
if len(opts) > 0 {
opt = opts[0]
}
// Ignore error here, we are sure our data is good.
f, _ := LoadSources(opt, []byte(""))
return f
}
// NewSection creates a new section.
func (f *File) NewSection(name string) (*Section, error) {
if len(name) == 0 {
return nil, errors.New("error creating new section: empty section name")
} else if f.options.Insensitive && name != DefaultSection {
return nil, errors.New("empty section name")
}
if f.options.Insensitive && name != DefaultSection {
name = strings.ToLower(name)
}
@@ -77,13 +91,20 @@ func (f *File) NewSection(name string) (*Section, error) {
defer f.lock.Unlock()
}
if inSlice(name, f.sectionList) {
return f.sections[name], nil
if !f.options.AllowNonUniqueSections && inSlice(name, f.sectionList) {
return f.sections[name][0], nil
}
f.sectionList = append(f.sectionList, name)
f.sections[name] = newSection(f, name)
return f.sections[name], nil
// NOTE: Append to indexes must happen before appending to sections,
// otherwise index will have off-by-one problem.
f.sectionIndexes = append(f.sectionIndexes, len(f.sections[name]))
sec := newSection(f, name)
f.sections[name] = append(f.sections[name], sec)
return sec, nil
}
// NewRawSection creates a new section with an unparseable body.
@@ -110,6 +131,16 @@ func (f *File) NewSections(names ...string) (err error) {
// GetSection returns section by given name.
func (f *File) GetSection(name string) (*Section, error) {
secs, err := f.SectionsByName(name)
if err != nil {
return nil, err
}
return secs[0], err
}
// SectionsByName returns all sections with given name.
func (f *File) SectionsByName(name string) ([]*Section, error) {
if len(name) == 0 {
name = DefaultSection
}
@@ -122,11 +153,12 @@ func (f *File) GetSection(name string) (*Section, error) {
defer f.lock.RUnlock()
}
sec := f.sections[name]
if sec == nil {
return nil, fmt.Errorf("section '%s' does not exist", name)
secs := f.sections[name]
if len(secs) == 0 {
return nil, fmt.Errorf("section %q does not exist", name)
}
return sec, nil
return secs, nil
}
// Section assumes named section exists and returns a zero-value when not.
@@ -141,6 +173,19 @@ func (f *File) Section(name string) *Section {
return sec
}
// SectionWithIndex assumes named section exists and returns a new section when not.
func (f *File) SectionWithIndex(name string, index int) *Section {
secs, err := f.SectionsByName(name)
if err != nil || len(secs) <= index {
// NOTE: It's OK here because the only possible error is empty section name,
// but if it's empty, this piece of code won't be executed.
newSec, _ := f.NewSection(name)
return newSec
}
return secs[index]
}
// Sections returns a list of Section stored in the current instance.
func (f *File) Sections() []*Section {
if f.BlockMode {
@@ -150,7 +195,7 @@ func (f *File) Sections() []*Section {
sections := make([]*Section, len(f.sectionList))
for i, name := range f.sectionList {
sections[i] = f.sections[name]
sections[i] = f.sections[name][f.sectionIndexes[i]]
}
return sections
}
@@ -167,24 +212,70 @@ func (f *File) SectionStrings() []string {
return list
}
// DeleteSection deletes a section.
// DeleteSection deletes a section or all sections with given name.
func (f *File) DeleteSection(name string) {
if f.BlockMode {
f.lock.Lock()
defer f.lock.Unlock()
secs, err := f.SectionsByName(name)
if err != nil {
return
}
for i := 0; i < len(secs); i++ {
// For non-unique sections, it is always needed to remove the first one so
// in the next iteration, the subsequent section continue having index 0.
// Ignoring the error as index 0 never returns an error.
_ = f.DeleteSectionWithIndex(name, 0)
}
}
// DeleteSectionWithIndex deletes a section with given name and index.
func (f *File) DeleteSectionWithIndex(name string, index int) error {
if !f.options.AllowNonUniqueSections && index != 0 {
return fmt.Errorf("delete section with non-zero index is only allowed when non-unique sections is enabled")
}
if len(name) == 0 {
name = DefaultSection
}
for i, s := range f.sectionList {
if s == name {
f.sectionList = append(f.sectionList[:i], f.sectionList[i+1:]...)
delete(f.sections, name)
return
}
if f.options.Insensitive {
name = strings.ToLower(name)
}
if f.BlockMode {
f.lock.Lock()
defer f.lock.Unlock()
}
// Count occurrences of the sections
occurrences := 0
sectionListCopy := make([]string, len(f.sectionList))
copy(sectionListCopy, f.sectionList)
for i, s := range sectionListCopy {
if s != name {
continue
}
if occurrences == index {
if len(f.sections[name]) <= 1 {
delete(f.sections, name) // The last one in the map
} else {
f.sections[name] = append(f.sections[name][:index], f.sections[name][index+1:]...)
}
// Fix section lists
f.sectionList = append(f.sectionList[:i], f.sectionList[i+1:]...)
f.sectionIndexes = append(f.sectionIndexes[:i], f.sectionIndexes[i+1:]...)
} else if occurrences > index {
// Fix the indices of all following sections with this name.
f.sectionIndexes[i-1]--
}
occurrences++
}
return nil
}
func (f *File) reload(s dataSource) error {
@@ -203,7 +294,7 @@ func (f *File) Reload() (err error) {
if err = f.reload(s); err != nil {
// In loose mode, we create an empty default section for nonexistent files.
if os.IsNotExist(err) && f.options.Loose {
f.parse(bytes.NewBuffer(nil))
_ = f.parse(bytes.NewBuffer(nil))
continue
}
return err
@@ -230,16 +321,16 @@ func (f *File) Append(source interface{}, others ...interface{}) error {
}
func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) {
equalSign := DefaultFormatLeft + "=" + DefaultFormatRight
equalSign := DefaultFormatLeft + f.options.KeyValueDelimiterOnWrite + DefaultFormatRight
if PrettyFormat || PrettyEqual {
equalSign = " = "
equalSign = fmt.Sprintf(" %s ", f.options.KeyValueDelimiterOnWrite)
}
// Use buffer to make sure target is safe until finish encoding.
buf := bytes.NewBuffer(nil)
for i, sname := range f.sectionList {
sec := f.Section(sname)
sec := f.SectionWithIndex(sname, f.sectionIndexes[i])
if len(sec.Comment) > 0 {
// Support multiline comments
lines := strings.Split(sec.Comment, LineBreak)
@@ -282,7 +373,7 @@ func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) {
}
// Count and generate alignment length and buffer spaces using the
// longest key. Keys may be modifed if they contain certain characters so
// longest key. Keys may be modified if they contain certain characters so
// we need to take that into account in our calculation.
alignLength := 0
if PrettyFormat {

16
vendor/gopkg.in/ini.v1/ini.go generated vendored
View File

@@ -18,8 +18,10 @@
package ini
import (
"os"
"regexp"
"runtime"
"strings"
)
const (
@@ -29,14 +31,8 @@ const (
// Maximum allowed depth when recursively substituing variable names.
depthValues = 99
version = "1.52.0"
)
// Version returns current package version literal.
func Version() string {
return version
}
var (
// LineBreak is the delimiter to determine or compose a new line.
// This variable will be changed to "\r\n" automatically on Windows at package init time.
@@ -61,8 +57,10 @@ var (
DefaultFormatRight = ""
)
var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test")
func init() {
if runtime.GOOS == "windows" {
if runtime.GOOS == "windows" && !inTest {
LineBreak = "\r\n"
}
}
@@ -109,12 +107,16 @@ type LoadOptions struct {
UnparseableSections []string
// KeyValueDelimiters is the sequence of delimiters that are used to separate key and value. By default, it is "=:".
KeyValueDelimiters string
// KeyValueDelimiters is the delimiter that are used to separate key and value output. By default, it is "=".
KeyValueDelimiterOnWrite string
// PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes).
PreserveSurroundedQuote bool
// DebugFunc is called to collect debug information (currently only useful to debug parsing Python-style multiline values).
DebugFunc DebugFunc
// ReaderBufferSize is the buffer size of the reader in bytes.
ReaderBufferSize int
// AllowNonUniqueSections indicates whether to allow sections with the same name multiple times.
AllowNonUniqueSections bool
}
// DebugFunc is the type of function called to log parse events.

120
vendor/gopkg.in/ini.v1/key.go generated vendored
View File

@@ -686,99 +686,127 @@ func (k *Key) StrictTimes(delim string) ([]time.Time, error) {
// parseBools transforms strings to bools.
func (k *Key) parseBools(strs []string, addInvalid, returnOnInvalid bool) ([]bool, error) {
vals := make([]bool, 0, len(strs))
for _, str := range strs {
parser := func(str string) (interface{}, error) {
val, err := parseBool(str)
if err != nil && returnOnInvalid {
return nil, err
}
if err == nil || addInvalid {
vals = append(vals, val)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(bool))
}
}
return vals, nil
return vals, err
}
// parseFloat64s transforms strings to float64s.
func (k *Key) parseFloat64s(strs []string, addInvalid, returnOnInvalid bool) ([]float64, error) {
vals := make([]float64, 0, len(strs))
for _, str := range strs {
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseFloat(str, 64)
if err != nil && returnOnInvalid {
return nil, err
}
if err == nil || addInvalid {
vals = append(vals, val)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(float64))
}
}
return vals, nil
return vals, err
}
// parseInts transforms strings to ints.
func (k *Key) parseInts(strs []string, addInvalid, returnOnInvalid bool) ([]int, error) {
vals := make([]int, 0, len(strs))
for _, str := range strs {
valInt64, err := strconv.ParseInt(str, 0, 64)
val := int(valInt64)
if err != nil && returnOnInvalid {
return nil, err
}
if err == nil || addInvalid {
vals = append(vals, val)
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseInt(str, 0, 64)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, int(val.(int64)))
}
}
return vals, nil
return vals, err
}
// parseInt64s transforms strings to int64s.
func (k *Key) parseInt64s(strs []string, addInvalid, returnOnInvalid bool) ([]int64, error) {
vals := make([]int64, 0, len(strs))
for _, str := range strs {
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseInt(str, 0, 64)
if err != nil && returnOnInvalid {
return nil, err
}
if err == nil || addInvalid {
vals = append(vals, val)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(int64))
}
}
return vals, nil
return vals, err
}
// parseUints transforms strings to uints.
func (k *Key) parseUints(strs []string, addInvalid, returnOnInvalid bool) ([]uint, error) {
vals := make([]uint, 0, len(strs))
for _, str := range strs {
val, err := strconv.ParseUint(str, 0, 0)
if err != nil && returnOnInvalid {
return nil, err
}
if err == nil || addInvalid {
vals = append(vals, uint(val))
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseUint(str, 0, 64)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, uint(val.(uint64)))
}
}
return vals, nil
return vals, err
}
// parseUint64s transforms strings to uint64s.
func (k *Key) parseUint64s(strs []string, addInvalid, returnOnInvalid bool) ([]uint64, error) {
vals := make([]uint64, 0, len(strs))
for _, str := range strs {
parser := func(str string) (interface{}, error) {
val, err := strconv.ParseUint(str, 0, 64)
if err != nil && returnOnInvalid {
return nil, err
}
if err == nil || addInvalid {
vals = append(vals, val)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(uint64))
}
}
return vals, nil
return vals, err
}
type Parser func(str string) (interface{}, error)
// parseTimesFormat transforms strings to times in given format.
func (k *Key) parseTimesFormat(format string, strs []string, addInvalid, returnOnInvalid bool) ([]time.Time, error) {
vals := make([]time.Time, 0, len(strs))
for _, str := range strs {
parser := func(str string) (interface{}, error) {
val, err := time.Parse(format, str)
return val, err
}
rawVals, err := k.doParse(strs, addInvalid, returnOnInvalid, parser)
if err == nil {
for _, val := range rawVals {
vals = append(vals, val.(time.Time))
}
}
return vals, err
}
// doParse transforms strings to different types
func (k *Key) doParse(strs []string, addInvalid, returnOnInvalid bool, parser Parser) ([]interface{}, error) {
vals := make([]interface{}, 0, len(strs))
for _, str := range strs {
val, err := parser(str)
if err != nil && returnOnInvalid {
return nil, err
}

21
vendor/gopkg.in/ini.v1/parser.go generated vendored
View File

@@ -84,7 +84,10 @@ func (p *parser) BOM() error {
case mask[0] == 254 && mask[1] == 255:
fallthrough
case mask[0] == 255 && mask[1] == 254:
p.buf.Read(mask)
_, err = p.buf.Read(mask)
if err != nil {
return err
}
case mask[0] == 239 && mask[1] == 187:
mask, err := p.buf.Peek(3)
if err != nil && err != io.EOF {
@@ -93,7 +96,10 @@ func (p *parser) BOM() error {
return nil
}
if mask[2] == 191 {
p.buf.Read(mask)
_, err = p.buf.Read(mask)
if err != nil {
return err
}
}
}
return nil
@@ -135,7 +141,7 @@ func readKeyName(delimiters string, in []byte) (string, int, error) {
}
// Get out key name
endIdx := -1
var endIdx int
if len(keyQuote) > 0 {
startIdx := len(keyQuote)
// FIXME: fail case -> """"""name"""=value
@@ -181,7 +187,7 @@ func (p *parser) readMultilines(line, val, valQuote string) (string, error) {
}
val += next
if p.isEOF {
return "", fmt.Errorf("missing closing key quote from '%s' to '%s'", line, next)
return "", fmt.Errorf("missing closing key quote from %q to %q", line, next)
}
}
return val, nil
@@ -413,7 +419,10 @@ func (f *File) parse(reader io.Reader) (err error) {
if f.options.AllowNestedValues &&
isLastValueEmpty && len(line) > 0 {
if line[0] == ' ' || line[0] == '\t' {
lastRegularKey.addNestedValue(string(bytes.TrimSpace(line)))
err = lastRegularKey.addNestedValue(string(bytes.TrimSpace(line)))
if err != nil {
return err
}
continue
}
}
@@ -460,7 +469,7 @@ func (f *File) parse(reader io.Reader) (err error) {
inUnparseableSection = false
for i := range f.options.UnparseableSections {
if f.options.UnparseableSections[i] == name ||
(f.options.Insensitive && strings.ToLower(f.options.UnparseableSections[i]) == strings.ToLower(name)) {
(f.options.Insensitive && strings.EqualFold(f.options.UnparseableSections[i], name)) {
inUnparseableSection = true
continue
}

4
vendor/gopkg.in/ini.v1/section.go generated vendored
View File

@@ -131,7 +131,7 @@ func (s *Section) GetKey(name string) (*Key, error) {
}
break
}
return nil, fmt.Errorf("error when getting key of section '%s': key '%s' not exists", s.name, name)
return nil, fmt.Errorf("error when getting key of section %q: key %q not exists", s.name, name)
}
return key, nil
}
@@ -249,7 +249,7 @@ func (s *Section) ChildSections() []*Section {
children := make([]*Section, 0, 3)
for _, name := range s.f.sectionList {
if strings.HasPrefix(name, prefix) {
children = append(children, s.f.sections[name])
children = append(children, s.f.sections[name]...)
}
}
return children

178
vendor/gopkg.in/ini.v1/struct.go generated vendored
View File

@@ -258,13 +258,13 @@ func setWithProperType(t reflect.Type, key *Key, field reflect.Value, delim stri
case reflect.Slice:
return setSliceWithProperType(key, field, delim, allowShadow, isStrict)
default:
return fmt.Errorf("unsupported type '%s'", t)
return fmt.Errorf("unsupported type %q", t)
}
return nil
}
func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool) {
opts := strings.SplitN(tag, ",", 3)
func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool, allowNonUnique bool) {
opts := strings.SplitN(tag, ",", 4)
rawName = opts[0]
if len(opts) > 1 {
omitEmpty = opts[1] == "omitempty"
@@ -272,10 +272,15 @@ func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bo
if len(opts) > 2 {
allowShadow = opts[2] == "allowshadow"
}
return rawName, omitEmpty, allowShadow
if len(opts) > 3 {
allowNonUnique = opts[3] == "nonunique"
}
return rawName, omitEmpty, allowShadow, allowNonUnique
}
func (s *Section) mapTo(val reflect.Value, isStrict bool) error {
// mapToField maps the given value to the matching field of the given section.
// The sectionIndex is the index (if non unique sections are enabled) to which the value should be added.
func (s *Section) mapToField(val reflect.Value, isStrict bool, sectionIndex int) error {
if val.Kind() == reflect.Ptr {
val = val.Elem()
}
@@ -290,7 +295,7 @@ func (s *Section) mapTo(val reflect.Value, isStrict bool) error {
continue
}
rawName, _, allowShadow := parseTagOptions(tag)
rawName, _, allowShadow, allowNonUnique := parseTagOptions(tag)
fieldName := s.parseFieldName(tpField.Name, rawName)
if len(fieldName) == 0 || !field.CanSet() {
continue
@@ -304,55 +309,96 @@ func (s *Section) mapTo(val reflect.Value, isStrict bool) error {
}
if isAnonymous || isStruct || isStructPtr {
if sec, err := s.f.GetSection(fieldName); err == nil {
if secs, err := s.f.SectionsByName(fieldName); err == nil {
if len(secs) <= sectionIndex {
return fmt.Errorf("there are not enough sections (%d <= %d) for the field %q", len(secs), sectionIndex, fieldName)
}
// Only set the field to non-nil struct value if we have a section for it.
// Otherwise, we end up with a non-nil struct ptr even though there is no data.
if isStructPtr && field.IsNil() {
field.Set(reflect.New(tpField.Type.Elem()))
}
if err = sec.mapTo(field, isStrict); err != nil {
return fmt.Errorf("error mapping field %q: %v", fieldName, err)
if err = secs[sectionIndex].mapToField(field, isStrict, sectionIndex); err != nil {
return fmt.Errorf("map to field %q: %v", fieldName, err)
}
continue
}
}
// Map non-unique sections
if allowNonUnique && tpField.Type.Kind() == reflect.Slice {
newField, err := s.mapToSlice(fieldName, field, isStrict)
if err != nil {
return fmt.Errorf("map to slice %q: %v", fieldName, err)
}
field.Set(newField)
continue
}
if key, err := s.GetKey(fieldName); err == nil {
delim := parseDelim(tpField.Tag.Get("delim"))
if err = setWithProperType(tpField.Type, key, field, delim, allowShadow, isStrict); err != nil {
return fmt.Errorf("error mapping field %q: %v", fieldName, err)
return fmt.Errorf("set field %q: %v", fieldName, err)
}
}
}
return nil
}
// MapTo maps section to given struct.
func (s *Section) MapTo(v interface{}) error {
// mapToSlice maps all sections with the same name and returns the new value.
// The type of the Value must be a slice.
func (s *Section) mapToSlice(secName string, val reflect.Value, isStrict bool) (reflect.Value, error) {
secs, err := s.f.SectionsByName(secName)
if err != nil {
return reflect.Value{}, err
}
typ := val.Type().Elem()
for i, sec := range secs {
elem := reflect.New(typ)
if err = sec.mapToField(elem, isStrict, i); err != nil {
return reflect.Value{}, fmt.Errorf("map to field from section %q: %v", secName, err)
}
val = reflect.Append(val, elem.Elem())
}
return val, nil
}
// mapTo maps a section to object v.
func (s *Section) mapTo(v interface{}, isStrict bool) error {
typ := reflect.TypeOf(v)
val := reflect.ValueOf(v)
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
val = val.Elem()
} else {
return errors.New("cannot map to non-pointer struct")
return errors.New("not a pointer to a struct")
}
return s.mapTo(val, false)
if typ.Kind() == reflect.Slice {
newField, err := s.mapToSlice(s.name, val, isStrict)
if err != nil {
return err
}
val.Set(newField)
return nil
}
return s.mapToField(val, isStrict, 0)
}
// MapTo maps section to given struct.
func (s *Section) MapTo(v interface{}) error {
return s.mapTo(v, false)
}
// StrictMapTo maps section to given struct in strict mode,
// which returns all possible error including value parsing error.
func (s *Section) StrictMapTo(v interface{}) error {
typ := reflect.TypeOf(v)
val := reflect.ValueOf(v)
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
val = val.Elem()
} else {
return errors.New("cannot map to non-pointer struct")
}
return s.mapTo(val, true)
return s.mapTo(v, true)
}
// MapTo maps file to given struct.
@@ -430,7 +476,7 @@ func reflectSliceWithProperType(key *Key, field reflect.Value, delim string, all
if i == 0 {
keyWithShadows = newKey(key.s, key.name, val)
} else {
keyWithShadows.AddShadow(val)
_ = keyWithShadows.AddShadow(val)
}
}
key = keyWithShadows
@@ -483,7 +529,7 @@ func reflectWithProperType(t reflect.Type, key *Key, field reflect.Value, delim
return reflectWithProperType(t.Elem(), key, field.Elem(), delim, allowShadow)
}
default:
return fmt.Errorf("unsupported type '%s'", t)
return fmt.Errorf("unsupported type %q", t)
}
return nil
}
@@ -523,6 +569,10 @@ func (s *Section) reflectFrom(val reflect.Value) error {
typ := val.Type()
for i := 0; i < typ.NumField(); i++ {
if !val.Field(i).CanInterface() {
continue
}
field := val.Field(i)
tpField := typ.Field(i)
@@ -531,7 +581,7 @@ func (s *Section) reflectFrom(val reflect.Value) error {
continue
}
rawName, omitEmpty, allowShadow := parseTagOptions(tag)
rawName, omitEmpty, allowShadow, allowNonUnique := parseTagOptions(tag)
if omitEmpty && isEmptyValue(field) {
continue
}
@@ -560,11 +610,41 @@ func (s *Section) reflectFrom(val reflect.Value) error {
}
if err = sec.reflectFrom(field); err != nil {
return fmt.Errorf("error reflecting field %q: %v", fieldName, err)
return fmt.Errorf("reflect from field %q: %v", fieldName, err)
}
continue
}
if allowNonUnique && tpField.Type.Kind() == reflect.Slice {
slice := field.Slice(0, field.Len())
if field.Len() == 0 {
return nil
}
sliceOf := field.Type().Elem().Kind()
for i := 0; i < field.Len(); i++ {
if sliceOf != reflect.Struct && sliceOf != reflect.Ptr {
return fmt.Errorf("field %q is not a slice of pointer or struct", fieldName)
}
sec, err := s.f.NewSection(fieldName)
if err != nil {
return err
}
// Add comment from comment tag
if len(sec.Comment) == 0 {
sec.Comment = tpField.Tag.Get("comment")
}
if err := sec.reflectFrom(slice.Index(i)); err != nil {
return fmt.Errorf("reflect from field %q: %v", fieldName, err)
}
}
continue
}
// Note: Same reason as section.
key, err := s.GetKey(fieldName)
if err != nil {
key, _ = s.NewKey(fieldName, "")
@@ -577,22 +657,56 @@ func (s *Section) reflectFrom(val reflect.Value) error {
delim := parseDelim(tpField.Tag.Get("delim"))
if err = reflectWithProperType(tpField.Type, key, field, delim, allowShadow); err != nil {
return fmt.Errorf("error reflecting field %q: %v", fieldName, err)
return fmt.Errorf("reflect field %q: %v", fieldName, err)
}
}
return nil
}
// ReflectFrom reflects secion from given struct.
// ReflectFrom reflects section from given struct. It overwrites existing ones.
func (s *Section) ReflectFrom(v interface{}) error {
typ := reflect.TypeOf(v)
val := reflect.ValueOf(v)
if s.name != DefaultSection && s.f.options.AllowNonUniqueSections &&
(typ.Kind() == reflect.Slice || typ.Kind() == reflect.Ptr) {
// Clear sections to make sure none exists before adding the new ones
s.f.DeleteSection(s.name)
if typ.Kind() == reflect.Ptr {
sec, err := s.f.NewSection(s.name)
if err != nil {
return err
}
return sec.reflectFrom(val.Elem())
}
slice := val.Slice(0, val.Len())
sliceOf := val.Type().Elem().Kind()
if sliceOf != reflect.Ptr {
return fmt.Errorf("not a slice of pointers")
}
for i := 0; i < slice.Len(); i++ {
sec, err := s.f.NewSection(s.name)
if err != nil {
return err
}
err = sec.reflectFrom(slice.Index(i))
if err != nil {
return fmt.Errorf("reflect from %dth field: %v", i, err)
}
}
return nil
}
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
val = val.Elem()
} else {
return errors.New("cannot reflect from non-pointer struct")
return errors.New("not a pointer to a struct")
}
return s.reflectFrom(val)