Add a storage layer for attachments (#11387)

* Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
2025-07-15 23:17:19 +00:00 · 2020-08-18 12:23:45 +08:00
parent 02fbe1e5dc
commit 62e6c9bc6c
330 changed files with 62099 additions and 331 deletions
--- a/vendor/gopkg.in/ini.v1/struct.go
+++ b/vendor/gopkg.in/ini.v1/struct.go
@@ -258,13 +258,13 @@ func setWithProperType(t reflect.Type, key *Key, field reflect.Value, delim stri
 	case reflect.Slice:
 		return setSliceWithProperType(key, field, delim, allowShadow, isStrict)
 	default:
-		return fmt.Errorf("unsupported type '%s'", t)
+		return fmt.Errorf("unsupported type %q", t)
 	}
 	return nil
 }

-func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool) {
-	opts := strings.SplitN(tag, ",", 3)
+func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool, allowNonUnique bool) {
+	opts := strings.SplitN(tag, ",", 4)
 	rawName = opts[0]
 	if len(opts) > 1 {
 		omitEmpty = opts[1] == "omitempty"
@@ -272,10 +272,15 @@ func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bo
 	if len(opts) > 2 {
 		allowShadow = opts[2] == "allowshadow"
 	}
-	return rawName, omitEmpty, allowShadow
+	if len(opts) > 3 {
+		allowNonUnique = opts[3] == "nonunique"
+	}
+	return rawName, omitEmpty, allowShadow, allowNonUnique
 }

-func (s *Section) mapTo(val reflect.Value, isStrict bool) error {
+// mapToField maps the given value to the matching field of the given section.
+// The sectionIndex is the index (if non unique sections are enabled) to which the value should be added.
+func (s *Section) mapToField(val reflect.Value, isStrict bool, sectionIndex int) error {
 	if val.Kind() == reflect.Ptr {
 		val = val.Elem()
 	}
@@ -290,7 +295,7 @@ func (s *Section) mapTo(val reflect.Value, isStrict bool) error {
 			continue
 		}

-		rawName, _, allowShadow := parseTagOptions(tag)
+		rawName, _, allowShadow, allowNonUnique := parseTagOptions(tag)
 		fieldName := s.parseFieldName(tpField.Name, rawName)
 		if len(fieldName) == 0 || !field.CanSet() {
 			continue
@@ -304,55 +309,96 @@ func (s *Section) mapTo(val reflect.Value, isStrict bool) error {
 		}

 		if isAnonymous || isStruct || isStructPtr {
-			if sec, err := s.f.GetSection(fieldName); err == nil {
+			if secs, err := s.f.SectionsByName(fieldName); err == nil {
+				if len(secs) <= sectionIndex {
+					return fmt.Errorf("there are not enough sections (%d <= %d) for the field %q", len(secs), sectionIndex, fieldName)
+				}
 				// Only set the field to non-nil struct value if we have a section for it.
 				// Otherwise, we end up with a non-nil struct ptr even though there is no data.
 				if isStructPtr && field.IsNil() {
 					field.Set(reflect.New(tpField.Type.Elem()))
 				}
-				if err = sec.mapTo(field, isStrict); err != nil {
-					return fmt.Errorf("error mapping field %q: %v", fieldName, err)
+				if err = secs[sectionIndex].mapToField(field, isStrict, sectionIndex); err != nil {
+					return fmt.Errorf("map to field %q: %v", fieldName, err)
 				}
 				continue
 			}
 		}
+
+		// Map non-unique sections
+		if allowNonUnique && tpField.Type.Kind() == reflect.Slice {
+			newField, err := s.mapToSlice(fieldName, field, isStrict)
+			if err != nil {
+				return fmt.Errorf("map to slice %q: %v", fieldName, err)
+			}
+
+			field.Set(newField)
+			continue
+		}
+
 		if key, err := s.GetKey(fieldName); err == nil {
 			delim := parseDelim(tpField.Tag.Get("delim"))
 			if err = setWithProperType(tpField.Type, key, field, delim, allowShadow, isStrict); err != nil {
-				return fmt.Errorf("error mapping field %q: %v", fieldName, err)
+				return fmt.Errorf("set field %q: %v", fieldName, err)
 			}
 		}
 	}
 	return nil
 }

-// MapTo maps section to given struct.
-func (s *Section) MapTo(v interface{}) error {
+// mapToSlice maps all sections with the same name and returns the new value.
+// The type of the Value must be a slice.
+func (s *Section) mapToSlice(secName string, val reflect.Value, isStrict bool) (reflect.Value, error) {
+	secs, err := s.f.SectionsByName(secName)
+	if err != nil {
+		return reflect.Value{}, err
+	}
+
+	typ := val.Type().Elem()
+	for i, sec := range secs {
+		elem := reflect.New(typ)
+		if err = sec.mapToField(elem, isStrict, i); err != nil {
+			return reflect.Value{}, fmt.Errorf("map to field from section %q: %v", secName, err)
+		}
+
+		val = reflect.Append(val, elem.Elem())
+	}
+	return val, nil
+}
+
+// mapTo maps a section to object v.
+func (s *Section) mapTo(v interface{}, isStrict bool) error {
 	typ := reflect.TypeOf(v)
 	val := reflect.ValueOf(v)
 	if typ.Kind() == reflect.Ptr {
 		typ = typ.Elem()
 		val = val.Elem()
 	} else {
-		return errors.New("cannot map to non-pointer struct")
+		return errors.New("not a pointer to a struct")
 	}

-	return s.mapTo(val, false)
+	if typ.Kind() == reflect.Slice {
+		newField, err := s.mapToSlice(s.name, val, isStrict)
+		if err != nil {
+			return err
+		}
+
+		val.Set(newField)
+		return nil
+	}
+
+	return s.mapToField(val, isStrict, 0)
+}
+
+// MapTo maps section to given struct.
+func (s *Section) MapTo(v interface{}) error {
+	return s.mapTo(v, false)
 }

 // StrictMapTo maps section to given struct in strict mode,
 // which returns all possible error including value parsing error.
 func (s *Section) StrictMapTo(v interface{}) error {
-	typ := reflect.TypeOf(v)
-	val := reflect.ValueOf(v)
-	if typ.Kind() == reflect.Ptr {
-		typ = typ.Elem()
-		val = val.Elem()
-	} else {
-		return errors.New("cannot map to non-pointer struct")
-	}
-
-	return s.mapTo(val, true)
+	return s.mapTo(v, true)
 }

 // MapTo maps file to given struct.
@@ -430,7 +476,7 @@ func reflectSliceWithProperType(key *Key, field reflect.Value, delim string, all
 			if i == 0 {
 				keyWithShadows = newKey(key.s, key.name, val)
 			} else {
-				keyWithShadows.AddShadow(val)
+				_ = keyWithShadows.AddShadow(val)
 			}
 		}
 		key = keyWithShadows
@@ -483,7 +529,7 @@ func reflectWithProperType(t reflect.Type, key *Key, field reflect.Value, delim
 			return reflectWithProperType(t.Elem(), key, field.Elem(), delim, allowShadow)
 		}
 	default:
-		return fmt.Errorf("unsupported type '%s'", t)
+		return fmt.Errorf("unsupported type %q", t)
 	}
 	return nil
 }
@@ -523,6 +569,10 @@ func (s *Section) reflectFrom(val reflect.Value) error {
 	typ := val.Type()

 	for i := 0; i < typ.NumField(); i++ {
+		if !val.Field(i).CanInterface() {
+			continue
+		}
+
 		field := val.Field(i)
 		tpField := typ.Field(i)

@@ -531,7 +581,7 @@ func (s *Section) reflectFrom(val reflect.Value) error {
 			continue
 		}

-		rawName, omitEmpty, allowShadow := parseTagOptions(tag)
+		rawName, omitEmpty, allowShadow, allowNonUnique := parseTagOptions(tag)
 		if omitEmpty && isEmptyValue(field) {
 			continue
 		}
@@ -560,11 +610,41 @@ func (s *Section) reflectFrom(val reflect.Value) error {
 			}

 			if err = sec.reflectFrom(field); err != nil {
-				return fmt.Errorf("error reflecting field %q: %v", fieldName, err)
+				return fmt.Errorf("reflect from field %q: %v", fieldName, err)
 			}
 			continue
 		}

+		if allowNonUnique && tpField.Type.Kind() == reflect.Slice {
+			slice := field.Slice(0, field.Len())
+			if field.Len() == 0 {
+				return nil
+			}
+			sliceOf := field.Type().Elem().Kind()
+
+			for i := 0; i < field.Len(); i++ {
+				if sliceOf != reflect.Struct && sliceOf != reflect.Ptr {
+					return fmt.Errorf("field %q is not a slice of pointer or struct", fieldName)
+				}
+
+				sec, err := s.f.NewSection(fieldName)
+				if err != nil {
+					return err
+				}
+
+				// Add comment from comment tag
+				if len(sec.Comment) == 0 {
+					sec.Comment = tpField.Tag.Get("comment")
+				}
+
+				if err := sec.reflectFrom(slice.Index(i)); err != nil {
+					return fmt.Errorf("reflect from field %q: %v", fieldName, err)
+				}
+			}
+			continue
+		}
+
+		// Note: Same reason as section.
 		key, err := s.GetKey(fieldName)
 		if err != nil {
 			key, _ = s.NewKey(fieldName, "")
@@ -577,22 +657,56 @@ func (s *Section) reflectFrom(val reflect.Value) error {

 		delim := parseDelim(tpField.Tag.Get("delim"))
 		if err = reflectWithProperType(tpField.Type, key, field, delim, allowShadow); err != nil {
-			return fmt.Errorf("error reflecting field %q: %v", fieldName, err)
+			return fmt.Errorf("reflect field %q: %v", fieldName, err)
 		}

 	}
 	return nil
 }

-// ReflectFrom reflects secion from given struct.
+// ReflectFrom reflects section from given struct. It overwrites existing ones.
 func (s *Section) ReflectFrom(v interface{}) error {
 	typ := reflect.TypeOf(v)
 	val := reflect.ValueOf(v)
+
+	if s.name != DefaultSection && s.f.options.AllowNonUniqueSections &&
+		(typ.Kind() == reflect.Slice || typ.Kind() == reflect.Ptr) {
+		// Clear sections to make sure none exists before adding the new ones
+		s.f.DeleteSection(s.name)
+
+		if typ.Kind() == reflect.Ptr {
+			sec, err := s.f.NewSection(s.name)
+			if err != nil {
+				return err
+			}
+			return sec.reflectFrom(val.Elem())
+		}
+
+		slice := val.Slice(0, val.Len())
+		sliceOf := val.Type().Elem().Kind()
+		if sliceOf != reflect.Ptr {
+			return fmt.Errorf("not a slice of pointers")
+		}
+
+		for i := 0; i < slice.Len(); i++ {
+			sec, err := s.f.NewSection(s.name)
+			if err != nil {
+				return err
+			}
+
+			err = sec.reflectFrom(slice.Index(i))
+			if err != nil {
+				return fmt.Errorf("reflect from %dth field: %v", i, err)
+			}
+		}
+
+		return nil
+	}
+
 	if typ.Kind() == reflect.Ptr {
-		typ = typ.Elem()
 		val = val.Elem()
 	} else {
-		return errors.New("cannot reflect from non-pointer struct")
+		return errors.New("not a pointer to a struct")
 	}

 	return s.reflectFrom(val)