mirror of
https://github.com/go-gitea/gitea
synced 2024-11-16 07:04:25 +00:00
62e6c9bc6c
* Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
318 lines
9.0 KiB
Go
Vendored
318 lines
9.0 KiB
Go
Vendored
/*
|
|
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
|
|
* Copyright 2015-2017 MinIO, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package signer
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/hmac"
|
|
"crypto/sha1"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/v7/pkg/s3utils"
|
|
)
|
|
|
|
// Signature and API related constants.
|
|
const (
|
|
signV2Algorithm = "AWS"
|
|
)
|
|
|
|
// Encode input URL path to URL encoded path.
|
|
func encodeURL2Path(req *http.Request, virtualHost bool) (path string) {
|
|
if virtualHost {
|
|
reqHost := getHostAddr(req)
|
|
dotPos := strings.Index(reqHost, ".")
|
|
if dotPos > -1 {
|
|
bucketName := reqHost[:dotPos]
|
|
path = "/" + bucketName
|
|
path += req.URL.Path
|
|
path = s3utils.EncodePath(path)
|
|
return
|
|
}
|
|
}
|
|
path = s3utils.EncodePath(req.URL.Path)
|
|
return
|
|
}
|
|
|
|
// PreSignV2 - presign the request in following style.
|
|
// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}.
|
|
func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64, virtualHost bool) *http.Request {
|
|
// Presign is not needed for anonymous credentials.
|
|
if accessKeyID == "" || secretAccessKey == "" {
|
|
return &req
|
|
}
|
|
|
|
d := time.Now().UTC()
|
|
// Find epoch expires when the request will expire.
|
|
epochExpires := d.Unix() + expires
|
|
|
|
// Add expires header if not present.
|
|
if expiresStr := req.Header.Get("Expires"); expiresStr == "" {
|
|
req.Header.Set("Expires", strconv.FormatInt(epochExpires, 10))
|
|
}
|
|
|
|
// Get presigned string to sign.
|
|
stringToSign := preStringToSignV2(req, virtualHost)
|
|
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
|
hm.Write([]byte(stringToSign))
|
|
|
|
// Calculate signature.
|
|
signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
|
|
|
|
query := req.URL.Query()
|
|
// Handle specially for Google Cloud Storage.
|
|
if strings.Contains(getHostAddr(&req), ".storage.googleapis.com") {
|
|
query.Set("GoogleAccessId", accessKeyID)
|
|
} else {
|
|
query.Set("AWSAccessKeyId", accessKeyID)
|
|
}
|
|
|
|
// Fill in Expires for presigned query.
|
|
query.Set("Expires", strconv.FormatInt(epochExpires, 10))
|
|
|
|
// Encode query and save.
|
|
req.URL.RawQuery = s3utils.QueryEncode(query)
|
|
|
|
// Save signature finally.
|
|
req.URL.RawQuery += "&Signature=" + s3utils.EncodePath(signature)
|
|
|
|
// Return.
|
|
return &req
|
|
}
|
|
|
|
// PostPresignSignatureV2 - presigned signature for PostPolicy
|
|
// request.
|
|
func PostPresignSignatureV2(policyBase64, secretAccessKey string) string {
|
|
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
|
hm.Write([]byte(policyBase64))
|
|
signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
|
|
return signature
|
|
}
|
|
|
|
// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
|
|
// Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) );
|
|
//
|
|
// StringToSign = HTTP-Verb + "\n" +
|
|
// Content-Md5 + "\n" +
|
|
// Content-Type + "\n" +
|
|
// Date + "\n" +
|
|
// CanonicalizedProtocolHeaders +
|
|
// CanonicalizedResource;
|
|
//
|
|
// CanonicalizedResource = [ "/" + Bucket ] +
|
|
// <HTTP-Request-URI, from the protocol name up to the query string> +
|
|
// [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
|
|
//
|
|
// CanonicalizedProtocolHeaders = <described below>
|
|
|
|
// SignV2 sign the request before Do() (AWS Signature Version 2).
|
|
func SignV2(req http.Request, accessKeyID, secretAccessKey string, virtualHost bool) *http.Request {
|
|
// Signature calculation is not needed for anonymous credentials.
|
|
if accessKeyID == "" || secretAccessKey == "" {
|
|
return &req
|
|
}
|
|
|
|
// Initial time.
|
|
d := time.Now().UTC()
|
|
|
|
// Add date if not present.
|
|
if date := req.Header.Get("Date"); date == "" {
|
|
req.Header.Set("Date", d.Format(http.TimeFormat))
|
|
}
|
|
|
|
// Calculate HMAC for secretAccessKey.
|
|
stringToSign := stringToSignV2(req, virtualHost)
|
|
hm := hmac.New(sha1.New, []byte(secretAccessKey))
|
|
hm.Write([]byte(stringToSign))
|
|
|
|
// Prepare auth header.
|
|
authHeader := new(bytes.Buffer)
|
|
authHeader.WriteString(fmt.Sprintf("%s %s:", signV2Algorithm, accessKeyID))
|
|
encoder := base64.NewEncoder(base64.StdEncoding, authHeader)
|
|
encoder.Write(hm.Sum(nil))
|
|
encoder.Close()
|
|
|
|
// Set Authorization header.
|
|
req.Header.Set("Authorization", authHeader.String())
|
|
|
|
return &req
|
|
}
|
|
|
|
// From the Amazon docs:
|
|
//
|
|
// StringToSign = HTTP-Verb + "\n" +
|
|
// Content-Md5 + "\n" +
|
|
// Content-Type + "\n" +
|
|
// Expires + "\n" +
|
|
// CanonicalizedProtocolHeaders +
|
|
// CanonicalizedResource;
|
|
func preStringToSignV2(req http.Request, virtualHost bool) string {
|
|
buf := new(bytes.Buffer)
|
|
// Write standard headers.
|
|
writePreSignV2Headers(buf, req)
|
|
// Write canonicalized protocol headers if any.
|
|
writeCanonicalizedHeaders(buf, req)
|
|
// Write canonicalized Query resources if any.
|
|
writeCanonicalizedResource(buf, req, virtualHost)
|
|
return buf.String()
|
|
}
|
|
|
|
// writePreSignV2Headers - write preSign v2 required headers.
|
|
func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) {
|
|
buf.WriteString(req.Method + "\n")
|
|
buf.WriteString(req.Header.Get("Content-Md5") + "\n")
|
|
buf.WriteString(req.Header.Get("Content-Type") + "\n")
|
|
buf.WriteString(req.Header.Get("Expires") + "\n")
|
|
}
|
|
|
|
// From the Amazon docs:
|
|
//
|
|
// StringToSign = HTTP-Verb + "\n" +
|
|
// Content-Md5 + "\n" +
|
|
// Content-Type + "\n" +
|
|
// Date + "\n" +
|
|
// CanonicalizedProtocolHeaders +
|
|
// CanonicalizedResource;
|
|
func stringToSignV2(req http.Request, virtualHost bool) string {
|
|
buf := new(bytes.Buffer)
|
|
// Write standard headers.
|
|
writeSignV2Headers(buf, req)
|
|
// Write canonicalized protocol headers if any.
|
|
writeCanonicalizedHeaders(buf, req)
|
|
// Write canonicalized Query resources if any.
|
|
writeCanonicalizedResource(buf, req, virtualHost)
|
|
return buf.String()
|
|
}
|
|
|
|
// writeSignV2Headers - write signV2 required headers.
|
|
func writeSignV2Headers(buf *bytes.Buffer, req http.Request) {
|
|
buf.WriteString(req.Method + "\n")
|
|
buf.WriteString(req.Header.Get("Content-Md5") + "\n")
|
|
buf.WriteString(req.Header.Get("Content-Type") + "\n")
|
|
buf.WriteString(req.Header.Get("Date") + "\n")
|
|
}
|
|
|
|
// writeCanonicalizedHeaders - write canonicalized headers.
|
|
func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) {
|
|
var protoHeaders []string
|
|
vals := make(map[string][]string)
|
|
for k, vv := range req.Header {
|
|
// All the AMZ headers should be lowercase
|
|
lk := strings.ToLower(k)
|
|
if strings.HasPrefix(lk, "x-amz") {
|
|
protoHeaders = append(protoHeaders, lk)
|
|
vals[lk] = vv
|
|
}
|
|
}
|
|
sort.Strings(protoHeaders)
|
|
for _, k := range protoHeaders {
|
|
buf.WriteString(k)
|
|
buf.WriteByte(':')
|
|
for idx, v := range vals[k] {
|
|
if idx > 0 {
|
|
buf.WriteByte(',')
|
|
}
|
|
if strings.Contains(v, "\n") {
|
|
// TODO: "Unfold" long headers that
|
|
// span multiple lines (as allowed by
|
|
// RFC 2616, section 4.2) by replacing
|
|
// the folding white-space (including
|
|
// new-line) by a single space.
|
|
buf.WriteString(v)
|
|
} else {
|
|
buf.WriteString(v)
|
|
}
|
|
}
|
|
buf.WriteByte('\n')
|
|
}
|
|
}
|
|
|
|
// AWS S3 Signature V2 calculation rule is give here:
|
|
// http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign
|
|
|
|
// Whitelist resource list that will be used in query string for signature-V2 calculation.
|
|
// The list should be alphabetically sorted
|
|
var resourceList = []string{
|
|
"acl",
|
|
"delete",
|
|
"lifecycle",
|
|
"location",
|
|
"logging",
|
|
"notification",
|
|
"partNumber",
|
|
"policy",
|
|
"replication",
|
|
"requestPayment",
|
|
"response-cache-control",
|
|
"response-content-disposition",
|
|
"response-content-encoding",
|
|
"response-content-language",
|
|
"response-content-type",
|
|
"response-expires",
|
|
"torrent",
|
|
"uploadId",
|
|
"uploads",
|
|
"versionId",
|
|
"versioning",
|
|
"versions",
|
|
"website",
|
|
}
|
|
|
|
// From the Amazon docs:
|
|
//
|
|
// CanonicalizedResource = [ "/" + Bucket ] +
|
|
// <HTTP-Request-URI, from the protocol name up to the query string> +
|
|
// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
|
|
func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, virtualHost bool) {
|
|
// Save request URL.
|
|
requestURL := req.URL
|
|
// Get encoded URL path.
|
|
buf.WriteString(encodeURL2Path(&req, virtualHost))
|
|
if requestURL.RawQuery != "" {
|
|
var n int
|
|
vals, _ := url.ParseQuery(requestURL.RawQuery)
|
|
// Verify if any sub resource queries are present, if yes
|
|
// canonicallize them.
|
|
for _, resource := range resourceList {
|
|
if vv, ok := vals[resource]; ok && len(vv) > 0 {
|
|
n++
|
|
// First element
|
|
switch n {
|
|
case 1:
|
|
buf.WriteByte('?')
|
|
// The rest
|
|
default:
|
|
buf.WriteByte('&')
|
|
}
|
|
buf.WriteString(resource)
|
|
// Request parameters
|
|
if len(vv[0]) > 0 {
|
|
buf.WriteByte('=')
|
|
buf.WriteString(vv[0])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|