From c0880e7695346997c6a93f05cd01634cb3ad03ee Mon Sep 17 00:00:00 2001 From: Rowan Bohde Date: Mon, 27 May 2024 07:56:04 -0500 Subject: [PATCH] feat: add support for a credentials chain for minio access (#31051) We wanted to be able to use the IAM role provided by the EC2 instance metadata in order to access S3 via the Minio configuration. To do this, a new credentials chain is added that will check the following locations for credentials when an access key is not provided. In priority order, they are: 1. MINIO_ prefixed environment variables 2. AWS_ prefixed environment variables 3. a minio credentials file 4. an aws credentials file 5. EC2 instance metadata --- custom/conf/app.example.ini | 10 +- .../config-cheat-sheet.en-us.md | 18 ++- modules/storage/minio.go | 31 +++++- modules/storage/minio_test.go | 104 ++++++++++++++++++ modules/storage/testdata/aws_credentials | 3 + modules/storage/testdata/minio.json | 12 ++ 6 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 modules/storage/testdata/aws_credentials create mode 100644 modules/storage/testdata/minio.json diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index afbd20eb56..7c05e7fefd 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1872,7 +1872,10 @@ LEVEL = Info ;; Minio endpoint to connect only available when STORAGE_TYPE is `minio` ;MINIO_ENDPOINT = localhost:9000 ;; -;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` +;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. +;; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known +;; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files +;; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata. ;MINIO_ACCESS_KEY_ID = ;; ;; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio` @@ -2573,7 +2576,10 @@ LEVEL = Info ;; Minio endpoint to connect only available when STORAGE_TYPE is `minio` ;MINIO_ENDPOINT = localhost:9000 ;; -;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` +;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. +;; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known +;; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files +;; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata. ;MINIO_ACCESS_KEY_ID = ;; ;; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio` diff --git a/docs/content/administration/config-cheat-sheet.en-us.md b/docs/content/administration/config-cheat-sheet.en-us.md index 1165a83e25..2c15d161ea 100644 --- a/docs/content/administration/config-cheat-sheet.en-us.md +++ b/docs/content/administration/config-cheat-sheet.en-us.md @@ -843,7 +843,7 @@ Default templates for project board view: - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing. - `PATH`: **attachments**: Path to store attachments only available when STORAGE_TYPE is `local`, relative paths will be resolved to `${AppDataPath}/${attachment.PATH}`. - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when STORAGE_TYPE is `minio` -- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` +- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. If not provided and STORAGE_TYPE is `minio`, will search for credentials in known environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata. - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio` - `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORAGE_TYPE is `minio` - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORAGE_TYPE is `minio` @@ -1274,7 +1274,7 @@ is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`. - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing. - `PATH`: **./data/lfs**: Where to store LFS files, only available when `STORAGE_TYPE` is `local`. If not set it fall back to deprecated LFS_CONTENT_PATH value in [server] section. - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio` -- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio` +- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. If not provided and STORAGE_TYPE is `minio`, will search for credentials in known environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata. - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio` - `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio` - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio` @@ -1290,7 +1290,7 @@ Default storage configuration for attachments, lfs, avatars, repo-avatars, repo- - `STORAGE_TYPE`: **local**: Storage type, `local` for local disk or `minio` for s3 compatible object storage service. - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing. - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio` -- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio` +- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. If not provided and STORAGE_TYPE is `minio`, will search for credentials in known environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata. - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio` - `MINIO_BUCKET`: **gitea**: Minio bucket to store the data only available when `STORAGE_TYPE` is `minio` - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio` @@ -1305,7 +1305,10 @@ The recommended storage configuration for minio like below: STORAGE_TYPE = minio ; Minio endpoint to connect only available when STORAGE_TYPE is `minio` MINIO_ENDPOINT = localhost:9000 -; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` +; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. +; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known +; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files +; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata. MINIO_ACCESS_KEY_ID = ; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio` MINIO_SECRET_ACCESS_KEY = @@ -1354,7 +1357,10 @@ STORAGE_TYPE = my_minio STORAGE_TYPE = minio ; Minio endpoint to connect only available when STORAGE_TYPE is `minio` MINIO_ENDPOINT = localhost:9000 -; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio` +; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. +; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known +; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files +; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata. MINIO_ACCESS_KEY_ID = ; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio` MINIO_SECRET_ACCESS_KEY = @@ -1380,7 +1386,7 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`. - `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing. - `PATH`: **./data/repo-archive**: Where to store archive files, only available when `STORAGE_TYPE` is `local`. - `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio` -- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio` +- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`. If not provided and STORAGE_TYPE is `minio`, will search for credentials in known environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata. - `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio` - `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio` - `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio` diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 986332dfed..1b32b2f54f 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -97,7 +97,7 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, } minioClient, err := minio.New(config.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""), + Creds: buildMinioCredentials(config, credentials.DefaultIAMRoleEndpoint), Secure: config.UseSSL, Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}}, Region: config.Location, @@ -164,6 +164,35 @@ func (m *MinioStorage) buildMinioDirPrefix(p string) string { return p } +func buildMinioCredentials(config setting.MinioStorageConfig, iamEndpoint string) *credentials.Credentials { + // If static credentials are provided, use those + if config.AccessKeyID != "" { + return credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "") + } + + // Otherwise, fallback to a credentials chain for S3 access + chain := []credentials.Provider{ + // configure based upon MINIO_ prefixed environment variables + &credentials.EnvMinio{}, + // configure based upon AWS_ prefixed environment variables + &credentials.EnvAWS{}, + // read credentials from MINIO_SHARED_CREDENTIALS_FILE + // environment variable, or default json config files + &credentials.FileMinioClient{}, + // read credentials from AWS_SHARED_CREDENTIALS_FILE + // environment variable, or default credentials file + &credentials.FileAWSCredentials{}, + // read IAM role from EC2 metadata endpoint if available + &credentials.IAM{ + Endpoint: iamEndpoint, + Client: &http.Client{ + Transport: http.DefaultTransport, + }, + }, + } + return credentials.NewChainCredentials(chain) +} + // Open opens a file func (m *MinioStorage) Open(path string) (Object, error) { opts := minio.GetObjectOptions{} diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go index c6fbb91ab4..ad11046dd6 100644 --- a/modules/storage/minio_test.go +++ b/modules/storage/minio_test.go @@ -6,6 +6,7 @@ package storage import ( "context" "net/http" + "net/http/httptest" "os" "testing" @@ -92,3 +93,106 @@ func TestS3StorageBadRequest(t *testing.T) { _, err := NewStorage(setting.MinioStorageType, cfg) assert.ErrorContains(t, err, message) } + +func TestMinioCredentials(t *testing.T) { + const ( + ExpectedAccessKey = "ExampleAccessKeyID" + ExpectedSecretAccessKey = "ExampleSecretAccessKeyID" + // Use a FakeEndpoint for IAM credentials to avoid logging any + // potential real IAM credentials when running in EC2. + FakeEndpoint = "http://localhost" + ) + + t.Run("Static Credentials", func(t *testing.T) { + cfg := setting.MinioStorageConfig{ + AccessKeyID: ExpectedAccessKey, + SecretAccessKey: ExpectedSecretAccessKey, + } + creds := buildMinioCredentials(cfg, FakeEndpoint) + v, err := creds.Get() + + assert.NoError(t, err) + assert.Equal(t, ExpectedAccessKey, v.AccessKeyID) + assert.Equal(t, ExpectedSecretAccessKey, v.SecretAccessKey) + }) + + t.Run("Chain", func(t *testing.T) { + cfg := setting.MinioStorageConfig{} + + t.Run("EnvMinio", func(t *testing.T) { + t.Setenv("MINIO_ACCESS_KEY", ExpectedAccessKey+"Minio") + t.Setenv("MINIO_SECRET_KEY", ExpectedSecretAccessKey+"Minio") + + creds := buildMinioCredentials(cfg, FakeEndpoint) + v, err := creds.Get() + + assert.NoError(t, err) + assert.Equal(t, ExpectedAccessKey+"Minio", v.AccessKeyID) + assert.Equal(t, ExpectedSecretAccessKey+"Minio", v.SecretAccessKey) + }) + + t.Run("EnvAWS", func(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY", ExpectedAccessKey+"AWS") + t.Setenv("AWS_SECRET_KEY", ExpectedSecretAccessKey+"AWS") + + creds := buildMinioCredentials(cfg, FakeEndpoint) + v, err := creds.Get() + + assert.NoError(t, err) + assert.Equal(t, ExpectedAccessKey+"AWS", v.AccessKeyID) + assert.Equal(t, ExpectedSecretAccessKey+"AWS", v.SecretAccessKey) + }) + + t.Run("FileMinio", func(t *testing.T) { + t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/minio.json") + // prevent loading any actual credentials files from the user + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake") + + creds := buildMinioCredentials(cfg, FakeEndpoint) + v, err := creds.Get() + + assert.NoError(t, err) + assert.Equal(t, ExpectedAccessKey+"MinioFile", v.AccessKeyID) + assert.Equal(t, ExpectedSecretAccessKey+"MinioFile", v.SecretAccessKey) + }) + + t.Run("FileAWS", func(t *testing.T) { + // prevent loading any actual credentials files from the user + t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json") + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/aws_credentials") + + creds := buildMinioCredentials(cfg, FakeEndpoint) + v, err := creds.Get() + + assert.NoError(t, err) + assert.Equal(t, ExpectedAccessKey+"AWSFile", v.AccessKeyID) + assert.Equal(t, ExpectedSecretAccessKey+"AWSFile", v.SecretAccessKey) + }) + + t.Run("IAM", func(t *testing.T) { + // prevent loading any actual credentials files from the user + t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json") + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake") + + // Spawn a server to emulate the EC2 Instance Metadata + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The client will actually make 3 requests here, + // first will be to get the IMDSv2 token, second to + // get the role, and third for the actual + // credentials. However, we can return credentials + // every request since we're not emulating a full + // IMDSv2 flow. + w.Write([]byte(`{"Code":"Success","AccessKeyId":"ExampleAccessKeyIDIAM","SecretAccessKey":"ExampleSecretAccessKeyIDIAM"}`)) + })) + defer server.Close() + + // Use the provided EC2 Instance Metadata server + creds := buildMinioCredentials(cfg, server.URL) + v, err := creds.Get() + + assert.NoError(t, err) + assert.Equal(t, ExpectedAccessKey+"IAM", v.AccessKeyID) + assert.Equal(t, ExpectedSecretAccessKey+"IAM", v.SecretAccessKey) + }) + }) +} diff --git a/modules/storage/testdata/aws_credentials b/modules/storage/testdata/aws_credentials new file mode 100644 index 0000000000..62a5488b51 --- /dev/null +++ b/modules/storage/testdata/aws_credentials @@ -0,0 +1,3 @@ +[default] +aws_access_key_id=ExampleAccessKeyIDAWSFile +aws_secret_access_key=ExampleSecretAccessKeyIDAWSFile diff --git a/modules/storage/testdata/minio.json b/modules/storage/testdata/minio.json new file mode 100644 index 0000000000..3876257626 --- /dev/null +++ b/modules/storage/testdata/minio.json @@ -0,0 +1,12 @@ +{ + "version": "10", + "aliases": { + "s3": { + "url": "https://s3.amazonaws.com", + "accessKey": "ExampleAccessKeyIDMinioFile", + "secretKey": "ExampleSecretAccessKeyIDMinioFile", + "api": "S3v4", + "path": "dns" + } + } +}