mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2024-11-23 11:01:49 +01:00
feat: add support for a credentials chain for minio access (#31051)
We wanted to be able to use the IAM role provided by the EC2 instance metadata in order to access S3 via the Minio configuration. To do this, a new credentials chain is added that will check the following locations for credentials when an access key is not provided. In priority order, they are: 1. MINIO_ prefixed environment variables 2. AWS_ prefixed environment variables 3. a minio credentials file 4. an aws credentials file 5. EC2 instance metadata (cherry picked from commit c0880e7695346997c6a93f05cd01634cb3ad03ee) Conflicts: docs/content/administration/config-cheat-sheet.en-us.md does not exist in Forgejo
This commit is contained in:
parent
a7591f9738
commit
73706ae26d
5 changed files with 157 additions and 3 deletions
|
@ -1924,7 +1924,10 @@ LEVEL = Info
|
|||
;; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
|
||||
;MINIO_ENDPOINT = localhost:9000
|
||||
;;
|
||||
;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
|
||||
;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`.
|
||||
;; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known
|
||||
;; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files
|
||||
;; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
|
||||
;MINIO_ACCESS_KEY_ID =
|
||||
;;
|
||||
;; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
|
||||
|
@ -2633,7 +2636,10 @@ LEVEL = Info
|
|||
;; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
|
||||
;MINIO_ENDPOINT = localhost:9000
|
||||
;;
|
||||
;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
|
||||
;; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`.
|
||||
;; If not provided and STORAGE_TYPE is `minio`, will search for credentials in known
|
||||
;; environment variables (MINIO_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), credentials files
|
||||
;; (~/.mc/config.json, ~/.aws/credentials), and EC2 instance metadata.
|
||||
;MINIO_ACCESS_KEY_ID =
|
||||
;;
|
||||
;; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
|
||||
|
|
|
@ -97,7 +97,7 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage,
|
|||
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
|
||||
|
||||
minioClient, err := minio.New(config.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
|
||||
Creds: buildMinioCredentials(config, credentials.DefaultIAMRoleEndpoint),
|
||||
Secure: config.UseSSL,
|
||||
Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
|
||||
Region: config.Location,
|
||||
|
@ -164,6 +164,35 @@ func (m *MinioStorage) buildMinioDirPrefix(p string) string {
|
|||
return p
|
||||
}
|
||||
|
||||
func buildMinioCredentials(config setting.MinioStorageConfig, iamEndpoint string) *credentials.Credentials {
|
||||
// If static credentials are provided, use those
|
||||
if config.AccessKeyID != "" {
|
||||
return credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "")
|
||||
}
|
||||
|
||||
// Otherwise, fallback to a credentials chain for S3 access
|
||||
chain := []credentials.Provider{
|
||||
// configure based upon MINIO_ prefixed environment variables
|
||||
&credentials.EnvMinio{},
|
||||
// configure based upon AWS_ prefixed environment variables
|
||||
&credentials.EnvAWS{},
|
||||
// read credentials from MINIO_SHARED_CREDENTIALS_FILE
|
||||
// environment variable, or default json config files
|
||||
&credentials.FileMinioClient{},
|
||||
// read credentials from AWS_SHARED_CREDENTIALS_FILE
|
||||
// environment variable, or default credentials file
|
||||
&credentials.FileAWSCredentials{},
|
||||
// read IAM role from EC2 metadata endpoint if available
|
||||
&credentials.IAM{
|
||||
Endpoint: iamEndpoint,
|
||||
Client: &http.Client{
|
||||
Transport: http.DefaultTransport,
|
||||
},
|
||||
},
|
||||
}
|
||||
return credentials.NewChainCredentials(chain)
|
||||
}
|
||||
|
||||
// Open opens a file
|
||||
func (m *MinioStorage) Open(path string) (Object, error) {
|
||||
opts := minio.GetObjectOptions{}
|
||||
|
|
|
@ -6,6 +6,7 @@ package storage
|
|||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
|
@ -109,3 +110,106 @@ func TestS3StorageBadRequest(t *testing.T) {
|
|||
_, err := NewStorage(setting.MinioStorageType, cfg)
|
||||
assert.ErrorContains(t, err, message)
|
||||
}
|
||||
|
||||
func TestMinioCredentials(t *testing.T) {
|
||||
const (
|
||||
ExpectedAccessKey = "ExampleAccessKeyID"
|
||||
ExpectedSecretAccessKey = "ExampleSecretAccessKeyID"
|
||||
// Use a FakeEndpoint for IAM credentials to avoid logging any
|
||||
// potential real IAM credentials when running in EC2.
|
||||
FakeEndpoint = "http://localhost"
|
||||
)
|
||||
|
||||
t.Run("Static Credentials", func(t *testing.T) {
|
||||
cfg := setting.MinioStorageConfig{
|
||||
AccessKeyID: ExpectedAccessKey,
|
||||
SecretAccessKey: ExpectedSecretAccessKey,
|
||||
}
|
||||
creds := buildMinioCredentials(cfg, FakeEndpoint)
|
||||
v, err := creds.Get()
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, ExpectedAccessKey, v.AccessKeyID)
|
||||
assert.Equal(t, ExpectedSecretAccessKey, v.SecretAccessKey)
|
||||
})
|
||||
|
||||
t.Run("Chain", func(t *testing.T) {
|
||||
cfg := setting.MinioStorageConfig{}
|
||||
|
||||
t.Run("EnvMinio", func(t *testing.T) {
|
||||
t.Setenv("MINIO_ACCESS_KEY", ExpectedAccessKey+"Minio")
|
||||
t.Setenv("MINIO_SECRET_KEY", ExpectedSecretAccessKey+"Minio")
|
||||
|
||||
creds := buildMinioCredentials(cfg, FakeEndpoint)
|
||||
v, err := creds.Get()
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, ExpectedAccessKey+"Minio", v.AccessKeyID)
|
||||
assert.Equal(t, ExpectedSecretAccessKey+"Minio", v.SecretAccessKey)
|
||||
})
|
||||
|
||||
t.Run("EnvAWS", func(t *testing.T) {
|
||||
t.Setenv("AWS_ACCESS_KEY", ExpectedAccessKey+"AWS")
|
||||
t.Setenv("AWS_SECRET_KEY", ExpectedSecretAccessKey+"AWS")
|
||||
|
||||
creds := buildMinioCredentials(cfg, FakeEndpoint)
|
||||
v, err := creds.Get()
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, ExpectedAccessKey+"AWS", v.AccessKeyID)
|
||||
assert.Equal(t, ExpectedSecretAccessKey+"AWS", v.SecretAccessKey)
|
||||
})
|
||||
|
||||
t.Run("FileMinio", func(t *testing.T) {
|
||||
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/minio.json")
|
||||
// prevent loading any actual credentials files from the user
|
||||
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")
|
||||
|
||||
creds := buildMinioCredentials(cfg, FakeEndpoint)
|
||||
v, err := creds.Get()
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, ExpectedAccessKey+"MinioFile", v.AccessKeyID)
|
||||
assert.Equal(t, ExpectedSecretAccessKey+"MinioFile", v.SecretAccessKey)
|
||||
})
|
||||
|
||||
t.Run("FileAWS", func(t *testing.T) {
|
||||
// prevent loading any actual credentials files from the user
|
||||
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
|
||||
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/aws_credentials")
|
||||
|
||||
creds := buildMinioCredentials(cfg, FakeEndpoint)
|
||||
v, err := creds.Get()
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, ExpectedAccessKey+"AWSFile", v.AccessKeyID)
|
||||
assert.Equal(t, ExpectedSecretAccessKey+"AWSFile", v.SecretAccessKey)
|
||||
})
|
||||
|
||||
t.Run("IAM", func(t *testing.T) {
|
||||
// prevent loading any actual credentials files from the user
|
||||
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
|
||||
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")
|
||||
|
||||
// Spawn a server to emulate the EC2 Instance Metadata
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// The client will actually make 3 requests here,
|
||||
// first will be to get the IMDSv2 token, second to
|
||||
// get the role, and third for the actual
|
||||
// credentials. However, we can return credentials
|
||||
// every request since we're not emulating a full
|
||||
// IMDSv2 flow.
|
||||
w.Write([]byte(`{"Code":"Success","AccessKeyId":"ExampleAccessKeyIDIAM","SecretAccessKey":"ExampleSecretAccessKeyIDIAM"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Use the provided EC2 Instance Metadata server
|
||||
creds := buildMinioCredentials(cfg, server.URL)
|
||||
v, err := creds.Get()
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, ExpectedAccessKey+"IAM", v.AccessKeyID)
|
||||
assert.Equal(t, ExpectedSecretAccessKey+"IAM", v.SecretAccessKey)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
3
modules/storage/testdata/aws_credentials
vendored
Normal file
3
modules/storage/testdata/aws_credentials
vendored
Normal file
|
@ -0,0 +1,3 @@
|
|||
[default]
|
||||
aws_access_key_id=ExampleAccessKeyIDAWSFile
|
||||
aws_secret_access_key=ExampleSecretAccessKeyIDAWSFile
|
12
modules/storage/testdata/minio.json
vendored
Normal file
12
modules/storage/testdata/minio.json
vendored
Normal file
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"version": "10",
|
||||
"aliases": {
|
||||
"s3": {
|
||||
"url": "https://s3.amazonaws.com",
|
||||
"accessKey": "ExampleAccessKeyIDMinioFile",
|
||||
"secretKey": "ExampleSecretAccessKeyIDMinioFile",
|
||||
"api": "S3v4",
|
||||
"path": "dns"
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue