summaryrefslogtreecommitdiffstats
path: root/modules/storage/minio.go
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--modules/storage/minio.go310
1 files changed, 310 insertions, 0 deletions
diff --git a/modules/storage/minio.go b/modules/storage/minio.go
new file mode 100644
index 0000000..d0c2dec
--- /dev/null
+++ b/modules/storage/minio.go
@@ -0,0 +1,310 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package storage
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/util"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+var (
+ _ ObjectStorage = &MinioStorage{}
+
+ quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+)
+
+type minioObject struct {
+ *minio.Object
+}
+
+func (m *minioObject) Stat() (os.FileInfo, error) {
+ oi, err := m.Object.Stat()
+ if err != nil {
+ return nil, convertMinioErr(err)
+ }
+
+ return &minioFileInfo{oi}, nil
+}
+
+// MinioStorage returns a minio bucket storage
+type MinioStorage struct {
+ cfg *setting.MinioStorageConfig
+ ctx context.Context
+ client *minio.Client
+ bucket string
+ basePath string
+}
+
+func convertMinioErr(err error) error {
+ if err == nil {
+ return nil
+ }
+ errResp, ok := err.(minio.ErrorResponse)
+ if !ok {
+ return err
+ }
+
+ // Convert two responses to standard analogues
+ switch errResp.Code {
+ case "NoSuchKey":
+ return os.ErrNotExist
+ case "AccessDenied":
+ return os.ErrPermission
+ }
+
+ return err
+}
+
+var getBucketVersioning = func(ctx context.Context, minioClient *minio.Client, bucket string) error {
+ _, err := minioClient.GetBucketVersioning(ctx, bucket)
+ return err
+}
+
+// NewMinioStorage returns a minio storage
+func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error) {
+ config := cfg.MinioConfig
+ if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
+ return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
+ }
+ var lookup minio.BucketLookupType
+ switch config.BucketLookup {
+ case "auto", "":
+ lookup = minio.BucketLookupAuto
+ case "dns":
+ lookup = minio.BucketLookupDNS
+ case "path":
+ lookup = minio.BucketLookupPath
+ default:
+ return nil, fmt.Errorf("invalid minio bucket lookup type %s", config.BucketLookup)
+ }
+
+ log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
+
+ minioClient, err := minio.New(config.Endpoint, &minio.Options{
+ Creds: buildMinioCredentials(config, credentials.DefaultIAMRoleEndpoint),
+ Secure: config.UseSSL,
+ Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
+ Region: config.Location,
+ BucketLookup: lookup,
+ })
+ if err != nil {
+ return nil, convertMinioErr(err)
+ }
+
+ // The GetBucketVersioning is only used for checking whether the Object Storage parameters are generally good. It doesn't need to succeed.
+ // The assumption is that if the API returns the HTTP code 400, then the parameters could be incorrect.
+ // Otherwise even if the request itself fails (403, 404, etc), the code should still continue because the parameters seem "good" enough.
+ // Keep in mind that GetBucketVersioning requires "owner" to really succeed, so it can't be used to check the existence.
+ // Not using "BucketExists (HeadBucket)" because it doesn't include detailed failure reasons.
+ err = getBucketVersioning(ctx, minioClient, config.Bucket)
+ if err != nil {
+ errResp, ok := err.(minio.ErrorResponse)
+ if !ok {
+ return nil, err
+ }
+ if errResp.StatusCode == http.StatusBadRequest {
+ log.Error("S3 storage connection failure at %s:%s with base path %s and region: %s", config.Endpoint, config.Bucket, config.Location, errResp.Message)
+ return nil, err
+ }
+ }
+
+ // Check to see if we already own this bucket
+ exists, err := minioClient.BucketExists(ctx, config.Bucket)
+ if err != nil {
+ return nil, convertMinioErr(err)
+ }
+
+ if !exists {
+ if err := minioClient.MakeBucket(ctx, config.Bucket, minio.MakeBucketOptions{
+ Region: config.Location,
+ }); err != nil {
+ return nil, convertMinioErr(err)
+ }
+ }
+
+ return &MinioStorage{
+ cfg: &config,
+ ctx: ctx,
+ client: minioClient,
+ bucket: config.Bucket,
+ basePath: config.BasePath,
+ }, nil
+}
+
+func (m *MinioStorage) buildMinioPath(p string) string {
+ p = strings.TrimPrefix(util.PathJoinRelX(m.basePath, p), "/") // object store doesn't use slash for root path
+ if p == "." {
+ p = "" // object store doesn't use dot as relative path
+ }
+ return p
+}
+
+func (m *MinioStorage) buildMinioDirPrefix(p string) string {
+ // ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo"
+ p = m.buildMinioPath(p) + "/"
+ if p == "/" {
+ p = "" // object store doesn't use slash for root path
+ }
+ return p
+}
+
+func buildMinioCredentials(config setting.MinioStorageConfig, iamEndpoint string) *credentials.Credentials {
+ // If static credentials are provided, use those
+ if config.AccessKeyID != "" {
+ return credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "")
+ }
+
+ // Otherwise, fallback to a credentials chain for S3 access
+ chain := []credentials.Provider{
+ // configure based upon MINIO_ prefixed environment variables
+ &credentials.EnvMinio{},
+ // configure based upon AWS_ prefixed environment variables
+ &credentials.EnvAWS{},
+ // read credentials from MINIO_SHARED_CREDENTIALS_FILE
+ // environment variable, or default json config files
+ &credentials.FileMinioClient{},
+ // read credentials from AWS_SHARED_CREDENTIALS_FILE
+ // environment variable, or default credentials file
+ &credentials.FileAWSCredentials{},
+ // read IAM role from EC2 metadata endpoint if available
+ &credentials.IAM{
+ Endpoint: iamEndpoint,
+ Client: &http.Client{
+ Transport: http.DefaultTransport,
+ },
+ },
+ }
+ return credentials.NewChainCredentials(chain)
+}
+
+// Open opens a file
+func (m *MinioStorage) Open(path string) (Object, error) {
+ opts := minio.GetObjectOptions{}
+ object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
+ if err != nil {
+ return nil, convertMinioErr(err)
+ }
+ return &minioObject{object}, nil
+}
+
+// Save saves a file to minio
+func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) {
+ uploadInfo, err := m.client.PutObject(
+ m.ctx,
+ m.bucket,
+ m.buildMinioPath(path),
+ r,
+ size,
+ minio.PutObjectOptions{
+ ContentType: "application/octet-stream",
+ // some storages like:
+ // * https://developers.cloudflare.com/r2/api/s3/api/
+ // * https://www.backblaze.com/b2/docs/s3_compatible_api.html
+ // do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum
+ SendContentMd5: m.cfg.ChecksumAlgorithm == "md5",
+ },
+ )
+ if err != nil {
+ return 0, convertMinioErr(err)
+ }
+ return uploadInfo.Size, nil
+}
+
+type minioFileInfo struct {
+ minio.ObjectInfo
+}
+
+func (m minioFileInfo) Name() string {
+ return path.Base(m.ObjectInfo.Key)
+}
+
+func (m minioFileInfo) Size() int64 {
+ return m.ObjectInfo.Size
+}
+
+func (m minioFileInfo) ModTime() time.Time {
+ return m.LastModified
+}
+
+func (m minioFileInfo) IsDir() bool {
+ return strings.HasSuffix(m.ObjectInfo.Key, "/")
+}
+
+func (m minioFileInfo) Mode() os.FileMode {
+ return os.ModePerm
+}
+
+func (m minioFileInfo) Sys() any {
+ return nil
+}
+
+// Stat returns the stat information of the object
+func (m *MinioStorage) Stat(path string) (os.FileInfo, error) {
+ info, err := m.client.StatObject(
+ m.ctx,
+ m.bucket,
+ m.buildMinioPath(path),
+ minio.StatObjectOptions{},
+ )
+ if err != nil {
+ return nil, convertMinioErr(err)
+ }
+ return &minioFileInfo{info}, nil
+}
+
+// Delete delete a file
+func (m *MinioStorage) Delete(path string) error {
+ err := m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})
+
+ return convertMinioErr(err)
+}
+
+// URL gets the redirect URL to a file. The presigned link is valid for 5 minutes.
+func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
+ reqParams := make(url.Values)
+ // TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we?
+ reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
+ u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
+ return u, convertMinioErr(err)
+}
+
+// IterateObjects iterates across the objects in the miniostorage
+func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
+ opts := minio.GetObjectOptions{}
+ for mObjInfo := range m.client.ListObjects(m.ctx, m.bucket, minio.ListObjectsOptions{
+ Prefix: m.buildMinioDirPrefix(dirName),
+ Recursive: true,
+ }) {
+ object, err := m.client.GetObject(m.ctx, m.bucket, mObjInfo.Key, opts)
+ if err != nil {
+ return convertMinioErr(err)
+ }
+ if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
+ defer object.Close()
+ return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
+ }(object, fn); err != nil {
+ return convertMinioErr(err)
+ }
+ }
+ return nil
+}
+
+func init() {
+ RegisterStorageType(setting.MinioStorageType, NewMinioStorage)
+}