refactor: separate backend domains and storage adapters

internal/data/storage/aliyun_storage.go

@@ -0,0 +1,105 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"kra/internal/biz"
+	"kra/internal/conf"
+)
+
+type aliyunStorage struct {
+	bucket          *oss.Bucket
+	baseURL, prefix string
+}
+
+func newAliyunStorage(config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
+	if config == nil || config.Endpoint == "" || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
+		return nil, fmt.Errorf("aliyun-oss storage configuration is incomplete")
+	}
+	client, err := oss.New(config.Endpoint, config.AccessKey, config.SecretKey)
+	if err != nil {
+		return nil, err
+	}
+	bucket, err := client.Bucket(config.Bucket)
+	if err != nil {
+		return nil, err
+	}
+	return &aliyunStorage{bucket: bucket, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
+}
+func (s *aliyunStorage) key(name string) string {
+	if s.prefix == "" {
+		return strings.TrimPrefix(name, "/")
+	}
+	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
+}
+func (s *aliyunStorage) unkey(key string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
+}
+func (s *aliyunStorage) file(key string, size int64) *biz.StoredFile {
+	name := s.unkey(key)
+	url := s.baseURL + "/" + key
+	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: url, Size: size}
+}
+func (s *aliyunStorage) Put(_ context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
+	key := s.key(name)
+	if err := s.bucket.PutObject(key, reader); err != nil {
+		return nil, err
+	}
+	meta, err := s.bucket.GetObjectDetailedMeta(key)
+	if err != nil {
+		return nil, err
+	}
+	size := int64(0)
+	_, _ = fmt.Sscan(meta.Get("Content-Length"), &size)
+	return s.file(key, size), nil
+}
+func (s *aliyunStorage) Open(_ context.Context, name string) (io.ReadCloser, error) {
+	return s.bucket.GetObject(s.key(name))
+}
+func (s *aliyunStorage) Delete(_ context.Context, name string) error {
+	return s.bucket.DeleteObject(s.key(name))
+}
+func (s *aliyunStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
+	return composeFiles(ctx, s, names, destination)
+}
+func (s *aliyunStorage) DeletePrefix(ctx context.Context, prefix string) error {
+	cursor := ""
+	for {
+		items, next, more, err := s.List(ctx, prefix, cursor, 1000)
+		if err != nil {
+			return err
+		}
+		for _, item := range items {
+			if err = s.Delete(ctx, item.Path); err != nil {
+				return err
+			}
+		}
+		if !more {
+			return nil
+		}
+		cursor = next
+	}
+}
+func (s *aliyunStorage) List(_ context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
+	if limit < 1 || limit > 1000 {
+		limit = 100
+	}
+	result, err := s.bucket.ListObjects(oss.Prefix(s.key(prefix)), oss.Marker(cursor), oss.MaxKeys(limit))
+	if err != nil {
+		return nil, "", false, err
+	}
+	items := make([]*biz.StoredFile, 0, len(result.Objects))
+	for _, object := range result.Objects {
+		items = append(items, s.file(object.Key, object.Size))
+	}
+	next := ""
+	if result.IsTruncated {
+		next = result.NextMarker
+	}
+	return items, next, result.IsTruncated, nil
+}

internal/data/storage/aws_storage.go

@@ -0,0 +1,152 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"kra/internal/biz"
+	"kra/internal/conf"
+)
+
+type awsStorage struct {
+	client                  *s3.Client
+	bucket, baseURL, prefix string
+}
+
+func newAWSStorage(provider string, config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
+	if config == nil || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
+		return nil, fmt.Errorf("%s storage configuration is incomplete", provider)
+	}
+	region := config.Region
+	endpoint := config.Endpoint
+	if provider == "cloudflare-r2" {
+		region = "auto"
+		if endpoint == "" {
+			if config.AccountId == "" {
+				return nil, fmt.Errorf("cloudflare-r2 account id is required")
+			}
+			endpoint = fmt.Sprintf("https://%s.r2.cloudflarestorage.com", config.AccountId)
+		}
+	}
+	if region == "" {
+		return nil, fmt.Errorf("%s region is required", provider)
+	}
+	loaded, err := awsconfig.LoadDefaultConfig(context.Background(), awsconfig.WithRegion(region), awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.AccessKey, config.SecretKey, "")))
+	if err != nil {
+		return nil, err
+	}
+	client := s3.NewFromConfig(loaded, func(options *s3.Options) {
+		if endpoint != "" {
+			if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
+				if config.UseSsl {
+					endpoint = "https://" + endpoint
+				} else {
+					endpoint = "http://" + endpoint
+				}
+			}
+			options.BaseEndpoint = aws.String(endpoint)
+		}
+		options.UsePathStyle = config.ForcePathStyle
+	})
+	return &awsStorage{client: client, bucket: config.Bucket, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
+}
+
+func (s *awsStorage) key(name string) string {
+	if s.prefix == "" {
+		return strings.TrimPrefix(name, "/")
+	}
+	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
+}
+func (s *awsStorage) unkey(key string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
+}
+func (s *awsStorage) file(key string, size int64) *biz.StoredFile {
+	name := s.unkey(key)
+	url := s.baseURL + "/" + key
+	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: url, Size: size}
+}
+func (s *awsStorage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
+	key := s.key(name)
+	result, err := manager.NewUploader(s.client).Upload(ctx, &s3.PutObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key), Body: reader})
+	if err != nil {
+		return nil, err
+	}
+	_ = result
+	head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
+	if err != nil {
+		return nil, err
+	}
+	size := int64(0)
+	if head.ContentLength != nil {
+		size = *head.ContentLength
+	}
+	return s.file(key, size), nil
+}
+func (s *awsStorage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
+	result, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(s.key(name))})
+	if err != nil {
+		return nil, err
+	}
+	return result.Body, nil
+}
+func (s *awsStorage) Delete(ctx context.Context, name string) error {
+	_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(s.key(name))})
+	return err
+}
+func (s *awsStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
+	return composeFiles(ctx, s, names, destination)
+}
+func (s *awsStorage) DeletePrefix(ctx context.Context, prefix string) error {
+	for {
+		items, _, more, err := s.List(ctx, prefix, "", 1000)
+		if err != nil {
+			return err
+		}
+		for _, item := range items {
+			if err = s.Delete(ctx, item.Path); err != nil {
+				return err
+			}
+		}
+		if !more || len(items) == 0 {
+			return nil
+		}
+	}
+}
+func (s *awsStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
+	if limit < 1 || limit > 1000 {
+		limit = 100
+	}
+	input := &s3.ListObjectsV2Input{Bucket: aws.String(s.bucket), Prefix: aws.String(s.key(prefix)), MaxKeys: aws.Int32(int32(limit))}
+	if cursor != "" {
+		input.ContinuationToken = aws.String(cursor)
+	}
+	result, err := s.client.ListObjectsV2(ctx, input)
+	if err != nil {
+		return nil, "", false, err
+	}
+	items := make([]*biz.StoredFile, 0, len(result.Contents))
+	for _, object := range result.Contents {
+		if object.Key == nil {
+			continue
+		}
+		size := int64(0)
+		if object.Size != nil {
+			size = *object.Size
+		}
+		items = append(items, s.file(*object.Key, size))
+	}
+	next := ""
+	if result.NextContinuationToken != nil {
+		next = *result.NextContinuationToken
+	}
+	more := result.IsTruncated != nil && *result.IsTruncated
+	return items, next, more, nil
+}

internal/data/storage/huawei_storage.go

@@ -0,0 +1,104 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+
+	"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
+	"kra/internal/biz"
+	"kra/internal/conf"
+)
+
+type huaweiStorage struct {
+	client                  *obs.ObsClient
+	bucket, baseURL, prefix string
+}
+
+func newHuaweiStorage(config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
+	if config == nil || config.Endpoint == "" || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
+		return nil, fmt.Errorf("huawei-obs storage configuration is incomplete")
+	}
+	client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &huaweiStorage{client: client, bucket: config.Bucket, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
+}
+func (s *huaweiStorage) key(name string) string {
+	if s.prefix == "" {
+		return strings.TrimPrefix(name, "/")
+	}
+	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
+}
+func (s *huaweiStorage) unkey(key string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
+}
+func (s *huaweiStorage) file(key string, size int64) *biz.StoredFile {
+	name := s.unkey(key)
+	fileURL := s.baseURL + "/" + key
+	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: fileURL, Size: size}
+}
+func (s *huaweiStorage) Put(_ context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
+	key := s.key(name)
+	if _, err := s.client.PutObject(&obs.PutObjectInput{PutObjectBasicInput: obs.PutObjectBasicInput{ObjectOperationInput: obs.ObjectOperationInput{Bucket: s.bucket, Key: key}}, Body: reader}); err != nil {
+		return nil, err
+	}
+	meta, err := s.client.GetObjectMetadata(&obs.GetObjectMetadataInput{Bucket: s.bucket, Key: key})
+	if err != nil {
+		return nil, err
+	}
+	return s.file(key, meta.ContentLength), nil
+}
+func (s *huaweiStorage) Open(_ context.Context, name string) (io.ReadCloser, error) {
+	result, err := s.client.GetObject(&obs.GetObjectInput{GetObjectMetadataInput: obs.GetObjectMetadataInput{Bucket: s.bucket, Key: s.key(name)}})
+	if err != nil {
+		return nil, err
+	}
+	return result.Body, nil
+}
+func (s *huaweiStorage) Delete(_ context.Context, name string) error {
+	_, err := s.client.DeleteObject(&obs.DeleteObjectInput{Bucket: s.bucket, Key: s.key(name)})
+	return err
+}
+func (s *huaweiStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
+	return composeFiles(ctx, s, names, destination)
+}
+func (s *huaweiStorage) DeletePrefix(ctx context.Context, prefix string) error {
+	cursor := ""
+	for {
+		items, next, more, err := s.List(ctx, prefix, cursor, 1000)
+		if err != nil {
+			return err
+		}
+		for _, item := range items {
+			if err = s.Delete(ctx, item.Path); err != nil {
+				return err
+			}
+		}
+		if !more {
+			return nil
+		}
+		cursor = next
+	}
+}
+func (s *huaweiStorage) List(_ context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
+	if limit < 1 || limit > 1000 {
+		limit = 100
+	}
+	result, err := s.client.ListObjects(&obs.ListObjectsInput{ListObjsInput: obs.ListObjsInput{Prefix: s.key(prefix), MaxKeys: limit}, Bucket: s.bucket, Marker: cursor})
+	if err != nil {
+		return nil, "", false, err
+	}
+	items := make([]*biz.StoredFile, 0, len(result.Contents))
+	for _, object := range result.Contents {
+		items = append(items, s.file(object.Key, object.Size))
+	}
+	next := result.NextMarker
+	if result.IsTruncated && next == "" && len(result.Contents) > 0 {
+		next = result.Contents[len(result.Contents)-1].Key
+	}
+	return items, next, result.IsTruncated, nil
+}

internal/data/storage/local.go

@@ -0,0 +1,219 @@
+package storage
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"kra/internal/biz"
+	"kra/internal/conf"
+)
+
+type fileStorage struct {
+	root      string
+	urlPrefix string
+}
+
+func New(config *conf.AdminBackend) (biz.FileStorage, error) {
+	storageType := "local"
+	if config != nil && config.Storage != nil && config.Storage.Type != "" {
+		storageType = strings.ToLower(config.Storage.Type)
+	}
+	if storageType == "qiniu" {
+		return newQiniuStorage(config.Storage.Qiniu)
+	}
+	if storageType != "local" {
+		var object *conf.AdminBackend_ObjectStore
+		switch storageType {
+		case "aliyun-oss":
+			return newAliyunStorage(config.Storage.AliyunOss)
+		case "huawei-obs":
+			return newHuaweiStorage(config.Storage.HuaweiObs)
+		case "tencent-cos":
+			return newTencentStorage(config.Storage.TencentCos)
+		case "aws-s3":
+			return newAWSStorage(storageType, config.Storage.AwsS3)
+		case "cloudflare-r2":
+			return newAWSStorage(storageType, config.Storage.CloudflareR2)
+		case "minio":
+			object = config.Storage.Minio
+		default:
+			return nil, fmt.Errorf("unsupported storage type %q", storageType)
+		}
+		return newS3Storage(storageType, object)
+	}
+	root, prefix := "uploads/file", "uploads/file"
+	if config != nil && config.Local != nil {
+		if config.Local.StorePath != "" {
+			root = config.Local.StorePath
+		}
+		if config.Local.PathPrefix != "" {
+			prefix = config.Local.PathPrefix
+		}
+	}
+	root = filepath.Clean(root)
+	if err := os.MkdirAll(root, 0o755); err != nil {
+		return nil, fmt.Errorf("create upload directory: %w", err)
+	}
+	return &fileStorage{root: root, urlPrefix: "/" + strings.Trim(prefix, "/")}, nil
+}
+
+func composeFiles(ctx context.Context, storage biz.FileStorage, names []string, destination string) (*biz.StoredFile, string, error) {
+	reader, writer := io.Pipe()
+	hash := md5.New()
+	errCh := make(chan error, 1)
+	go func() {
+		defer writer.Close()
+		for _, name := range names {
+			if ctx.Err() != nil {
+				errCh <- ctx.Err()
+				return
+			}
+			file, err := storage.Open(ctx, name)
+			if err != nil {
+				errCh <- err
+				return
+			}
+			_, err = io.Copy(io.MultiWriter(writer, hash), file)
+			_ = file.Close()
+			if err != nil {
+				errCh <- err
+				return
+			}
+		}
+		errCh <- nil
+	}()
+	stored, err := storage.Put(ctx, destination, reader)
+	composeErr := <-errCh
+	if err != nil {
+		return nil, "", err
+	}
+	if composeErr != nil {
+		_ = storage.Delete(ctx, destination)
+		return nil, "", composeErr
+	}
+	return stored, hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+func (s *fileStorage) resolve(name string) (string, error) {
+	clean := filepath.Clean(strings.TrimPrefix(name, "/"))
+	if clean == "." || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
+		return "", fmt.Errorf("invalid storage path")
+	}
+	return filepath.Join(s.root, clean), nil
+}
+
+func (s *fileStorage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
+	path, err := s.resolve(name)
+	if err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return nil, err
+	}
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+	if err != nil {
+		return nil, err
+	}
+	size, copyErr := io.Copy(file, reader)
+	closeErr := file.Close()
+	if copyErr != nil {
+		return nil, copyErr
+	}
+	if closeErr != nil {
+		return nil, closeErr
+	}
+	return &biz.StoredFile{Name: filepath.Base(name), Path: name, URL: s.urlPrefix + "/" + strings.TrimPrefix(filepath.ToSlash(name), "/"), Size: size}, nil
+}
+
+func (s *fileStorage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
+	path, err := s.resolve(name)
+	if err != nil {
+		return nil, err
+	}
+	return os.Open(path)
+}
+
+func (s *fileStorage) Delete(ctx context.Context, name string) error {
+	path, err := s.resolve(name)
+	if err != nil {
+		return err
+	}
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *fileStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
+	return composeFiles(ctx, s, names, destination)
+}
+func (s *fileStorage) DeletePrefix(ctx context.Context, prefix string) error {
+	path, err := s.resolve(strings.TrimSuffix(prefix, "/") + "/placeholder")
+	if err != nil {
+		return err
+	}
+	dir := filepath.Dir(path)
+	if !strings.HasPrefix(dir, s.root+string(filepath.Separator)) {
+		return fmt.Errorf("invalid storage path")
+	}
+	if err = os.RemoveAll(dir); err != nil {
+		return err
+	}
+	return nil
+}
+func (s *fileStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
+	if limit < 1 || limit > 1000 {
+		limit = 100
+	}
+	root, err := s.resolve(strings.TrimSuffix(prefix, "/") + "/placeholder")
+	if err != nil {
+		return nil, "", false, err
+	}
+	walkRoot := filepath.Dir(root)
+	items := []*biz.StoredFile{}
+	err = filepath.WalkDir(walkRoot, func(path string, entry os.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			if os.IsNotExist(walkErr) {
+				return nil
+			}
+			return walkErr
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if entry.IsDir() || entry.Type()&os.ModeSymlink != 0 {
+			return nil
+		}
+		relative, _ := filepath.Rel(s.root, path)
+		relative = filepath.ToSlash(relative)
+		if relative <= cursor {
+			return nil
+		}
+		info, infoErr := entry.Info()
+		if infoErr != nil {
+			return infoErr
+		}
+		items = append(items, &biz.StoredFile{Name: entry.Name(), Path: relative, URL: s.urlPrefix + "/" + relative, Size: info.Size()})
+		return nil
+	})
+	if err != nil {
+		return nil, "", false, err
+	}
+	sort.Slice(items, func(i, j int) bool { return items[i].Path < items[j].Path })
+	hasMore := len(items) > limit
+	if hasMore {
+		items = items[:limit]
+	}
+	next := ""
+	if len(items) > 0 {
+		next = items[len(items)-1].Path
+	}
+	return items, next, hasMore, nil
+}

internal/data/storage/qiniu_storage.go

@@ -0,0 +1,126 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/qiniu/go-sdk/v7/auth/qbox"
+	qstorage "github.com/qiniu/go-sdk/v7/storage"
+	"kra/internal/biz"
+	"kra/internal/conf"
+)
+
+type qiniuStorage struct {
+	config  *conf.AdminBackend_Qiniu
+	upload  *qstorage.FormUploader
+	manager *qstorage.BucketManager
+	mac     *qbox.Mac
+	token   string
+}
+
+func newQiniuStorage(config *conf.AdminBackend_Qiniu) (biz.FileStorage, error) {
+	if config == nil || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
+		return nil, fmt.Errorf("qiniu storage configuration is incomplete")
+	}
+	cfg := qstorage.Config{UseHTTPS: config.UseHttps, UseCdnDomains: config.UseCdnDomains}
+	switch config.Zone {
+	case "ZoneHuadong":
+		cfg.Zone = &qstorage.ZoneHuadong
+	case "ZoneHuabei":
+		cfg.Zone = &qstorage.ZoneHuabei
+	case "ZoneHuanan":
+		cfg.Zone = &qstorage.ZoneHuanan
+	case "ZoneBeimei":
+		cfg.Zone = &qstorage.ZoneBeimei
+	case "ZoneXinjiapo":
+		cfg.Zone = &qstorage.ZoneXinjiapo
+	}
+	mac := qbox.NewMac(config.AccessKey, config.SecretKey)
+	policy := qstorage.PutPolicy{Scope: config.Bucket}
+	token := policy.UploadToken(mac)
+	return &qiniuStorage{config: config, upload: qstorage.NewFormUploader(&cfg), manager: qstorage.NewBucketManager(mac, &cfg), mac: mac, token: token}, nil
+}
+func (s *qiniuStorage) file(key string, size int64) *biz.StoredFile {
+	return &biz.StoredFile{Name: path.Base(key), Path: key, URL: strings.TrimSuffix(s.config.BaseUrl, "/") + "/" + key, Size: size}
+}
+func (s *qiniuStorage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
+	temporary, err := os.CreateTemp("", "kra-qiniu-upload-*")
+	if err != nil {
+		return nil, err
+	}
+	temporaryName := temporary.Name()
+	defer os.Remove(temporaryName)
+	size, err := io.Copy(temporary, reader)
+	if err != nil {
+		_ = temporary.Close()
+		return nil, err
+	}
+	if _, err = temporary.Seek(0, io.SeekStart); err != nil {
+		_ = temporary.Close()
+		return nil, err
+	}
+	defer temporary.Close()
+	ret := qstorage.PutRet{}
+	if err = s.upload.Put(ctx, &ret, s.token, strings.TrimPrefix(name, "/"), temporary, size, &qstorage.PutExtra{}); err != nil {
+		return nil, err
+	}
+	return s.file(ret.Key, size), nil
+}
+func (s *qiniuStorage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
+	downloadURL := qstorage.MakePrivateURL(s.mac, strings.TrimSuffix(s.config.BaseUrl, "/"), strings.TrimPrefix(name, "/"), time.Now().Add(time.Hour).Unix())
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode >= 300 {
+		_ = resp.Body.Close()
+		return nil, fmt.Errorf("qiniu download failed: %s", resp.Status)
+	}
+	return resp.Body, nil
+}
+func (s *qiniuStorage) Delete(ctx context.Context, name string) error {
+	return s.manager.Delete(s.config.Bucket, strings.TrimPrefix(name, "/"))
+}
+func (s *qiniuStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
+	return composeFiles(ctx, s, names, destination)
+}
+func (s *qiniuStorage) DeletePrefix(ctx context.Context, prefix string) error {
+	for {
+		items, _, more, err := s.List(ctx, prefix, "", 1000)
+		if err != nil {
+			return err
+		}
+		for _, item := range items {
+			if err = s.Delete(ctx, item.Path); err != nil {
+				return err
+			}
+		}
+		if !more {
+			return nil
+		}
+	}
+}
+func (s *qiniuStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
+	if limit < 1 || limit > 1000 {
+		limit = 100
+	}
+	entries, _, marker, more, err := s.manager.ListFiles(s.config.Bucket, prefix, "", cursor, limit)
+	if err != nil {
+		return nil, "", false, err
+	}
+	out := make([]*biz.StoredFile, 0, len(entries))
+	for _, entry := range entries {
+		out = append(out, s.file(entry.Key, entry.Fsize))
+	}
+	return out, marker, more, nil
+}

internal/data/storage/s3_storage.go

@@ -0,0 +1,135 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"kra/internal/biz"
+	"kra/internal/conf"
+)
+
+type s3Storage struct {
+	client                  *minio.Client
+	bucket, baseURL, prefix string
+}
+
+func newS3Storage(provider string, config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
+	if config == nil || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
+		return nil, fmt.Errorf("%s storage configuration is incomplete", provider)
+	}
+	endpoint := config.Endpoint
+	if endpoint == "" {
+		switch provider {
+		case "aws-s3":
+			endpoint = "s3." + config.Region + ".amazonaws.com"
+		case "cloudflare-r2":
+			endpoint = config.AccountId + ".r2.cloudflarestorage.com"
+		case "aliyun-oss":
+			endpoint = "oss-" + config.Region + ".aliyuncs.com"
+		case "huawei-obs":
+			endpoint = "obs." + config.Region + ".myhuaweicloud.com"
+		case "tencent-cos":
+			endpoint = "cos." + config.Region + ".myqcloud.com"
+		}
+	}
+	secure := config.UseSsl
+	if parsed, err := url.Parse(endpoint); err == nil && parsed.Host != "" {
+		secure = parsed.Scheme == "https"
+		endpoint = parsed.Host
+	}
+	if endpoint == "" {
+		return nil, fmt.Errorf("%s endpoint is required", provider)
+	}
+	client, err := minio.New(strings.TrimSuffix(endpoint, "/"), &minio.Options{Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""), Secure: secure, Region: config.Region, BucketLookup: func() minio.BucketLookupType {
+		if config.ForcePathStyle {
+			return minio.BucketLookupPath
+		}
+		return minio.BucketLookupAuto
+	}()})
+	if err != nil {
+		return nil, err
+	}
+	return &s3Storage{client: client, bucket: config.Bucket, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
+}
+func (s *s3Storage) key(name string) string {
+	if s.prefix == "" {
+		return strings.TrimPrefix(name, "/")
+	}
+	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
+}
+func (s *s3Storage) unkey(key string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
+}
+func (s *s3Storage) file(key string, size int64) *biz.StoredFile {
+	name := s.unkey(key)
+	rawURL := s.baseURL + "/" + key
+	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: rawURL, Size: size}
+}
+func (s *s3Storage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
+	key := s.key(name)
+	info, err := s.client.PutObject(ctx, s.bucket, key, reader, -1, minio.PutObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return s.file(key, info.Size), nil
+}
+func (s *s3Storage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
+	obj, err := s.client.GetObject(ctx, s.bucket, s.key(name), minio.GetObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+	if _, err = obj.Stat(); err != nil {
+		_ = obj.Close()
+		return nil, err
+	}
+	return obj, nil
+}
+func (s *s3Storage) Delete(ctx context.Context, name string) error {
+	return s.client.RemoveObject(ctx, s.bucket, s.key(name), minio.RemoveObjectOptions{})
+}
+func (s *s3Storage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
+	return composeFiles(ctx, s, names, destination)
+}
+func (s *s3Storage) DeletePrefix(ctx context.Context, prefix string) error {
+	items := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: s.key(prefix), Recursive: true})
+	for item := range items {
+		if item.Err != nil {
+			return item.Err
+		}
+		if err := s.client.RemoveObject(ctx, s.bucket, item.Key, minio.RemoveObjectOptions{}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+func (s *s3Storage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
+	if limit < 1 || limit > 1000 {
+		limit = 100
+	}
+	items := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: s.key(prefix), Recursive: true, StartAfter: s.key(cursor)})
+	out := make([]*biz.StoredFile, 0, limit+1)
+	for item := range items {
+		if item.Err != nil {
+			return nil, "", false, item.Err
+		}
+		out = append(out, s.file(item.Key, item.Size))
+		if len(out) > limit {
+			break
+		}
+	}
+	more := len(out) > limit
+	if more {
+		out = out[:limit]
+	}
+	next := ""
+	if len(out) > 0 {
+		next = out[len(out)-1].Path
+	}
+	return out, next, more, nil
+}

internal/data/storage/tencent_storage.go

@@ -0,0 +1,117 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	cos "github.com/tencentyun/cos-go-sdk-v5"
+	"kra/internal/biz"
+	"kra/internal/conf"
+)
+
+type tencentStorage struct {
+	client          *cos.Client
+	baseURL, prefix string
+}
+
+func newTencentStorage(config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
+	if config == nil || config.Bucket == "" || config.Region == "" || config.AccessKey == "" || config.SecretKey == "" {
+		return nil, fmt.Errorf("tencent-cos storage configuration is incomplete")
+	}
+	endpoint := config.Endpoint
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.cos.%s.myqcloud.com", config.Bucket, config.Region)
+	} else if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
+		if config.UseSsl {
+			endpoint = "https://" + endpoint
+		} else {
+			endpoint = "http://" + endpoint
+		}
+	}
+	bucketURL, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	client := cos.NewClient(&cos.BaseURL{BucketURL: bucketURL}, &http.Client{Transport: &cos.AuthorizationTransport{SecretID: config.AccessKey, SecretKey: config.SecretKey}})
+	return &tencentStorage{client: client, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
+}
+func (s *tencentStorage) key(name string) string {
+	if s.prefix == "" {
+		return strings.TrimPrefix(name, "/")
+	}
+	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
+}
+func (s *tencentStorage) unkey(key string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
+}
+func (s *tencentStorage) file(key string, size int64) *biz.StoredFile {
+	name := s.unkey(key)
+	fileURL := s.baseURL + "/" + key
+	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: fileURL, Size: size}
+}
+func (s *tencentStorage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
+	key := s.key(name)
+	if _, err := s.client.Object.Put(ctx, key, reader, nil); err != nil {
+		return nil, err
+	}
+	response, err := s.client.Object.Head(ctx, key, nil)
+	if err != nil {
+		return nil, err
+	}
+	return s.file(key, response.ContentLength), nil
+}
+func (s *tencentStorage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
+	response, err := s.client.Object.Get(ctx, s.key(name), nil)
+	if err != nil {
+		return nil, err
+	}
+	return response.Body, nil
+}
+func (s *tencentStorage) Delete(ctx context.Context, name string) error {
+	_, err := s.client.Object.Delete(ctx, s.key(name))
+	return err
+}
+func (s *tencentStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
+	return composeFiles(ctx, s, names, destination)
+}
+func (s *tencentStorage) DeletePrefix(ctx context.Context, prefix string) error {
+	cursor := ""
+	for {
+		items, next, more, err := s.List(ctx, prefix, cursor, 1000)
+		if err != nil {
+			return err
+		}
+		for _, item := range items {
+			if err = s.Delete(ctx, item.Path); err != nil {
+				return err
+			}
+		}
+		if !more {
+			return nil
+		}
+		cursor = next
+	}
+}
+func (s *tencentStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
+	if limit < 1 || limit > 1000 {
+		limit = 100
+	}
+	result, _, err := s.client.Bucket.Get(ctx, &cos.BucketGetOptions{Prefix: s.key(prefix), Marker: cursor, MaxKeys: limit})
+	if err != nil {
+		return nil, "", false, err
+	}
+	items := make([]*biz.StoredFile, 0, len(result.Contents))
+	for _, object := range result.Contents {
+		items = append(items, s.file(object.Key, object.Size))
+	}
+	next := result.NextMarker
+	if result.IsTruncated && next == "" && len(result.Contents) > 0 {
+		next = result.Contents[len(result.Contents)-1].Key
+	}
+	return items, next, result.IsTruncated, nil
+}