优化结构

internal/integration/storage/aliyun_storage.go

@@ -1,107 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"path"
-	"strings"
-
-	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-	"kra/internal/biz"
-	"kra/internal/conf"
-)
-
-type aliyunStorage struct {
-	bucket          *oss.Bucket
-	baseURL, prefix string
-}
-
-func newAliyunStorage(config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
-	if config == nil || config.Endpoint == "" || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
-		return nil, fmt.Errorf("aliyun-oss storage configuration is incomplete")
-	}
-	client, err := oss.New(config.Endpoint, config.AccessKey, config.SecretKey)
-	if err != nil {
-		return nil, err
-	}
-	bucket, err := client.Bucket(config.Bucket)
-	if err != nil {
-		return nil, err
-	}
-	return &aliyunStorage{bucket: bucket, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
-}
-func (s *aliyunStorage) key(name string) string {
-	if s.prefix == "" {
-		return strings.TrimPrefix(name, "/")
-	}
-	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
-}
-func (s *aliyunStorage) unkey(key string) string {
-	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
-}
-func (s *aliyunStorage) file(key string, size int64) *biz.StoredFile {
-	name := s.unkey(key)
-	url := s.baseURL + "/" + key
-	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: url, Size: size}
-}
-func (s *aliyunStorage) Put(_ context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
-	key := s.key(name)
-	if err := s.bucket.PutObject(key, reader); err != nil {
-		return nil, err
-	}
-	meta, err := s.bucket.GetObjectDetailedMeta(key)
-	if err != nil {
-		return nil, err
-	}
-	size := int64(0)
-	_, _ = fmt.Sscan(meta.Get("Content-Length"), &size)
-	return s.file(key, size), nil
-}
-func (s *aliyunStorage) Open(_ context.Context, name string) (io.ReadCloser, error) {
-	return s.bucket.GetObject(s.key(name))
-}
-func (s *aliyunStorage) Delete(_ context.Context, name string) error {
-	return s.bucket.DeleteObject(s.key(name))
-}
-func (s *aliyunStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
-	return composeFiles(ctx, s, names, destination)
-}
-func (s *aliyunStorage) DeletePrefix(ctx context.Context, prefix string) error {
-	cursor := ""
-	for {
-		items, next, more, err := s.List(ctx, prefix, cursor, 1000)
-		if err != nil {
-			return err
-		}
-		for _, item := range items {
-			if err = s.Delete(ctx, item.Path); err != nil {
-				return err
-			}
-		}
-		if !more {
-			return nil
-		}
-		cursor = next
-	}
-}
-func (s *aliyunStorage) List(_ context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
-	if limit <= 0 {
-		limit = 100
-	}
-	result, err := s.bucket.ListObjects(oss.Prefix(s.key(prefix)), oss.Marker(cursor), oss.MaxKeys(limit))
-	if err != nil {
-		return nil, "", false, err
-	}
-	items := make([]*biz.StoredFile, 0, len(result.Objects))
-	for _, object := range result.Objects {
-		item := s.file(object.Key, object.Size)
-		item.LastModified = object.LastModified
-		items = append(items, item)
-	}
-	next := ""
-	if result.IsTruncated {
-		next = result.NextMarker
-	}
-	return items, next, result.IsTruncated, nil
-}

internal/integration/storage/aws_storage.go

@@ -1,156 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"path"
-	"strings"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	awsconfig "github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"kra/internal/biz"
-	"kra/internal/conf"
-)
-
-type awsStorage struct {
-	client                  *s3.Client
-	bucket, baseURL, prefix string
-}
-
-func newAWSStorage(provider string, config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
-	if config == nil || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
-		return nil, fmt.Errorf("%s storage configuration is incomplete", provider)
-	}
-	region := config.Region
-	endpoint := config.Endpoint
-	if provider == "cloudflare-r2" {
-		region = "auto"
-		if endpoint == "" {
-			if config.AccountId == "" {
-				return nil, fmt.Errorf("cloudflare-r2 account id is required")
-			}
-			endpoint = fmt.Sprintf("https://%s.r2.cloudflarestorage.com", config.AccountId)
-		}
-	}
-	if region == "" {
-		return nil, fmt.Errorf("%s region is required", provider)
-	}
-	loaded, err := awsconfig.LoadDefaultConfig(context.Background(), awsconfig.WithRegion(region), awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.AccessKey, config.SecretKey, "")))
-	if err != nil {
-		return nil, err
-	}
-	client := s3.NewFromConfig(loaded, func(options *s3.Options) {
-		if endpoint != "" {
-			if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
-				if config.UseSsl {
-					endpoint = "https://" + endpoint
-				} else {
-					endpoint = "http://" + endpoint
-				}
-			}
-			options.BaseEndpoint = aws.String(endpoint)
-		}
-		options.UsePathStyle = config.ForcePathStyle
-	})
-	return &awsStorage{client: client, bucket: config.Bucket, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
-}
-
-func (s *awsStorage) key(name string) string {
-	if s.prefix == "" {
-		return strings.TrimPrefix(name, "/")
-	}
-	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
-}
-func (s *awsStorage) unkey(key string) string {
-	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
-}
-func (s *awsStorage) file(key string, size int64) *biz.StoredFile {
-	name := s.unkey(key)
-	url := s.baseURL + "/" + key
-	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: url, Size: size}
-}
-func (s *awsStorage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
-	key := s.key(name)
-	result, err := manager.NewUploader(s.client).Upload(ctx, &s3.PutObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key), Body: reader})
-	if err != nil {
-		return nil, err
-	}
-	_ = result
-	head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
-	if err != nil {
-		return nil, err
-	}
-	size := int64(0)
-	if head.ContentLength != nil {
-		size = *head.ContentLength
-	}
-	return s.file(key, size), nil
-}
-func (s *awsStorage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
-	result, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(s.key(name))})
-	if err != nil {
-		return nil, err
-	}
-	return result.Body, nil
-}
-func (s *awsStorage) Delete(ctx context.Context, name string) error {
-	_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(s.key(name))})
-	return err
-}
-func (s *awsStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
-	return composeFiles(ctx, s, names, destination)
-}
-func (s *awsStorage) DeletePrefix(ctx context.Context, prefix string) error {
-	for {
-		items, _, more, err := s.List(ctx, prefix, "", 1000)
-		if err != nil {
-			return err
-		}
-		for _, item := range items {
-			if err = s.Delete(ctx, item.Path); err != nil {
-				return err
-			}
-		}
-		if !more || len(items) == 0 {
-			return nil
-		}
-	}
-}
-func (s *awsStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
-	if limit <= 0 {
-		limit = 100
-	}
-	input := &s3.ListObjectsV2Input{Bucket: aws.String(s.bucket), Prefix: aws.String(s.key(prefix)), MaxKeys: aws.Int32(int32(limit))}
-	if cursor != "" {
-		input.ContinuationToken = aws.String(cursor)
-	}
-	result, err := s.client.ListObjectsV2(ctx, input)
-	if err != nil {
-		return nil, "", false, err
-	}
-	items := make([]*biz.StoredFile, 0, len(result.Contents))
-	for _, object := range result.Contents {
-		if object.Key == nil {
-			continue
-		}
-		size := int64(0)
-		if object.Size != nil {
-			size = *object.Size
-		}
-		item := s.file(*object.Key, size)
-		if object.LastModified != nil {
-			item.LastModified = *object.LastModified
-		}
-		items = append(items, item)
-	}
-	next := ""
-	if result.NextContinuationToken != nil {
-		next = *result.NextContinuationToken
-	}
-	more := result.IsTruncated != nil && *result.IsTruncated
-	return items, next, more, nil
-}

internal/integration/storage/compose.go

@@ -1,56 +0,0 @@
-package storage
-
-import (
-	"context"
-	"crypto/md5"
-	"encoding/hex"
-	"io"
-)
-
-// composeStreams concatenates objects into destination while calculating the
-// MD5 of the exact bytes written. Provider-specific storage clients stay out
-// of this package; callbacks keep this helper independent from biz and SDKs.
-func composeStreams(
-	ctx context.Context,
-	names []string,
-	open func(context.Context, string) (io.ReadCloser, error),
-	put func(context.Context, string, io.Reader) error,
-	remove func(context.Context, string) error,
-	destination string,
-) (string, error) {
-	reader, writer := io.Pipe()
-	hash := md5.New()
-	errCh := make(chan error, 1)
-	go func() {
-		defer writer.Close()
-		for _, name := range names {
-			if err := ctx.Err(); err != nil {
-				errCh <- err
-				return
-			}
-			file, err := open(ctx, name)
-			if err != nil {
-				errCh <- err
-				return
-			}
-			_, copyErr := io.Copy(io.MultiWriter(writer, hash), file)
-			_ = file.Close()
-			if copyErr != nil {
-				errCh <- copyErr
-				return
-			}
-		}
-		errCh <- nil
-	}()
-	putErr := put(ctx, destination, reader)
-	composeErr := <-errCh
-	if putErr != nil {
-		_ = remove(ctx, destination)
-		return "", putErr
-	}
-	if composeErr != nil {
-		_ = remove(ctx, destination)
-		return "", composeErr
-	}
-	return hex.EncodeToString(hash.Sum(nil)), nil
-}

internal/integration/storage/compose_test.go

@@ -1,50 +0,0 @@
-package storage
-
-import (
-	"context"
-	"crypto/md5"
-	"encoding/hex"
-	"errors"
-	"io"
-	"strings"
-	"testing"
-)
-
-func TestComposeStreams(t *testing.T) {
-	objects := map[string]string{"a": "hello ", "b": "world"}
-	var stored string
-	hash, err := composeStreams(context.Background(), []string{"a", "b"}, func(_ context.Context, name string) (io.ReadCloser, error) {
-		value, ok := objects[name]
-		if !ok {
-			return nil, errors.New("missing object")
-		}
-		return io.NopCloser(strings.NewReader(value)), nil
-	}, func(_ context.Context, _ string, reader io.Reader) error {
-		value, err := io.ReadAll(reader)
-		stored = string(value)
-		return err
-	}, func(context.Context, string) error { return nil }, "out")
-	if err != nil || stored != "hello world" {
-		t.Fatalf("compose = %q, %v", stored, err)
-	}
-	want := md5.Sum([]byte(stored))
-	if hash != hex.EncodeToString(want[:]) {
-		t.Fatalf("hash = %q", hash)
-	}
-}
-
-func TestComposeStreamsRemovesPartialDestination(t *testing.T) {
-	removed := false
-	_, err := composeStreams(context.Background(), []string{"missing"}, func(context.Context, string) (io.ReadCloser, error) {
-		return nil, errors.New("open failed")
-	}, func(_ context.Context, _ string, reader io.Reader) error {
-		_, _ = io.ReadAll(reader)
-		return nil
-	}, func(context.Context, string) error {
-		removed = true
-		return nil
-	}, "out")
-	if err == nil || !removed {
-		t.Fatalf("compose error = %v, removed = %v", err, removed)
-	}
-}

internal/integration/storage/huawei_storage.go

@@ -1,106 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"path"
-	"strings"
-
-	"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
-	"kra/internal/biz"
-	"kra/internal/conf"
-)
-
-type huaweiStorage struct {
-	client                  *obs.ObsClient
-	bucket, baseURL, prefix string
-}
-
-func newHuaweiStorage(config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
-	if config == nil || config.Endpoint == "" || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
-		return nil, fmt.Errorf("huawei-obs storage configuration is incomplete")
-	}
-	client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint)
-	if err != nil {
-		return nil, err
-	}
-	return &huaweiStorage{client: client, bucket: config.Bucket, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
-}
-func (s *huaweiStorage) key(name string) string {
-	if s.prefix == "" {
-		return strings.TrimPrefix(name, "/")
-	}
-	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
-}
-func (s *huaweiStorage) unkey(key string) string {
-	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
-}
-func (s *huaweiStorage) file(key string, size int64) *biz.StoredFile {
-	name := s.unkey(key)
-	fileURL := s.baseURL + "/" + key
-	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: fileURL, Size: size}
-}
-func (s *huaweiStorage) Put(_ context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
-	key := s.key(name)
-	if _, err := s.client.PutObject(&obs.PutObjectInput{PutObjectBasicInput: obs.PutObjectBasicInput{ObjectOperationInput: obs.ObjectOperationInput{Bucket: s.bucket, Key: key}}, Body: reader}); err != nil {
-		return nil, err
-	}
-	meta, err := s.client.GetObjectMetadata(&obs.GetObjectMetadataInput{Bucket: s.bucket, Key: key})
-	if err != nil {
-		return nil, err
-	}
-	return s.file(key, meta.ContentLength), nil
-}
-func (s *huaweiStorage) Open(_ context.Context, name string) (io.ReadCloser, error) {
-	result, err := s.client.GetObject(&obs.GetObjectInput{GetObjectMetadataInput: obs.GetObjectMetadataInput{Bucket: s.bucket, Key: s.key(name)}})
-	if err != nil {
-		return nil, err
-	}
-	return result.Body, nil
-}
-func (s *huaweiStorage) Delete(_ context.Context, name string) error {
-	_, err := s.client.DeleteObject(&obs.DeleteObjectInput{Bucket: s.bucket, Key: s.key(name)})
-	return err
-}
-func (s *huaweiStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
-	return composeFiles(ctx, s, names, destination)
-}
-func (s *huaweiStorage) DeletePrefix(ctx context.Context, prefix string) error {
-	cursor := ""
-	for {
-		items, next, more, err := s.List(ctx, prefix, cursor, 1000)
-		if err != nil {
-			return err
-		}
-		for _, item := range items {
-			if err = s.Delete(ctx, item.Path); err != nil {
-				return err
-			}
-		}
-		if !more {
-			return nil
-		}
-		cursor = next
-	}
-}
-func (s *huaweiStorage) List(_ context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
-	if limit <= 0 {
-		limit = 100
-	}
-	result, err := s.client.ListObjects(&obs.ListObjectsInput{ListObjsInput: obs.ListObjsInput{Prefix: s.key(prefix), MaxKeys: limit}, Bucket: s.bucket, Marker: cursor})
-	if err != nil {
-		return nil, "", false, err
-	}
-	items := make([]*biz.StoredFile, 0, len(result.Contents))
-	for _, object := range result.Contents {
-		item := s.file(object.Key, object.Size)
-		item.LastModified = object.LastModified
-		items = append(items, item)
-	}
-	next := result.NextMarker
-	if result.IsTruncated && next == "" && len(result.Contents) > 0 {
-		next = result.Contents[len(result.Contents)-1].Key
-	}
-	return items, next, result.IsTruncated, nil
-}

internal/integration/storage/local.go

@@ -1,202 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"mime"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-
-	"kra/internal/biz"
-	"kra/internal/conf"
-)
-
-type fileStorage struct {
-	root      string
-	urlPrefix string
-}
-
-func New(config *conf.AdminBackend) (biz.FileStorage, error) {
-	storageType := "local"
-	if config != nil && config.Storage != nil && config.Storage.Type != "" {
-		storageType = strings.ToLower(config.Storage.Type)
-	}
-	if storageType == "qiniu" {
-		return newQiniuStorage(config.Storage.Qiniu)
-	}
-	if storageType != "local" {
-		var object *conf.AdminBackend_ObjectStore
-		switch storageType {
-		case "aliyun-oss":
-			return newAliyunStorage(config.Storage.AliyunOss)
-		case "huawei-obs":
-			return newHuaweiStorage(config.Storage.HuaweiObs)
-		case "tencent-cos":
-			return newTencentStorage(config.Storage.TencentCos)
-		case "aws-s3":
-			return newAWSStorage(storageType, config.Storage.AwsS3)
-		case "cloudflare-r2":
-			return newAWSStorage(storageType, config.Storage.CloudflareR2)
-		case "minio":
-			object = config.Storage.Minio
-		default:
-			return nil, fmt.Errorf("unsupported storage type %q", storageType)
-		}
-		return newS3Storage(storageType, object)
-	}
-	root, prefix := "uploads/file", "uploads/file"
-	if config != nil && config.Local != nil {
-		if config.Local.StorePath != "" {
-			root = config.Local.StorePath
-		}
-		if config.Local.PathPrefix != "" {
-			prefix = config.Local.PathPrefix
-		}
-	}
-	root = filepath.Clean(root)
-	if err := os.MkdirAll(root, 0o755); err != nil {
-		return nil, fmt.Errorf("create upload directory: %w", err)
-	}
-	return &fileStorage{root: root, urlPrefix: "/" + strings.Trim(prefix, "/")}, nil
-}
-
-func composeFiles(ctx context.Context, storage biz.FileStorage, names []string, destination string) (*biz.StoredFile, string, error) {
-	var stored *biz.StoredFile
-	hash, err := composeStreams(ctx, names, storage.Open, func(ctx context.Context, destination string, reader io.Reader) error {
-		var putErr error
-		stored, putErr = storage.Put(ctx, destination, reader)
-		return putErr
-	}, storage.Delete, destination)
-	if err != nil {
-		return nil, "", err
-	}
-	return stored, hash, nil
-}
-
-func (s *fileStorage) resolve(name string) (string, error) {
-	if name == "" {
-		return "", fmt.Errorf("key不能为空")
-	}
-	clean := filepath.Clean(strings.TrimPrefix(name, "/"))
-	if clean == "." || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
-		return "", fmt.Errorf("invalid storage path")
-	}
-	return filepath.Join(s.root, clean), nil
-}
-
-func (s *fileStorage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
-	path, err := s.resolve(name)
-	if err != nil {
-		return nil, err
-	}
-	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
-		return nil, err
-	}
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
-	if err != nil {
-		return nil, err
-	}
-	size, copyErr := io.Copy(file, reader)
-	closeErr := file.Close()
-	if copyErr != nil {
-		return nil, copyErr
-	}
-	if closeErr != nil {
-		return nil, closeErr
-	}
-	return &biz.StoredFile{Name: filepath.Base(name), Path: name, URL: s.urlPrefix + "/" + strings.TrimPrefix(filepath.ToSlash(name), "/"), Size: size}, nil
-}
-
-func (s *fileStorage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
-	path, err := s.resolve(name)
-	if err != nil {
-		return nil, err
-	}
-	return os.Open(path)
-}
-
-func (s *fileStorage) Delete(ctx context.Context, name string) error {
-	path, err := s.resolve(name)
-	if err != nil {
-		return err
-	}
-	// Match the compatible Local.DeleteFile behavior: a missing object is an error and the
-	// media DB record is retained when the physical file cannot be removed.
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return fmt.Errorf("文件不存在")
-	}
-	if err := os.Remove(path); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (s *fileStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
-	return composeFiles(ctx, s, names, destination)
-}
-func (s *fileStorage) DeletePrefix(ctx context.Context, prefix string) error {
-	path, err := s.resolve(strings.TrimSuffix(prefix, "/") + "/placeholder")
-	if err != nil {
-		return err
-	}
-	dir := filepath.Dir(path)
-	if !strings.HasPrefix(dir, s.root+string(filepath.Separator)) {
-		return fmt.Errorf("invalid storage path")
-	}
-	if err = os.RemoveAll(dir); err != nil {
-		return err
-	}
-	return nil
-}
-func (s *fileStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
-	if limit <= 0 {
-		limit = 100
-	}
-	root, err := s.resolve(strings.TrimSuffix(prefix, "/") + "/placeholder")
-	if err != nil {
-		return nil, "", false, err
-	}
-	walkRoot := filepath.Dir(root)
-	items := []*biz.StoredFile{}
-	err = filepath.WalkDir(walkRoot, func(path string, entry os.DirEntry, walkErr error) error {
-		if walkErr != nil {
-			if os.IsNotExist(walkErr) {
-				return nil
-			}
-			return walkErr
-		}
-		if ctx.Err() != nil {
-			return ctx.Err()
-		}
-		if entry.IsDir() || entry.Type()&os.ModeSymlink != 0 {
-			return nil
-		}
-		relative, _ := filepath.Rel(s.root, path)
-		relative = filepath.ToSlash(relative)
-		if relative <= cursor {
-			return nil
-		}
-		info, infoErr := entry.Info()
-		if infoErr != nil {
-			return infoErr
-		}
-		items = append(items, &biz.StoredFile{Name: entry.Name(), Path: relative, URL: s.urlPrefix + "/" + relative, Size: info.Size(), LastModified: info.ModTime(), ContentType: mime.TypeByExtension(filepath.Ext(entry.Name()))})
-		return nil
-	})
-	if err != nil {
-		return nil, "", false, err
-	}
-	sort.Slice(items, func(i, j int) bool { return items[i].Path < items[j].Path })
-	hasMore := len(items) > limit
-	if hasMore {
-		items = items[:limit]
-	}
-	next := ""
-	if len(items) > 0 {
-		next = items[len(items)-1].Path
-	}
-	return items, next, hasMore, nil
-}

internal/integration/storage/qiniu_storage.go

@@ -1,129 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/qiniu/go-sdk/v7/auth/qbox"
-	qstorage "github.com/qiniu/go-sdk/v7/storage"
-	"kra/internal/biz"
-	"kra/internal/conf"
-)
-
-type qiniuStorage struct {
-	config  *conf.AdminBackend_Qiniu
-	upload  *qstorage.FormUploader
-	manager *qstorage.BucketManager
-	mac     *qbox.Mac
-	token   string
-}
-
-func newQiniuStorage(config *conf.AdminBackend_Qiniu) (biz.FileStorage, error) {
-	if config == nil || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
-		return nil, fmt.Errorf("qiniu storage configuration is incomplete")
-	}
-	cfg := qstorage.Config{UseHTTPS: config.UseHttps, UseCdnDomains: config.UseCdnDomains}
-	switch config.Zone {
-	case "ZoneHuadong":
-		cfg.Zone = &qstorage.ZoneHuadong
-	case "ZoneHuabei":
-		cfg.Zone = &qstorage.ZoneHuabei
-	case "ZoneHuanan":
-		cfg.Zone = &qstorage.ZoneHuanan
-	case "ZoneBeimei":
-		cfg.Zone = &qstorage.ZoneBeimei
-	case "ZoneXinjiapo":
-		cfg.Zone = &qstorage.ZoneXinjiapo
-	}
-	mac := qbox.NewMac(config.AccessKey, config.SecretKey)
-	policy := qstorage.PutPolicy{Scope: config.Bucket}
-	token := policy.UploadToken(mac)
-	return &qiniuStorage{config: config, upload: qstorage.NewFormUploader(&cfg), manager: qstorage.NewBucketManager(mac, &cfg), mac: mac, token: token}, nil
-}
-func (s *qiniuStorage) file(key string, size int64) *biz.StoredFile {
-	return &biz.StoredFile{Name: path.Base(key), Path: key, URL: strings.TrimSuffix(s.config.BaseUrl, "/") + "/" + key, Size: size}
-}
-func (s *qiniuStorage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
-	temporary, err := os.CreateTemp("", "kra-qiniu-upload-*")
-	if err != nil {
-		return nil, err
-	}
-	temporaryName := temporary.Name()
-	defer os.Remove(temporaryName)
-	size, err := io.Copy(temporary, reader)
-	if err != nil {
-		_ = temporary.Close()
-		return nil, err
-	}
-	if _, err = temporary.Seek(0, io.SeekStart); err != nil {
-		_ = temporary.Close()
-		return nil, err
-	}
-	defer temporary.Close()
-	ret := qstorage.PutRet{}
-	if err = s.upload.Put(ctx, &ret, s.token, strings.TrimPrefix(name, "/"), temporary, size, &qstorage.PutExtra{}); err != nil {
-		return nil, err
-	}
-	return s.file(ret.Key, size), nil
-}
-func (s *qiniuStorage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
-	downloadURL := qstorage.MakePrivateURL(s.mac, strings.TrimSuffix(s.config.BaseUrl, "/"), strings.TrimPrefix(name, "/"), time.Now().Add(time.Hour).Unix())
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode >= 300 {
-		_ = resp.Body.Close()
-		return nil, fmt.Errorf("qiniu download failed: %s", resp.Status)
-	}
-	return resp.Body, nil
-}
-func (s *qiniuStorage) Delete(ctx context.Context, name string) error {
-	return s.manager.Delete(s.config.Bucket, strings.TrimPrefix(name, "/"))
-}
-func (s *qiniuStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
-	return composeFiles(ctx, s, names, destination)
-}
-func (s *qiniuStorage) DeletePrefix(ctx context.Context, prefix string) error {
-	for {
-		items, _, more, err := s.List(ctx, prefix, "", 1000)
-		if err != nil {
-			return err
-		}
-		for _, item := range items {
-			if err = s.Delete(ctx, item.Path); err != nil {
-				return err
-			}
-		}
-		if !more {
-			return nil
-		}
-	}
-}
-func (s *qiniuStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
-	if limit <= 0 {
-		limit = 100
-	}
-	entries, _, marker, more, err := s.manager.ListFiles(s.config.Bucket, prefix, "", cursor, limit)
-	if err != nil {
-		return nil, "", false, err
-	}
-	out := make([]*biz.StoredFile, 0, len(entries))
-	for _, entry := range entries {
-		item := s.file(entry.Key, entry.Fsize)
-		item.LastModified = time.Unix(0, entry.PutTime*100)
-		item.ContentType = entry.MimeType
-		out = append(out, item)
-	}
-	return out, marker, more, nil
-}

internal/integration/storage/reloadable.go

@@ -1,68 +0,0 @@
-package storage
-
-import (
-	"context"
-	"io"
-	"sync"
-
-	"kra/internal/biz"
-	"kra/internal/conf"
-)
-
-type Reloadable struct {
-	mu      sync.RWMutex
-	current biz.FileStorage
-}
-
-func NewFileStorage(runtime *conf.Runtime) (*Reloadable, error) {
-	var config *conf.AdminBackend
-	if runtime != nil {
-		config = runtime.Admin()
-	}
-	return NewReloadable(config)
-}
-
-func NewReloadable(config *conf.AdminBackend) (*Reloadable, error) {
-	current, err := New(config)
-	if err != nil {
-		return nil, err
-	}
-	return &Reloadable{current: current}, nil
-}
-
-func (s *Reloadable) Replace(current biz.FileStorage) {
-	s.mu.Lock()
-	s.current = current
-	s.mu.Unlock()
-}
-
-func (s *Reloadable) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.current.Put(ctx, name, reader)
-}
-func (s *Reloadable) Open(ctx context.Context, name string) (io.ReadCloser, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.current.Open(ctx, name)
-}
-func (s *Reloadable) Delete(ctx context.Context, name string) error {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.current.Delete(ctx, name)
-}
-func (s *Reloadable) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.current.Compose(ctx, names, destination)
-}
-func (s *Reloadable) DeletePrefix(ctx context.Context, prefix string) error {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.current.DeletePrefix(ctx, prefix)
-}
-func (s *Reloadable) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-	return s.current.List(ctx, prefix, cursor, limit)
-}

internal/integration/storage/s3_storage.go

@@ -1,138 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/url"
-	"path"
-	"strings"
-
-	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
-	"kra/internal/biz"
-	"kra/internal/conf"
-)
-
-type s3Storage struct {
-	client                  *minio.Client
-	bucket, baseURL, prefix string
-}
-
-func newS3Storage(provider string, config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
-	if config == nil || config.Bucket == "" || config.AccessKey == "" || config.SecretKey == "" {
-		return nil, fmt.Errorf("%s storage configuration is incomplete", provider)
-	}
-	endpoint := config.Endpoint
-	if endpoint == "" {
-		switch provider {
-		case "aws-s3":
-			endpoint = "s3." + config.Region + ".amazonaws.com"
-		case "cloudflare-r2":
-			endpoint = config.AccountId + ".r2.cloudflarestorage.com"
-		case "aliyun-oss":
-			endpoint = "oss-" + config.Region + ".aliyuncs.com"
-		case "huawei-obs":
-			endpoint = "obs." + config.Region + ".myhuaweicloud.com"
-		case "tencent-cos":
-			endpoint = "cos." + config.Region + ".myqcloud.com"
-		}
-	}
-	secure := config.UseSsl
-	if parsed, err := url.Parse(endpoint); err == nil && parsed.Host != "" {
-		secure = parsed.Scheme == "https"
-		endpoint = parsed.Host
-	}
-	if endpoint == "" {
-		return nil, fmt.Errorf("%s endpoint is required", provider)
-	}
-	client, err := minio.New(strings.TrimSuffix(endpoint, "/"), &minio.Options{Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""), Secure: secure, Region: config.Region, BucketLookup: func() minio.BucketLookupType {
-		if config.ForcePathStyle {
-			return minio.BucketLookupPath
-		}
-		return minio.BucketLookupAuto
-	}()})
-	if err != nil {
-		return nil, err
-	}
-	return &s3Storage{client: client, bucket: config.Bucket, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
-}
-func (s *s3Storage) key(name string) string {
-	if s.prefix == "" {
-		return strings.TrimPrefix(name, "/")
-	}
-	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
-}
-func (s *s3Storage) unkey(key string) string {
-	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
-}
-func (s *s3Storage) file(key string, size int64) *biz.StoredFile {
-	name := s.unkey(key)
-	rawURL := s.baseURL + "/" + key
-	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: rawURL, Size: size}
-}
-func (s *s3Storage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
-	key := s.key(name)
-	info, err := s.client.PutObject(ctx, s.bucket, key, reader, -1, minio.PutObjectOptions{})
-	if err != nil {
-		return nil, err
-	}
-	return s.file(key, info.Size), nil
-}
-func (s *s3Storage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
-	obj, err := s.client.GetObject(ctx, s.bucket, s.key(name), minio.GetObjectOptions{})
-	if err != nil {
-		return nil, err
-	}
-	if _, err = obj.Stat(); err != nil {
-		_ = obj.Close()
-		return nil, err
-	}
-	return obj, nil
-}
-func (s *s3Storage) Delete(ctx context.Context, name string) error {
-	return s.client.RemoveObject(ctx, s.bucket, s.key(name), minio.RemoveObjectOptions{})
-}
-func (s *s3Storage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
-	return composeFiles(ctx, s, names, destination)
-}
-func (s *s3Storage) DeletePrefix(ctx context.Context, prefix string) error {
-	items := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: s.key(prefix), Recursive: true})
-	for item := range items {
-		if item.Err != nil {
-			return item.Err
-		}
-		if err := s.client.RemoveObject(ctx, s.bucket, item.Key, minio.RemoveObjectOptions{}); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-func (s *s3Storage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
-	if limit <= 0 {
-		limit = 100
-	}
-	items := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: s.key(prefix), Recursive: true, StartAfter: s.key(cursor)})
-	out := make([]*biz.StoredFile, 0, limit+1)
-	for item := range items {
-		if item.Err != nil {
-			return nil, "", false, item.Err
-		}
-		file := s.file(item.Key, item.Size)
-		file.LastModified = item.LastModified
-		file.ContentType = item.ContentType
-		out = append(out, file)
-		if len(out) > limit {
-			break
-		}
-	}
-	more := len(out) > limit
-	if more {
-		out = out[:limit]
-	}
-	next := ""
-	if len(out) > 0 {
-		next = out[len(out)-1].Path
-	}
-	return out, next, more, nil
-}

internal/integration/storage/tencent_storage.go

@@ -1,125 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"path"
-	"strings"
-	"time"
-
-	cos "github.com/tencentyun/cos-go-sdk-v5"
-	"kra/internal/biz"
-	"kra/internal/conf"
-)
-
-type tencentStorage struct {
-	client          *cos.Client
-	baseURL, prefix string
-}
-
-func newTencentStorage(config *conf.AdminBackend_ObjectStore) (biz.FileStorage, error) {
-	if config == nil || config.Bucket == "" || config.Region == "" || config.AccessKey == "" || config.SecretKey == "" {
-		return nil, fmt.Errorf("tencent-cos storage configuration is incomplete")
-	}
-	endpoint := config.Endpoint
-	if endpoint == "" {
-		endpoint = fmt.Sprintf("https://%s.cos.%s.myqcloud.com", config.Bucket, config.Region)
-	} else if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
-		if config.UseSsl {
-			endpoint = "https://" + endpoint
-		} else {
-			endpoint = "http://" + endpoint
-		}
-	}
-	bucketURL, err := url.Parse(endpoint)
-	if err != nil {
-		return nil, err
-	}
-	client := cos.NewClient(&cos.BaseURL{BucketURL: bucketURL}, &http.Client{Transport: &cos.AuthorizationTransport{SecretID: config.AccessKey, SecretKey: config.SecretKey}})
-	return &tencentStorage{client: client, baseURL: strings.TrimSuffix(config.BaseUrl, "/"), prefix: strings.Trim(config.PathPrefix, "/")}, nil
-}
-func (s *tencentStorage) key(name string) string {
-	if s.prefix == "" {
-		return strings.TrimPrefix(name, "/")
-	}
-	return path.Join(s.prefix, strings.TrimPrefix(name, "/"))
-}
-func (s *tencentStorage) unkey(key string) string {
-	return strings.TrimPrefix(strings.TrimPrefix(key, s.prefix), "/")
-}
-func (s *tencentStorage) file(key string, size int64) *biz.StoredFile {
-	name := s.unkey(key)
-	fileURL := s.baseURL + "/" + key
-	return &biz.StoredFile{Name: path.Base(name), Path: name, URL: fileURL, Size: size}
-}
-func (s *tencentStorage) Put(ctx context.Context, name string, reader io.Reader) (*biz.StoredFile, error) {
-	key := s.key(name)
-	if _, err := s.client.Object.Put(ctx, key, reader, nil); err != nil {
-		return nil, err
-	}
-	response, err := s.client.Object.Head(ctx, key, nil)
-	if err != nil {
-		return nil, err
-	}
-	return s.file(key, response.ContentLength), nil
-}
-func (s *tencentStorage) Open(ctx context.Context, name string) (io.ReadCloser, error) {
-	response, err := s.client.Object.Get(ctx, s.key(name), nil)
-	if err != nil {
-		return nil, err
-	}
-	return response.Body, nil
-}
-func (s *tencentStorage) Delete(ctx context.Context, name string) error {
-	_, err := s.client.Object.Delete(ctx, s.key(name))
-	return err
-}
-func (s *tencentStorage) Compose(ctx context.Context, names []string, destination string) (*biz.StoredFile, string, error) {
-	return composeFiles(ctx, s, names, destination)
-}
-func (s *tencentStorage) DeletePrefix(ctx context.Context, prefix string) error {
-	cursor := ""
-	for {
-		items, next, more, err := s.List(ctx, prefix, cursor, 1000)
-		if err != nil {
-			return err
-		}
-		for _, item := range items {
-			if err = s.Delete(ctx, item.Path); err != nil {
-				return err
-			}
-		}
-		if !more {
-			return nil
-		}
-		cursor = next
-	}
-}
-func (s *tencentStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*biz.StoredFile, string, bool, error) {
-	if limit <= 0 {
-		limit = 100
-	}
-	result, _, err := s.client.Bucket.Get(ctx, &cos.BucketGetOptions{Prefix: s.key(prefix), Marker: cursor, MaxKeys: limit})
-	if err != nil {
-		return nil, "", false, err
-	}
-	items := make([]*biz.StoredFile, 0, len(result.Contents))
-	for _, object := range result.Contents {
-		item := s.file(object.Key, object.Size)
-		for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05.000Z"} {
-			if parsed, parseErr := time.Parse(layout, object.LastModified); parseErr == nil {
-				item.LastModified = parsed
-				break
-			}
-		}
-		items = append(items, item)
-	}
-	next := result.NextMarker
-	if result.IsTruncated && next == "" && len(result.Contents) > 0 {
-		next = result.Contents[len(result.Contents)-1].Key
-	}
-	return items, next, result.IsTruncated, nil
-}