yvan/kra-new
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

优化结构

Browse Source
This commit is contained in:
Yvan
2026-08-22 22:34:33 +08:00
parent 0126995090
commit b18baf1547
26 changed files with 501 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/integration/storage/aliyun_storage.go
+8 -1
View File
@@ -69,6 +69,10 @@ func (s *aliyunStorage) Compose(ctx context.Context, names []string, destination
return composeFiles(ctx, s, names, destination)
}
func (s *aliyunStorage) DeletePrefix(ctx context.Context, prefix string) error {
prefix, err := normalizeDeletePrefix(prefix)
if err != nil {
return err
}
cursor := ""
for {
items, next, more, err := s.List(ctx, prefix, cursor, 1000)
@@ -83,7 +87,10 @@ func (s *aliyunStorage) DeletePrefix(ctx context.Context, prefix string) error {
if !more {
return nil
}
cursor = next
cursor, err = advanceDeletePrefixCursor(cursor, next, more)
if err != nil {
return err
}
}
}
func (s *aliyunStorage) List(_ context.Context, prefix, cursor string, limit int) ([]*system.StoredFile, string, bool, error) {
internal/integration/storage/aws_storage.go
+11 -2
View File
@@ -106,8 +106,13 @@ func (s *awsStorage) Compose(ctx context.Context, names []string, destination st
return composeFiles(ctx, s, names, destination)
}
func (s *awsStorage) DeletePrefix(ctx context.Context, prefix string) error {
prefix, err := normalizeDeletePrefix(prefix)
if err != nil {
return err
}
cursor := ""
for {
items, _, more, err := s.List(ctx, prefix, "", 1000)
items, next, more, err := s.List(ctx, prefix, cursor, 1000)
if err != nil {
return err
}
@@ -116,9 +121,13 @@ func (s *awsStorage) DeletePrefix(ctx context.Context, prefix string) error {
return err
}
}
if !more || len(items) == 0 {
if !more {
return nil
}
cursor, err = advanceDeletePrefixCursor(cursor, next, more)
if err != nil {
return err
}
}
}
func (s *awsStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*system.StoredFile, string, bool, error) {
internal/integration/storage/compose.go
+8
View File
@@ -43,6 +43,14 @@ func composeStreams(
errCh <- nil
}()
putErr := put(ctx, destination, reader)
// A failed destination may stop reading before the producer reaches EOF.
// Close the pipe reader in that case so the producer's next write returns
// instead of leaving the goroutine blocked forever.
if putErr != nil {
_ = reader.CloseWithError(putErr)
} else {
_ = reader.Close()
}
composeErr := <-errCh
if putErr != nil {
_ = remove(ctx, destination)
internal/integration/storage/compose_test.go
+60
View File
@@ -8,6 +8,7 @@ import (
"io"
"strings"
"testing"
"time"
)
func TestComposeStreams(t *testing.T) {
@@ -48,3 +49,62 @@ func TestComposeStreamsRemovesPartialDestination(t *testing.T) {
t.Fatalf("compose error = %v, removed = %v", err, removed)
}
}
func TestComposeStreamsUnblocksProducerWhenDestinationFails(t *testing.T) {
putErr := errors.New("destination failed")
done := make(chan error, 1)
go func() {
_, err := composeStreams(context.Background(), []string{"large"}, func(context.Context, string) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader(strings.Repeat("x", 1<<20))), nil
}, func(context.Context, string, io.Reader) error {
return putErr
}, func(context.Context, string) error { return nil }, "out")
done <- err
}()
select {
case err := <-done:
if !errors.Is(err, putErr) {
t.Fatalf("compose error = %v, want %v", err, putErr)
}
case <-time.After(time.Second):
t.Fatal("compose remained blocked after destination failure")
}
}
func TestNormalizeDeletePrefix(t *testing.T) {
tests := []struct {
name string
value string
valid string
wantErr bool
}{
{name: "canonicalizes", value: "/uploads/chunks/", valid: "uploads/chunks"},
{name: "cleans duplicate separators", value: "uploads//chunks", valid: "uploads/chunks"},
{name: "rejects empty", value: " / ", wantErr: true},
{name: "rejects traversal", value: "uploads/../", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := normalizeDeletePrefix(tt.value)
if tt.wantErr {
if err == nil {
t.Fatalf("normalizeDeletePrefix(%q) succeeded with %q", tt.value, got)
}
return
}
if err != nil || got != tt.valid {
t.Fatalf("normalizeDeletePrefix(%q) = %q, %v; want %q", tt.value, got, err, tt.valid)
}
})
}
}
func TestAdvanceDeletePrefixCursor(t *testing.T) {
if _, err := advanceDeletePrefixCursor("cursor", "cursor", true); err == nil {
t.Fatal("same cursor should fail")
}
if next, err := advanceDeletePrefixCursor("", "next", true); err != nil || next != "next" {
t.Fatalf("advance cursor = %q, %v", next, err)
}
}
internal/integration/storage/huawei_storage.go
+8 -1
View File
@@ -68,6 +68,10 @@ func (s *huaweiStorage) Compose(ctx context.Context, names []string, destination
return composeFiles(ctx, s, names, destination)
}
func (s *huaweiStorage) DeletePrefix(ctx context.Context, prefix string) error {
prefix, err := normalizeDeletePrefix(prefix)
if err != nil {
return err
}
cursor := ""
for {
items, next, more, err := s.List(ctx, prefix, cursor, 1000)
@@ -82,7 +86,10 @@ func (s *huaweiStorage) DeletePrefix(ctx context.Context, prefix string) error {
if !more {
return nil
}
cursor = next
cursor, err = advanceDeletePrefixCursor(cursor, next, more)
if err != nil {
return err
}
}
}
func (s *huaweiStorage) List(_ context.Context, prefix, cursor string, limit int) ([]*system.StoredFile, string, bool, error) {
internal/integration/storage/local.go
+5 -1
View File
@@ -138,7 +138,11 @@ func (s *fileStorage) Compose(ctx context.Context, names []string, destination s
return composeFiles(ctx, s, names, destination)
}
func (s *fileStorage) DeletePrefix(ctx context.Context, prefix string) error {
path, err := s.resolve(strings.TrimSuffix(prefix, "/") + "/placeholder")
prefix, err := normalizeDeletePrefix(prefix)
if err != nil {
return err
}
path, err := s.resolve(prefix + "/placeholder")
if err != nil {
return err
}
internal/integration/storage/local_test.go
+24
View File
@@ -0,0 +1,24 @@
package storage
import (
"context"
"os"
"path/filepath"
"testing"
)
func TestLocalDeletePrefixRejectsEmptyPrefix(t *testing.T) {
root := t.TempDir()
sentinel := filepath.Join(root, "keep.txt")
if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil {
t.Fatalf("write sentinel: %v", err)
}
storage := &fileStorage{root: root, urlPrefix: "/files"}
if err := storage.DeletePrefix(context.Background(), " /"); err == nil {
t.Fatal("DeletePrefix() accepted an empty prefix")
}
if _, err := os.Stat(sentinel); err != nil {
t.Fatalf("sentinel was removed: %v", err)
}
}
internal/integration/storage/prefix.go
+41
View File
@@ -0,0 +1,41 @@
package storage
import (
"errors"
"path"
"strings"
)
// normalizeDeletePrefix rejects requests that could accidentally target the
// storage root and returns the canonical key form used by all backends.
func normalizeDeletePrefix(prefix string) (string, error) {
prefix = strings.TrimSpace(strings.ReplaceAll(prefix, "\\", "/"))
prefix = strings.Trim(prefix, "/")
if prefix == "" {
return "", errors.New("storage delete prefix is required")
}
for _, part := range strings.Split(prefix, "/") {
if part == ".." {
return "", errors.New("invalid storage delete prefix")
}
}
clean := path.Clean(prefix)
if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") {
return "", errors.New("invalid storage delete prefix")
}
return clean, nil
}
// advanceDeletePrefixCursor prevents a backend that reports a truncated page
// without a new cursor from making DeletePrefix loop forever.
func advanceDeletePrefixCursor(current, next string, more bool) (string, error) {
if !more {
return "", nil
}
current = strings.TrimSpace(current)
next = strings.TrimSpace(next)
if next == "" || next == current {
return "", errors.New("storage delete prefix pagination made no progress")
}
return next, nil
}
internal/integration/storage/qiniu_storage.go
+10 -1
View File
@@ -96,8 +96,13 @@ func (s *qiniuStorage) Compose(ctx context.Context, names []string, destination
return composeFiles(ctx, s, names, destination)
}
func (s *qiniuStorage) DeletePrefix(ctx context.Context, prefix string) error {
prefix, err := normalizeDeletePrefix(prefix)
if err != nil {
return err
}
cursor := ""
for {
items, _, more, err := s.List(ctx, prefix, "", 1000)
items, next, more, err := s.List(ctx, prefix, cursor, 1000)
if err != nil {
return err
}
@@ -109,6 +114,10 @@ func (s *qiniuStorage) DeletePrefix(ctx context.Context, prefix string) error {
if !more {
return nil
}
cursor, err = advanceDeletePrefixCursor(cursor, next, more)
if err != nil {
return err
}
}
}
func (s *qiniuStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*system.StoredFile, string, bool, error) {
internal/integration/storage/s3_storage.go
+4
View File
@@ -98,6 +98,10 @@ func (s *s3Storage) Compose(ctx context.Context, names []string, destination str
return composeFiles(ctx, s, names, destination)
}
func (s *s3Storage) DeletePrefix(ctx context.Context, prefix string) error {
prefix, err := normalizeDeletePrefix(prefix)
if err != nil {
return err
}
items := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: s.key(prefix), Recursive: true})
for item := range items {
if item.Err != nil {
internal/integration/storage/tencent_storage.go
+8 -1
View File
@@ -82,6 +82,10 @@ func (s *tencentStorage) Compose(ctx context.Context, names []string, destinatio
return composeFiles(ctx, s, names, destination)
}
func (s *tencentStorage) DeletePrefix(ctx context.Context, prefix string) error {
prefix, err := normalizeDeletePrefix(prefix)
if err != nil {
return err
}
cursor := ""
for {
items, next, more, err := s.List(ctx, prefix, cursor, 1000)
@@ -96,7 +100,10 @@ func (s *tencentStorage) DeletePrefix(ctx context.Context, prefix string) error
if !more {
return nil
}
cursor = next
cursor, err = advanceDeletePrefixCursor(cursor, next, more)
if err != nil {
return err
}
}
}
func (s *tencentStorage) List(ctx context.Context, prefix, cursor string, limit int) ([]*system.StoredFile, string, bool, error) {