优化结构
This commit is contained in:
@@ -1,12 +0,0 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"kra/internal/config"
|
||||
)
|
||||
|
||||
func cloneAdminConfig(value *config.Admin) *config.Admin {
|
||||
if value == nil {
|
||||
return &config.Admin{}
|
||||
}
|
||||
return config.CloneAdmin(value)
|
||||
}
|
||||
+53
-235
@@ -3,144 +3,24 @@ package data
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"kra/internal/config"
|
||||
dataintegration "kra/internal/data/integration"
|
||||
"kra/internal/integration/storage"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func configMap(input any) (map[string]any, error) {
|
||||
raw, err := yaml.Marshal(input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var output map[string]any
|
||||
if err = yaml.Unmarshal(raw, &output); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
// Writing config.yaml goes through config.Document, which owns the file format
|
||||
// for both directions. This file only decides which sections to write.
|
||||
|
||||
func setYAMLMapping(node *yaml.Node, key string, value any) error {
|
||||
if node.Kind == yaml.DocumentNode {
|
||||
node = node.Content[0]
|
||||
// cloneAdminConfig copies an admin section, normalizing nil to an empty value so
|
||||
// callers can edit the result without a nil check.
|
||||
func cloneAdminConfig(value *config.Admin) *config.Admin {
|
||||
if value == nil {
|
||||
return &config.Admin{}
|
||||
}
|
||||
if node.Kind != yaml.MappingNode {
|
||||
return fmt.Errorf("configuration root is not a mapping")
|
||||
}
|
||||
raw, err := yaml.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var replacement yaml.Node
|
||||
if err = yaml.Unmarshal(raw, &replacement); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
if node.Content[i].Value == key {
|
||||
mergeYAMLNode(node.Content[i+1], replacement.Content[0])
|
||||
return nil
|
||||
}
|
||||
}
|
||||
node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, replacement.Content[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
// mergeYAMLNode updates values produced from the runtime configuration while
|
||||
// retaining keys that are not represented by the typed configuration and
|
||||
// comments that were already present in the user's config file.
|
||||
func mergeYAMLNode(dst, src *yaml.Node) {
|
||||
if dst.Kind == yaml.MappingNode && src.Kind == yaml.MappingNode {
|
||||
for i := 0; i+1 < len(src.Content); i += 2 {
|
||||
key := src.Content[i].Value
|
||||
found := false
|
||||
for j := 0; j+1 < len(dst.Content); j += 2 {
|
||||
if dst.Content[j].Value == key {
|
||||
mergeYAMLNode(dst.Content[j+1], src.Content[i+1])
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
dst.Content = append(dst.Content, src.Content[i], src.Content[i+1])
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Keep comments attached to a scalar/sequence node when its value changes.
|
||||
head, line, foot := dst.HeadComment, dst.LineComment, dst.FootComment
|
||||
*dst = *src
|
||||
dst.HeadComment, dst.LineComment, dst.FootComment = head, line, foot
|
||||
}
|
||||
|
||||
func yamlMappingValue(node *yaml.Node, keys ...string) *yaml.Node {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
|
||||
node = node.Content[0]
|
||||
}
|
||||
for _, key := range keys {
|
||||
if node.Kind != yaml.MappingNode {
|
||||
return nil
|
||||
}
|
||||
var next *yaml.Node
|
||||
for i := 0; i+1 < len(node.Content); i += 2 {
|
||||
if node.Content[i].Value == key {
|
||||
next = node.Content[i+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
if next == nil {
|
||||
return nil
|
||||
}
|
||||
node = next
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func deleteYAMLMapping(node *yaml.Node, keys ...string) {
|
||||
if len(keys) == 0 || node == nil {
|
||||
return
|
||||
}
|
||||
if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
|
||||
node = node.Content[0]
|
||||
}
|
||||
if node.Kind != yaml.MappingNode {
|
||||
return
|
||||
}
|
||||
key := keys[0]
|
||||
for i := 0; i+1 < len(node.Content); i += 2 {
|
||||
if node.Content[i].Value != key {
|
||||
continue
|
||||
}
|
||||
if len(keys) == 1 {
|
||||
node.Content = append(node.Content[:i], node.Content[i+2:]...)
|
||||
return
|
||||
}
|
||||
deleteYAMLMapping(node.Content[i+1], keys[1:]...)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func setServerHTTPPort(document *yaml.Node, port int) error {
|
||||
if port <= 0 {
|
||||
return nil
|
||||
}
|
||||
host := "0.0.0.0"
|
||||
if addr := yamlMappingValue(document, "server", "http", "addr"); addr != nil {
|
||||
if currentHost, _, err := net.SplitHostPort(addr.Value); err == nil && currentHost != "" {
|
||||
host = currentHost
|
||||
}
|
||||
}
|
||||
return setYAMLMapping(document, "server", map[string]any{"http": map[string]any{"addr": net.JoinHostPort(host, strconv.Itoa(int(port)))}})
|
||||
return config.CloneAdmin(value)
|
||||
}
|
||||
|
||||
func (d *Data) persistConfig() error {
|
||||
@@ -158,40 +38,28 @@ func (d *Data) persistConfigValuesLocked(dataConfig *config.Data, adminConfig *c
|
||||
if adminConfig == nil || adminConfig.ConfigPath == "" {
|
||||
return nil
|
||||
}
|
||||
configPath := adminConfig.ConfigPath
|
||||
raw, err := os.ReadFile(configPath)
|
||||
document, err := config.OpenDocument(adminConfig.ConfigPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var document yaml.Node
|
||||
if err = yaml.Unmarshal(raw, &document); err != nil {
|
||||
return err
|
||||
}
|
||||
dataValue, err := configMap(dataConfig)
|
||||
if err != nil {
|
||||
if err = document.Set("data", dataConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
// Storage and email live in the database; the file must not shadow them.
|
||||
fileAdmin := cloneAdminConfig(adminConfig)
|
||||
fileAdmin.Storage = nil
|
||||
fileAdmin.Email = nil
|
||||
adminValue, err := configMap(fileAdmin)
|
||||
if err != nil {
|
||||
if err = document.Set("admin", fileAdmin); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = setYAMLMapping(&document, "data", dataValue); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = setYAMLMapping(&document, "admin", adminValue); err != nil {
|
||||
return err
|
||||
}
|
||||
deleteYAMLMapping(&document, "admin", "storage")
|
||||
deleteYAMLMapping(&document, "admin", "email")
|
||||
document.Delete("admin", "storage")
|
||||
document.Delete("admin", "email")
|
||||
if adminConfig.System != nil {
|
||||
if err = setServerHTTPPort(&document, adminConfig.System.Addr); err != nil {
|
||||
if err = document.SetServerHTTPPort(adminConfig.System.Addr); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writeConfigDocument(configPath, &document)
|
||||
return document.Save()
|
||||
}
|
||||
|
||||
// persistDatabaseConfig writes only the database selected on the init page and
|
||||
@@ -207,14 +75,10 @@ func (d *Data) persistDatabaseConfig(database *config.Database, signingKey strin
|
||||
if configPath == "" {
|
||||
return nil
|
||||
}
|
||||
raw, err := os.ReadFile(configPath)
|
||||
document, err := config.OpenDocument(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var document yaml.Node
|
||||
if err = yaml.Unmarshal(raw, &document); err != nil {
|
||||
return err
|
||||
}
|
||||
source, err := databaseDSN(database, "")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -232,17 +96,17 @@ func (d *Data) persistDatabaseConfig(database *config.Database, signingKey strin
|
||||
"config": database.Config,
|
||||
"path": database.Path,
|
||||
}
|
||||
if err = setYAMLMapping(&document, "data", map[string]any{"database": databaseValue}); err != nil {
|
||||
if err = document.Set("data", map[string]any{"database": databaseValue}); err != nil {
|
||||
return err
|
||||
}
|
||||
if signingKey != "" {
|
||||
if err = setYAMLMapping(&document, "admin", map[string]any{"jwt": map[string]any{"signing_key": signingKey}}); err != nil {
|
||||
if err = document.Set("admin", map[string]any{"jwt": map[string]any{"signing_key": signingKey}}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
deleteYAMLMapping(&document, "admin", "storage")
|
||||
deleteYAMLMapping(&document, "admin", "email")
|
||||
return writeConfigDocument(configPath, &document)
|
||||
document.Delete("admin", "storage")
|
||||
document.Delete("admin", "email")
|
||||
return document.Save()
|
||||
}
|
||||
|
||||
func (d *Data) removeIntegrationConfigFromFile() error {
|
||||
@@ -252,48 +116,16 @@ func (d *Data) removeIntegrationConfigFromFile() error {
|
||||
if configPath == "" {
|
||||
return nil
|
||||
}
|
||||
raw, err := os.ReadFile(configPath)
|
||||
document, err := config.OpenDocument(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var document yaml.Node
|
||||
if err = yaml.Unmarshal(raw, &document); err != nil {
|
||||
return err
|
||||
}
|
||||
if yamlMappingValue(&document, "admin", "storage") == nil && yamlMappingValue(&document, "admin", "email") == nil {
|
||||
if !document.Has("admin", "storage") && !document.Has("admin", "email") {
|
||||
return nil
|
||||
}
|
||||
deleteYAMLMapping(&document, "admin", "storage")
|
||||
deleteYAMLMapping(&document, "admin", "email")
|
||||
return writeConfigDocument(configPath, &document)
|
||||
}
|
||||
|
||||
func writeConfigDocument(configPath string, document *yaml.Node) error {
|
||||
output, err := yaml.Marshal(document)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Dir(configPath), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
temporary, err := os.CreateTemp(filepath.Dir(configPath), ".kra-config-*.yaml")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tempName := temporary.Name()
|
||||
defer os.Remove(tempName)
|
||||
if _, err = temporary.Write(output); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err = temporary.Chmod(0o600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err = temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tempName, configPath)
|
||||
document.Delete("admin", "storage")
|
||||
document.Delete("admin", "email")
|
||||
return document.Save()
|
||||
}
|
||||
|
||||
func (d *Data) reloadConfig(ctx context.Context) error {
|
||||
@@ -311,6 +143,11 @@ func (d *Data) reloadConfig(ctx context.Context) error {
|
||||
return fmt.Errorf("data.database and admin configuration are required")
|
||||
}
|
||||
next.Admin.ConfigPath = configPath
|
||||
// Every client below is opened before anything is published, so a failure at
|
||||
// any step leaves the running process untouched. undo closes whatever this
|
||||
// call opened; commit at the end hands all of it over instead.
|
||||
var undo rollback
|
||||
defer undo.run()
|
||||
databaseReady := databaseConnectionConfigured(next.Data.Database)
|
||||
var candidateDB *gorm.DB
|
||||
if databaseReady {
|
||||
@@ -324,14 +161,7 @@ func (d *Data) reloadConfig(ctx context.Context) error {
|
||||
return fmt.Errorf("reload bootstrap database: %w", err)
|
||||
}
|
||||
}
|
||||
closeCandidate := true
|
||||
defer func() {
|
||||
if closeCandidate {
|
||||
if sqlDB, closeErr := candidateDB.DB(); closeErr == nil {
|
||||
_ = sqlDB.Close()
|
||||
}
|
||||
}
|
||||
}()
|
||||
undo.add(func() { closeGormDB(candidateDB) })
|
||||
if databaseReady {
|
||||
if sqlDB, dbErr := candidateDB.DB(); dbErr != nil {
|
||||
return dbErr
|
||||
@@ -355,12 +185,12 @@ func (d *Data) reloadConfig(ctx context.Context) error {
|
||||
if legacyEmail == nil && currentAdmin != nil {
|
||||
legacyEmail = currentAdmin.Email
|
||||
}
|
||||
storageConfig, err := resolveStorageIntegrationConfig(candidateDB.WithContext(ctx), legacyStorage)
|
||||
storageConfig, err := dataintegration.ResolveStorageConfig(candidateDB.WithContext(ctx), legacyStorage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reload storage configuration: %w", err)
|
||||
}
|
||||
next.Admin.Storage = storageConfig
|
||||
emailConfig, err := resolveEmailIntegrationConfig(candidateDB.WithContext(ctx), legacyEmail)
|
||||
emailConfig, err := dataintegration.ResolveEmailConfig(candidateDB.WithContext(ctx), legacyEmail)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reload email configuration: %w", err)
|
||||
}
|
||||
@@ -371,47 +201,37 @@ func (d *Data) reloadConfig(ctx context.Context) error {
|
||||
}
|
||||
useRedis := next.Admin.System != nil && next.Admin.System.UseRedis
|
||||
candidateRedis := openRedis(next.Data.Redis, useRedis, d.logger())
|
||||
candidateRedisAccepted := false
|
||||
defer func() {
|
||||
if !candidateRedisAccepted && candidateRedis != nil {
|
||||
_ = candidateRedis.Close()
|
||||
undo.add(func() {
|
||||
if candidateRedis != nil {
|
||||
closeRedisClient(candidateRedis)
|
||||
}
|
||||
}()
|
||||
})
|
||||
useRedisList := useRedis && next.Admin.System.UseMultipoint
|
||||
candidateRedisList := openRedisList(next.Data.RedisList, useRedisList, d.logger())
|
||||
candidateRedisListAccepted := false
|
||||
defer func() {
|
||||
if !candidateRedisListAccepted {
|
||||
closeRedisList(candidateRedisList)
|
||||
}
|
||||
}()
|
||||
undo.add(func() { closeRedisList(candidateRedisList) })
|
||||
useMongo := next.Admin.System != nil && next.Admin.System.UseMongo
|
||||
// openMongo always reports a nil client alongside its error, so an unreachable
|
||||
// Mongo only warns: it must not fail an otherwise valid reload.
|
||||
candidateMongo, mongoErr := openMongo(next.Data.Mongo, useMongo)
|
||||
if mongoErr != nil {
|
||||
d.logger().Error("mongo unavailable during configuration reload", "mod", "mongo", "error", mongoErr)
|
||||
}
|
||||
mongoAccepted := false
|
||||
defer func() {
|
||||
if !mongoAccepted && candidateMongo != nil {
|
||||
_ = candidateMongo.Disconnect(context.Background())
|
||||
undo.add(func() {
|
||||
if candidateMongo != nil {
|
||||
closeMongoClient(candidateMongo)
|
||||
}
|
||||
}()
|
||||
})
|
||||
candidateDBList, err := openDatabaseList(next.Data.DatabaseList, d.logger())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidateDBListAccepted := false
|
||||
defer func() {
|
||||
if !candidateDBListAccepted {
|
||||
closeDatabaseList(candidateDBList)
|
||||
}
|
||||
}()
|
||||
integrationConfigs, err := readIntegrationRuntime(candidateDB.WithContext(ctx))
|
||||
undo.add(func() { closeDatabaseList(candidateDBList) })
|
||||
integrationConfigs, err := dataintegration.ReadRuntime(candidateDB.WithContext(ctx))
|
||||
if err != nil {
|
||||
return fmt.Errorf("reload integration runtime: %w", err)
|
||||
}
|
||||
|
||||
d.gormDB.replace(candidateDB, d.enqueueDataScopeAudit)
|
||||
d.replacePrimaryDB(candidateDB)
|
||||
d.databaseReady.Store(databaseReady)
|
||||
for _, item := range candidateDBList {
|
||||
registerDataScopeCallbacks(item, d.enqueueDataScopeAudit)
|
||||
@@ -431,14 +251,12 @@ func (d *Data) reloadConfig(ctx context.Context) error {
|
||||
} else {
|
||||
d.logger().Warn("keeping the previous redis list because the reloaded configuration failed to connect", "mod", "redis")
|
||||
}
|
||||
// A nil client here means Mongo is switched off in the reloaded configuration,
|
||||
// which must still retire the running client; only an outright failure keeps it.
|
||||
if mongoErr == nil {
|
||||
d.mongo.replace(candidateMongo)
|
||||
mongoAccepted = true
|
||||
}
|
||||
closeCandidate = false
|
||||
candidateDBListAccepted = true
|
||||
candidateRedisAccepted = true
|
||||
candidateRedisListAccepted = true
|
||||
undo.commit()
|
||||
d.runtime.Replace(next)
|
||||
if d.integrations != nil {
|
||||
d.integrations.Replace(integrationConfigs)
|
||||
|
||||
+38
-10
@@ -19,6 +19,7 @@ import (
|
||||
datatask "kra/internal/data/task"
|
||||
"kra/internal/integration/runtimeconfig"
|
||||
"kra/internal/integration/storage"
|
||||
"kra/pkg/database/migration"
|
||||
"kra/pkg/module"
|
||||
)
|
||||
|
||||
@@ -47,11 +48,11 @@ type Data struct {
|
||||
initMu sync.Mutex
|
||||
configMu sync.Mutex
|
||||
databaseReady atomic.Bool
|
||||
gormDB *reloadableDB
|
||||
redis *reloadableRedis
|
||||
gormDB *reloadable[*gorm.DB]
|
||||
redis *reloadable[redis.UniversalClient]
|
||||
redisListMu sync.RWMutex
|
||||
redisList map[string]redis.UniversalClient
|
||||
mongo *reloadableMongo
|
||||
mongo *reloadable[*mongo.Client]
|
||||
runtime *config.Store
|
||||
integrations *runtimeconfig.Store
|
||||
storage *storage.Reloadable
|
||||
@@ -71,7 +72,7 @@ func (d *Data) DB() *gorm.DB {
|
||||
if d == nil || d.gormDB == nil {
|
||||
return nil
|
||||
}
|
||||
return d.gormDB.DB()
|
||||
return d.gormDB.load()
|
||||
}
|
||||
|
||||
// DatabaseReady reports whether the configured primary database has been
|
||||
@@ -103,7 +104,7 @@ func (d *Data) IntegrationRuntime() *runtimeconfig.Store {
|
||||
// the system export module.
|
||||
func (d *Data) Database(name string) (*gorm.DB, error) {
|
||||
if name == "" {
|
||||
return d.gormDB.DB(), nil
|
||||
return d.gormDB.load(), nil
|
||||
}
|
||||
d.dbListMu.RLock()
|
||||
db := d.dbList[name]
|
||||
@@ -306,11 +307,11 @@ func NewData(runtime *config.Store, appLogger *slog.Logger, storageManager *stor
|
||||
}
|
||||
}
|
||||
if !usingFallback {
|
||||
storageConfig, storageErr := resolveStorageIntegrationConfig(db, admin.Storage)
|
||||
storageConfig, storageErr := dataintegration.ResolveStorageConfig(db, admin.Storage)
|
||||
if storageErr != nil {
|
||||
return nil, nil, fmt.Errorf("load storage integration configuration: %w", storageErr)
|
||||
}
|
||||
emailConfig, emailErr := resolveEmailIntegrationConfig(db, admin.Email)
|
||||
emailConfig, emailErr := dataintegration.ResolveEmailConfig(db, admin.Email)
|
||||
if emailErr != nil {
|
||||
return nil, nil, fmt.Errorf("load email integration configuration: %w", emailErr)
|
||||
}
|
||||
@@ -337,7 +338,7 @@ func NewData(runtime *config.Store, appLogger *slog.Logger, storageManager *stor
|
||||
return nil, nil, fmt.Errorf("storage manager is nil")
|
||||
}
|
||||
useRedis := admin != nil && admin.System != nil && admin.System.UseRedis
|
||||
d.redis = newReloadableRedis(openRedis(c.Redis, useRedis, appLogger))
|
||||
d.redis = newReloadable(openRedis(c.Redis, useRedis, appLogger), closeRedisClient)
|
||||
useRedisList := useRedis && admin.System.UseMultipoint
|
||||
d.redisList = openRedisList(c.RedisList, useRedisList, appLogger)
|
||||
useMongo := admin != nil && admin.System != nil && admin.System.UseMongo
|
||||
@@ -346,7 +347,7 @@ func NewData(runtime *config.Store, appLogger *slog.Logger, storageManager *stor
|
||||
appLogger.Error("mongo unavailable", "mod", "mongo", "error", err)
|
||||
mongoClient = nil
|
||||
}
|
||||
d.mongo = newReloadableMongo(mongoClient)
|
||||
d.mongo = newReloadable(mongoClient, closeMongoClient)
|
||||
initialized = true
|
||||
return d, cleanup, nil
|
||||
}
|
||||
@@ -437,8 +438,35 @@ func (d *Data) replaceRedisList(clients map[string]redis.UniversalClient) {
|
||||
}
|
||||
}
|
||||
|
||||
// replacePrimaryDB swaps in a hot-reloaded pool, attaching the row-level data
|
||||
// scope callbacks before the handle becomes readable.
|
||||
func (d *Data) replacePrimaryDB(db *gorm.DB) {
|
||||
registerDataScopeCallbacks(db, d.enqueueDataScopeAudit)
|
||||
d.gormDB.replace(db)
|
||||
}
|
||||
|
||||
func (d *Data) activateDatabase(db *gorm.DB, config *config.Database) {
|
||||
d.gormDB.replace(db, d.enqueueDataScopeAudit)
|
||||
d.replacePrimaryDB(db)
|
||||
d.runtime.UpdateDatabase(config)
|
||||
d.databaseReady.Store(true)
|
||||
}
|
||||
|
||||
// migrateAll is the single data-layer migration entry point. Every module
|
||||
// must register its migrations through the application catalog; there is no
|
||||
// hidden system/payment fallback that could silently omit a new module.
|
||||
func migrateAll(db *gorm.DB, catalog module.Catalog) error {
|
||||
return migration.Run(db, catalog.MigrationSteps())
|
||||
}
|
||||
|
||||
// loadIntegrationRuntime republishes the database-backed integration
|
||||
// configuration into the live store the long-lived adapters read.
|
||||
func (d *Data) loadIntegrationRuntime(db *gorm.DB) error {
|
||||
configs, err := dataintegration.ReadRuntime(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.integrations != nil {
|
||||
d.integrations.Replace(configs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ func (w *dataScopeAuditWriter) run() {
|
||||
if len(batch) == 0 || w.data == nil || w.data.gormDB == nil {
|
||||
return
|
||||
}
|
||||
db := w.data.gormDB.DB()
|
||||
db := w.data.gormDB.load()
|
||||
if db == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ func TestDataScopeAuditWriterUsesReloadedDatabase(t *testing.T) {
|
||||
t.Cleanup(w.Close)
|
||||
|
||||
d.enqueueDataScopeAudit(dataAccessLogPO{EventType: "no_identity", TargetTable: "example"})
|
||||
d.gormDB.replace(second, nil)
|
||||
d.gormDB.replace(second)
|
||||
waitForDataScopeAuditCount(t, second, 1)
|
||||
|
||||
var firstCount int64
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"kra/internal/biz/system"
|
||||
configpkg "kra/internal/config"
|
||||
dataintegration "kra/internal/data/integration"
|
||||
"kra/internal/integration/storage"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -172,7 +173,7 @@ func (d *Data) InitializeDatabase(ctx context.Context, input *system.DatabaseCon
|
||||
if currentAdmin != nil {
|
||||
legacyStorage = currentAdmin.Storage
|
||||
}
|
||||
storageConfig, err := resolveStorageIntegrationConfig(candidate.WithContext(ctx), legacyStorage)
|
||||
storageConfig, err := dataintegration.ResolveStorageConfig(candidate.WithContext(ctx), legacyStorage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize storage integration configuration: %w", err)
|
||||
}
|
||||
@@ -180,7 +181,7 @@ func (d *Data) InitializeDatabase(ctx context.Context, input *system.DatabaseCon
|
||||
if currentAdmin != nil {
|
||||
legacyEmail = currentAdmin.Email
|
||||
}
|
||||
emailConfig, err := resolveEmailIntegrationConfig(candidate.WithContext(ctx), legacyEmail)
|
||||
emailConfig, err := dataintegration.ResolveEmailConfig(candidate.WithContext(ctx), legacyEmail)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize email integration configuration: %w", err)
|
||||
}
|
||||
@@ -188,7 +189,7 @@ func (d *Data) InitializeDatabase(ctx context.Context, input *system.DatabaseCon
|
||||
if err := d.persistDatabaseConfig(database, signingKey); err != nil {
|
||||
return fmt.Errorf("persist database configuration: %w", err)
|
||||
}
|
||||
integrationConfigs, err := readIntegrationRuntime(candidate)
|
||||
integrationConfigs, err := dataintegration.ReadRuntime(candidate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize integration runtime: %w", err)
|
||||
}
|
||||
|
||||
@@ -26,6 +26,73 @@ type ConfigPO struct {
|
||||
|
||||
func (ConfigPO) TableName() string { return "sys_integration_configs" }
|
||||
|
||||
// ConfigRow is one integration configuration row without its PO: the parent
|
||||
// data package owns the storage and email upgrade paths, but the table shape
|
||||
// stays private here.
|
||||
type ConfigRow struct {
|
||||
Provider string
|
||||
Enabled bool
|
||||
Config string
|
||||
}
|
||||
|
||||
// HasConfigTable reports whether the integration configuration table exists,
|
||||
// which callers use to tell "not migrated yet" from "no rows".
|
||||
func HasConfigTable(db *gorm.DB) bool {
|
||||
return db != nil && cleanSession(db).Migrator().HasTable(&ConfigPO{})
|
||||
}
|
||||
|
||||
// ListConfigs returns every row of a kind in insertion order.
|
||||
func ListConfigs(db *gorm.DB, kind string) ([]ConfigRow, error) {
|
||||
var rows []ConfigPO
|
||||
if err := cleanSession(db).Where("kind = ?", kind).Order("id ASC").Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]ConfigRow, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
result = append(result, ConfigRow{Provider: row.Provider, Enabled: row.Enabled, Config: row.Config})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// FindConfig returns one row and whether it exists. A missing row is not an
|
||||
// error because every caller treats it as "fall back to the legacy value".
|
||||
func FindConfig(db *gorm.DB, kind, provider string) (ConfigRow, bool, error) {
|
||||
var row ConfigPO
|
||||
err := cleanSession(db).Where("kind = ? AND provider = ?", kind, provider).First(&row).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return ConfigRow{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return ConfigRow{}, false, err
|
||||
}
|
||||
return ConfigRow{Provider: row.Provider, Enabled: row.Enabled, Config: row.Config}, true, nil
|
||||
}
|
||||
|
||||
// UpsertConfigs writes every given row of a kind in one transaction, inserting
|
||||
// the ones that do not exist yet and updating the rest in place.
|
||||
func UpsertConfigs(db *gorm.DB, kind string, rows []ConfigRow) error {
|
||||
return cleanSession(db).Transaction(func(tx *gorm.DB) error {
|
||||
for _, item := range rows {
|
||||
var current ConfigPO
|
||||
err := tx.Where("kind = ? AND provider = ?", kind, item.Provider).First(¤t).Error
|
||||
switch {
|
||||
case errors.Is(err, gorm.ErrRecordNotFound):
|
||||
err = tx.Create(&ConfigPO{Kind: kind, Provider: item.Provider, Enabled: item.Enabled, Config: item.Config}).Error
|
||||
case err == nil:
|
||||
err = tx.Model(¤t).Updates(map[string]any{"enabled": item.Enabled, "config": item.Config}).Error
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// cleanSession drops conditions inherited from the caller's handle so a scoped
|
||||
// query cannot leak a WHERE clause into these statements.
|
||||
func cleanSession(db *gorm.DB) *gorm.DB { return db.Session(&gorm.Session{NewDB: true}) }
|
||||
|
||||
type integrationConfigRepo struct{ data Provider }
|
||||
|
||||
type paymentConfigReader struct{ data Provider }
|
||||
|
||||
@@ -29,7 +29,7 @@ func ensureCommunicationIntegrationConfigs(db *gorm.DB) error {
|
||||
for _, item := range defaults {
|
||||
var row ConfigPO
|
||||
err := db.Where("kind = ? AND provider = ?", item.kind, item.provider).First(&row).Error
|
||||
if gorm.ErrRecordNotFound == err {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
values, marshalErr := json.Marshal(integrationbiz.DefaultIntegrationConfig(item.kind, item.provider))
|
||||
if marshalErr != nil {
|
||||
return marshalErr
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"kra/internal/config"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
kindStorage = "storage"
|
||||
kindEmail = "email"
|
||||
providerSMTP = "smtp"
|
||||
)
|
||||
|
||||
var storageProviders = []string{
|
||||
"local", "qiniu", "aliyun-oss", "huawei-obs", "tencent-cos", "aws-s3", "cloudflare-r2", "minio",
|
||||
}
|
||||
|
||||
func normalizeStorageType(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value == "" {
|
||||
return "local"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func storageProviderValue(storage *config.Storage, provider string) any {
|
||||
if storage == nil {
|
||||
storage = &config.Storage{}
|
||||
}
|
||||
switch provider {
|
||||
case "qiniu":
|
||||
if storage.Qiniu == nil {
|
||||
storage.Qiniu = &config.Qiniu{}
|
||||
}
|
||||
return storage.Qiniu
|
||||
case "aliyun-oss":
|
||||
return ensureObjectStore(&storage.AliyunOSS)
|
||||
case "huawei-obs":
|
||||
return ensureObjectStore(&storage.HuaweiOBS)
|
||||
case "tencent-cos":
|
||||
return ensureObjectStore(&storage.TencentCOS)
|
||||
case "aws-s3":
|
||||
return ensureObjectStore(&storage.AWSS3)
|
||||
case "cloudflare-r2":
|
||||
return ensureObjectStore(&storage.CloudflareR2)
|
||||
case "minio":
|
||||
return ensureObjectStore(&storage.Minio)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func ensureObjectStore(value **config.ObjectStore) *config.ObjectStore {
|
||||
if *value == nil {
|
||||
*value = &config.ObjectStore{}
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func marshalStorageProvider(storage *config.Storage, provider string) (string, error) {
|
||||
value := storageProviderValue(storage, provider)
|
||||
if value == nil {
|
||||
return "{}", nil
|
||||
}
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
func unmarshalStorageProvider(storage *config.Storage, provider, value string) error {
|
||||
if provider == "local" || strings.TrimSpace(value) == "" {
|
||||
return nil
|
||||
}
|
||||
target := storageProviderValue(storage, provider)
|
||||
if target == nil {
|
||||
return nil
|
||||
}
|
||||
if !json.Valid([]byte(value)) {
|
||||
return fmt.Errorf("invalid %s integration configuration", provider)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(value), target); err != nil {
|
||||
return fmt.Errorf("decode %s integration configuration: %w", provider, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveStorageConfig(db *gorm.DB, storage *config.Storage) error {
|
||||
if storage == nil {
|
||||
storage = &config.Storage{}
|
||||
}
|
||||
active := normalizeStorageType(storage.Type)
|
||||
known := false
|
||||
for _, provider := range storageProviders {
|
||||
if provider == active {
|
||||
known = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
return fmt.Errorf("unsupported storage type %q", active)
|
||||
}
|
||||
|
||||
rows := make([]ConfigRow, 0, len(storageProviders))
|
||||
for _, provider := range storageProviders {
|
||||
value, err := marshalStorageProvider(storage, provider)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode %s integration configuration: %w", provider, err)
|
||||
}
|
||||
rows = append(rows, ConfigRow{Provider: provider, Enabled: provider == active, Config: value})
|
||||
}
|
||||
return UpsertConfigs(db, kindStorage, rows)
|
||||
}
|
||||
|
||||
func LoadStorageConfig(db *gorm.DB) (*config.Storage, bool, error) {
|
||||
rows, err := ListConfigs(db, kindStorage)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
storage := &config.Storage{Type: "local"}
|
||||
for _, row := range rows {
|
||||
if err = unmarshalStorageProvider(storage, row.Provider, row.Config); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if row.Enabled {
|
||||
storage.Type = row.Provider
|
||||
}
|
||||
}
|
||||
return storage, true, nil
|
||||
}
|
||||
|
||||
// ResolveStorageConfig imports a legacy YAML value only when the database has
|
||||
// no storage configuration yet. The database is authoritative afterwards.
|
||||
func ResolveStorageConfig(db *gorm.DB, legacy *config.Storage) (*config.Storage, error) {
|
||||
if !HasConfigTable(db) {
|
||||
if legacy == nil {
|
||||
return &config.Storage{Type: "local"}, nil
|
||||
}
|
||||
return config.CloneStorage(legacy), nil
|
||||
}
|
||||
storage, found, err := LoadStorageConfig(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
return storage, nil
|
||||
}
|
||||
if legacy == nil {
|
||||
legacy = &config.Storage{Type: "local"}
|
||||
}
|
||||
if err = SaveStorageConfig(db, legacy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storage, _, err = LoadStorageConfig(db)
|
||||
return storage, err
|
||||
}
|
||||
|
||||
func defaultEmailConfig() *config.Email { return &config.Email{Port: 465, IsSSL: true} }
|
||||
|
||||
func SaveEmailConfig(db *gorm.DB, email *config.Email) error {
|
||||
if email == nil {
|
||||
email = defaultEmailConfig()
|
||||
}
|
||||
raw, err := json.Marshal(email)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode smtp integration configuration: %w", err)
|
||||
}
|
||||
enabled := email.Host != "" && email.From != "" && email.Secret != "" && email.Port > 0
|
||||
return UpsertConfigs(db, kindEmail, []ConfigRow{{Provider: providerSMTP, Enabled: enabled, Config: string(raw)}})
|
||||
}
|
||||
|
||||
func LoadEmailConfig(db *gorm.DB) (*config.Email, bool, error) {
|
||||
row, found, err := FindConfig(db, kindEmail, providerSMTP)
|
||||
if err != nil || !found {
|
||||
return nil, false, err
|
||||
}
|
||||
if !json.Valid([]byte(row.Config)) {
|
||||
return nil, false, errors.New("invalid smtp integration configuration")
|
||||
}
|
||||
email := defaultEmailConfig()
|
||||
if err = json.Unmarshal([]byte(row.Config), email); err != nil {
|
||||
return nil, false, fmt.Errorf("decode smtp integration configuration: %w", err)
|
||||
}
|
||||
return email, true, nil
|
||||
}
|
||||
|
||||
func ResolveEmailConfig(db *gorm.DB, legacy *config.Email) (*config.Email, error) {
|
||||
if !HasConfigTable(db) {
|
||||
if legacy == nil {
|
||||
return defaultEmailConfig(), nil
|
||||
}
|
||||
return config.CloneEmail(legacy), nil
|
||||
}
|
||||
email, found, err := LoadEmailConfig(db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
return email, nil
|
||||
}
|
||||
if legacy == nil {
|
||||
legacy = defaultEmailConfig()
|
||||
}
|
||||
if err = SaveEmailConfig(db, legacy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
email, _, err = LoadEmailConfig(db)
|
||||
return email, err
|
||||
}
|
||||
@@ -2,292 +2,30 @@ package data
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"kra/internal/config"
|
||||
dataintegration "kra/internal/data/integration"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
integrationKindStorage = "storage"
|
||||
integrationKindEmail = "email"
|
||||
)
|
||||
|
||||
var storageProviderNames = []string{
|
||||
"local",
|
||||
"qiniu",
|
||||
"aliyun-oss",
|
||||
"huawei-obs",
|
||||
"tencent-cos",
|
||||
"aws-s3",
|
||||
"cloudflare-r2",
|
||||
"minio",
|
||||
}
|
||||
|
||||
func normalizeStorageType(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value == "" {
|
||||
return "local"
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func storageProviderValue(storage *config.Storage, provider string) any {
|
||||
if storage == nil {
|
||||
storage = &config.Storage{}
|
||||
}
|
||||
switch provider {
|
||||
case "qiniu":
|
||||
if storage.Qiniu == nil {
|
||||
storage.Qiniu = &config.Qiniu{}
|
||||
}
|
||||
return storage.Qiniu
|
||||
case "aliyun-oss":
|
||||
return ensureObjectStore(&storage.AliyunOSS)
|
||||
case "huawei-obs":
|
||||
return ensureObjectStore(&storage.HuaweiOBS)
|
||||
case "tencent-cos":
|
||||
return ensureObjectStore(&storage.TencentCOS)
|
||||
case "aws-s3":
|
||||
return ensureObjectStore(&storage.AWSS3)
|
||||
case "cloudflare-r2":
|
||||
return ensureObjectStore(&storage.CloudflareR2)
|
||||
case "minio":
|
||||
return ensureObjectStore(&storage.Minio)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func ensureObjectStore(value **config.ObjectStore) *config.ObjectStore {
|
||||
if *value == nil {
|
||||
*value = &config.ObjectStore{}
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func marshalStorageProvider(storage *config.Storage, provider string) (string, error) {
|
||||
value := storageProviderValue(storage, provider)
|
||||
if value == nil {
|
||||
return "{}", nil
|
||||
}
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
func unmarshalStorageProvider(storage *config.Storage, provider, value string) error {
|
||||
if provider == "local" || strings.TrimSpace(value) == "" {
|
||||
return nil
|
||||
}
|
||||
target := storageProviderValue(storage, provider)
|
||||
if target == nil {
|
||||
return nil
|
||||
}
|
||||
if !json.Valid([]byte(value)) {
|
||||
return fmt.Errorf("invalid %s integration configuration", provider)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(value), target); err != nil {
|
||||
return fmt.Errorf("decode %s integration configuration: %w", provider, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func saveStorageIntegrationConfig(db *gorm.DB, storage *config.Storage) error {
|
||||
if storage == nil {
|
||||
storage = &config.Storage{}
|
||||
}
|
||||
active := normalizeStorageType(storage.Type)
|
||||
known := false
|
||||
for _, provider := range storageProviderNames {
|
||||
if provider == active {
|
||||
known = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
return fmt.Errorf("unsupported storage type %q", active)
|
||||
}
|
||||
|
||||
return db.Session(&gorm.Session{NewDB: true}).Transaction(func(tx *gorm.DB) error {
|
||||
for _, provider := range storageProviderNames {
|
||||
value, err := marshalStorageProvider(storage, provider)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode %s integration configuration: %w", provider, err)
|
||||
}
|
||||
var current dataintegration.ConfigPO
|
||||
err = tx.Where("kind = ? AND provider = ?", integrationKindStorage, provider).First(¤t).Error
|
||||
switch {
|
||||
case errors.Is(err, gorm.ErrRecordNotFound):
|
||||
current = dataintegration.ConfigPO{Kind: integrationKindStorage, Provider: provider}
|
||||
current.Enabled, current.Config = provider == active, value
|
||||
if err = tx.Create(¤t).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
if err = tx.Model(¤t).Updates(map[string]any{"enabled": provider == active, "config": value}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func loadStorageIntegrationConfig(db *gorm.DB) (*config.Storage, bool, error) {
|
||||
var rows []dataintegration.ConfigPO
|
||||
err := db.Session(&gorm.Session{NewDB: true}).
|
||||
Where("kind = ?", integrationKindStorage).
|
||||
Order("id ASC").
|
||||
Find(&rows).Error
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
storage := &config.Storage{Type: "local"}
|
||||
for _, row := range rows {
|
||||
if err = unmarshalStorageProvider(storage, row.Provider, row.Config); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if row.Enabled {
|
||||
storage.Type = row.Provider
|
||||
}
|
||||
}
|
||||
return storage, true, nil
|
||||
}
|
||||
|
||||
// resolveStorageIntegrationConfig upgrades a legacy YAML configuration only
|
||||
// when the database has no storage rows yet. From then on the database is the
|
||||
// sole source of truth.
|
||||
func resolveStorageIntegrationConfig(db *gorm.DB, legacy *config.Storage) (*config.Storage, error) {
|
||||
clean := db.Session(&gorm.Session{NewDB: true})
|
||||
if !clean.Migrator().HasTable(&dataintegration.ConfigPO{}) {
|
||||
if legacy == nil {
|
||||
return &config.Storage{Type: "local"}, nil
|
||||
}
|
||||
return config.CloneStorage(legacy), nil
|
||||
}
|
||||
storage, found, err := loadStorageIntegrationConfig(clean)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
return storage, nil
|
||||
}
|
||||
if legacy == nil {
|
||||
legacy = &config.Storage{Type: "local"}
|
||||
}
|
||||
if err = saveStorageIntegrationConfig(clean, legacy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storage, _, err = loadStorageIntegrationConfig(clean)
|
||||
return storage, err
|
||||
}
|
||||
|
||||
func (d *Data) persistStorageIntegrationConfig(ctx context.Context, storage *config.Storage) error {
|
||||
if !d.databaseReady.Load() {
|
||||
return errors.New("database is not initialized")
|
||||
}
|
||||
db := d.gormDB.WithContext(ctx)
|
||||
if !db.Migrator().HasTable(&dataintegration.ConfigPO{}) {
|
||||
db := d.gormDB.load().WithContext(ctx)
|
||||
if !dataintegration.HasConfigTable(db) {
|
||||
return errors.New("integration configuration table does not exist")
|
||||
}
|
||||
return saveStorageIntegrationConfig(db, storage)
|
||||
}
|
||||
|
||||
func defaultEmailIntegrationConfig() *config.Email {
|
||||
return &config.Email{Port: 465, IsSSL: true}
|
||||
}
|
||||
|
||||
func saveEmailIntegrationConfig(db *gorm.DB, email *config.Email) error {
|
||||
if email == nil {
|
||||
email = defaultEmailIntegrationConfig()
|
||||
}
|
||||
raw, err := json.Marshal(email)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode smtp integration configuration: %w", err)
|
||||
}
|
||||
enabled := email.Host != "" && email.From != "" && email.Secret != "" && email.Port > 0
|
||||
clean := db.Session(&gorm.Session{NewDB: true})
|
||||
var current dataintegration.ConfigPO
|
||||
err = clean.Where("kind = ? AND provider = ?", integrationKindEmail, "smtp").First(¤t).Error
|
||||
switch {
|
||||
case errors.Is(err, gorm.ErrRecordNotFound):
|
||||
return clean.Create(&dataintegration.ConfigPO{
|
||||
Kind: integrationKindEmail, Provider: "smtp", Enabled: enabled, Config: string(raw),
|
||||
}).Error
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
return clean.Model(¤t).Updates(map[string]any{"enabled": enabled, "config": string(raw)}).Error
|
||||
}
|
||||
}
|
||||
|
||||
func loadEmailIntegrationConfig(db *gorm.DB) (*config.Email, bool, error) {
|
||||
var row dataintegration.ConfigPO
|
||||
err := db.Session(&gorm.Session{NewDB: true}).
|
||||
Where("kind = ? AND provider = ?", integrationKindEmail, "smtp").
|
||||
First(&row).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !json.Valid([]byte(row.Config)) {
|
||||
return nil, false, errors.New("invalid smtp integration configuration")
|
||||
}
|
||||
email := defaultEmailIntegrationConfig()
|
||||
if err = json.Unmarshal([]byte(row.Config), email); err != nil {
|
||||
return nil, false, fmt.Errorf("decode smtp integration configuration: %w", err)
|
||||
}
|
||||
return email, true, nil
|
||||
}
|
||||
|
||||
func resolveEmailIntegrationConfig(db *gorm.DB, legacy *config.Email) (*config.Email, error) {
|
||||
clean := db.Session(&gorm.Session{NewDB: true})
|
||||
if !clean.Migrator().HasTable(&dataintegration.ConfigPO{}) {
|
||||
if legacy == nil {
|
||||
return defaultEmailIntegrationConfig(), nil
|
||||
}
|
||||
return config.CloneEmail(legacy), nil
|
||||
}
|
||||
email, found, err := loadEmailIntegrationConfig(clean)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if found {
|
||||
return email, nil
|
||||
}
|
||||
if legacy == nil {
|
||||
legacy = defaultEmailIntegrationConfig()
|
||||
}
|
||||
if err = saveEmailIntegrationConfig(clean, legacy); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
email, _, err = loadEmailIntegrationConfig(clean)
|
||||
return email, err
|
||||
return dataintegration.SaveStorageConfig(db, storage)
|
||||
}
|
||||
|
||||
func (d *Data) persistEmailIntegrationConfig(ctx context.Context, email *config.Email) error {
|
||||
if !d.databaseReady.Load() {
|
||||
return errors.New("database is not initialized")
|
||||
}
|
||||
db := d.gormDB.WithContext(ctx)
|
||||
if !db.Migrator().HasTable(&dataintegration.ConfigPO{}) {
|
||||
db := d.gormDB.load().WithContext(ctx)
|
||||
if !dataintegration.HasConfigTable(db) {
|
||||
return errors.New("integration configuration table does not exist")
|
||||
}
|
||||
return saveEmailIntegrationConfig(db, email)
|
||||
return dataintegration.SaveEmailConfig(db, email)
|
||||
}
|
||||
|
||||
@@ -64,21 +64,21 @@ func TestStorageIntegrationConfigRoundTrip(t *testing.T) {
|
||||
},
|
||||
Minio: &config.ObjectStore{Endpoint: "127.0.0.1:9000", Bucket: "local", ForcePathStyle: true},
|
||||
}
|
||||
if err := saveStorageIntegrationConfig(db, storage); err != nil {
|
||||
if err := dataintegration.SaveStorageConfig(db, storage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
email := &config.Email{
|
||||
To: "ops@example.com", From: "mailer@example.com", Host: "smtp.example.com",
|
||||
Secret: "smtp-secret", Nickname: "Kra", Port: 465, IsSSL: true,
|
||||
}
|
||||
if err := saveEmailIntegrationConfig(db, email); err != nil {
|
||||
if err := dataintegration.SaveEmailConfig(db, email); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&dataintegration.ConfigPO{Kind: integrationbiz.IntegrationKindPayment, Provider: "wechat-pay", Config: `{"merchant_id":"123"}`}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
loaded, found, err := loadStorageIntegrationConfig(db)
|
||||
loaded, found, err := dataintegration.LoadStorageConfig(db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -95,7 +95,7 @@ func TestStorageIntegrationConfigRoundTrip(t *testing.T) {
|
||||
t.Fatalf("qiniu configuration = %#v", loaded.Qiniu)
|
||||
}
|
||||
|
||||
loadedEmail, found, err := loadEmailIntegrationConfig(db)
|
||||
loadedEmail, found, err := dataintegration.LoadEmailConfig(db)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("email configuration found=%v, err=%v", found, err)
|
||||
}
|
||||
@@ -104,17 +104,17 @@ func TestStorageIntegrationConfigRoundTrip(t *testing.T) {
|
||||
}
|
||||
|
||||
var storageCount, emailCount, paymentCount int64
|
||||
if err = db.Model(&dataintegration.ConfigPO{}).Where("kind = ?", integrationKindStorage).Count(&storageCount).Error; err != nil {
|
||||
if err = db.Model(&dataintegration.ConfigPO{}).Where("kind = ?", "storage").Count(&storageCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.Model(&dataintegration.ConfigPO{}).Where("kind = ?", integrationbiz.IntegrationKindPayment).Count(&paymentCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = db.Model(&dataintegration.ConfigPO{}).Where("kind = ?", integrationKindEmail).Count(&emailCount).Error; err != nil {
|
||||
if err = db.Model(&dataintegration.ConfigPO{}).Where("kind = ?", "email").Count(&emailCount).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if storageCount != int64(len(storageProviderNames)) {
|
||||
t.Fatalf("storage row count = %d, want %d", storageCount, len(storageProviderNames))
|
||||
if storageCount != 8 {
|
||||
t.Fatalf("storage row count = %d, want 8", storageCount)
|
||||
}
|
||||
if paymentCount != 1 {
|
||||
t.Fatalf("payment row count = %d, want 1", paymentCount)
|
||||
@@ -130,7 +130,7 @@ func TestResolveStorageIntegrationConfigMigratesLegacyOnlyOnce(t *testing.T) {
|
||||
Type: "qiniu",
|
||||
Qiniu: &config.Qiniu{Bucket: "legacy", SecretKey: "legacy-secret"},
|
||||
}
|
||||
loaded, err := resolveStorageIntegrationConfig(db, legacy)
|
||||
loaded, err := dataintegration.ResolveStorageConfig(db, legacy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -142,7 +142,7 @@ func TestResolveStorageIntegrationConfigMigratesLegacyOnlyOnce(t *testing.T) {
|
||||
Type: "minio",
|
||||
Minio: &config.ObjectStore{Bucket: "must-not-replace-database"},
|
||||
}
|
||||
loaded, err = resolveStorageIntegrationConfig(db, other)
|
||||
loaded, err = dataintegration.ResolveStorageConfig(db, other)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -169,14 +169,14 @@ func TestMaskStorageSecretsLeavesUnconfiguredProvidersEmpty(t *testing.T) {
|
||||
func TestResolveEmailIntegrationConfigMigratesLegacyOnlyOnce(t *testing.T) {
|
||||
db := openIntegrationConfigTestDB(t)
|
||||
legacy := &config.Email{To: "ops@example.com", From: "old@example.com", Host: "smtp.old.example.com", Secret: "old-secret", Port: 465, IsSSL: true}
|
||||
loaded, err := resolveEmailIntegrationConfig(db, legacy)
|
||||
loaded, err := dataintegration.ResolveEmailConfig(db, legacy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.Host != legacy.Host || loaded.Secret != legacy.Secret {
|
||||
t.Fatalf("migrated email = %#v", loaded)
|
||||
}
|
||||
loaded, err = resolveEmailIntegrationConfig(db, &config.Email{Host: "must-not-replace.example.com"})
|
||||
loaded, err = dataintegration.ResolveEmailConfig(db, &config.Email{Host: "must-not-replace.example.com"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -245,11 +245,9 @@ func TestPersistRuntimeConfigReplacesActiveStorage(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reloadableDB := &reloadableDB{}
|
||||
reloadableDB.current.Store(db)
|
||||
d := &Data{
|
||||
runtime: config.NewStore(&config.Config{Data: &config.Data{}, Admin: currentAdmin}),
|
||||
gormDB: reloadableDB,
|
||||
gormDB: newReloadableDB(db, nil),
|
||||
storage: currentStorage,
|
||||
}
|
||||
d.databaseReady.Store(true)
|
||||
@@ -274,11 +272,11 @@ func TestPersistRuntimeConfigReplacesActiveStorage(t *testing.T) {
|
||||
if _, err = os.Stat(filepath.Join(newRoot, "active.txt")); err != nil {
|
||||
t.Fatalf("active storage did not write to the new root: %v", err)
|
||||
}
|
||||
loaded, found, err := loadStorageIntegrationConfig(db)
|
||||
loaded, found, err := dataintegration.LoadStorageConfig(db)
|
||||
if err != nil || !found || loaded.Type != "local" {
|
||||
t.Fatalf("database storage config = %#v, found=%v, err=%v", loaded, found, err)
|
||||
}
|
||||
loadedEmail, found, err := loadEmailIntegrationConfig(db)
|
||||
loadedEmail, found, err := dataintegration.LoadEmailConfig(db)
|
||||
if err != nil || !found || loadedEmail.Secret != "runtime-secret" {
|
||||
t.Fatalf("database email config = %#v, found=%v, err=%v", loadedEmail, found, err)
|
||||
}
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"gorm.io/gorm"
|
||||
dataintegration "kra/internal/data/integration"
|
||||
"kra/internal/integration/runtimeconfig"
|
||||
)
|
||||
|
||||
func readIntegrationRuntime(db *gorm.DB) ([]runtimeconfig.Config, error) {
|
||||
return dataintegration.ReadRuntime(db)
|
||||
}
|
||||
|
||||
func (d *Data) loadIntegrationRuntime(db *gorm.DB) error {
|
||||
configs, err := readIntegrationRuntime(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.integrations != nil {
|
||||
d.integrations.Replace(configs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package data
|
||||
|
||||
import (
|
||||
"kra/pkg/database/migration"
|
||||
"kra/pkg/module"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// migrateAll is the single data-layer migration entry point. Every module
|
||||
// must register its migrations through the application catalog; there is no
|
||||
// hidden system/payment fallback that could silently omit a new module.
|
||||
func migrateAll(db *gorm.DB, catalog module.Catalog) error {
|
||||
return migration.Run(db, catalog.MigrationSteps())
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package data
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
@@ -48,6 +47,21 @@ func (s *retiredSet[T]) take(item T) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// forget removes every pending retirement for item without closing it. A
|
||||
// reload can legitimately publish a previously retired handle again (for
|
||||
// example A -> B -> A); its old timers must no longer own that live handle.
|
||||
func (s *retiredSet[T]) forget(item T) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for index := 0; index < len(s.items); {
|
||||
if s.items[index] != item {
|
||||
index++
|
||||
continue
|
||||
}
|
||||
s.items = append(s.items[:index], s.items[index+1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *retiredSet[T]) drain() []T {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
@@ -56,138 +70,116 @@ func (s *retiredSet[T]) drain() []T {
|
||||
return items
|
||||
}
|
||||
|
||||
// reloadableDB makes the pointer swap atomic. Replaced pools stay open for
|
||||
// retireGrace so in-flight GORM operations remain valid.
|
||||
type reloadableDB struct {
|
||||
current atomic.Pointer[gorm.DB]
|
||||
retired retiredSet[*gorm.DB]
|
||||
// reloadable owns one hot-swappable storage client. The database, Redis and
|
||||
// Mongo handles all need the same three things — a racy-free read on every
|
||||
// request, an atomic swap on reload, and a close that also drains whatever the
|
||||
// grace period has not reaped yet — so they share this one implementation.
|
||||
type reloadable[T comparable] struct {
|
||||
mu sync.RWMutex
|
||||
current T
|
||||
retired retiredSet[T]
|
||||
closeOne func(T)
|
||||
}
|
||||
|
||||
func newReloadableDB(db *gorm.DB, enqueue dataScopeAuditEnqueue) *reloadableDB {
|
||||
r := &reloadableDB{}
|
||||
registerDataScopeCallbacks(db, enqueue)
|
||||
r.current.Store(db)
|
||||
return r
|
||||
func newReloadable[T comparable](current T, closeOne func(T)) *reloadable[T] {
|
||||
return &reloadable[T]{current: current, closeOne: closeOne}
|
||||
}
|
||||
|
||||
func (r *reloadableDB) WithContext(ctx context.Context) *gorm.DB {
|
||||
return r.current.Load().WithContext(ctx)
|
||||
// load returns the active client, or the zero value when the client was never
|
||||
// configured, so optional backends need no separate presence flag.
|
||||
func (r *reloadable[T]) load() T {
|
||||
var zero T
|
||||
if r == nil {
|
||||
return zero
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.current
|
||||
}
|
||||
|
||||
func (r *reloadableDB) DB() *gorm.DB { return r.current.Load() }
|
||||
|
||||
func (r *reloadableDB) replace(db *gorm.DB, enqueue dataScopeAuditEnqueue) {
|
||||
registerDataScopeCallbacks(db, enqueue)
|
||||
old := r.current.Swap(db)
|
||||
if old != nil && old != db {
|
||||
r.retired.retire(old, closeGormDB)
|
||||
// replace publishes next and retires the handle it displaced.
|
||||
func (r *reloadable[T]) replace(next T) {
|
||||
var zero T
|
||||
if next != zero {
|
||||
r.retired.forget(next)
|
||||
}
|
||||
r.mu.Lock()
|
||||
old := r.current
|
||||
r.current = next
|
||||
r.mu.Unlock()
|
||||
if old != zero && old != next {
|
||||
r.retired.retire(old, func(item T) {
|
||||
// Serialize the final liveness check with replace: a handle that was
|
||||
// re-published while its timer fired must never be closed underneath
|
||||
// the new active configuration.
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.current != item {
|
||||
r.closeOne(item)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reloadableDB) close() {
|
||||
all := append([]*gorm.DB{r.current.Load()}, r.retired.drain()...)
|
||||
seen := map[*gorm.DB]struct{}{}
|
||||
for _, db := range all {
|
||||
if db == nil {
|
||||
func (r *reloadable[T]) close() {
|
||||
var zero T
|
||||
r.mu.Lock()
|
||||
current := r.current
|
||||
r.current = zero
|
||||
r.mu.Unlock()
|
||||
seen := make(map[T]struct{})
|
||||
for _, item := range append([]T{current}, r.retired.drain()...) {
|
||||
if item == zero {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[db]; ok {
|
||||
if _, ok := seen[item]; ok {
|
||||
continue
|
||||
}
|
||||
seen[db] = struct{}{}
|
||||
closeGormDB(db)
|
||||
seen[item] = struct{}{}
|
||||
r.closeOne(item)
|
||||
}
|
||||
}
|
||||
|
||||
// rollback collects the cleanup for every client a multi-step reload opened
|
||||
// before it knows whether the reload will succeed. Any early return closes
|
||||
// exactly what was opened, in reverse order; commit hands every handle over to
|
||||
// the process and cancels all of it. One flag replaces one accepted-boolean plus
|
||||
// one deferred closure per client.
|
||||
type rollback struct {
|
||||
cleanups []func()
|
||||
committed bool
|
||||
}
|
||||
|
||||
func (r *rollback) add(cleanup func()) { r.cleanups = append(r.cleanups, cleanup) }
|
||||
|
||||
func (r *rollback) commit() { r.committed = true }
|
||||
|
||||
func (r *rollback) run() {
|
||||
if r.committed {
|
||||
return
|
||||
}
|
||||
for index := len(r.cleanups) - 1; index >= 0; index-- {
|
||||
r.cleanups[index]()
|
||||
}
|
||||
}
|
||||
|
||||
// newReloadableDB attaches the row-level data scope callbacks the pool needs
|
||||
// before it becomes readable. Data.replacePrimaryDB is the reload counterpart.
|
||||
func newReloadableDB(db *gorm.DB, enqueue dataScopeAuditEnqueue) *reloadable[*gorm.DB] {
|
||||
registerDataScopeCallbacks(db, enqueue)
|
||||
return newReloadable(db, closeGormDB)
|
||||
}
|
||||
|
||||
func closeGormDB(db *gorm.DB) {
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
_ = sqlDB.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type reloadableMongo struct {
|
||||
mu sync.RWMutex
|
||||
current *mongo.Client
|
||||
retired retiredSet[*mongo.Client]
|
||||
}
|
||||
|
||||
func newReloadableMongo(client *mongo.Client) *reloadableMongo {
|
||||
return &reloadableMongo{current: client}
|
||||
}
|
||||
|
||||
func (r *reloadableMongo) replace(client *mongo.Client) {
|
||||
r.mu.Lock()
|
||||
old := r.current
|
||||
r.current = client
|
||||
r.mu.Unlock()
|
||||
if old != nil && old != client {
|
||||
r.retired.retire(old, closeMongoClient)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reloadableMongo) load() *mongo.Client {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.current
|
||||
}
|
||||
|
||||
func (r *reloadableMongo) close() {
|
||||
r.mu.Lock()
|
||||
current := r.current
|
||||
r.current = nil
|
||||
r.mu.Unlock()
|
||||
for _, client := range append([]*mongo.Client{current}, r.retired.drain()...) {
|
||||
if client != nil {
|
||||
closeMongoClient(client)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func closeMongoClient(client *mongo.Client) {
|
||||
_ = client.Disconnect(context.Background())
|
||||
}
|
||||
|
||||
type reloadableRedis struct {
|
||||
mu sync.RWMutex
|
||||
current redis.UniversalClient
|
||||
retired retiredSet[redis.UniversalClient]
|
||||
}
|
||||
|
||||
func newReloadableRedis(client redis.UniversalClient) *reloadableRedis {
|
||||
return &reloadableRedis{current: client}
|
||||
}
|
||||
|
||||
func (r *reloadableRedis) load() redis.UniversalClient {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.current
|
||||
}
|
||||
|
||||
func (r *reloadableRedis) replace(client redis.UniversalClient) {
|
||||
r.mu.Lock()
|
||||
old := r.current
|
||||
r.current = client
|
||||
r.mu.Unlock()
|
||||
if old != nil && old != client {
|
||||
r.retired.retire(old, closeRedisClient)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reloadableRedis) close() {
|
||||
r.mu.Lock()
|
||||
current := r.current
|
||||
r.current = nil
|
||||
r.mu.Unlock()
|
||||
for _, client := range append([]redis.UniversalClient{current}, r.retired.drain()...) {
|
||||
if client != nil {
|
||||
closeRedisClient(client)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func closeRedisClient(client redis.UniversalClient) {
|
||||
_ = client.Close()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package data
|
||||
|
||||
import "testing"
|
||||
|
||||
type reloadableTestClient struct{ id int }
|
||||
|
||||
func TestReloadableReclaimsRepublishedClient(t *testing.T) {
|
||||
first := &reloadableTestClient{id: 1}
|
||||
second := &reloadableTestClient{id: 2}
|
||||
closed := map[*reloadableTestClient]int{}
|
||||
clients := newReloadable(first, func(client *reloadableTestClient) { closed[client]++ })
|
||||
|
||||
clients.replace(second)
|
||||
clients.replace(first)
|
||||
|
||||
if clients.retired.take(first) {
|
||||
t.Fatal("republished client remained scheduled for retirement")
|
||||
}
|
||||
clients.close()
|
||||
if closed[first] != 1 {
|
||||
t.Fatalf("active client closed %d times, want 1", closed[first])
|
||||
}
|
||||
if closed[second] != 1 {
|
||||
t.Fatalf("retired client closed %d times, want 1", closed[second])
|
||||
}
|
||||
}
|
||||
|
||||
func TestReloadableCloseDeduplicatesHandles(t *testing.T) {
|
||||
client := &reloadableTestClient{id: 1}
|
||||
closed := 0
|
||||
clients := newReloadable(client, func(*reloadableTestClient) { closed++ })
|
||||
clients.retired.items = append(clients.retired.items, client, client)
|
||||
|
||||
clients.close()
|
||||
if closed != 1 {
|
||||
t.Fatalf("client closed %d times, want 1", closed)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user