优化结构

This commit is contained in:
Yvan
2026-08-21 21:37:30 +08:00
parent a2ce3ae218
commit e4ba0dced9
41 changed files with 3646 additions and 364 deletions
+2 -2
View File
@@ -5,10 +5,10 @@
- `app`:组合根、system 模块定义和任务/路由运行时组合
- `biz`:系统领域对象、用例和仓储接口
- `conf`:配置 proto 与运行时配置解析
- `conf`:基础配置 proto 与运行时配置解析
- `data`:数据库生命周期、系统仓储、系统表和支付持久化
- `initialize`:数据库首次初始化和系统种子数据编排
- `integration`:Redis、邮件、对象存储、支付、WebSocket 和 EMQX 适配器
- `integration`:Redis、邮件、对象存储、支付、WebSocket、EMQX 和 RabbitMQ 适配器
- `security`:后台 JWT 等安全实现
- `server`:Gin server 组合与生命周期;横切 HTTP 代码按子包维护:
`server/handler`、`server/middleware`、`server/router`、`server/httpx`
+13 -1
View File
@@ -9,10 +9,22 @@ import (
// Definition describes the built-in system contribution to the application
// catalog. Other business modules can expose the same shape independently.
func Definition() module.Definition {
surface := datapayment.AdminSurface()
communication := module.Surface{
Menus: []module.Menu{{Name: "integrationConfig", Path: "integrationConfig", ParentName: "extensions", Component: "view/systemTools/integration/config.vue", Title: "通信集成", Icon: "connection", Sort: 8}},
APIs: []module.API{
{Path: "/integration/configs/:kind", Method: "GET", Group: "集成配置", Description: "按类型获取集成配置"},
{Path: "/integration/configs/:kind/:provider", Method: "GET", Group: "集成配置", Description: "获取指定集成配置"},
{Path: "/integration/configs/:kind/:provider", Method: "PUT", Group: "集成配置", Description: "保存集成配置"},
{Path: "/integration/configs/:kind/:provider", Method: "DELETE", Group: "集成配置", Description: "删除集成配置"},
},
}
surface.Menus = append(surface.Menus, communication.Menus...)
surface.APIs = append(surface.APIs, communication.APIs...)
return module.Definition{
Name: "system",
Migrations: append(datasystem.Migrations(), datapayment.Migrations()...),
Surface: datapayment.AdminSurface(),
Surface: surface,
TimedTasks: []module.TimedTask{
{Name: "ClearDB", Description: "定时清理数据库过期日志(操作记录/JWT黑名单/定时任务执行日志)", Spec: "@daily", MethodName: "ClearDB", Enabled: true},
{Name: "CleanStaleUploads", Description: "定时清理过期大文件上传会话", Spec: "@hourly", MethodName: "CleanStaleUploads", Enabled: true},
+27
View File
@@ -0,0 +1,27 @@
package app
import "testing"
func TestDefinitionIncludesCommunicationIntegrationSurface(t *testing.T) {
surface := Definition().Surface
menuFound := false
for _, menu := range surface.Menus {
if menu.Name == "integrationConfig" {
menuFound = menu.ParentName == "extensions" && menu.Component == "view/systemTools/integration/config.vue"
break
}
}
if !menuFound {
t.Fatal("communication integration menu is missing")
}
apiFound := false
for _, api := range surface.APIs {
if api.Method == "PUT" && api.Path == "/integration/configs/:kind/:provider" {
apiFound = true
break
}
}
if !apiFound {
t.Fatal("communication integration save API is missing")
}
}
+3 -2
View File
@@ -8,6 +8,7 @@ import (
"sort"
"strconv"
"strings"
"time"
)
const (
@@ -195,8 +196,8 @@ func validateCommunicationIntegrationConfig(kind, provider string, values map[st
return errors.New("rabbitmq port 必须在 1-65535 之间")
}
exchangeType := strings.ToLower(integrationText(values, "exchange_type"))
if exchangeType != "direct" && exchangeType != "fanout" && exchangeType != "topic" && exchangeType != "headers" {
return errors.New("rabbitmq exchange_type 必须是 direct、fanout、topic 或 headers")
if exchangeType != "direct" && exchangeType != "fanout" && exchangeType != "topic" {
return errors.New("rabbitmq exchange_type 必须是 direct、fanout 或 topic")
}
if integrationInt64(values, "prefetch_count", -1) < 0 {
return errors.New("rabbitmq prefetch_count 不能小于 0")
@@ -0,0 +1,32 @@
package biz
import "testing"
func TestCommunicationIntegrationDefinitionsAndValidation(t *testing.T) {
for _, target := range []struct{ kind, provider string }{
{IntegrationKindMQ, "emqx"},
{IntegrationKindMQ, "rabbitmq"},
{IntegrationKindWebSocket, "melody"},
} {
values := DefaultIntegrationConfig(target.kind, target.provider)
if len(values) == 0 {
t.Fatalf("default config missing for %s/%s", target.kind, target.provider)
}
if err := ValidateIntegrationConfig(target.kind, target.provider, values); err != nil {
t.Fatalf("default config invalid for %s/%s: %v", target.kind, target.provider, err)
}
}
}
func TestCommunicationIntegrationValidationRejectsInvalidValues(t *testing.T) {
rabbit := DefaultIntegrationConfig(IntegrationKindMQ, "rabbitmq")
rabbit["port"] = 0
if err := ValidateIntegrationConfig(IntegrationKindMQ, "rabbitmq", rabbit); err == nil {
t.Fatal("invalid rabbitmq port was accepted")
}
websocket := DefaultIntegrationConfig(IntegrationKindWebSocket, "melody")
websocket["path"] = "ws"
if err := ValidateIntegrationConfig(IntegrationKindWebSocket, "melody", websocket); err == nil {
t.Fatal("invalid websocket path was accepted")
}
}
@@ -113,7 +113,7 @@ var integrationDefinitions = map[string][]IntegrationConfigDefinition{
{Key: "password", Label: "密码", Type: "password", Required: true, Secret: true},
{Key: "vhost", Label: "Virtual Host", Type: "text", Required: true, Placeholder: "/"},
{Key: "exchange", Label: "Exchange", Type: "text", Required: true, Placeholder: "kra"},
{Key: "exchange_type", Label: "Exchange 类型", Type: "select", Required: true, Options: []IntegrationConfigOption{{Label: "topic", Value: "topic"}, {Label: "direct", Value: "direct"}, {Label: "fanout", Value: "fanout"}, {Label: "headers", Value: "headers"}}},
{Key: "exchange_type", Label: "Exchange 类型", Type: "select", Required: true, Options: []IntegrationConfigOption{{Label: "topic", Value: "topic"}, {Label: "direct", Value: "direct"}, {Label: "fanout", Value: "fanout"}}},
{Key: "queue", Label: "Queue", Type: "text", Required: true, Placeholder: "kra"},
{Key: "routing_key", Label: "默认 Routing Key", Type: "text", Required: true, Placeholder: "#", Description: "业务未指定订阅键时使用;topic 类型支持 * 和 #。"},
{Key: "durable", Label: "持久化", Type: "switch"},
+1 -7
View File
@@ -194,8 +194,6 @@ func (d *Data) persistConfigValuesLocked(dataConfig *conf.Data, adminConfig *con
}
deleteYAMLMapping(&document, "admin", "storage")
deleteYAMLMapping(&document, "admin", "email")
deleteYAMLMapping(&document, "admin", "websocket")
deleteYAMLMapping(&document, "admin", "mq")
if adminConfig.System != nil {
if err = setServerHTTPPort(&document, adminConfig.System.Addr); err != nil {
return err
@@ -255,8 +253,6 @@ func (d *Data) persistDatabaseConfig(database *conf.Data_Database, signingKey st
}
deleteYAMLMapping(&document, "admin", "storage")
deleteYAMLMapping(&document, "admin", "email")
deleteYAMLMapping(&document, "admin", "websocket")
deleteYAMLMapping(&document, "admin", "mq")
return writeConfigDocument(configPath, &document)
}
@@ -275,13 +271,11 @@ func (d *Data) removeIntegrationConfigFromFile() error {
if err = yaml.Unmarshal(raw, &document); err != nil {
return err
}
if yamlMappingValue(&document, "admin", "storage") == nil && yamlMappingValue(&document, "admin", "email") == nil && yamlMappingValue(&document, "admin", "websocket") == nil && yamlMappingValue(&document, "admin", "mq") == nil {
if yamlMappingValue(&document, "admin", "storage") == nil && yamlMappingValue(&document, "admin", "email") == nil {
return nil
}
deleteYAMLMapping(&document, "admin", "storage")
deleteYAMLMapping(&document, "admin", "email")
deleteYAMLMapping(&document, "admin", "websocket")
deleteYAMLMapping(&document, "admin", "mq")
return writeConfigDocument(configPath, &document)
}
+12 -4
View File
@@ -14,13 +14,14 @@ import (
"kra/internal/conf"
datapayment "kra/internal/data/payment"
datasystem "kra/internal/data/repository"
"kra/internal/integration/runtimeconfig"
"kra/internal/integration/storage"
"kra/internal/integrationruntime"
"kra/pkg/module"
)
var ProviderSet = wire.NewSet(
NewData,
NewIntegrationRuntime,
wire.Bind(new(datasystem.Provider), new(*Data)),
wire.Bind(new(datasystem.DatabaseProvider), new(*Data)),
wire.Bind(new(datapayment.Provider), new(*Data)),
@@ -34,6 +35,13 @@ var ProviderSet = wire.NewSet(
datasystem.NewIntegrationConfigRepo,
)
func NewIntegrationRuntime(data *Data) *runtimeconfig.Store {
if data == nil {
return runtimeconfig.NewStore()
}
return data.IntegrationRuntime()
}
type Data struct {
initMu sync.Mutex
configMu sync.Mutex
@@ -42,7 +50,7 @@ type Data struct {
redis *reloadableRedis
mongo *reloadableMongo
runtime *conf.Runtime
integrations *integrationruntime.Store
integrations *runtimeconfig.Store
storage *storage.Reloadable
dbListMu sync.RWMutex
dbList map[string]*gorm.DB
@@ -77,7 +85,7 @@ func (d *Data) Runtime() *conf.Runtime {
// IntegrationRuntime exposes database-backed integration configuration to
// long-lived adapters without making config.yaml part of their lifecycle.
func (d *Data) IntegrationRuntime() *integrationruntime.Store {
func (d *Data) IntegrationRuntime() *runtimeconfig.Store {
if d == nil {
return nil
}
@@ -167,7 +175,7 @@ func NewData(runtime *conf.Runtime, appLogger *slog.Logger, storageManager *stor
// and /init/initdb remain available.
c.Database = &conf.Data_Database{}
}
d := &Data{runtime: runtime, integrations: integrationruntime.NewStore(), appLogger: appLogger, storage: storageManager, catalog: catalog}
d := &Data{runtime: runtime, integrations: runtimeconfig.NewStore(), appLogger: appLogger, storage: storageManager, catalog: catalog}
usingFallback := !databaseConnectionConfigured(c.Database)
var db *gorm.DB
var err error
+1 -24
View File
@@ -123,25 +123,6 @@ func TestStorageIntegrationConfigRoundTrip(t *testing.T) {
}
}
func TestMQIntegrationConfigRoundTrip(t *testing.T) {
db := openIntegrationConfigTestDB(t)
legacy := &conf.AdminBackend_MQ{Enabled: true, Broker: "mqtt://emqx.example.com:1883", ClientId: "system", Username: "app", Password: "secret", KeepAlive: 45, CleanSession: true, ConnectTimeout: 12}
if err := saveMQIntegrationConfig(db, legacy); err != nil {
t.Fatal(err)
}
loaded, found, err := loadMQIntegrationConfig(db)
if err != nil || !found {
t.Fatalf("found=%v err=%v", found, err)
}
if loaded.Broker != legacy.Broker || loaded.Password != legacy.Password || loaded.KeepAlive != 45 {
t.Fatalf("loaded mq = %#v", loaded)
}
loaded, err = resolveMQIntegrationConfig(db, &conf.AdminBackend_MQ{Broker: "must-not-replace"})
if err != nil || loaded.Broker != legacy.Broker {
t.Fatalf("database mq was replaced: %#v err=%v", loaded, err)
}
}
func TestResolveStorageIntegrationConfigMigratesLegacyOnlyOnce(t *testing.T) {
db := openIntegrationConfigTestDB(t)
legacy := &conf.AdminBackend_Storage{
@@ -205,7 +186,7 @@ func TestResolveEmailIntegrationConfigMigratesLegacyOnlyOnce(t *testing.T) {
func TestPersistConfigValuesRemovesStorageFromYAML(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.yaml")
input := []byte("data: {}\nadmin:\n router_prefix: /old\n storage:\n type: qiniu\n qiniu:\n secret_key: legacy-secret\n email:\n host: smtp.legacy.example.com\n secret: legacy-email-secret\n mq:\n enabled: true\n broker: tcp://localhost:1883\n password: legacy-mq-secret\n extension_key: retained\n")
input := []byte("data: {}\nadmin:\n router_prefix: /old\n storage:\n type: qiniu\n qiniu:\n secret_key: legacy-secret\n email:\n host: smtp.legacy.example.com\n secret: legacy-email-secret\n extension_key: retained\n")
if err := os.WriteFile(path, input, 0o600); err != nil {
t.Fatal(err)
}
@@ -218,7 +199,6 @@ func TestPersistConfigValuesRemovesStorageFromYAML(t *testing.T) {
Qiniu: &conf.AdminBackend_Qiniu{SecretKey: "database-only-secret"},
},
Email: &conf.AdminBackend_Email{Host: "smtp.database.example.com", Secret: "database-only-email-secret"},
Mq: &conf.AdminBackend_MQ{Enabled: true, Broker: "tcp://emqx:1883", Password: "database-only-mq-secret"},
}
if err := d.persistConfigValues(&conf.Data{}, admin); err != nil {
t.Fatal(err)
@@ -242,9 +222,6 @@ func TestPersistConfigValuesRemovesStorageFromYAML(t *testing.T) {
if _, exists := adminValue["email"]; exists {
t.Fatalf("email remained in YAML: %s", raw)
}
if _, exists := adminValue["mq"]; exists {
t.Fatalf("mq remained in YAML: %s", raw)
}
if adminValue["extension_key"] != "retained" {
t.Fatalf("extension key was not retained: %#v", adminValue)
}
+39
View File
@@ -0,0 +1,39 @@
package data
import (
"encoding/json"
"errors"
"kra/internal/biz"
"gorm.io/gorm"
)
func ensureCommunicationIntegrationConfigs(db *gorm.DB) error {
defaults := []struct {
kind string
provider string
}{
{kind: biz.IntegrationKindMQ, provider: "emqx"},
{kind: biz.IntegrationKindMQ, provider: "rabbitmq"},
{kind: biz.IntegrationKindWebSocket, provider: "melody"},
}
for _, item := range defaults {
var row integrationConfigPO
err := db.Where("kind = ? AND provider = ?", item.kind, item.provider).First(&row).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
values, marshalErr := json.Marshal(biz.DefaultIntegrationConfig(item.kind, item.provider))
if marshalErr != nil {
return marshalErr
}
if err = db.Create(&integrationConfigPO{Kind: item.kind, Provider: item.provider, Enabled: false, Config: string(values)}).Error; err != nil {
return err
}
continue
}
if err != nil {
return err
}
}
return nil
}
+4 -4
View File
@@ -3,12 +3,12 @@ package data
import (
"errors"
"kra/internal/integrationruntime"
"kra/internal/integration/runtimeconfig"
"gorm.io/gorm"
)
func readIntegrationRuntime(db *gorm.DB) ([]integrationruntime.Config, error) {
func readIntegrationRuntime(db *gorm.DB) ([]runtimeconfig.Config, error) {
if db == nil || !db.Migrator().HasTable(&integrationConfigPO{}) {
return nil, nil
}
@@ -19,9 +19,9 @@ func readIntegrationRuntime(db *gorm.DB) ([]integrationruntime.Config, error) {
Find(&rows).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, err
}
configs := make([]integrationruntime.Config, 0, len(rows))
configs := make([]runtimeconfig.Config, 0, len(rows))
for _, row := range rows {
configs = append(configs, integrationruntime.Config{Kind: row.Kind, Provider: row.Provider, Enabled: row.Enabled, Values: []byte(row.Config)})
configs = append(configs, runtimeconfig.Config{Kind: row.Kind, Provider: row.Provider, Enabled: row.Enabled, Values: []byte(row.Config)})
}
return configs, nil
}
+8 -5
View File
@@ -10,12 +10,15 @@ import (
)
func InfrastructureMigrations() []migration.Step {
return []migration.Step{{
ID: "202608200001_data_infrastructure",
Migrate: func(db *gorm.DB) error {
return migration.CreateMissingTables(db, &integrationConfigPO{})
return []migration.Step{
{
ID: "202608200001_data_infrastructure",
Migrate: func(db *gorm.DB) error {
return migration.CreateMissingTables(db, &integrationConfigPO{})
},
},
}}
{ID: "202608210001_communication_integration_defaults", Migrate: ensureCommunicationIntegrationConfigs},
}
}
// migrateAll is the single data-layer migration entry point. Module-specific
+14 -2
View File
@@ -26,8 +26,20 @@ func TestMigrateAllRunsModuleSchemasWithoutBootstrapSeed(t *testing.T) {
if err = db.Table(migration.TableName).Count(&versions).Error; err != nil {
t.Fatal(err)
}
if versions != 4 {
t.Fatalf("migration versions = %d, want 4", versions)
if versions != 6 {
t.Fatalf("migration versions = %d, want 6", versions)
}
var communicationRows []integrationConfigPO
if err = db.Where("kind IN ?", []string{"mq", "websocket"}).Order("kind, provider").Find(&communicationRows).Error; err != nil {
t.Fatal(err)
}
if len(communicationRows) != 3 {
t.Fatalf("communication integration rows = %d, want 3", len(communicationRows))
}
for _, row := range communicationRows {
if row.Enabled || row.Config == "" {
t.Fatalf("default communication integration = %#v", row)
}
}
for _, table := range []string{"sys_users", "sys_base_menus", "sys_apis"} {
var count int64
+8 -4
View File
@@ -26,12 +26,16 @@ func AdminSurface() platformmodule.Surface {
{Name: "paymentConfig", Path: "paymentConfig", ParentName: "extensions", Component: "view/systemTools/payment/config.vue", Title: "支付配置", Icon: "credit-card", Sort: 7},
},
APIs: []platformmodule.API{
{Path: "/integration/configs/:kind", Method: "GET", Group: "集成配置", Description: "按类型获取集成配置"},
{Path: "/integration/configs/:kind/:provider", Method: "GET", Group: "集成配置", Description: "获取指定集成配置"},
{Path: "/integration/configs/:kind/:provider", Method: "PUT", Group: "集成配置", Description: "保存集成配置"},
{Path: "/integration/configs/:kind/:provider", Method: "DELETE", Group: "集成配置", Description: "删除集成配置"},
{Path: "/payment/orders", Method: "GET", Group: "支付", Description: "分页查询支付订单"},
{Path: "/payment/order", Method: "POST", Group: "支付", Description: "查询支付订单"},
{Path: "/payment/orders/:provider/:tradeNo", Method: "GET", Group: "支付", Description: "按路径查询支付订单"},
{Path: "/payment/create", Method: "POST", Group: "支付", Description: "创建支付订单"},
{Path: "/payment/query", Method: "POST", Group: "支付", Description: "同步支付订单状态"},
{Path: "/payment/refund", Method: "POST", Group: "支付", Description: "申请支付订单退款"},
{Path: "/payment/orders/:provider/:tradeNo/refund", Method: "POST", Group: "支付", Description: "按路径申请支付订单退款"},
{Path: "/payment/fulfill", Method: "POST", Group: "支付", Description: "重试支付订单发货"},
{Path: "/payment/orders/:provider/:tradeNo/fulfill", Method: "POST", Group: "支付", Description: "按路径重试支付订单发货"},
{Path: "/payment/providers/:provider/test", Method: "POST", Group: "支付", Description: "测试支付渠道"},
},
}
}
+15 -4
View File
@@ -8,7 +8,7 @@ import (
"time"
"kra/internal/biz"
"kra/internal/integrationruntime"
"kra/internal/integration/runtimeconfig"
"gorm.io/gorm"
)
@@ -27,6 +27,10 @@ func (integrationConfigPO) TableName() string { return "sys_integration_configs"
type integrationConfigRepo struct{ data Provider }
type integrationRuntimeProvider interface {
IntegrationRuntime() *runtimeconfig.Store
}
func NewIntegrationConfigRepo(data Provider) biz.IntegrationConfigRepo {
return &integrationConfigRepo{data: data}
}
@@ -93,18 +97,25 @@ func (r *integrationConfigRepo) DeleteIntegrationConfig(ctx context.Context, kin
if err := r.data.DB().WithContext(ctx).Where("kind = ? AND provider = ?", kind, provider).Delete(&integrationConfigPO{}).Error; err != nil {
return err
}
if runtime := r.data.IntegrationRuntime(); runtime != nil {
if runtime := integrationRuntime(r.data); runtime != nil {
runtime.Delete(kind, provider)
}
return nil
}
func (r *integrationConfigRepo) publish(kind, provider string, enabled bool, values []byte) {
if runtime := r.data.IntegrationRuntime(); runtime != nil {
runtime.Set(integrationruntime.Config{Kind: kind, Provider: provider, Enabled: enabled, Values: values})
if runtime := integrationRuntime(r.data); runtime != nil {
runtime.Set(runtimeconfig.Config{Kind: kind, Provider: provider, Enabled: enabled, Values: values})
}
}
func integrationRuntime(provider Provider) *runtimeconfig.Store {
if value, ok := provider.(integrationRuntimeProvider); ok {
return value.IntegrationRuntime()
}
return nil
}
func integrationConfigFromPO(row integrationConfigPO) *biz.IntegrationConfig {
values := integrationObject(json.RawMessage(row.Config))
maskIntegrationSecrets(row.Kind, row.Provider, values)
@@ -0,0 +1,53 @@
package system
import (
"context"
"encoding/json"
"testing"
"kra/internal/biz"
"kra/internal/integration/runtimeconfig"
)
type integrationRuntimeTestProvider struct {
*Data
store *runtimeconfig.Store
}
func (p *integrationRuntimeTestProvider) IntegrationRuntime() *runtimeconfig.Store { return p.store }
func TestIntegrationConfigSavePublishesUnmaskedRuntimeValues(t *testing.T) {
db, err := openWithDriver("sqlite", "file:"+t.Name()+"?mode=memory&cache=shared")
if err != nil {
t.Fatal(err)
}
if err = db.AutoMigrate(&integrationConfigPO{}); err != nil {
t.Fatal(err)
}
provider := &integrationRuntimeTestProvider{Data: &Data{gormDB: newReloadableDB(db, nil)}, store: runtimeconfig.NewStore()}
repo := &integrationConfigRepo{data: provider}
values := biz.DefaultIntegrationConfig(biz.IntegrationKindMQ, "rabbitmq")
values["password"] = "runtime-secret"
raw, _ := json.Marshal(values)
if err = repo.SaveIntegrationConfig(context.Background(), &biz.IntegrationConfig{Kind: biz.IntegrationKindMQ, Provider: "rabbitmq", Enabled: true, Values: raw}); err != nil {
t.Fatal(err)
}
values["password"] = "******"
raw, _ = json.Marshal(values)
if err = repo.SaveIntegrationConfig(context.Background(), &biz.IntegrationConfig{Kind: biz.IntegrationKindMQ, Provider: "rabbitmq", Enabled: true, Values: raw}); err != nil {
t.Fatal(err)
}
active, ok := provider.store.Get(biz.IntegrationKindMQ, "rabbitmq")
if !ok || !active.Enabled {
t.Fatalf("runtime config = %#v, ok=%v", active, ok)
}
stored := map[string]any{}
if err = json.Unmarshal(active.Values, &stored); err != nil {
t.Fatal(err)
}
if stored["password"] != "runtime-secret" {
t.Fatalf("runtime password = %#v", stored["password"])
}
}
-2
View File
@@ -2,7 +2,6 @@ package system
import (
"kra/internal/conf"
"kra/internal/integrationruntime"
"gorm.io/gorm"
)
@@ -14,5 +13,4 @@ type Provider interface {
Database(name string) (*gorm.DB, error)
DatabaseReady() bool
Runtime() *conf.Runtime
IntegrationRuntime() *integrationruntime.Store
}
+6 -5
View File
@@ -8,7 +8,7 @@ state, or translate provider-specific protocols into `biz` interfaces.
- `cache`: Redis-backed cache with an in-memory fallback.
- `email`: SMTP email repository.
- `mq`: reloadable EMQX/MQTT client exposed through the shared `pkg/mq` interface.
- `mq`: reloadable EMQX/MQTT and RabbitMQ/AMQP clients exposed through the shared `pkg/mq` interface.
- `payment`: payment-channel SDKs and callback/signature handling.
- `storage`: local and object-storage implementations of `biz.FileStorage`.
- `websocket`: reloadable Melody endpoint exposed through the shared
@@ -28,7 +28,7 @@ Business modules depend on `websocket.Hub` and `mq.Client` from the shared
`pkg/websocket` and `pkg/mq` packages; they do not construct
Melody or Paho clients and do not read system configuration directly. The
system integration packages own runtime refresh and shutdown. Registered
WebSocket handlers and MQTT subscriptions are retained when database-backed
WebSocket handlers and broker subscriptions are retained when database-backed
configuration replaces a live client.
```go
@@ -45,7 +45,8 @@ Integration configuration is stored in `sys_integration_configs`:
- WebSocket: `kind=websocket`, `provider=melody`
- EMQX: `kind=mq`, `provider=emqx`
- RabbitMQ: `kind=mq`, `provider=rabbitmq`
The YAML values are only migration/bootstrap inputs. After the integration
table exists, the database is authoritative and runtime updates are applied
without restarting the process. The WebSocket public path defaults to `/ws`.
These three integrations are stored only in `sys_integration_configs`; they do
not come from `config.yaml`. Runtime updates are applied without restarting
the process. The WebSocket public path defaults to `/ws`.
+230 -71
View File
@@ -2,140 +2,294 @@ package mq
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"strconv"
"strings"
"sync"
"time"
"kra/internal/conf"
"kra/pkg/mq"
"kra/internal/integration/runtimeconfig"
platformmq "kra/pkg/mq"
)
// Reloadable follows the runtime snapshot and keeps a single shared EMQX
// connection for all modules in this process.
const (
ProviderEMQX = "emqx"
ProviderRabbitMQ = "rabbitmq"
)
// Reloadable owns the process-wide message clients. Configuration comes only
// from sys_integration_configs through runtimeconfig.Store.
type Reloadable struct {
mu sync.RWMutex
opMu sync.Mutex
current mq.Client
subscriptions map[string]subscription
stop func()
clients map[string]platformmq.Client
subscriptions map[string]map[string]subscription
stop []func()
logger *slog.Logger
closed bool
}
type subscription struct {
qos byte
handler mq.Handler
handler platformmq.Handler
}
func New(runtime *conf.Runtime, logger *slog.Logger) (*Reloadable, func(), error) {
type namedClient struct {
owner *Reloadable
provider string
}
func New(store *runtimeconfig.Store, logger *slog.Logger) (*Reloadable, func(), error) {
if logger == nil {
logger = slog.Default()
}
r := &Reloadable{logger: logger, subscriptions: make(map[string]subscription)}
if runtime != nil {
var config *conf.AdminBackend_MQ
if admin := runtime.Admin(); admin != nil {
config = admin.GetMq()
}
r.replace(config)
r.stop = runtime.Subscribe(func(_ *conf.Data, admin *conf.AdminBackend) {
if admin != nil {
r.replace(admin.GetMq())
}
})
r := &Reloadable{
clients: make(map[string]platformmq.Client),
subscriptions: make(map[string]map[string]subscription),
logger: logger,
}
if store != nil {
r.apply(ProviderEMQX, storeConfig(store, ProviderEMQX))
r.apply(ProviderRabbitMQ, storeConfig(store, ProviderRabbitMQ))
r.stop = append(r.stop,
store.Subscribe("mq", ProviderEMQX, func(config runtimeconfig.Config) { r.apply(ProviderEMQX, config) }),
store.Subscribe("mq", ProviderRabbitMQ, func(config runtimeconfig.Config) { r.apply(ProviderRabbitMQ, config) }),
)
}
cleanup := func() {
if r.stop != nil {
r.stop()
for _, stop := range r.stop {
stop()
}
_ = r.Close()
}
return r, cleanup, nil
}
func (r *Reloadable) replace(config *conf.AdminBackend_MQ) {
func storeConfig(store *runtimeconfig.Store, provider string) runtimeconfig.Config {
config, _ := store.Get("mq", provider)
return config
}
func (r *Reloadable) apply(provider string, config runtimeconfig.Config) {
r.opMu.Lock()
defer r.opMu.Unlock()
if r.closed {
return
}
if config == nil {
config = &conf.AdminBackend_MQ{}
}
cfg := mq.Config{Enabled: config.Enabled, Broker: config.Broker, ClientID: config.ClientId, Username: config.Username, Password: config.Password, CleanSession: config.CleanSession}
if config.KeepAlive > 0 {
cfg.KeepAlive = time.Duration(config.KeepAlive) * time.Second
}
if config.ConnectTimeout > 0 {
cfg.ConnectTimeout = time.Duration(config.ConnectTimeout) * time.Second
}
client, err := mq.NewMQTT(cfg)
if err != nil {
r.logger.Warn("emqx unavailable", "mod", "mq", "error", err)
if !config.Enabled {
r.replaceClientLocked(provider, nil)
return
}
if config.Enabled {
if err = r.restoreSubscriptions(context.Background(), client); err != nil {
_ = client.Close()
r.logger.Warn("restore emqx subscriptions failed", "mod", "mq", "error", err)
return
}
client, err := newProviderClient(provider, config.Values)
if err != nil {
r.logger.Warn("message integration unavailable", "mod", "mq", "provider", provider, "error", err)
return
}
if err = r.restoreSubscriptionsLocked(provider, client); err != nil {
_ = client.Close()
r.logger.Warn("restore message subscriptions failed", "mod", "mq", "provider", provider, "error", err)
return
}
r.replaceClientLocked(provider, client)
}
func newProviderClient(provider string, raw json.RawMessage) (platformmq.Client, error) {
values := map[string]any{}
if err := json.Unmarshal(raw, &values); err != nil {
return nil, fmt.Errorf("decode %s configuration: %w", provider, err)
}
switch provider {
case ProviderEMQX:
return platformmq.NewMQTT(platformmq.Config{
Enabled: true,
Broker: configText(values, "broker"),
ClientID: configText(values, "client_id"),
Username: configText(values, "username"),
Password: configText(values, "password"),
KeepAlive: configSeconds(values, "keep_alive"),
CleanSession: configBool(values, "clean_session"),
ConnectTimeout: configSeconds(values, "connect_timeout"),
})
case ProviderRabbitMQ:
return platformmq.NewRabbitMQ(platformmq.RabbitMQConfig{
Enabled: true,
Host: configText(values, "host"),
Port: configInt(values, "port"),
Username: configText(values, "username"),
Password: configText(values, "password"),
VHost: configText(values, "vhost"),
Exchange: configText(values, "exchange"),
ExchangeType: configText(values, "exchange_type"),
Queue: configText(values, "queue"),
RoutingKey: configText(values, "routing_key"),
Durable: configBool(values, "durable"),
AutoDelete: configBool(values, "auto_delete"),
PrefetchCount: configInt(values, "prefetch_count"),
Heartbeat: configSeconds(values, "heartbeat"),
ConnectTimeout: configSeconds(values, "connect_timeout"),
TLS: configBool(values, "tls"),
})
default:
return nil, fmt.Errorf("unsupported message provider %q", provider)
}
}
func configText(values map[string]any, key string) string {
value, ok := values[key]
if !ok || value == nil {
return ""
}
return strings.TrimSpace(fmt.Sprint(value))
}
func configInt(values map[string]any, key string) int {
switch value := values[key].(type) {
case float64:
return int(value)
case int:
return value
case json.Number:
parsed, _ := strconv.Atoi(string(value))
return parsed
default:
parsed, _ := strconv.Atoi(configText(values, key))
return parsed
}
}
func configSeconds(values map[string]any, key string) time.Duration {
seconds := configInt(values, key)
if seconds <= 0 {
return 0
}
return time.Duration(seconds) * time.Second
}
func configBool(values map[string]any, key string) bool {
value, _ := values[key].(bool)
return value
}
func (r *Reloadable) replaceClientLocked(provider string, next platformmq.Client) {
r.mu.Lock()
old := r.current
r.current = client
old := r.clients[provider]
if next == nil {
delete(r.clients, provider)
} else {
r.clients[provider] = next
}
r.mu.Unlock()
if old != nil {
_ = old.Close()
}
}
func (r *Reloadable) restoreSubscriptions(ctx context.Context, client mq.Client) error {
for topic, item := range r.subscriptions {
if err := client.Subscribe(ctx, topic, item.qos, item.handler); err != nil {
func (r *Reloadable) restoreSubscriptionsLocked(provider string, client platformmq.Client) error {
for topic, item := range r.subscriptions[provider] {
if err := client.Subscribe(context.Background(), topic, item.qos, item.handler); err != nil {
return err
}
}
return nil
}
func (r *Reloadable) client() mq.Client { r.mu.RLock(); defer r.mu.RUnlock(); return r.current }
func (r *Reloadable) Publish(ctx context.Context, topic string, payload []byte, qos byte, retain bool) error {
c := r.client()
if c == nil {
return mq.ErrUnavailable
}
return c.Publish(ctx, topic, payload, qos, retain)
func (r *Reloadable) client(provider string) platformmq.Client {
provider = strings.ToLower(strings.TrimSpace(provider))
r.mu.RLock()
defer r.mu.RUnlock()
return r.clients[provider]
}
func (r *Reloadable) Subscribe(ctx context.Context, topic string, qos byte, handler mq.Handler) error {
func (r *Reloadable) Client(provider string) platformmq.Client {
provider = strings.ToLower(strings.TrimSpace(provider))
if provider != ProviderEMQX && provider != ProviderRabbitMQ {
return nil
}
return &namedClient{owner: r, provider: provider}
}
func (c *namedClient) Publish(ctx context.Context, topic string, payload []byte, qos byte, retain bool) error {
return c.owner.PublishTo(ctx, c.provider, topic, payload, qos, retain)
}
func (c *namedClient) Subscribe(ctx context.Context, topic string, qos byte, handler platformmq.Handler) error {
return c.owner.SubscribeTo(ctx, c.provider, topic, qos, handler)
}
func (c *namedClient) Unsubscribe(ctx context.Context, topics ...string) error {
return c.owner.UnsubscribeFrom(ctx, c.provider, topics...)
}
func (c *namedClient) Connected() bool { return c.owner.ConnectedTo(c.provider) }
func (*namedClient) Close() error { return nil }
func (r *Reloadable) Publish(ctx context.Context, topic string, payload []byte, qos byte, retain bool) error {
return r.PublishTo(ctx, ProviderEMQX, topic, payload, qos, retain)
}
func (r *Reloadable) PublishTo(ctx context.Context, provider, topic string, payload []byte, qos byte, retain bool) error {
provider = strings.ToLower(strings.TrimSpace(provider))
r.mu.RLock()
defer r.mu.RUnlock()
client := r.clients[provider]
if client == nil {
return platformmq.ErrUnavailable
}
return client.Publish(ctx, topic, payload, qos, retain)
}
func (r *Reloadable) Subscribe(ctx context.Context, topic string, qos byte, handler platformmq.Handler) error {
return r.SubscribeTo(ctx, ProviderEMQX, topic, qos, handler)
}
func (r *Reloadable) SubscribeTo(ctx context.Context, provider, topic string, qos byte, handler platformmq.Handler) error {
provider = strings.ToLower(strings.TrimSpace(provider))
r.opMu.Lock()
defer r.opMu.Unlock()
c := r.client()
if c == nil {
return mq.ErrUnavailable
client := r.client(provider)
if client == nil {
return platformmq.ErrUnavailable
}
if err := c.Subscribe(ctx, topic, qos, handler); err != nil {
if err := client.Subscribe(ctx, topic, qos, handler); err != nil {
return err
}
r.subscriptions[topic] = subscription{qos: qos, handler: handler}
if r.subscriptions[provider] == nil {
r.subscriptions[provider] = make(map[string]subscription)
}
r.subscriptions[provider][topic] = subscription{qos: qos, handler: handler}
return nil
}
func (r *Reloadable) Unsubscribe(ctx context.Context, topics ...string) error {
return r.UnsubscribeFrom(ctx, ProviderEMQX, topics...)
}
func (r *Reloadable) UnsubscribeFrom(ctx context.Context, provider string, topics ...string) error {
provider = strings.ToLower(strings.TrimSpace(provider))
r.opMu.Lock()
defer r.opMu.Unlock()
c := r.client()
if c == nil {
return mq.ErrUnavailable
client := r.client(provider)
if client == nil {
return platformmq.ErrUnavailable
}
if err := c.Unsubscribe(ctx, topics...); err != nil {
if err := client.Unsubscribe(ctx, topics...); err != nil {
return err
}
for _, topic := range topics {
delete(r.subscriptions, topic)
delete(r.subscriptions[provider], topic)
}
return nil
}
func (r *Reloadable) Connected() bool { c := r.client(); return c != nil && c.Connected() }
func (r *Reloadable) Connected() bool { return r.ConnectedTo(ProviderEMQX) }
func (r *Reloadable) ConnectedTo(provider string) bool {
provider = strings.ToLower(strings.TrimSpace(provider))
r.mu.RLock()
defer r.mu.RUnlock()
client := r.clients[provider]
return client != nil && client.Connected()
}
func (r *Reloadable) Close() error {
r.opMu.Lock()
defer r.opMu.Unlock()
@@ -144,11 +298,16 @@ func (r *Reloadable) Close() error {
}
r.closed = true
r.mu.Lock()
old := r.current
r.current = nil
clients := make([]platformmq.Client, 0, len(r.clients))
for provider, client := range r.clients {
clients = append(clients, client)
delete(r.clients, provider)
}
r.mu.Unlock()
if old != nil {
return old.Close()
for _, client := range clients {
if client != nil {
_ = client.Close()
}
}
return nil
}
+11 -6
View File
@@ -26,28 +26,33 @@ func (*fakeClient) Close() error { return nil }
func TestReloadableTracksSubscriptions(t *testing.T) {
client := &fakeClient{}
r := &Reloadable{current: client, subscriptions: make(map[string]subscription)}
r := &Reloadable{
clients: map[string]platformmq.Client{ProviderEMQX: client},
subscriptions: make(map[string]map[string]subscription),
}
handler := func(context.Context, platformmq.Message) {}
if err := r.Subscribe(context.Background(), "orders/+/paid", platformmq.AtLeastOnce, handler); err != nil {
t.Fatal(err)
}
if _, ok := r.subscriptions["orders/+/paid"]; !ok {
if _, ok := r.subscriptions[ProviderEMQX]["orders/+/paid"]; !ok {
t.Fatal("subscription was not retained for configuration reload")
}
if err := r.Unsubscribe(context.Background(), "orders/+/paid"); err != nil {
t.Fatal(err)
}
if _, ok := r.subscriptions["orders/+/paid"]; ok {
if _, ok := r.subscriptions[ProviderEMQX]["orders/+/paid"]; ok {
t.Fatal("unsubscribed topic remained in the reload registry")
}
}
func TestReloadableRestoresSubscriptions(t *testing.T) {
client := &fakeClient{}
r := &Reloadable{subscriptions: map[string]subscription{
"orders/+/paid": {qos: platformmq.AtLeastOnce, handler: func(context.Context, platformmq.Message) {}},
r := &Reloadable{subscriptions: map[string]map[string]subscription{
ProviderEMQX: {
"orders/+/paid": {qos: platformmq.AtLeastOnce, handler: func(context.Context, platformmq.Message) {}},
},
}}
if err := r.restoreSubscriptions(context.Background(), client); err != nil {
if err := r.restoreSubscriptionsLocked(ProviderEMQX, client); err != nil {
t.Fatal(err)
}
if len(client.subscribed) != 1 || client.subscribed[0] != "orders/+/paid" {
+1
View File
@@ -22,6 +22,7 @@ var ProviderSet = wire.NewSet(
storage.NewFileStorage,
mqintegration.New,
wire.Bind(new(mq.Client), new(*mqintegration.Reloadable)),
wire.Bind(new(mq.Registry), new(*mqintegration.Reloadable)),
websocketintegration.New,
wire.Bind(new(platformws.Hub), new(*websocketintegration.Server)),
wire.Bind(new(biz.FileStorage), new(*storage.Reloadable)),
@@ -1,6 +1,6 @@
// Package integrationruntime keeps the active database-backed integration
// settings and notifies long-lived provider clients when they change.
package integrationruntime
// Package runtimeconfig keeps the active database-backed integration settings
// and notifies long-lived provider clients when they change.
package runtimeconfig
import (
"encoding/json"
@@ -1,4 +1,4 @@
package integrationruntime
package runtimeconfig
import (
"encoding/json"
+109 -64
View File
@@ -1,19 +1,23 @@
package websocket
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"sync"
"time"
melody "github.com/olahol/melody"
"kra/internal/conf"
"kra/internal/integration/runtimeconfig"
platformws "kra/pkg/websocket"
)
// Server is the system-owned WebSocket endpoint. Business modules can attach
// handlers and publish messages without depending on Gin or Melody directly.
const ProviderMelody = "melody"
// Server owns the database-configured WebSocket endpoint.
type Server struct {
mu sync.RWMutex
current *platformws.Server
@@ -22,24 +26,25 @@ type Server struct {
binaryHandlers []func(*melody.Session, []byte)
connectHandlers []func(*melody.Session)
disconnectHandlers []func(*melody.Session)
stop func()
closed bool
}
func New(runtime *conf.Runtime) (*Server, func(), error) {
func New(store *runtimeconfig.Store) (*Server, func(), error) {
s := &Server{}
s.Replace(runtime)
var unsubscribe func()
if runtime != nil {
unsubscribe = runtime.Subscribe(func(_ *conf.Data, _ *conf.AdminBackend) { s.Replace(runtime) })
if store != nil {
s.apply(storeConfig(store))
s.stop = store.Subscribe("websocket", ProviderMelody, func(config runtimeconfig.Config) { s.apply(config) })
}
return s, func() {
if unsubscribe != nil {
unsubscribe()
if s.stop != nil {
s.stop()
}
s.mu.Lock()
s.closed = true
current := s.current
s.current = nil
s.path = ""
s.mu.Unlock()
if current != nil {
_ = current.Close()
@@ -47,13 +52,20 @@ func New(runtime *conf.Runtime) (*Server, func(), error) {
}, nil
}
func (s *Server) Replace(runtime *conf.Runtime) {
var config *conf.AdminBackend_WebSocket
if runtime != nil && runtime.Admin() != nil {
config = runtime.Admin().Websocket
func storeConfig(store *runtimeconfig.Store) runtimeconfig.Config {
config, _ := store.Get("websocket", ProviderMelody)
return config
}
func (s *Server) apply(config runtimeconfig.Config) {
if s == nil {
return
}
if config == nil {
config = &conf.AdminBackend_WebSocket{}
values := map[string]any{}
if len(config.Values) > 0 {
if err := json.Unmarshal(config.Values, &values); err != nil {
return
}
}
s.mu.Lock()
if s.closed {
@@ -70,14 +82,21 @@ func (s *Server) Replace(runtime *conf.Runtime) {
}
return
}
path := text(values, "path")
if path == "" {
path = "/ws"
}
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
next := platformws.New(platformws.Config{
WriteWait: duration(config.WriteWait, 10*time.Second),
PongWait: duration(config.PongWait, 60*time.Second),
PingPeriod: duration(config.PingPeriod, 54*time.Second),
MaxMessageSize: config.MaxMessageSize,
MessageBufferSize: int(config.MessageBufferSize),
ConcurrentMessageHandling: config.ConcurrentMessageHandling,
AllowOrigins: config.AllowOrigins,
WriteWait: durationValue(values, "write_wait", 10*time.Second),
PongWait: durationValue(values, "pong_wait", 60*time.Second),
PingPeriod: durationValue(values, "ping_period", 54*time.Second),
MaxMessageSize: int64Value(values, "max_message_size"),
MessageBufferSize: int(intValue(values, "message_buffer_size")),
ConcurrentMessageHandling: boolValue(values, "concurrent_message_handling"),
AllowOrigins: stringList(values, "allow_origins"),
})
for _, handler := range s.messageHandlers {
next.OnMessage(handler)
@@ -91,27 +110,65 @@ func (s *Server) Replace(runtime *conf.Runtime) {
for _, handler := range s.disconnectHandlers {
next.OnDisconnect(handler)
}
s.path = strings.TrimSpace(config.Path)
if s.path == "" {
s.path = "/ws"
} else if !strings.HasPrefix(s.path, "/") {
s.path = "/" + s.path
}
s.current = next
s.path = path
s.mu.Unlock()
if previous != nil {
_ = previous.Close()
}
}
func duration(value interface{ AsDuration() time.Duration }, fallback time.Duration) time.Duration {
if value == nil {
return fallback
func text(values map[string]any, key string) string {
value, ok := values[key]
if !ok || value == nil {
return ""
}
if result := value.AsDuration(); result > 0 {
return strings.TrimSpace(fmt.Sprint(value))
}
func intValue(values map[string]any, key string) int64 {
switch value := values[key].(type) {
case float64:
return int64(value)
case int:
return int64(value)
case json.Number:
parsed, _ := strconv.ParseInt(string(value), 10, 64)
return parsed
default:
parsed, _ := strconv.ParseInt(text(values, key), 10, 64)
return parsed
}
}
func int64Value(values map[string]any, key string) int64 { return intValue(values, key) }
func boolValue(values map[string]any, key string) bool { value, _ := values[key].(bool); return value }
func stringList(values map[string]any, key string) []string {
value, ok := values[key].([]any)
if ok {
result := make([]string, 0, len(value))
for _, item := range value {
if item != nil && strings.TrimSpace(fmt.Sprint(item)) != "" {
result = append(result, strings.TrimSpace(fmt.Sprint(item)))
}
}
return result
}
return fallback
if value, ok := values[key].([]string); ok {
return append([]string(nil), value...)
}
return nil
}
func durationValue(values map[string]any, key string, fallback time.Duration) time.Duration {
value := text(values, key)
if value == "" {
return fallback
}
parsed, err := time.ParseDuration(value)
if err != nil || parsed <= 0 {
return fallback
}
return parsed
}
func (s *Server) Enabled() bool {
@@ -131,14 +188,9 @@ func (s *Server) Path() string {
return s.path
}
func (s *Server) HandleRequest(w http.ResponseWriter, r *http.Request) error {
if s == nil {
return errors.New("websocket server is disabled")
}
s.mu.RLock()
current := s.current
s.mu.RUnlock()
if current == nil {
return errors.New("websocket server is disabled")
current, err := s.active()
if err != nil {
return err
}
return current.HandleRequest(w, r)
}
@@ -150,26 +202,16 @@ func (s *Server) HandleRequestWithKeys(w http.ResponseWriter, r *http.Request, k
return current.HandleRequestWithKeys(w, r, keys)
}
func (s *Server) Broadcast(message []byte) error {
if s == nil {
return errors.New("websocket server is disabled")
}
s.mu.RLock()
current := s.current
s.mu.RUnlock()
if current == nil {
return errors.New("websocket server is disabled")
current, err := s.active()
if err != nil {
return err
}
return current.Broadcast(message)
}
func (s *Server) BroadcastBinary(message []byte) error {
if s == nil {
return errors.New("websocket server is disabled")
}
s.mu.RLock()
current := s.current
s.mu.RUnlock()
if current == nil {
return errors.New("websocket server is disabled")
current, err := s.active()
if err != nil {
return err
}
return current.BroadcastBinary(message)
}
@@ -231,8 +273,9 @@ func (s *Server) OnBinaryMessage(handler func(*melody.Session, []byte)) {
}
s.mu.Lock()
s.binaryHandlers = append(s.binaryHandlers, handler)
if s.current != nil {
s.current.OnBinaryMessage(handler)
current := s.current
if current != nil {
current.OnBinaryMessage(handler)
}
s.mu.Unlock()
}
@@ -242,8 +285,9 @@ func (s *Server) OnConnect(handler func(*melody.Session)) {
}
s.mu.Lock()
s.connectHandlers = append(s.connectHandlers, handler)
if s.current != nil {
s.current.OnConnect(handler)
current := s.current
if current != nil {
current.OnConnect(handler)
}
s.mu.Unlock()
}
@@ -253,8 +297,9 @@ func (s *Server) OnDisconnect(handler func(*melody.Session)) {
}
s.mu.Lock()
s.disconnectHandlers = append(s.disconnectHandlers, handler)
if s.current != nil {
s.current.OnDisconnect(handler)
current := s.current
if current != nil {
current.OnDisconnect(handler)
}
s.mu.Unlock()
}
+6
View File
@@ -65,6 +65,12 @@ func NewGinEngineWithRuntime(runtime *conf.Runtime, access *service.AccessContro
}
registerSwagger(engine, prefix, version, logger)
registerLocalStorage(engine, runtime)
if logger != nil {
for _, route := range engine.Routes() {
logger.Info("router registered", "method", route.Method, "path", route.Path)
}
logger.Info("router register success", "route_count", len(engine.Routes()))
}
engine.NoRoute(func(c *gin.Context) {
if ws != nil && ws.Enabled() && c.Request.Method == http.MethodGet && c.Request.URL.Path == ws.Path() {
+95 -12
View File
@@ -2,10 +2,14 @@ package middleware
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"log/slog"
"mime"
"net/http"
"strconv"
"strings"
"time"
@@ -67,14 +71,26 @@ func AccessLog(runtime *conf.Runtime, logger *slog.Logger, version string) gin.H
if config != nil && config.Zap != nil && config.Zap.AccessLogMaxBytes > 0 {
logLimit = int(config.Zap.AccessLogMaxBytes)
}
paymentCallback := isPaymentCallbackPath(c.Request.URL.Path)
paymentConfigWrite := isPaymentIntegrationConfigWrite(c.Request.Method, c.Request.URL.Path)
requestText := ""
if multipart {
if paymentCallback {
requestText = paymentCallbackSummary(requestBody, c.GetHeader("Content-Type"))
} else if paymentConfigWrite {
requestText = paymentConfigSummary(requestBody)
} else if multipart {
requestText = "[文件]"
} else {
requestText = redactJSON(requestBody, c.GetHeader("Content-Type"), logLimit)
}
c.Set(ctxReqBodyKey, requestText)
c.Set(ctxRespBufferKey, &writer.body)
if paymentCallback {
// Callback acknowledgements and provider payloads must not flow into
// the generic response/error audit pipeline.
c.Set(ctxRespBufferKey, &bytes.Buffer{})
} else {
c.Set(ctxRespBufferKey, &writer.body)
}
if !requestReadFailed {
c.Next()
}
@@ -82,6 +98,9 @@ func AccessLog(runtime *conf.Runtime, logger *slog.Logger, version string) gin.H
return
}
responseText := redactJSON(writer.body.Bytes(), c.Writer.Header().Get("Content-Type"), logLimit)
if paymentCallback {
responseText = "[支付回调响应已省略]"
}
userID, authorityID := uint(0), uint(0)
if claims := Claims(c); claims != nil {
userID, authorityID = claims.ID, claims.AuthorityID
@@ -95,22 +114,36 @@ func AccessLog(runtime *conf.Runtime, logger *slog.Logger, version string) gin.H
bytesOut = 0
}
privateErrors := strings.TrimRight(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
attributes := []any{
"mod", "http", "ip", c.ClientIP(), "method", c.Request.Method, "http_path", c.Request.URL.Path, "http_route", route,
"http_status", c.Writer.Status(), "latency_ms", time.Since(started).Milliseconds(),
"request_id", stringValueFromContext(c, "request_id"), "trace_id", stringValueFromContext(c, "trace_id"),
"bytes_in", bytesIn, "bytes_out", bytesOut, "user_id", userID, "authority_id", authorityID,
"error", c.Writer.Status() >= http.StatusInternalServerError || privateErrors != "", "ua", c.Request.UserAgent(), "req_query", c.Request.URL.RawQuery}
if config != nil && config.Zap != nil && config.Zap.AccessReqHeaders {
var attributes []any
if paymentCallback {
attributes = []any{
"mod", "payment-callback", "payment_provider", paymentCallbackProvider(c.Request.URL.Path),
"http_status", c.Writer.Status(), "latency_ms", time.Since(started).Milliseconds(),
"request_id", stringValueFromContext(c, "request_id"), "trace_id", stringValueFromContext(c, "trace_id"),
"bytes_in", bytesIn, "bytes_out", bytesOut,
"error", c.Writer.Status() >= http.StatusInternalServerError || privateErrors != "",
"payment_callback", true, "payment_callback_summary", requestText,
}
} else {
attributes = []any{
"mod", "http", "ip", c.ClientIP(), "method", c.Request.Method, "http_path", c.Request.URL.Path, "http_route", route,
"http_status", c.Writer.Status(), "latency_ms", time.Since(started).Milliseconds(),
"request_id", stringValueFromContext(c, "request_id"), "trace_id", stringValueFromContext(c, "trace_id"),
"bytes_in", bytesIn, "bytes_out", bytesOut, "user_id", userID, "authority_id", authorityID,
"error", c.Writer.Status() >= http.StatusInternalServerError || privateErrors != "", "ua", c.Request.UserAgent(),
"req_query", c.Request.URL.RawQuery,
}
}
if !paymentCallback && config != nil && config.Zap != nil && config.Zap.AccessReqHeaders {
attributes = append(attributes, "req_headers", redactHeaders(c.Request.Header))
}
if config != nil && config.Zap != nil && config.Zap.AccessReqBody {
if !paymentCallback && config != nil && config.Zap != nil && config.Zap.AccessReqBody {
attributes = append(attributes, "req_body", requestText)
}
if config != nil && config.Zap != nil && config.Zap.AccessRespData {
if !paymentCallback && config != nil && config.Zap != nil && config.Zap.AccessRespData {
attributes = append(attributes, "resp_data", responseText)
}
if privateErrors != "" {
if !paymentCallback && privateErrors != "" {
attributes = append(attributes, "error_msg", privateErrors)
}
logger.InfoContext(c.Request.Context(), "请求完成", attributes...)
@@ -122,6 +155,56 @@ func isMediaUploadRoute(route string) bool {
strings.HasSuffix(route, "/mediaUpload/chunk")
}
func isPaymentCallbackPath(path string) bool {
parts := strings.Split(strings.Trim(path, "/"), "/")
for index := 0; index+1 < len(parts); index++ {
if parts[index] == "payment" && parts[index+1] == "callback" {
return true
}
}
return false
}
func paymentCallbackProvider(path string) string {
parts := strings.Split(strings.Trim(path, "/"), "/")
for index := 0; index+2 < len(parts); index++ {
if parts[index] == "payment" && parts[index+1] == "callback" {
return parts[index+2]
}
}
return "unknown"
}
func paymentCallbackSummary(body []byte, contentType string) string {
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil || mediaType == "" {
mediaType = strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])
}
if mediaType == "" {
mediaType = "unknown"
}
digest := sha256.Sum256(body)
return "[支付回调正文已省略 body_bytes=" + strconv.Itoa(len(body)) + " body_sha256=" + hex.EncodeToString(digest[:]) + " content_type=" + mediaType + "]"
}
func isPaymentIntegrationConfigWrite(method, path string) bool {
if method != http.MethodPut {
return false
}
parts := strings.Split(strings.Trim(path, "/"), "/")
for index := 0; index+3 < len(parts); index++ {
if parts[index] == "integration" && parts[index+1] == "configs" && parts[index+2] == "payment" && parts[index+3] != "" {
return true
}
}
return false
}
func paymentConfigSummary(body []byte) string {
digest := sha256.Sum256(body)
return "[支付配置正文已省略 body_bytes=" + strconv.Itoa(len(body)) + " body_sha256=" + hex.EncodeToString(digest[:]) + "]"
}
func redactHeaders(headers map[string][]string) map[string]string {
out := make(map[string]string, len(headers))
for key, values := range headers {
@@ -1,7 +1,9 @@
package middleware
import (
"bytes"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
@@ -63,3 +65,60 @@ func TestAccessLogAllowsMediaLimitOnlyOnUploadRoute(t *testing.T) {
t.Fatalf("status=%d called=%v body=%s", response.Code, called, response.Body.String())
}
}
func TestAccessLogRedactsPaymentCallbackPayloadAndHeaders(t *testing.T) {
gin.SetMode(gin.TestMode)
var logs bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo}))
runtime := conf.NewRuntime(nil, &conf.AdminBackend{Zap: &conf.AdminBackend_Zap{AccessReqBody: true, AccessReqHeaders: true, AccessRespData: true}})
engine := gin.New()
engine.Use(AccessLog(runtime, logger, "test"))
engine.POST("/api/payment/callback/:provider", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"body": "callback-response-secret"})
})
request := httptest.NewRequest(http.MethodPost, "/api/payment/callback/alipay?signature=query-secret", strings.NewReader("payment-body-secret"))
request.Header.Set("Content-Type", "application/json; boundary=credential-secret")
request.Header.Set("Authorization", "Bearer header-secret")
request.Header.Set("X-Alipay-Signature", "signature-secret")
response := httptest.NewRecorder()
engine.ServeHTTP(response, request)
logText := logs.String()
for _, secret := range []string{"payment-body-secret", "query-secret", "header-secret", "signature-secret", "callback-response-secret", "credential-secret"} {
if strings.Contains(logText, secret) {
t.Fatalf("payment callback secret leaked into access log: %q in %s", secret, logText)
}
}
for _, marker := range []string{"payment_callback=true", "payment_provider=alipay", "body_sha256=", "http_status=200"} {
if !strings.Contains(logText, marker) {
t.Fatalf("payment callback access summary missing %q: %s", marker, logText)
}
}
}
func TestAccessLogOmitsPaymentIntegrationConfigBody(t *testing.T) {
gin.SetMode(gin.TestMode)
var logs bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo}))
runtime := conf.NewRuntime(nil, &conf.AdminBackend{Zap: &conf.AdminBackend_Zap{AccessReqBody: true, AccessReqHeaders: true, AccessRespData: true}})
engine := gin.New()
engine.Use(AccessLog(runtime, logger, "test"))
engine.PUT("/api/integration/configs/:kind/:provider", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"code": 0}) })
body := `{"enabled":true,"config":{"key":"saobei-secret","certificate_blob":"certificate-secret","unknown_credential":"credential-secret"}}`
request := httptest.NewRequest(http.MethodPut, "/api/integration/configs/payment/saobei", strings.NewReader(body))
request.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
engine.ServeHTTP(response, request)
logText := logs.String()
for _, secret := range []string{"saobei-secret", "certificate-secret", "credential-secret"} {
if strings.Contains(logText, secret) {
t.Fatalf("payment configuration secret leaked into access log: %q in %s", secret, logText)
}
}
if !strings.Contains(logText, "支付配置正文已省略") || !strings.Contains(logText, "body_sha256=") {
t.Fatalf("payment configuration summary missing: %s", logText)
}
}
+45 -7
View File
@@ -81,7 +81,11 @@ func OperationAudit(runtime *conf.Runtime, recorder *service.AuditRecorder) gin.
responseBody = "[超出记录长度]"
}
errorMessage := c.Errors.ByType(gin.ErrorTypePrivate).String()
if err := recorder.RecordOperationRequest(c.Request.Context(), &dto.OperationRecordRequest{IP: c.ClientIP(), Method: c.Request.Method, Path: path, Status: status, LatencyMS: time.Since(started).Milliseconds(), Agent: c.Request.UserAgent(), ErrorMessage: errorMessage, Body: operationRequestBody(requestBody, c.GetHeader("Content-Type"), maxBytes), Response: responseBody, UserID: userID, RequestID: stringValue(requestID), TraceID: stringValueFromContext(c, "trace_id"), DeviceID: c.GetHeader("X-Device-Id")}); err != nil {
operationBody := operationRequestBody(requestBody, c.GetHeader("Content-Type"), maxBytes)
if isPaymentIntegrationConfigWrite(c.Request.Method, path) {
operationBody = paymentConfigSummary(requestBody)
}
if err := recorder.RecordOperationRequest(c.Request.Context(), &dto.OperationRecordRequest{IP: c.ClientIP(), Method: c.Request.Method, Path: path, Status: status, LatencyMS: time.Since(started).Milliseconds(), Agent: c.Request.UserAgent(), ErrorMessage: errorMessage, Body: operationBody, Response: responseBody, UserID: userID, RequestID: stringValue(requestID), TraceID: stringValueFromContext(c, "trace_id"), DeviceID: c.GetHeader("X-Device-Id")}); err != nil {
// Preserve the business response, but expose audit persistence failures
// to the global access/error logging pipeline.
c.Set(ctxOperationAuditPersistFailedKey, true)
@@ -128,7 +132,7 @@ func maskOperationBody(value any) {
case map[string]any:
for key, item := range current {
normalized := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(key, "_", ""), "-", ""))
if normalized == "password" || normalized == "newpassword" || normalized == "oldpassword" || normalized == "confirmpassword" || normalized == "passwd" || normalized == "pwd" || normalized == "token" || normalized == "accesstoken" || normalized == "refreshtoken" || normalized == "secret" || normalized == "clientsecret" || normalized == "apikey" || normalized == "privatekey" || normalized == "idcard" {
if normalized == "password" || normalized == "newpassword" || normalized == "oldpassword" || normalized == "confirmpassword" || normalized == "passwd" || normalized == "pwd" || normalized == "token" || normalized == "accesstoken" || normalized == "refreshtoken" || normalized == "secret" || normalized == "clientsecret" || normalized == "apikey" || normalized == "privatekey" || normalized == "idcard" || normalized == "appkey" || normalized == "mchkey" || normalized == "apiv3key" || normalized == "clientcert" || normalized == "clientkey" || normalized == "platformcert" || normalized == "platformserialno" || normalized == "credentialcode" || normalized == "certfile" || normalized == "keyfile" || normalized == "publickey" || normalized == "rootcert" || normalized == "appcert" || normalized == "webhookid" {
current[key] = "***"
continue
}
@@ -157,17 +161,49 @@ func isDownloadResponse(c *gin.Context) bool {
// recordsOperation mirrors the routes on which operation records are enabled.
// Matching by suffix keeps the behavior stable when router-prefix is configured.
func recordsOperation(method, path string) bool {
_, ok := operationRoutes[method+" "+routeSuffix(path)]
return ok
for route := range operationRoutes {
parts := strings.SplitN(route, " ", 2)
if len(parts) != 2 || parts[0] != method || !operationPathMatches(parts[1], path) {
continue
}
return true
}
return false
}
func operationPathMatches(pattern, path string) bool {
patternParts := strings.Split(strings.Trim(pattern, "/"), "/")
pathParts := strings.Split(strings.Trim(path, "/"), "/")
if len(pathParts) < len(patternParts) {
return false
}
pathParts = pathParts[len(pathParts)-len(patternParts):]
for index, patternPart := range patternParts {
if strings.HasPrefix(patternPart, "*") {
return index <= len(pathParts)
}
if index >= len(pathParts) || (strings.HasPrefix(patternPart, ":") == false && patternPart != pathParts[index]) {
return false
}
}
return len(patternParts) == len(pathParts)
}
func routeSuffix(path string) string {
for _, marker := range []string{"/user/", "/api/", "/casbin/", "/authority/", "/menu/", "/department/", "/position/", "/sysDictionary/", "/sysDictionaryDetail/", "/sysParams/", "/securityConfig/", "/system/", "/sysApiToken/", "/sysVersion/", "/sysExportTemplate/", "/sysError/", "/sysLoginLog/", "/sysOperationRecord/", "/dataAccessLog/", "/timedTask/", "/info/", "/email/"} {
bestIndex := -1
bestPath := path
for _, marker := range []string{"/user/", "/api/", "/casbin/", "/authority/", "/menu/", "/department/", "/position/", "/sysDictionary/", "/sysDictionaryDetail/", "/sysParams/", "/securityConfig/", "/system/", "/sysApiToken/", "/sysVersion/", "/sysExportTemplate/", "/sysError/", "/sysLoginLog/", "/sysOperationRecord/", "/dataAccessLog/", "/timedTask/", "/info/", "/email/", "/integration/", "/payment/"} {
if index := strings.Index(path, marker); index >= 0 {
return path[index:]
// Router prefixes may themselves contain a registered route marker
// (for example /api/integration/...). Keep the deepest match so the
// policy and audit route remain the actual application endpoint.
if index > bestIndex {
bestIndex = index
bestPath = path[index:]
}
}
}
return path
return bestPath
}
var operationRoutes = func() map[string]struct{} {
@@ -189,6 +225,8 @@ var operationRoutes = func() map[string]struct{} {
"DELETE /sysLoginLog/deleteLoginLog", "DELETE /sysLoginLog/deleteLoginLogByIds", "DELETE /dataAccessLog/deleteDataAccessLogByIds",
"POST /timedTask/createTimedTask", "PUT /timedTask/updateTimedTask", "DELETE /timedTask/deleteTimedTask", "POST /timedTask/toggleTimedTask", "POST /timedTask/triggerTimedTask",
"POST /info/createInfo", "DELETE /info/deleteInfo", "DELETE /info/deleteInfoByIds", "PUT /info/updateInfo", "POST /email/emailTest", "POST /email/sendEmail",
"PUT /integration/configs/:kind/:provider", "DELETE /integration/configs/:kind/:provider",
"POST /payment/create", "POST /payment/query", "POST /payment/refund", "POST /payment/orders/:provider/:tradeNo/refund", "POST /payment/fulfill", "POST /payment/orders/:provider/:tradeNo/fulfill", "POST /payment/providers/:provider/test",
}
out := make(map[string]struct{}, len(values))
for _, value := range values {
@@ -0,0 +1,56 @@
package middleware
import (
"encoding/json"
"strings"
"testing"
)
func TestPaymentIntegrationSecretsAreRedacted(t *testing.T) {
raw := []byte(`{"enabled":true,"config":{"app_id":"app","mch_key":"merchant-secret","api_v3_key":"v3-secret","client_cert":"certificate","client_key":"private-key","platform_cert":"platform-certificate","credential_code":"credential","webhook_id":"webhook"}}`)
redacted := redactJSON(raw, "application/json", 4096)
var payload struct {
Config map[string]string `json:"config"`
}
if err := json.Unmarshal([]byte(redacted), &payload); err != nil {
t.Fatalf("decode redacted payload: %v", err)
}
for _, key := range []string{"mch_key", "api_v3_key", "client_cert", "client_key", "platform_cert", "credential_code", "webhook_id"} {
if payload.Config[key] != "***" {
t.Fatalf("payment secret %q was not redacted: %s", key, redacted)
}
}
if !strings.Contains(redacted, `"app_id":"app"`) {
t.Fatalf("non-secret integration field was removed: %s", redacted)
}
}
func TestPaymentOperationsAreAuditedWithRouterPrefix(t *testing.T) {
for _, route := range []struct {
method string
path string
}{
{method: "PUT", path: "/api/integration/configs/payment/alipay"},
{method: "POST", path: "/api/payment/refund"},
{method: "POST", path: "/api/payment/fulfill"},
{method: "POST", path: "/api/payment/providers/alipay/test"},
} {
if !recordsOperation(route.method, route.path) {
t.Fatalf("payment operation was not audited: %s %s", route.method, route.path)
}
}
}
func TestPaymentIntegrationConfigUsesRouteLevelSummary(t *testing.T) {
raw := []byte(`{"enabled":true,"config":{"key":"secret","custom_certificate":"certificate"}}`)
summary := paymentConfigSummary(raw)
if strings.Contains(summary, "secret") || strings.Contains(summary, "certificate") {
t.Fatalf("payment configuration summary leaked payload: %s", summary)
}
if !isPaymentIntegrationConfigWrite("PUT", "/api/integration/configs/payment/saobei") {
t.Fatal("payment configuration write route was not recognized")
}
if isPaymentIntegrationConfigWrite("PUT", "/api/integration/configs/mq/emqx") {
t.Fatal("non-payment integration was treated as payment configuration")
}
}
+7
View File
@@ -47,6 +47,13 @@ var apiMetadata = map[string]apiMetadataValue{
"GET /integration/configs/:kind": {group: "集成配置", description: "按类型获取集成配置"},
"GET /integration/configs/:kind/:provider": {group: "集成配置", description: "获取指定集成配置"},
"GET /payment/orders": {group: "支付", description: "分页查询支付订单"},
"GET /payment/orders/:provider/:tradeNo": {group: "支付", description: "按路径查询支付订单"},
"POST /payment/create": {group: "支付", description: "创建支付订单"},
"POST /payment/query": {group: "支付", description: "同步支付订单状态"},
"POST /payment/refund": {group: "支付", description: "申请支付订单退款"},
"POST /payment/orders/:provider/:tradeNo/refund": {group: "支付", description: "按路径申请支付订单退款"},
"POST /payment/fulfill": {group: "支付", description: "重试支付订单发货"},
"POST /payment/orders/:provider/:tradeNo/fulfill": {group: "支付", description: "按路径重试支付订单发货"},
"POST /payment/providers/:provider/test": {group: "支付", description: "测试支付渠道配置与沙箱交易链路"},
"GET /securityConfig/getSecurityConfig": {group: "安全配置", description: "获取安全配置"},
"GET /sysDictionary/exportSysDictionary": {group: "系统字典", description: "导出字典JSON"},