优化结构

This commit is contained in:
Yvan
2026-08-22 08:25:25 +08:00
parent 626f505b5d
commit 2b78b195c1
10 changed files with 815 additions and 115 deletions
+352 -79
View File
@@ -20,8 +20,8 @@ const (
retryTick = time.Second
)
// Reloadable owns the process-wide message clients. Configuration comes only
// from sys_integration_configs through runtimeconfig.Store.
// Reloadable owns process-wide message clients and the logical subscription
// declarations used to restore them after reconnects or configuration reloads.
type Reloadable struct {
mu sync.RWMutex
opMu sync.Mutex
@@ -31,10 +31,10 @@ type Reloadable struct {
bindings map[string]map[string]byte
pending map[string]bool
nextRetry map[string]time.Time
legacySeq uint64
stop []func()
retryStop chan struct{}
retryDone chan struct{}
closeOnce sync.Once
logger *slog.Logger
closed bool
}
@@ -139,19 +139,22 @@ func (r *Reloadable) apply(provider string, config runtimeconfig.Config) {
if r.closed {
return
}
r.ensureStateLocked()
config.Kind = "mq"
config.Provider = provider
config.Values = append(json.RawMessage(nil), config.Values...)
r.configs[provider] = config
r.replaceClientLocked(provider, nil)
if !config.Enabled {
delete(r.pending, provider)
delete(r.nextRetry, provider)
r.replaceClientLocked(provider, nil)
return
}
if err := r.activateLocked(provider, config); err != nil {
r.pending[provider] = true
r.nextRetry[provider] = time.Now().Add(configRetryInterval(config.Values))
r.logger.Warn("message integration unavailable", "mod", "mq", "provider", provider, "error", err)
r.scheduleRetryLocked(provider, config.Values)
if r.logger != nil {
r.logger.Warn("message integration unavailable", "mod", "mq", "provider", provider, "error", err)
}
}
}
@@ -167,6 +170,9 @@ func (r *Reloadable) activateLocked(provider string, config runtimeconfig.Config
}
r.replaceClientLocked(provider, client)
r.mu.Lock()
if r.bindings == nil {
r.bindings = make(map[string]map[string]byte)
}
r.bindings[provider] = bindings
r.mu.Unlock()
delete(r.pending, provider)
@@ -253,8 +259,89 @@ func configBool(values map[string]any, key string) bool {
return value
}
func (r *Reloadable) retryLoop() {
defer close(r.retryDone)
ticker := time.NewTicker(retryTick)
defer ticker.Stop()
for {
select {
case <-r.retryStop:
return
case <-ticker.C:
r.retryOnce()
}
}
}
func (r *Reloadable) retryOnce() {
r.opMu.Lock()
defer r.opMu.Unlock()
if r.closed {
return
}
r.ensureStateLocked()
now := time.Now()
for _, provider := range []string{ProviderEMQX, ProviderRabbitMQ} {
config, exists := r.configs[provider]
if !exists || !config.Enabled {
continue
}
client := r.clientLocked(provider)
if client != nil && client.Connected() {
if r.pending[provider] {
if err := r.reconcileProviderLocked(provider); err != nil {
r.scheduleRetryLocked(provider, config.Values)
}
}
continue
}
if client != nil {
if recovering, ok := client.(interface{ Reconnecting() bool }); ok && recovering.Reconnecting() {
continue
}
}
if retryAt := r.nextRetry[provider]; !retryAt.IsZero() && now.Before(retryAt) {
continue
}
r.replaceClientLocked(provider, nil)
if err := r.activateLocked(provider, config); err != nil {
r.scheduleRetryLocked(provider, config.Values)
if r.logger != nil {
r.logger.Warn("message integration reconnect failed", "mod", "mq", "provider", provider, "error", err)
}
}
}
}
func (r *Reloadable) scheduleRetryLocked(provider string, raw json.RawMessage) {
if r.pending == nil {
r.pending = make(map[string]bool)
}
if r.nextRetry == nil {
r.nextRetry = make(map[string]time.Time)
}
r.pending[provider] = true
r.nextRetry[provider] = time.Now().Add(configRetryInterval(raw))
}
func configRetryInterval(raw json.RawMessage) time.Duration {
values := map[string]any{}
_ = json.Unmarshal(raw, &values)
interval := configSeconds(values, "reconnect_interval")
if interval <= 0 {
return 5 * time.Second
}
return interval
}
func (r *Reloadable) replaceClientLocked(provider string, next platformmq.Client) {
r.mu.Lock()
if r.clients == nil {
r.clients = make(map[string]platformmq.Client)
}
if r.bindings == nil {
r.bindings = make(map[string]map[string]byte)
}
old := r.clients[provider]
if next == nil {
delete(r.clients, provider)
@@ -263,7 +350,7 @@ func (r *Reloadable) replaceClientLocked(provider string, next platformmq.Client
r.clients[provider] = next
}
r.mu.Unlock()
if old != nil {
if old != nil && old != next {
_ = old.Close()
}
}
@@ -280,23 +367,241 @@ func (r *Reloadable) restoreSubscriptionsLocked(provider string, client platform
return bindings, nil
}
func configRetryInterval(raw json.RawMessage) time.Duration {
values := map[string]any{}
_ = json.Unmarshal(raw, &values)
interval := configSeconds(values, "reconnect_interval")
if interval <= 0 {
return 5 * time.Second
func (r *Reloadable) desiredSubscriptions(provider string) map[string]byte {
result := make(map[string]byte)
r.mu.RLock()
defer r.mu.RUnlock()
for topic, owners := range r.subscriptions[provider] {
for _, item := range owners {
if qos, exists := result[topic]; !exists || item.qos > qos {
result[topic] = item.qos
}
}
}
return interval
return result
}
func (r *Reloadable) client(provider string) platformmq.Client {
provider = strings.ToLower(strings.TrimSpace(provider))
func (r *Reloadable) dispatcher(provider, topic string) platformmq.Handler {
return func(ctx context.Context, message platformmq.Message) {
r.mu.RLock()
owners := r.subscriptions[provider][topic]
handlers := make([]platformmq.Handler, 0, len(owners))
for _, item := range owners {
handlers = append(handlers, item.handler)
}
r.mu.RUnlock()
for _, handler := range handlers {
handler(ctx, message)
}
}
}
func (r *Reloadable) clientLocked(provider string) platformmq.Client {
r.mu.RLock()
defer r.mu.RUnlock()
return r.clients[provider]
}
func (r *Reloadable) reconcileProviderLocked(provider string) error {
client := r.clientLocked(provider)
desired := r.desiredSubscriptions(provider)
if client == nil || !client.Connected() {
if len(desired) == 0 {
delete(r.pending, provider)
delete(r.nextRetry, provider)
return nil
}
return platformmq.ErrUnavailable
}
r.mu.RLock()
current := make(map[string]byte, len(r.bindings[provider]))
for topic, qos := range r.bindings[provider] {
current[topic] = qos
}
r.mu.RUnlock()
for topic := range current {
if _, exists := desired[topic]; exists {
continue
}
if err := client.Unsubscribe(context.Background(), topic); err != nil {
return err
}
delete(current, topic)
}
for topic, qos := range desired {
if oldQoS, exists := current[topic]; exists && oldQoS == qos {
continue
}
if _, exists := current[topic]; exists {
if err := client.Unsubscribe(context.Background(), topic); err != nil {
return err
}
}
if err := client.Subscribe(context.Background(), topic, qos, r.dispatcher(provider, topic)); err != nil {
return err
}
current[topic] = qos
}
r.mu.Lock()
r.bindings[provider] = current
r.mu.Unlock()
delete(r.pending, provider)
delete(r.nextRetry, provider)
return nil
}
func (r *Reloadable) Register(set platformmq.SubscriptionSet) error {
normalized, err := platformmq.NormalizeSubscriptionSet(set)
if err != nil {
return err
}
r.opMu.Lock()
defer r.opMu.Unlock()
if r.closed {
return platformmq.ErrUnavailable
}
r.ensureStateLocked()
r.mu.Lock()
if r.subscriptions[normalized.Provider] == nil {
r.subscriptions[normalized.Provider] = make(map[string]map[string]subscription)
}
for topic, owners := range r.subscriptions[normalized.Provider] {
delete(owners, normalized.Owner)
if len(owners) == 0 {
delete(r.subscriptions[normalized.Provider], topic)
}
}
for _, item := range normalized.Topics {
if r.subscriptions[normalized.Provider][item.Topic] == nil {
r.subscriptions[normalized.Provider][item.Topic] = make(map[string]subscription)
}
r.subscriptions[normalized.Provider][item.Topic][normalized.Owner] = subscription{qos: item.QoS, handler: item.Handler}
}
r.mu.Unlock()
if err := r.reconcileProviderLocked(normalized.Provider); err != nil {
config := r.configs[normalized.Provider]
r.scheduleRetryLocked(normalized.Provider, config.Values)
if r.logger != nil {
r.logger.Warn("message subscription bind deferred", "mod", "mq", "provider", normalized.Provider, "owner", normalized.Owner, "error", err)
}
}
return nil
}
func (r *Reloadable) Unregister(owner string) error {
owner = strings.TrimSpace(owner)
if owner == "" {
return fmt.Errorf("mq subscription owner is empty")
}
r.opMu.Lock()
defer r.opMu.Unlock()
if r.closed {
return nil
}
r.ensureStateLocked()
r.mu.Lock()
providers := make([]string, 0, len(r.subscriptions))
for provider, topics := range r.subscriptions {
providers = append(providers, provider)
for topic, owners := range topics {
delete(owners, owner)
if len(owners) == 0 {
delete(topics, topic)
}
}
}
r.mu.Unlock()
for _, provider := range providers {
if err := r.reconcileProviderLocked(provider); err != nil {
if config, ok := r.configs[provider]; ok {
r.scheduleRetryLocked(provider, config.Values)
}
}
}
return nil
}
func (r *Reloadable) Publish(ctx context.Context, topic string, payload []byte, qos byte, retain bool) error {
return r.PublishTo(ctx, ProviderEMQX, topic, payload, qos, retain)
}
func (r *Reloadable) PublishTo(ctx context.Context, provider, topic string, payload []byte, qos byte, retain bool) error {
provider = strings.ToLower(strings.TrimSpace(provider))
client := r.clientLocked(provider)
if client == nil {
return platformmq.ErrUnavailable
}
return client.Publish(ctx, topic, payload, qos, retain)
}
func (r *Reloadable) Subscribe(ctx context.Context, topic string, qos byte, handler platformmq.Handler) error {
return r.SubscribeTo(ctx, ProviderEMQX, topic, qos, handler)
}
func (r *Reloadable) SubscribeTo(ctx context.Context, provider, topic string, qos byte, handler platformmq.Handler) error {
if ctx != nil {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
}
provider = strings.ToLower(strings.TrimSpace(provider))
r.opMu.Lock()
r.legacySeq++
owner := fmt.Sprintf("legacy/%s/%d", provider, r.legacySeq)
r.opMu.Unlock()
if err := r.Register(platformmq.SubscriptionSet{Owner: owner, Provider: provider, Topics: []platformmq.TopicSubscription{{Topic: topic, QoS: qos, Handler: handler}}}); err != nil {
return err
}
if !r.ConnectedTo(provider) {
return platformmq.ErrUnavailable
}
return nil
}
func (r *Reloadable) Unsubscribe(ctx context.Context, topics ...string) error {
return r.UnsubscribeFrom(ctx, ProviderEMQX, topics...)
}
func (r *Reloadable) UnsubscribeFrom(ctx context.Context, provider string, topics ...string) error {
if ctx != nil {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
}
provider = strings.ToLower(strings.TrimSpace(provider))
if len(topics) == 0 {
return fmt.Errorf("mq topics are empty")
}
r.opMu.Lock()
defer r.opMu.Unlock()
r.ensureStateLocked()
r.mu.Lock()
for _, rawTopic := range topics {
topic := strings.TrimSpace(rawTopic)
owners := r.subscriptions[provider][topic]
for owner := range owners {
if strings.HasPrefix(owner, "legacy/") {
delete(owners, owner)
}
}
if len(owners) == 0 {
delete(r.subscriptions[provider], topic)
}
}
r.mu.Unlock()
if err := r.reconcileProviderLocked(provider); err != nil {
if config, ok := r.configs[provider]; ok {
r.scheduleRetryLocked(provider, config.Values)
}
return err
}
return nil
}
func (r *Reloadable) Client(provider string) platformmq.Client {
provider = strings.ToLower(strings.TrimSpace(provider))
if provider != ProviderEMQX && provider != ProviderRabbitMQ {
@@ -317,81 +622,24 @@ func (c *namedClient) Unsubscribe(ctx context.Context, topics ...string) error {
func (c *namedClient) Connected() bool { return c.owner.ConnectedTo(c.provider) }
func (*namedClient) Close() error { return nil }
func (r *Reloadable) Publish(ctx context.Context, topic string, payload []byte, qos byte, retain bool) error {
return r.PublishTo(ctx, ProviderEMQX, topic, payload, qos, retain)
}
func (r *Reloadable) PublishTo(ctx context.Context, provider, topic string, payload []byte, qos byte, retain bool) error {
provider = strings.ToLower(strings.TrimSpace(provider))
r.mu.RLock()
defer r.mu.RUnlock()
client := r.clients[provider]
if client == nil {
return platformmq.ErrUnavailable
}
return client.Publish(ctx, topic, payload, qos, retain)
}
func (r *Reloadable) Subscribe(ctx context.Context, topic string, qos byte, handler platformmq.Handler) error {
return r.SubscribeTo(ctx, ProviderEMQX, topic, qos, handler)
}
func (r *Reloadable) SubscribeTo(ctx context.Context, provider, topic string, qos byte, handler platformmq.Handler) error {
provider = strings.ToLower(strings.TrimSpace(provider))
r.opMu.Lock()
defer r.opMu.Unlock()
client := r.client(provider)
if client == nil {
return platformmq.ErrUnavailable
}
if err := client.Subscribe(ctx, topic, qos, handler); err != nil {
return err
}
if r.subscriptions[provider] == nil {
r.subscriptions[provider] = make(map[string]subscription)
}
r.subscriptions[provider][topic] = subscription{qos: qos, handler: handler}
return nil
}
func (r *Reloadable) Unsubscribe(ctx context.Context, topics ...string) error {
return r.UnsubscribeFrom(ctx, ProviderEMQX, topics...)
}
func (r *Reloadable) UnsubscribeFrom(ctx context.Context, provider string, topics ...string) error {
provider = strings.ToLower(strings.TrimSpace(provider))
r.opMu.Lock()
defer r.opMu.Unlock()
client := r.client(provider)
if client == nil {
return platformmq.ErrUnavailable
}
if err := client.Unsubscribe(ctx, topics...); err != nil {
return err
}
for _, topic := range topics {
delete(r.subscriptions[provider], topic)
}
return nil
}
func (r *Reloadable) Connected() bool { return r.ConnectedTo(ProviderEMQX) }
func (r *Reloadable) ConnectedTo(provider string) bool {
provider = strings.ToLower(strings.TrimSpace(provider))
r.mu.RLock()
defer r.mu.RUnlock()
client := r.clients[provider]
client := r.clientLocked(provider)
return client != nil && client.Connected()
}
func (r *Reloadable) Close() error {
r.opMu.Lock()
defer r.opMu.Unlock()
if r.closed {
r.opMu.Unlock()
return nil
}
r.closed = true
if r.retryStop != nil {
close(r.retryStop)
}
r.mu.Lock()
clients := make([]platformmq.Client, 0, len(r.clients))
for provider, client := range r.clients {
@@ -399,6 +647,10 @@ func (r *Reloadable) Close() error {
delete(r.clients, provider)
}
r.mu.Unlock()
r.opMu.Unlock()
if r.retryDone != nil {
<-r.retryDone
}
for _, client := range clients {
if client != nil {
_ = client.Close()
@@ -406,3 +658,24 @@ func (r *Reloadable) Close() error {
}
return nil
}
func (r *Reloadable) ensureStateLocked() {
if r.clients == nil {
r.clients = make(map[string]platformmq.Client)
}
if r.configs == nil {
r.configs = make(map[string]runtimeconfig.Config)
}
if r.subscriptions == nil {
r.subscriptions = make(map[string]map[string]map[string]subscription)
}
if r.bindings == nil {
r.bindings = make(map[string]map[string]byte)
}
if r.pending == nil {
r.pending = make(map[string]bool)
}
if r.nextRetry == nil {
r.nextRetry = make(map[string]time.Time)
}
}
+94 -5
View File
@@ -2,19 +2,27 @@ package mq
import (
"context"
"log/slog"
"testing"
"time"
"kra/internal/integration/runtimeconfig"
platformmq "kra/pkg/mq"
)
type fakeClient struct {
subscribed []string
unsubscribed []string
handlers map[string]platformmq.Handler
}
func (*fakeClient) Publish(context.Context, string, []byte, byte, bool) error { return nil }
func (f *fakeClient) Subscribe(_ context.Context, topic string, _ byte, _ platformmq.Handler) error {
func (f *fakeClient) Subscribe(_ context.Context, topic string, _ byte, handler platformmq.Handler) error {
f.subscribed = append(f.subscribed, topic)
if f.handlers == nil {
f.handlers = make(map[string]platformmq.Handler)
}
f.handlers[topic] = handler
return nil
}
func (f *fakeClient) Unsubscribe(_ context.Context, topics ...string) error {
@@ -28,7 +36,11 @@ func TestReloadableTracksSubscriptions(t *testing.T) {
client := &fakeClient{}
r := &Reloadable{
clients: map[string]platformmq.Client{ProviderEMQX: client},
subscriptions: make(map[string]map[string]subscription),
configs: map[string]runtimeconfig.Config{ProviderEMQX: {Kind: "mq", Provider: ProviderEMQX, Enabled: true}},
subscriptions: make(map[string]map[string]map[string]subscription),
bindings: make(map[string]map[string]byte),
pending: make(map[string]bool),
nextRetry: make(map[string]time.Time),
}
handler := func(context.Context, platformmq.Message) {}
if err := r.Subscribe(context.Background(), "orders/+/paid", platformmq.AtLeastOnce, handler); err != nil {
@@ -47,15 +59,92 @@ func TestReloadableTracksSubscriptions(t *testing.T) {
func TestReloadableRestoresSubscriptions(t *testing.T) {
client := &fakeClient{}
r := &Reloadable{subscriptions: map[string]map[string]subscription{
r := &Reloadable{subscriptions: map[string]map[string]map[string]subscription{
ProviderEMQX: {
"orders/+/paid": {qos: platformmq.AtLeastOnce, handler: func(context.Context, platformmq.Message) {}},
"orders/+/paid": {
"orders": {qos: platformmq.AtLeastOnce, handler: func(context.Context, platformmq.Message) {}},
},
},
}}
if err := r.restoreSubscriptionsLocked(ProviderEMQX, client); err != nil {
bindings, err := r.restoreSubscriptionsLocked(ProviderEMQX, client)
if err != nil {
t.Fatal(err)
}
if len(client.subscribed) != 1 || client.subscribed[0] != "orders/+/paid" {
t.Fatalf("restored subscriptions = %v", client.subscribed)
}
if bindings["orders/+/paid"] != platformmq.AtLeastOnce {
t.Fatalf("restored binding qos = %d", bindings["orders/+/paid"])
}
}
func TestReloadableRegistersWhileOfflineAndRestoresLater(t *testing.T) {
client := &fakeClient{}
r := &Reloadable{
clients: make(map[string]platformmq.Client),
configs: map[string]runtimeconfig.Config{ProviderEMQX: {Kind: "mq", Provider: ProviderEMQX, Enabled: true}},
subscriptions: make(map[string]map[string]map[string]subscription),
bindings: make(map[string]map[string]byte),
pending: make(map[string]bool),
nextRetry: make(map[string]time.Time),
logger: slog.Default(),
}
called := make(chan struct{}, 1)
err := r.Register(platformmq.SubscriptionSet{
Owner: "orders",
Provider: ProviderEMQX,
Topics: []platformmq.TopicSubscription{{Topic: "orders.created", QoS: platformmq.AtLeastOnce, Handler: func(context.Context, platformmq.Message) { called <- struct{}{} }}},
})
if err != nil {
t.Fatal(err)
}
if !r.pending[ProviderEMQX] {
t.Fatal("offline subscription was not marked pending")
}
r.clients[ProviderEMQX] = client
if err = r.reconcileProviderLocked(ProviderEMQX); err != nil {
t.Fatal(err)
}
if len(client.subscribed) != 1 || client.subscribed[0] != "orders.created" {
t.Fatalf("restored subscriptions = %v", client.subscribed)
}
}
func TestReloadableDispatchesSameTopicToMultipleOwners(t *testing.T) {
client := &fakeClient{}
r := &Reloadable{
clients: map[string]platformmq.Client{ProviderEMQX: client},
configs: map[string]runtimeconfig.Config{ProviderEMQX: {Kind: "mq", Provider: ProviderEMQX, Enabled: true}},
subscriptions: make(map[string]map[string]map[string]subscription),
bindings: make(map[string]map[string]byte),
pending: make(map[string]bool),
nextRetry: make(map[string]time.Time),
logger: slog.Default(),
}
first, second := make(chan struct{}, 1), make(chan struct{}, 1)
for _, set := range []platformmq.SubscriptionSet{
{Owner: "orders", Provider: ProviderEMQX, Topics: []platformmq.TopicSubscription{{Topic: "events.created", Handler: func(context.Context, platformmq.Message) { first <- struct{}{} }}}},
{Owner: "audit", Provider: ProviderEMQX, Topics: []platformmq.TopicSubscription{{Topic: "events.created", Handler: func(context.Context, platformmq.Message) { second <- struct{}{} }}}},
} {
if err := r.Register(set); err != nil {
t.Fatal(err)
}
}
if err := r.reconcileProviderLocked(ProviderEMQX); err != nil {
t.Fatalf("explicit reconcile failed: %v", err)
}
if len(client.subscribed) != 1 {
t.Fatalf("broker subscriptions = %v, want one shared topic; desired=%#v bindings=%#v clients=%#v", client.subscribed, r.subscriptions, r.bindings, r.clients)
}
client.handlers["events.created"](context.Background(), platformmq.Message{Topic: "events.created"})
select {
case <-first:
default:
t.Fatal("orders handler was not called")
}
select {
case <-second:
default:
t.Fatal("audit handler was not called")
}
}
+1
View File
@@ -25,6 +25,7 @@ var ProviderSet = wire.NewSet(
mqintegration.New,
wire.Bind(new(mq.Client), new(*mqintegration.Reloadable)),
wire.Bind(new(mq.Registry), new(*mqintegration.Reloadable)),
wire.Bind(new(mq.SubscriptionRegistrar), new(*mqintegration.Reloadable)),
websocketintegration.New,
wire.Bind(new(platformws.Hub), new(*websocketintegration.Server)),
wire.Bind(new(biz.FileStorage), new(*storage.Reloadable)),