fix: 完成商务迁移复核与接口优化并清理历史资料

This commit is contained in:
Yvan
2026-11-04 14:45:02 +08:00
parent 5a96be6a79
commit 297e80827d
176 changed files with 6803 additions and 6002 deletions
+10 -6
View File
@@ -37,9 +37,13 @@ func NewActivationRepo(data Provider, databases PlatformProvider) business.Activ
return &activationRepo{data: data, databases: databases}
}
func (r *activationRepo) OwnAccounts(ctx context.Context, id uint) ([]string, error) {
accounts := []string{}
err := r.data.DB().WithContext(ctx).Table("sys_user_bind_platform").Where("user_id = ? AND status = ? AND platform_busier_account IS NOT NULL", id, "0").Distinct("platform_busier_account").Pluck("platform_busier_account", &accounts).Error
func (r *activationRepo) OwnAccounts(ctx context.Context, id uint) ([]business.ActivationAccount, error) {
var rows []struct{ SourceDB, Account string }
err := r.data.DB().WithContext(ctx).Table("sys_user_bind_platform").Where("user_id = ? AND status = ? AND platform_busier_account IS NOT NULL", id, "0").Distinct("source_db,platform_busier_account AS account").Scan(&rows).Error
accounts := make([]business.ActivationAccount, 0, len(rows))
for _, row := range rows {
accounts = append(accounts, business.ActivationAccount{SourceDB: row.SourceDB, Account: row.Account})
}
return accounts, storageError(err)
}
@@ -68,11 +72,11 @@ func (r *activationRepo) List(ctx context.Context, scope business.ActivationScop
}
q := db.Model(&activationPO{}).Where("deleted_at IS NULL")
if !scope.All {
accounts := make([]string, 0, len(scope.Accounts))
accounts := make([][]any, 0, len(scope.Accounts))
for _, account := range scope.Accounts {
accounts = append(accounts, strings.ToLower(account))
accounts = append(accounts, []any{strings.ToLower(account.SourceDB), strings.ToLower(account.Account)})
}
q = q.Where("LOWER(busier_account) IN ?", accounts)
q = q.Where("(LOWER(platform_name),LOWER(busier_account)) IN ?", accounts)
}
if len(filter.PlatformNames) > 0 {
q = q.Where("platform_name IN ?", filter.PlatformNames)
+3 -3
View File
@@ -59,11 +59,11 @@ func TestActivationListsReadOAAndRestrictAccounts(t *testing.T) {
db, _, repo := prepareActivations(t)
ctx := context.Background()
accounts, err := repo.OwnAccounts(ctx, 7)
if err != nil || len(accounts) != 1 || accounts[0] != "Own" {
if err != nil || len(accounts) != 1 || accounts[0] != (business.ActivationAccount{SourceDB: "lx", Account: "Own"}) {
t.Fatalf("本人已通过账号范围错误: %+v %v", accounts, err)
}
page, err := repo.List(ctx, business.ActivationScope{Accounts: []string{"OWN"}}, business.ActivationFilter{})
if err != nil || page.Total != 2 || page.Rows[0].ID != 2 {
page, err := repo.List(ctx, business.ActivationScope{Accounts: []business.ActivationAccount{{SourceDB: "lx", Account: "OWN"}}}, business.ActivationFilter{})
if err != nil || page.Total != 1 || page.Rows[0].ID != 1 {
t.Fatalf("OA 列表或排序错误: %+v %v", page, err)
}
page, err = repo.List(ctx, business.ActivationScope{All: true}, business.ActivationFilter{PlatformNames: []string{"lx"}, Statuses: []string{"reached"}})
+168
View File
@@ -0,0 +1,168 @@
package business
import (
"context"
"errors"
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"kra/internal/biz/business"
)
type alertTriggerRepo struct{ data Provider }
func NewAlertTriggerRepo(data Provider) business.AlertTriggerRepo {
return &alertTriggerRepo{data: data}
}
type alertTriggerLockPO struct {
SourceDB string `gorm:"column:source_db;primaryKey;size:50"`
PromoteID uint64 `gorm:"primaryKey;autoIncrement:false"`
RuleType string `gorm:"primaryKey;size:50"`
}
func (alertTriggerLockPO) TableName() string { return "sys_business_alert_locks" }
func (r *alertTriggerRepo) Input(ctx context.Context) (*business.AlertTriggerInput, error) {
db := r.data.DB().WithContext(ctx)
out := &business.AlertTriggerInput{}
var rules []alertRulePO
if err := db.Where("enabled=1 AND is_deleted=0").Order("id").Find(&rules).Error; err != nil {
return nil, business.ErrStorage
}
for _, row := range rules {
out.Rules = append(out.Rules, toBizAlertRule(row))
}
if len(rules) == 0 {
return out, nil
}
var accounts []accountPO
if err := db.Select("id,username,level,parent_id,id_path,authority_id").Find(&accounts).Error; err != nil {
return nil, business.ErrStorage
}
for _, row := range accounts {
out.Accounts = append(out.Accounts, toBizAccount(row))
}
var bindings []bindingPO
if err := db.Where("status=?", "0").Find(&bindings).Error; err != nil {
return nil, business.ErrStorage
}
for _, row := range bindings {
out.Bindings = append(out.Bindings, toBizBinding(row))
}
if len(bindings) == 0 {
return out, nil
}
available, err := businessTableAvailable(db, "tab_promote")
if err != nil {
return nil, err
}
if !available {
return nil, business.ErrAlertChannelsUnavailable
}
var channels []struct {
ID, BusierID uint64
SourceDB string `gorm:"column:source_db"`
Account string
LastLoginTime, CreateTime *int64
}
bound := db.Table("sys_user_bind_platform b").Select("1").Where("b.status=? AND b.source_db=p.source_db AND b.platform_busier_id=p.busier_id", "0")
if err := db.Table("tab_promote p").Select("p.id,p.source_db,p.account,p.busier_id,p.last_login_time,p.create_time").Where("p.promote_level=1 AND p.status=1").Where("EXISTS (?)", bound).Order("p.source_db,p.id").Scan(&channels).Error; err != nil {
return nil, business.ErrStorage
}
for _, row := range channels {
channel := &business.AlertTriggerChannel{ChannelKey: business.ChannelKey{SourceDB: row.SourceDB, ID: row.ID}, Account: row.Account, BusierID: row.BusierID}
if row.LastLoginTime != nil {
channel.LastLoginUnix = *row.LastLoginTime
}
if row.CreateTime != nil {
channel.CreatedUnix = *row.CreateTime
}
out.Channels = append(out.Channels, channel)
}
return out, nil
}
func (r *alertTriggerRepo) Comparison(ctx context.Context, metric int, channels []business.ChannelKey, today time.Time, days int) (map[business.ChannelKey]business.AlertComparison, error) {
db := r.data.DB().WithContext(ctx)
table, stamp, amount, condition := "tab_user", "register_time", "1", "(puid=0 OR puid IS NULL)"
if metric == 2 {
table, stamp, amount, condition = "tab_spend", "pay_time", "pay_amount", "pay_status=1"
} else if metric != 1 {
return nil, business.ErrAlertRuleInput
}
available, err := businessTableAvailable(db, table)
if err != nil {
return nil, err
}
if !available {
return nil, business.ErrReportBuildUnavailable
}
result := map[business.ChannelKey]business.AlertComparison{}
currentStart, previousStart := today.AddDate(0, 0, -days).Unix(), today.AddDate(0, 0, -2*days).Unix()
for start := 0; start < len(channels); start += 500 {
var rows []struct {
SourceDB string `gorm:"column:source_db"`
PromoteID uint64
FirstUnix *int64
CurrentValue, PreviousValue string
}
// 原始主账号与成功支付口径,不扩展下级渠道,也不使用身份证偏好。
selection := "source_db,promote_id,MIN(" + stamp + ") AS first_unix," +
"COALESCE(SUM(CASE WHEN " + stamp + ">=? AND " + stamp + "<? THEN " + amount + " ELSE 0 END),0) AS current_value," +
"COALESCE(SUM(CASE WHEN " + stamp + ">=? AND " + stamp + "<? THEN " + amount + " ELSE 0 END),0) AS previous_value"
if err := db.Table(table).Select(selection, currentStart, today.Unix(), previousStart, currentStart).Where(condition).Where("(source_db,promote_id) IN ?", channelPairs(channels[start:min(start+500, len(channels))])).Group("source_db,promote_id").Scan(&rows).Error; err != nil {
return nil, business.ErrStorage
}
for _, row := range rows {
value := business.AlertComparison{Current: row.CurrentValue, Previous: row.PreviousValue}
if row.FirstUnix != nil {
value.FirstUnix = *row.FirstUnix
}
result[business.ChannelKey{SourceDB: row.SourceDB, ID: row.PromoteID}] = value
}
}
return result, nil
}
func (r *alertTriggerRepo) StoreHit(ctx context.Context, hit *business.AlertRecord, open []int, shouldWrite func(*business.AlertRecord) bool) (bool, error) {
written := false
err := r.data.DB().WithContext(ctx).Transaction(func(tx *gorm.DB) error {
key := alertTriggerLockPO{SourceDB: hit.PlatformCode, PromoteID: hit.TargetPromoteID, RuleType: hit.RuleType}
if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&key).Error; err != nil {
return business.ErrStorage
}
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("source_db=? AND promote_id=? AND rule_type=?", key.SourceDB, key.PromoteID, key.RuleType).Take(&key).Error; err != nil {
return business.ErrStorage
}
var previous alertRecordPO
err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("platform_code=? AND target_promote_id=? AND rule_type=? AND status IN ?", key.SourceDB, key.PromoteID, key.RuleType, open).Order("id").Take(&previous).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return business.ErrStorage
}
var current *business.AlertRecord
if err == nil {
current = toBizAlertRecord(previous)
}
if !shouldWrite(current) {
return nil
}
day := time.Date(hit.CreatedAt.Year(), hit.CreatedAt.Month(), hit.CreatedAt.Day(), 0, 0, 0, 0, hit.CreatedAt.Location())
row := alertRecordPO{PlatformCode: hit.PlatformCode, PlatformName: hit.PlatformName, RuleID: hit.RuleID, RuleName: hit.RuleName, RuleType: hit.RuleType,
MonitorMetric: hit.MonitorMetric, TargetPromoteID: hit.TargetPromoteID, TargetPromoteName: hit.TargetPromoteName, BusinessID: hit.BusinessID, BusinessName: hit.BusinessName,
AlertContent: hit.AlertContent, Status: hit.Status, AlertDate: &day, CreatedAt: hit.CreatedAt, UpdatedAt: hit.UpdatedAt}
if current == nil {
err = tx.Create(&row).Error
} else {
row.ID = previous.ID
err = tx.Model(&alertRecordPO{}).Where("id=?", previous.ID).Select("platform_name", "rule_id", "rule_name", "rule_type", "monitor_metric", "target_promote_name", "business_id", "business_name", "alert_content", "status", "alert_date", "created_at", "updated_at").Updates(&row).Error
}
if err != nil {
return business.ErrStorage
}
written = true
return nil
})
return written, err
}
@@ -0,0 +1,110 @@
package business
import (
"context"
"testing"
"time"
"kra/internal/biz/business"
)
func TestAlertTriggerStorageUsesRawMetricsAndPreservesWorkflow(t *testing.T) {
db := openTestDB(t)
if err := db.AutoMigrate(&accountPO{}); err != nil {
t.Fatal(err)
}
now := time.Date(2026, 11, 4, 12, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600))
for _, sql := range []string{
"CREATE TABLE tab_promote(id INTEGER,source_db TEXT,busier_id INTEGER,account TEXT,promote_level INTEGER,status INTEGER,last_login_time INTEGER,create_time INTEGER)",
"CREATE TABLE tab_user(id INTEGER,source_db TEXT,promote_id INTEGER,puid INTEGER,register_time TEXT)",
"CREATE TABLE tab_spend(source_db TEXT,promote_id INTEGER,pay_time INTEGER,pay_status INTEGER,pay_amount DECIMAL(20,2))",
"INSERT INTO sys_users(id,username,level,parent_id,id_path,authority_id) VALUES(10,'一级',1,0,'0',901),(20,'二级',2,10,'0,10',902),(30,'三级',3,20,'0,10,20',903)",
"INSERT INTO sys_user_bind_platform(user_id,source_db,platform_busier_id,platform_busier_account,status) VALUES(30,'lx',7,'绑定','0')",
"INSERT INTO tab_promote VALUES(10,'lx',7,'正常渠道',1,1,0,1),(11,'lx',7,'停用渠道',1,0,0,1),(12,'lx',7,'下级渠道',2,1,0,1)",
} {
if err := db.Exec(sql).Error; err != nil {
t.Fatal(err)
}
}
today := time.Date(2026, 11, 4, 0, 0, 0, 0, now.Location())
if err := db.Exec("INSERT INTO tab_user VALUES(1,'lx',10,0,?),(2,'lx',10,0,?),(3,'lx',10,NULL,?),(4,'lx',10,1,?),(5,'hw',10,0,?)", today.AddDate(0, 0, -2).Unix(), today.AddDate(0, 0, -1).Unix(), today.AddDate(0, 0, -1).Unix(), today.AddDate(0, 0, -1).Unix(), today.AddDate(0, 0, -1).Unix()).Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO tab_spend VALUES('lx',10,?,1,100),('lx',10,?,1,110),('lx',10,?,0,999),('hw',10,?,1,999)", today.AddDate(0, 0, -2).Unix(), today.AddDate(0, 0, -1).Unix(), today.AddDate(0, 0, -1).Unix(), today.AddDate(0, 0, -1).Unix()).Error; err != nil {
t.Fatal(err)
}
day, direction := 1, 1
threshold := "10"
rule := alertRulePO{ID: 1, RuleName: "充值规则", RuleType: "充值", MonitorMetric: 2, CreatorID: 10, StatPeriodDays: &day, MomDirection: &direction, MomThreshold: &threshold, Enabled: 1}
if err := db.Create(&rule).Error; err != nil {
t.Fatal(err)
}
r := NewAlertTriggerRepo(testData{db})
input, err := r.Input(context.Background())
if err != nil || len(input.Channels) != 1 {
t.Fatalf("渠道查询错误: %+v %v", input, err)
}
metrics, err := r.Comparison(context.Background(), 1, []business.ChannelKey{{SourceDB: "lx", ID: 10}}, today, 1)
if err != nil || metrics[business.ChannelKey{SourceDB: "lx", ID: 10}].Current != "2" {
t.Fatalf("主账号、NULL父账号或跨平台口径错误: %+v %v", metrics, err)
}
uc := business.NewAlertTriggerUsecase(r)
result, err := uc.Detect(context.Background(), now)
if err != nil || result.Written != 1 {
t.Fatalf("首次命中: %+v %v", result, err)
}
var first alertRecordPO
if err := db.First(&first).Error; err != nil {
t.Fatal(err)
}
if first.BusinessID != 10 || first.TargetPromoteID != 10 || first.PlatformCode != "lx" {
t.Fatalf("归属错误: %+v", first)
}
result, err = uc.Detect(context.Background(), now.Add(time.Hour))
var count int64
db.Model(&alertRecordPO{}).Count(&count)
if err != nil || result.Written != 1 || count != 1 {
t.Fatalf("重复命中必须刷新同一未处理记录: %+v count=%d %v", result, count, err)
}
if err := db.Model(&alertRecordPO{}).Where("id=?", first.ID).Update("status", 2).Error; err != nil {
t.Fatal(err)
}
result, err = uc.Detect(context.Background(), now.Add(2*time.Hour))
if err != nil || result.Written != 0 {
t.Fatalf("处理中预警被覆盖: %+v %v", result, err)
}
if err := db.Model(&alertRecordPO{}).Where("id=?", first.ID).Update("status", 4).Error; err != nil {
t.Fatal(err)
}
result, err = uc.Detect(context.Background(), now.Add(3*time.Hour))
db.Model(&alertRecordPO{}).Count(&count)
if err != nil || result.Written != 0 || count != 1 {
t.Fatalf("驳回仍是未关闭记录,不应重复生成: %+v count=%d %v", result, count, err)
}
if err := db.Model(&alertRecordPO{}).Where("id=?", first.ID).Update("status", 5).Error; err != nil {
t.Fatal(err)
}
result, err = uc.Detect(context.Background(), now.Add(3*time.Hour))
db.Model(&alertRecordPO{}).Count(&count)
if err != nil || result.Written != 1 || count != 2 {
t.Fatalf("已关闭后应允许新一轮预警: %+v count=%d %v", result, count, err)
}
}
func TestAlertTriggerWriteFailureRollsBack(t *testing.T) {
db := openTestDB(t)
if err := db.Exec("CREATE TRIGGER reject_alert BEFORE INSERT ON sys_alert_record BEGIN SELECT RAISE(ABORT,'test write failure'); END").Error; err != nil {
t.Fatal(err)
}
r := NewAlertTriggerRepo(testData{db})
hit := &business.AlertRecord{PlatformCode: "lx", TargetPromoteID: 1, RuleType: "注册", Status: 1, CreatedAt: time.Now(), UpdatedAt: time.Now()}
_, err := r.StoreHit(context.Background(), hit, []int{1, 2, 3, 4}, func(*business.AlertRecord) bool { return true })
if err == nil {
t.Fatal("写入失败应返回错误")
}
var count int64
db.Model(&alertTriggerLockPO{}).Count(&count)
if count != 0 {
t.Fatalf("事务未回滚键锁: %d", count)
}
}
+52 -25
View File
@@ -4,6 +4,7 @@ import (
"context"
"sort"
"golang.org/x/sync/errgroup"
"gorm.io/gorm"
"kra/internal/biz/business"
)
@@ -80,6 +81,9 @@ func analyticsRoot(db *gorm.DB, kind business.AnalyticsKind, scope business.Anal
if kind == business.ChannelAnalytics {
id = "r.promote_id"
if scope.IncludeSub {
if pay {
q = analyticsPayLeaves(db, q)
}
q = q.Joins("LEFT JOIN tab_promote root_p ON root_p.source_db=r.source_db AND root_p.id=r.promote_id")
id = "COALESCE(NULLIF(root_p.top_promote_id,0),r.promote_id)"
if pay {
@@ -89,6 +93,12 @@ func analyticsRoot(db *gorm.DB, kind business.AnalyticsKind, scope business.Anal
}
return q, id
}
func analyticsPayLeaves(db, q *gorm.DB) *gorm.DB {
// 金额可先按原渠道加总再归并树,减少按每天/游戏重复关联渠道资料。
leaves := q.Select("r.source_db,r.promote_id,COALESCE(SUM(r.total_pay),0) AS total_pay").Group("r.source_db,r.promote_id")
return db.Table("(?) r", leaves)
}
func putAnalyticsCount(counts *business.PortraitCounts, metric int, count int64) {
switch metric {
case 1:
@@ -134,44 +144,61 @@ func (r *analyticsRepo) Aggregate(ctx context.Context, kind business.AnalyticsKi
}
return rowMap[key]
}
var pays []analyticsAggPO
q, id := analyticsRoot(db, kind, scope, f, "tab_sw_report_day", 0, true)
if err := q.Select("r.source_db," + id + " AS entity_id,COALESCE(SUM(r.total_pay),0) AS amount").Group("r.source_db," + id).Scan(&pays).Error; err != nil {
return nil, business.ErrStorage
var pays, counts []analyticsAggPO
var global []portraitCountPO
globalPay := "0"
withGlobal := kind == business.ChannelAnalytics && scope.Viewer.All
group, queryContext := errgroup.WithContext(ctx)
// 每个请求最多两个并行查询,复用同一存储客户端;结果读取完后再组装 DO。
group.SetLimit(2)
queryDB := db.WithContext(queryContext)
group.Go(func() error {
q, id := analyticsRoot(queryDB, kind, scope, f, "tab_sw_report_day", 0, true)
if err := q.Select("r.source_db," + id + " AS entity_id,COALESCE(SUM(r.total_pay),0) AS amount").Group("r.source_db," + id).Scan(&pays).Error; err != nil {
return business.ErrStorage
}
return nil
})
group.Go(func() error {
q, id := analyticsRoot(queryDB, kind, scope, f, "tab_sw_report_day_key", mode, false)
if err := q.Select("r.source_db," + id + " AS entity_id,r.metric,COUNT(DISTINCT r.identity_key) AS count").Group("r.source_db," + id + ",r.metric").Scan(&counts).Error; err != nil {
return business.ErrStorage
}
return nil
})
if withGlobal {
group.Go(func() error {
q := analyticsQuery(queryDB, scope, f, "tab_sw_report_day_key", mode)
if err := q.Select("r.metric,COUNT(DISTINCT " + portraitIdentity(queryDB, scope.IDCardDedup) + ") AS count").Group("r.metric").Scan(&global).Error; err != nil {
return business.ErrStorage
}
if err := analyticsQuery(queryDB, scope, f, "tab_sw_report_day", 0).Select("COALESCE(SUM(r.total_pay),0)").Scan(&globalPay).Error; err != nil {
return business.ErrStorage
}
return nil
})
}
if err := group.Wait(); err != nil {
return nil, err
}
for _, po := range pays {
if po.ID == nil {
continue
if po.ID != nil {
rowFor(po).Pay = po.Amount
}
rowFor(po).Pay = po.Amount
}
var counts []analyticsAggPO
q, id = analyticsRoot(db, kind, scope, f, "tab_sw_report_day_key", mode, false)
if err := q.Select("r.source_db," + id + " AS entity_id,r.metric,COUNT(DISTINCT r.identity_key) AS count").Group("r.source_db," + id + ",r.metric").Scan(&counts).Error; err != nil {
return nil, business.ErrStorage
}
for _, po := range counts {
if po.ID == nil {
continue
if po.ID != nil {
putAnalyticsCount(&rowFor(po).Counts, po.Metric, po.Count)
}
putAnalyticsCount(&rowFor(po).Counts, po.Metric, po.Count)
}
if kind == business.ChannelAnalytics && !scope.Viewer.All {
if err := filterAnalyticsRows(db, scope, f, rowMap); err != nil {
return nil, err
}
}
if kind == business.ChannelAnalytics && scope.Viewer.All {
var global []portraitCountPO
q := analyticsQuery(db, scope, f, "tab_sw_report_day_key", mode)
if err := q.Select("r.metric,COUNT(DISTINCT " + portraitIdentity(db, scope.IDCardDedup) + ") AS count").Group("r.metric").Scan(&global).Error; err != nil {
return nil, business.ErrStorage
}
if withGlobal {
totals := toPortraitCounts(global)
result.GlobalCounts = &totals
if err := analyticsQuery(db, scope, f, "tab_sw_report_day", 0).Select("COALESCE(SUM(r.total_pay),0)").Scan(&result.GlobalPay).Error; err != nil {
return nil, business.ErrStorage
}
result.GlobalCounts, result.GlobalPay = &totals, globalPay
}
if len(rowMap) == 0 {
return result, nil
+10 -2
View File
@@ -100,7 +100,9 @@ func (r *channelRepo) List(ctx context.Context, scope business.UserScope, filter
}
var pos []channelPO
o := business.NewListOptions(options...)
if err := q.Select("p.id,p.source_db,p.account,p.nickname,p.parent_name,p.real_name,p.promote_level,p.register_type,p.status,p.lock_status,p.parent_id,p.top_promote_id,p.busier_id").Order("p.create_time DESC,p.id DESC,p.source_db ASC").Offset(o.Offset).Limit(o.Limit).Scan(&pos).Error; err != nil {
keysQuery := q.Select("p.id,p.source_db").Order("p.create_time DESC,p.id DESC,p.source_db ASC").Offset(o.Offset).Limit(o.Limit)
pageQuery := db.Table("tab_promote p").Joins("JOIN (?) page_ids ON page_ids.id=p.id AND page_ids.source_db=p.source_db", keysQuery)
if err := pageQuery.Select("p.id,p.source_db,p.account,p.nickname,p.parent_name,p.real_name,p.promote_level,p.register_type,p.status,p.lock_status,p.parent_id,p.top_promote_id,p.busier_id").Order("p.create_time DESC,p.id DESC,p.source_db ASC").Scan(&pos).Error; err != nil {
return nil, business.ErrStorage
}
if len(pos) == 0 {
@@ -184,7 +186,13 @@ func (r *channelRepo) StatChannels(ctx context.Context, roots []*business.Channe
for start := 0; start < len(keys); start += 200 {
pairs := channelPairs(keys[start:min(start+200, len(keys))])
var pos []channelPO
err := r.data.DB().WithContext(ctx).Table("tab_promote").Select("id,source_db,promote_level,parent_id,top_promote_id").Where("(source_db,parent_id) IN ? OR (source_db,top_promote_id) IN ?", pairs, pairs).Scan(&pos).Error
db := r.data.DB().WithContext(ctx)
columns := "id,source_db,promote_level,parent_id,top_promote_id"
parents := db.Table("tab_promote").Select(columns).Where("(source_db,parent_id) IN ?", pairs)
tops := db.Table("tab_promote").Select(columns).Where("(source_db,top_promote_id) IN ?", pairs)
// 分别走 parent_id / top_promote_id 索引,避免跨列 OR 退化为全表扫描。
// 同时命中两条关系的渠道继续由下面的 ChannelKey 集合去重。
err := db.Raw("? UNION ALL ?", parents, tops).Scan(&pos).Error
if err != nil {
return nil, business.ErrStorage
}
+3 -1
View File
@@ -75,7 +75,9 @@ func (r *channelWriteRepo) Create(ctx context.Context, input *business.ChannelWr
conn.WithContext(cleanup).Exec("DO RELEASE_LOCK(?)", lockName)
}()
}
return conn.Transaction(func(tx *gorm.DB) error {
// GET_LOCK 的标量 Scan 会修改固定连接上的 Statement.Model。
// 清空查询状态后再开事务,仍复用持锁连接,避免后续结构体 Take 读取到标量模型。
return conn.Session(&gorm.Session{NewDB: true}).Transaction(func(tx *gorm.DB) error {
var count int64
if err := tx.Model(&platformChannelPO{}).Where("account = ?", input.Account).Count(&count).Error; err != nil {
return err
+27 -8
View File
@@ -87,11 +87,24 @@ type registrationPO struct {
PromoteID *int64
}
func registrationQuery(db *gorm.DB, scope business.LedgerScope, filter business.LedgerFilter) *gorm.DB {
func registrationTimeExpression(db *gorm.DB) (string, error) {
cast := "CAST(r.register_time AS INTEGER)"
if db.Dialector.Name() == "mysql" {
cast = "CAST(r.register_time AS UNSIGNED)"
var count int64
if err := db.Raw("SELECT COUNT(*) FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='tab_user' AND column_name='sw_register_time_unix'").Scan(&count).Error; err != nil {
return "", business.ErrStorage
}
if count > 0 {
// 直接使用等价生成列,使 COUNT 和分页主键查询走覆盖索引。
// CAST 虽可用于范围定位,但 MySQL 5.7 仍会回表读取原字符串。
cast = "r.sw_register_time_unix"
}
}
return cast, nil
}
func registrationQuery(db *gorm.DB, scope business.LedgerScope, filter business.LedgerFilter, cast string) *gorm.DB {
q := ledgerScopeQuery(db, scope, filter, "tab_user").Where("r.puid = 0").Where(cast+" BETWEEN ? AND ?", filter.BeginUnix, filter.EndUnix)
if filter.UserID != nil {
q = q.Where("r.id = ?", *filter.UserID)
@@ -135,7 +148,11 @@ func (r *ledgerRepo) Registrations(ctx context.Context, scope business.LedgerSco
page.Available = false
return page, nil
}
q := registrationQuery(db, scope, filter)
timeExpression, err := registrationTimeExpression(db)
if err != nil {
return nil, err
}
q := registrationQuery(db, scope, filter, timeExpression)
if err := q.Count(&page.Total).Error; err != nil {
return nil, business.ErrStorage
}
@@ -146,15 +163,14 @@ func (r *ledgerRepo) Registrations(ctx context.Context, scope business.LedgerSco
if filter.Ascending {
direction = " ASC"
}
order := "CAST(r.register_time AS INTEGER)"
if db.Dialector.Name() == "mysql" {
order = "CAST(r.register_time AS UNSIGNED)"
}
order := timeExpression
if filter.SortField == "totalPayAmount" {
order = "(SELECT COALESCE(SUM(s.pay_amount),0) FROM tab_spend s WHERE s.source_db=r.source_db AND s.user_id=r.id AND s.pay_status=1)"
}
var pos []registrationPO
if err := q.Select("r.id,r.source_db,r.register_time,r.fgame_id,r.fgame_name,r.register_ip,r.device_name,r.promote_id").Order(order + direction).Order("r.id DESC,r.source_db ASC").Offset(options.Offset).Limit(options.Limit).Scan(&pos).Error; err != nil {
keysQuery := q.Select("r.id,r.source_db").Order(order + direction).Order("r.id DESC,r.source_db ASC").Offset(options.Offset).Limit(options.Limit)
pageQuery := db.Table("tab_user r").Joins("JOIN (?) page_ids ON page_ids.id=r.id AND page_ids.source_db=r.source_db", keysQuery)
if err := pageQuery.Select("r.id,r.source_db,r.register_time,r.fgame_id,r.fgame_name,r.register_ip,r.device_name,r.promote_id").Order(order + direction).Order("r.id DESC,r.source_db ASC").Scan(&pos).Error; err != nil {
return nil, business.ErrStorage
}
meta := []ledgerMetaTarget{}
@@ -280,7 +296,10 @@ func (r *ledgerRepo) Spends(ctx context.Context, scope business.LedgerScope, fil
order += " DESC"
}
var pos []spendPO
if err := q.Select("r.id,r.source_db,r.user_id,r.pay_order_number,r.game_id,r.game_name,r.server_id,r.server_name,r.game_player_name,r.pay_amount,r.pay_time,r.pay_way,r.pay_status,r.spend_ip,r.promote_id,r.promote_account").Order(order).Order("r.id DESC,r.source_db ASC").Offset(options.Offset).Limit(options.Limit).Scan(&pos).Error; err != nil {
// 先在窄查询中选出当前页主键,再读取明细列,避免为排序读取整段流水。
keysQuery := q.Select("r.id,r.source_db").Order(order).Order("r.id DESC,r.source_db ASC").Offset(options.Offset).Limit(options.Limit)
pageQuery := db.Table("tab_spend r").Joins("JOIN (?) page_ids ON page_ids.id=r.id AND page_ids.source_db=r.source_db", keysQuery)
if err := pageQuery.Select("r.id,r.source_db,r.user_id,r.pay_order_number,r.game_id,r.game_name,r.server_id,r.server_name,r.game_player_name,r.pay_amount,r.pay_time,r.pay_way,r.pay_status,r.spend_ip,r.promote_id,r.promote_account").Order(order).Order("r.id DESC,r.source_db ASC").Scan(&pos).Error; err != nil {
return nil, business.ErrStorage
}
meta := []ledgerMetaTarget{}
+1 -1
View File
@@ -6,7 +6,7 @@ import (
)
func Migrations() []migration.Step {
return []migration.Step{{ID: "202611040009_business_menu_icons", Migrate: migrateBusinessMenuIcons}, {ID: "202611040008_business_report_build_locks", Migrate: func(db *gorm.DB) error { return migration.CreateMissingTables(db, &reportBuildLockPO{}) }}, {ID: "202611030007_business_alert_attachments", Migrate: migrateAlertAttachments}, {ID: "202611030006_business_alert_schema", Migrate: migrateAlertSchema}, {ID: "202611030005_business_password_reset", Migrate: func(db *gorm.DB) error {
return []migration.Step{{ID: "202611040013_business_query_indexes", Migrate: migrateBusinessQueryIndexes}, {ID: "202611040012_business_server_report_index", Migrate: migrateServerReportIndex}, {ID: "202611040011_business_registration_index", Migrate: migrateRegistrationIndex}, {ID: "202611040010_business_alert_trigger_locks", Migrate: func(db *gorm.DB) error { return migration.CreateMissingTables(db, &alertTriggerLockPO{}) }}, {ID: "202611040009_business_menu_icons", Migrate: migrateBusinessMenuIcons}, {ID: "202611040008_business_report_build_locks", Migrate: func(db *gorm.DB) error { return migration.CreateMissingTables(db, &reportBuildLockPO{}) }}, {ID: "202611030007_business_alert_attachments", Migrate: migrateAlertAttachments}, {ID: "202611030006_business_alert_schema", Migrate: migrateAlertSchema}, {ID: "202611030005_business_password_reset", Migrate: func(db *gorm.DB) error {
return migration.CreateMissingTables(db, &resetCodePO{}, &resetTokenPO{})
}}, {ID: "202611030004_business_binding_locks", Migrate: func(db *gorm.DB) error {
return migration.CreateMissingTables(db, &bindingLockPO{})
@@ -0,0 +1,279 @@
package business
import (
"context"
"database/sql"
"fmt"
"os"
"strings"
"sync"
"testing"
"time"
drivermysql "github.com/go-sql-driver/mysql"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"kra/internal/biz/business"
)
// 显式设置本地隔离 MySQL DSN 才执行;每项测试只创建并清理自己的随机数据库。
func openRehearsalMySQL(t *testing.T) *gorm.DB {
t.Helper()
dsn := os.Getenv("SW_TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("未配置隔离 MySQL")
}
cfg, err := drivermysql.ParseDSN(dsn)
if err != nil {
t.Fatal(err)
}
if cfg.Net != "tcp" || !strings.HasPrefix(cfg.Addr, "127.0.0.1:") {
t.Fatal("MySQL 演练测试只接受本地隔离实例")
}
cfg.DBName = ""
cfg.ParseTime = true
cfg.Loc = time.FixedZone("Asia/Shanghai", 8*3600)
admin, err := sql.Open("mysql", cfg.FormatDSN())
if err != nil {
t.Fatal(err)
}
name := fmt.Sprintf("sw_test_business_%d", time.Now().UnixNano())
if _, err := admin.Exec("CREATE DATABASE `" + name + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"); err != nil {
_ = admin.Close()
t.Fatal(err)
}
t.Cleanup(func() {
_, err := admin.Exec("DROP DATABASE `" + name + "`")
if err != nil {
t.Error(err)
}
_ = admin.Close()
})
cfg.DBName = name
db, err := gorm.Open(mysql.Open(cfg.FormatDSN()), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
pool, _ := db.DB()
pool.SetMaxOpenConns(20)
t.Cleanup(func() { _ = pool.Close() })
for _, step := range Migrations() {
if err := step.Migrate(db); err != nil {
t.Fatal(err)
}
}
return db
}
func TestMySQLAlertTriggerConcurrentDeduplication(t *testing.T) {
db := openRehearsalMySQL(t)
if err := db.Exec("ALTER TABLE sys_alert_record ADD COLUMN unclosed_flag TINYINT GENERATED ALWAYS AS (CASE WHEN status=5 THEN NULL ELSE 1 END) VIRTUAL, ADD UNIQUE INDEX uk_alert_record_unclosed(platform_code,target_promote_id,rule_type,unclosed_flag)").Error; err != nil {
t.Fatal(err)
}
r := NewAlertTriggerRepo(testData{db})
now := time.Now().Truncate(time.Millisecond)
hit := &business.AlertRecord{RuleID: 1, PlatformCode: "lx", PlatformName: "龙翔", TargetPromoteID: 10, RuleType: "注册", Status: 1, BusinessID: 10, CreatedAt: now, UpdatedAt: now}
decide := func(current *business.AlertRecord) bool { return current == nil || current.Status == 1 }
errors := make(chan error, 16)
var group sync.WaitGroup
for i := 0; i < 16; i++ {
group.Add(1)
go func() {
defer group.Done()
_, err := r.StoreHit(context.Background(), hit, []int{1, 2, 3, 4}, decide)
errors <- err
}()
}
group.Wait()
close(errors)
for err := range errors {
if err != nil {
t.Fatal(err)
}
}
var count int64
if err := db.Model(&alertRecordPO{}).Count(&count).Error; err != nil || count != 1 {
t.Fatalf("并发重复预警 count=%d error=%v", count, err)
}
if err := db.Model(&alertRecordPO{}).Where("platform_code='lx'").Update("status", 2).Error; err != nil {
t.Fatal(err)
}
changed := *hit
changed.AlertContent = "不应覆盖处理中记录"
written, err := r.StoreHit(context.Background(), &changed, []int{1, 2, 3, 4}, decide)
if err != nil || written {
t.Fatalf("在途记录被刷新: %v %v", written, err)
}
var row alertRecordPO
if err := db.First(&row).Error; err != nil {
t.Fatal(err)
}
if row.Status != 2 || row.AlertContent != "" {
t.Fatalf("状态或快照发生变化: %+v", row)
}
if err := db.Model(&row).Update("status", 4).Error; err != nil {
t.Fatal(err)
}
written, err = r.StoreHit(context.Background(), &changed, []int{1, 2, 3, 4}, decide)
if err != nil || written {
t.Fatalf("状态 4 是驳回,不能另建或覆盖未关闭记录: %v %v", written, err)
}
if err := db.Model(&row).Update("status", 5).Error; err != nil {
t.Fatal(err)
}
written, err = r.StoreHit(context.Background(), hit, []int{1, 2, 3, 4}, decide)
db.Model(&alertRecordPO{}).Count(&count)
if err != nil || !written || count != 2 {
t.Fatalf("关闭后新一轮预警失败: %v count=%d %v", written, count, err)
}
}
func TestMySQLRegistrationIndexPreservesCastAndUsesRange(t *testing.T) {
db := openRehearsalMySQL(t)
for _, statement := range []string{
"CREATE TABLE tab_user(id INT,source_db VARCHAR(10),puid INT,register_time VARCHAR(11),PRIMARY KEY(id,source_db),KEY(register_time))",
"INSERT INTO tab_user VALUES(1,'lx',0,'1793635200'),(2,'lx',0,'01793635200'),(3,'lx',0,'1793635200 '),(4,'lx',1,'1793635200'),(5,'hw',0,'1793635200'),(6,'lx',0,'1793000000')",
} {
if err := db.Exec(statement).Error; err != nil {
t.Fatal(err)
}
}
// 让命中比例接近实际分页查询,避免微型表上优化器合理选择全表扫描。
outside := make([]map[string]any, 1000)
for i := range outside {
outside[i] = map[string]any{"id": 100 + i, "source_db": "lx", "puid": 0, "register_time": "1793000000"}
}
if err := db.Table("tab_user").CreateInBatches(outside, 200).Error; err != nil {
t.Fatal(err)
}
pool, _ := db.DB()
pool.SetMaxOpenConns(1)
var beforeMode string
if err := db.Raw("SELECT @@session.sql_mode").Scan(&beforeMode).Error; err != nil {
t.Fatal(err)
}
query := "SELECT id,register_time FROM tab_user r WHERE r.source_db='lx' AND r.puid=0 AND CAST(r.register_time AS UNSIGNED) BETWEEN 1793635200 AND 1793721599 ORDER BY id"
var before, after []struct {
ID int
RegisterTime string
}
if err := db.Raw(query).Scan(&before).Error; err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
if err := migrateRegistrationIndex(db); err != nil {
t.Fatal(err)
}
}
var afterMode string
if err := db.Raw("SELECT @@session.sql_mode").Scan(&afterMode).Error; err != nil || beforeMode != afterMode {
t.Fatalf("索引创建未恢复会话模式: %v", err)
}
if err := db.Raw(query).Scan(&after).Error; err != nil {
t.Fatal(err)
}
if len(before) != 3 || fmt.Sprint(before) != fmt.Sprint(after) {
t.Fatalf("时间转换或原字符串改变: before=%+v after=%+v", before, after)
}
var plan []struct{ Key, Type string }
if err := db.Raw("EXPLAIN SELECT COUNT(*) FROM tab_user r WHERE r.source_db='lx' AND r.puid=0 AND CAST(r.register_time AS UNSIGNED) BETWEEN 1793635200 AND 1793721599").Scan(&plan).Error; err != nil {
t.Fatal(err)
}
if len(plan) != 1 || plan[0].Key != "idx_sw_user_registration" || plan[0].Type != "range" {
t.Fatalf("未使用时间范围索引: %+v", plan)
}
queryAll, err := reportRegistrationQuery(db)
if err != nil {
t.Fatal(err)
}
var allIDs []int
if err := queryAll.Select("id").Where("puid=0 AND CAST(register_time AS UNSIGNED) BETWEEN ? AND ?", 1793635200, 1793721599).Order("id").Scan(&allIDs).Error; err != nil {
t.Fatal(err)
}
if fmt.Sprint(allIDs) != "[1 2 3 5]" {
t.Fatalf("跨平台重建筛选丢失记录: %v", allIDs)
}
}
func TestMySQLChannelCreationSerializesAccount(t *testing.T) {
db := openRehearsalMySQL(t)
if err := db.AutoMigrate(&platformChannelPO{}, &channelSettlementPO{}); err != nil {
t.Fatal(err)
}
for _, statement := range []string{"CREATE TABLE tab_promote_business(id BIGINT PRIMARY KEY,promote_ids TEXT)", "INSERT INTO tab_promote_business VALUES(7,'5,6')"} {
if err := db.Exec(statement).Error; err != nil {
t.Fatal(err)
}
}
repo := NewChannelWriteRepo(channelDatabases{db})
input := &business.ChannelWrite{ChannelCreate: business.ChannelCreate{SourceDB: "lx", Account: "concurrentChannel", Password: "encoded", BusierID: 7},
Status: -3, Level: 1, SettlementDays: 7, CreatedUnix: 1000, NextCountUnix: 605800}
results := make(chan error, 8)
var group sync.WaitGroup
for i := 0; i < 8; i++ {
group.Add(1)
go func() {
defer group.Done()
_, err := repo.Create(context.Background(), input)
results <- err
}()
}
group.Wait()
close(results)
created, duplicates := 0, 0
for err := range results {
if err == nil {
created++
} else if err == business.ErrChannelExists {
duplicates++
} else {
t.Fatal(err)
}
}
var channelCount, settlementCount int64
db.Model(&platformChannelPO{}).Count(&channelCount)
db.Model(&channelSettlementPO{}).Count(&settlementCount)
if created != 1 || duplicates != 7 || channelCount != 1 || settlementCount != 1 {
t.Fatalf("并发创建结果错误: created=%d duplicate=%d channels=%d settlements=%d", created, duplicates, channelCount, settlementCount)
}
var relation string
db.Table("tab_promote_business").Select("promote_ids").Where("id=7").Scan(&relation)
if relation != "5,6,1" {
t.Fatalf("商务渠道关系未完整保留: %s", relation)
}
if err := db.Exec("CREATE TRIGGER fail_settlement BEFORE INSERT ON tab_promote_settlement_time FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='test failure'").Error; err != nil {
t.Fatal(err)
}
failed := *input
failed.Account = "rollbackChannel"
if _, err := repo.Create(context.Background(), &failed); err == nil {
t.Fatal("结算写入失败应该回滚")
}
db.Model(&platformChannelPO{}).Count(&channelCount)
if channelCount != 1 {
t.Fatal("失败的渠道记录没有回滚")
}
if err := db.Exec("DROP TRIGGER fail_settlement").Error; err != nil {
t.Fatal(err)
}
if _, err := repo.Create(context.Background(), &failed); err != nil {
t.Fatalf("失败后命名锁未释放或无法重试: %v", err)
}
}
func TestMySQLServerReportIndexIsAdditiveAndIdempotent(t *testing.T) {
db := openRehearsalMySQL(t)
if err := db.Exec("CREATE TABLE tab_user_play_info(id INT PRIMARY KEY,create_time INT,play_time INT,KEY(play_time))").Error; err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
if err := migrateServerReportIndex(db); err != nil {
t.Fatal(err)
}
}
var count int64
db.Raw("SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='tab_user_play_info' AND column_name='create_time' AND seq_in_index=1").Scan(&count)
if count != 1 || !db.Migrator().HasIndex("tab_user_play_info", "play_time") {
t.Fatalf("重复创建索引或覆盖旧索引: %d", count)
}
}
+70
View File
@@ -0,0 +1,70 @@
package business
import (
"fmt"
"strings"
"gorm.io/gorm"
)
type businessQueryIndex struct {
table, name string
columns []string
}
var businessQueryIndexes = []businessQueryIndex{
{"tab_promote", "idx_sw_promote_created", []string{"source_db", "create_time", "id"}},
{"tab_spend", "idx_sw_spend_time_summary", []string{"pay_status", "pay_time", "source_db", "pay_amount"}},
{"tab_spend", "idx_sw_spend_promote_summary", []string{"promote_id", "pay_status", "source_db", "pay_amount"}},
{"tab_sw_report_day", "idx_sw_report_pay_scan", []string{"dedup_mode", "stat_date", "source_db", "promote_id", "game_id", "total_pay"}},
{"tab_sw_report_day_key", "idx_sw_report_key_scan", []string{"dedup_mode", "stat_date", "source_db", "promote_id", "game_id", "metric", "identity_key"}},
{"tab_sw_report_server_day", "idx_sw_server_pay_game_scan", []string{"game_name", "dedup_mode", "stat_date", "source_db", "game_id", "server_num", "total_pay"}},
{"tab_sw_report_server_day_key", "idx_sw_server_key_game_scan", []string{"game_name", "dedup_mode", "stat_date", "source_db", "game_id", "server_num", "metric", "identity_key"}},
}
// 只为已经接入的业务表增加索引,不创建平台表、不改写记录。
// 按实际筛选前缀排列,并覆盖聚合读取列,避免大范围回表。
func migrateBusinessQueryIndexes(db *gorm.DB) error {
if db.Dialector.Name() != "mysql" {
return nil
}
groups := make(map[string][]businessQueryIndex)
tables := []string{}
for _, index := range businessQueryIndexes {
if _, exists := groups[index.table]; !exists {
tables = append(tables, index.table)
}
groups[index.table] = append(groups[index.table], index)
}
for _, table := range tables {
available, err := businessTableAvailable(db, table)
if err != nil {
return err
}
if !available {
continue
}
additions := []string{}
for _, index := range groups[table] {
var existing []string
if err := db.Raw("SELECT column_name FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name=? AND index_name=? ORDER BY seq_in_index", index.table, index.name).Scan(&existing).Error; err != nil {
return err
}
if len(existing) > 0 {
if strings.Join(existing, ",") != strings.Join(index.columns, ",") {
return fmt.Errorf("%s.%s 的现有索引列不符", index.table, index.name)
}
continue
}
columns := "`" + strings.Join(index.columns, "`,`") + "`"
additions = append(additions, "ADD INDEX `"+index.name+"` ("+columns+")")
}
// 同一大表的多个索引合并构建,减少重复扫描。
if len(additions) > 0 {
if err := db.Exec("ALTER TABLE `" + table + "` " + strings.Join(additions, ",") + ", ALGORITHM=INPLACE, LOCK=NONE").Error; err != nil {
return fmt.Errorf("add query indexes on %s: %w", table, err)
}
}
}
return nil
}
@@ -0,0 +1,50 @@
package business
import (
"strings"
"testing"
)
func TestMySQLBusinessQueryIndexesAreAdditiveAndIdempotent(t *testing.T) {
db := openRehearsalMySQL(t)
definitions := map[string]string{
"tab_promote": "id BIGINT NOT NULL,source_db VARCHAR(10) NOT NULL,create_time BIGINT,PRIMARY KEY(id,source_db)",
"tab_spend": "id BIGINT NOT NULL,source_db VARCHAR(10) NOT NULL,promote_id BIGINT,pay_status INT,pay_time BIGINT,pay_amount DECIMAL(20,2),PRIMARY KEY(id,source_db)",
"tab_sw_report_day": "dedup_mode INT,stat_date DATE,source_db VARCHAR(10),promote_id BIGINT,game_id BIGINT,total_pay DECIMAL(20,2)",
"tab_sw_report_day_key": "dedup_mode INT,stat_date DATE,source_db VARCHAR(10),promote_id BIGINT,game_id BIGINT,metric INT,identity_key VARCHAR(64)",
"tab_sw_report_server_day": "game_name VARCHAR(64),dedup_mode INT,stat_date DATE,source_db VARCHAR(10),game_id BIGINT,server_num VARCHAR(32),total_pay DECIMAL(20,2)",
"tab_sw_report_server_day_key": "game_name VARCHAR(64),dedup_mode INT,stat_date DATE,source_db VARCHAR(10),game_id BIGINT,server_num VARCHAR(32),metric INT,identity_key VARCHAR(64)",
}
for table, columns := range definitions {
if err := db.Exec("CREATE TABLE `" + table + "` (" + columns + ") ENGINE=InnoDB").Error; err != nil {
t.Fatal(err)
}
}
if err := db.Exec("INSERT INTO tab_spend VALUES(77,'lx',55,1,123456,12.30)").Error; err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < 2; attempt++ {
if err := migrateBusinessQueryIndexes(db); err != nil {
t.Fatal(err)
}
}
for _, index := range businessQueryIndexes {
var columns []string
if err := db.Raw("SELECT column_name FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name=? AND index_name=? ORDER BY seq_in_index", index.table, index.name).Scan(&columns).Error; err != nil {
t.Fatal(err)
}
if strings.Join(columns, ",") != strings.Join(index.columns, ",") {
t.Fatalf("index %s has unexpected columns: %v", index.name, columns)
}
}
var amount string
if err := db.Raw("SELECT pay_amount FROM tab_spend WHERE id=77 AND source_db='lx'").Scan(&amount).Error; err != nil || amount != "12.30" {
t.Fatalf("index migration changed business data: %s, %v", amount, err)
}
if err := db.Exec("ALTER TABLE tab_spend DROP INDEX idx_sw_spend_time_summary, ADD INDEX idx_sw_spend_time_summary(pay_time)").Error; err != nil {
t.Fatal(err)
}
if err := migrateBusinessQueryIndexes(db); err == nil || !strings.Contains(err.Error(), "idx_sw_spend_time_summary") {
t.Fatalf("must reject a conflicting existing index: %v", err)
}
}
+61 -44
View File
@@ -2,6 +2,8 @@ package business
import (
"context"
"golang.org/x/sync/errgroup"
"gorm.io/gorm"
"kra/internal/biz/business"
)
@@ -44,7 +46,10 @@ func rankingReportQuery(db *gorm.DB, kind business.RankingKind, scope business.A
return analyticsQuery(db, scope, f, table, mode)
}
all := business.AnalyticsScope{Viewer: business.UserScope{All: true}, IncludeSub: true}
return analyticsQuery(db, all, f, table, mode).Where("EXISTS (?)", rankingBusinessPromotes(db, scope, f.Sources).Select("1").Where("p.source_db=r.source_db AND p.id=r.promote_id"))
// 同一渠道会出现在多级商务的范围中,先去重并一次性展开可见渠道。
// 避免对每一条报表身份记录重复扫描整棵商务关系树。
visible := rankingBusinessPromotes(db, scope, f.Sources).Select("DISTINCT p.source_db,p.id")
return analyticsQuery(db, all, f, table, mode).Joins("JOIN (?) visible_promotes ON visible_promotes.source_db=r.source_db AND visible_promotes.id=r.promote_id", visible)
}
func (r *rankingRepo) Collect(ctx context.Context, kind business.RankingKind, scope business.AnalyticsScope, f business.RankingRange) (*business.RankingData, error) {
db := r.data.DB().WithContext(ctx)
@@ -109,60 +114,72 @@ func (r *rankingRepo) Collect(ctx context.Context, kind business.RankingKind, sc
return out, nil
}
}
group, queryContext := errgroup.WithContext(ctx)
group.SetLimit(2)
queryDB := db.WithContext(queryContext)
for _, period := range []struct {
begin, end string
target map[business.AnalyticsKey]*business.RankingMetrics
}{{f.Begin, f.End, out.Current}, {f.PreviousBegin, f.PreviousEnd, out.Previous}} {
query := f.AnalyticsFilter
query.Begin, query.End = period.begin, period.end
rowFor := func(po analyticsAggPO) *business.RankingMetrics {
key := business.AnalyticsKey{SourceDB: po.SourceDB, ID: *po.ID}
if period.target[key] == nil {
period.target[key] = &business.RankingMetrics{}
}
return period.target[key]
}
for _, pay := range []bool{true, false} {
table, mode := "tab_sw_report_day", 0
if !pay {
table = "tab_sw_report_day_key"
if scope.IDCardDedup {
mode = 1
group.Go(func() error {
query := f.AnalyticsFilter
query.Begin, query.End = period.begin, period.end
rowFor := func(po analyticsAggPO) *business.RankingMetrics {
key := business.AnalyticsKey{SourceDB: po.SourceDB, ID: *po.ID}
if period.target[key] == nil {
period.target[key] = &business.RankingMetrics{}
}
return period.target[key]
}
q := rankingReportQuery(db, kind, scope, query, table, mode)
id := "r.promote_id"
if kind == business.GameRanking {
id = "r.game_id"
}
if kind == business.ChannelRanking {
q = q.Joins("LEFT JOIN tab_promote root_p ON root_p.source_db=r.source_db AND root_p.id=r.promote_id")
id = "COALESCE(NULLIF(root_p.top_promote_id,0),r.promote_id)"
}
selects := "r.source_db," + id + " AS entity_id"
group := "r.source_db," + id
if pay {
selects += ",COALESCE(SUM(r.total_pay),0) AS amount"
} else {
selects += ",r.metric,COUNT(DISTINCT r.identity_key) AS count"
group += ",r.metric"
}
var rows []analyticsAggPO
if err := q.Select(selects).Group(group).Scan(&rows).Error; err != nil {
return nil, business.ErrStorage
}
for _, po := range rows {
if po.ID == nil || (pay && *po.ID < 0) {
continue
for _, pay := range []bool{true, false} {
table, mode := "tab_sw_report_day", 0
if !pay {
table = "tab_sw_report_day_key"
if scope.IDCardDedup {
mode = 1
}
}
row := rowFor(po)
q := rankingReportQuery(queryDB, kind, scope, query, table, mode)
id := "r.promote_id"
if kind == business.GameRanking {
id = "r.game_id"
}
if kind == business.ChannelRanking {
if pay {
q = analyticsPayLeaves(queryDB, q)
}
q = q.Joins("LEFT JOIN tab_promote root_p ON root_p.source_db=r.source_db AND root_p.id=r.promote_id")
id = "COALESCE(NULLIF(root_p.top_promote_id,0),r.promote_id)"
}
selects := "r.source_db," + id + " AS entity_id"
group := "r.source_db," + id
if pay {
row.Pay = po.Amount
selects += ",COALESCE(SUM(r.total_pay),0) AS amount"
} else {
putAnalyticsCount(&row.Counts, po.Metric, po.Count)
selects += ",r.metric,COUNT(DISTINCT r.identity_key) AS count"
group += ",r.metric"
}
var rows []analyticsAggPO
if err := q.Select(selects).Group(group).Scan(&rows).Error; err != nil {
return business.ErrStorage
}
for _, po := range rows {
if po.ID == nil || (pay && *po.ID < 0) {
continue
}
row := rowFor(po)
if pay {
row.Pay = po.Amount
} else {
putAnalyticsCount(&row.Counts, po.Metric, po.Count)
}
}
}
}
return nil
})
}
if err := group.Wait(); err != nil {
return nil, err
}
if kind != business.BusinessRanking && len(out.Current) > 0 {
table, column := "tab_promote", "account"
@@ -0,0 +1,70 @@
package business
import (
"errors"
"fmt"
"strings"
"gorm.io/gorm"
)
// 保留旧 register_time 字符串及其 CAST 语义,用虚拟列索引避免逐行扫描主账号。
// MySQL 可将查询中相同的 CAST 表达式匹配到该索引,既有查询无需更改统计口径。
func migrateRegistrationIndex(db *gorm.DB) error {
if db.Dialector.Name() != "mysql" || !db.Migrator().HasTable("tab_user") {
return nil
}
const column = "sw_register_time_unix"
const index = "idx_sw_user_registration"
if !db.Migrator().HasColumn("tab_user", column) {
if err := db.Exec("ALTER TABLE tab_user ADD COLUMN sw_register_time_unix BIGINT UNSIGNED GENERATED ALWAYS AS (CAST(register_time AS UNSIGNED)) VIRTUAL, ALGORITHM=INPLACE, LOCK=NONE").Error; err != nil {
return err
}
} else {
var expression string
if err := db.Raw("SELECT generation_expression FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='tab_user' AND column_name=?", column).Scan(&expression).Error; err != nil {
return err
}
normalized := strings.NewReplacer("`", "", " ", "", "\n", "", "\t", "").Replace(strings.ToLower(expression))
if normalized != "cast(register_timeasunsigned)" {
return fmt.Errorf("tab_user.%s 的现有定义与注册时间转换不符", column)
}
}
if !db.Migrator().HasIndex("tab_user", index) {
return addRegistrationIndex(db)
}
var columns []string
if err := db.Raw("SELECT column_name FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='tab_user' AND index_name=? ORDER BY seq_in_index", index).Scan(&columns).Error; err != nil {
return err
}
if strings.Join(columns, ",") != "source_db,puid,sw_register_time_unix,id" {
return fmt.Errorf("tab_user.%s 的现有索引列不符", index)
}
return nil
}
func addRegistrationIndex(db *gorm.DB) error {
// 旧 SELECT CAST 会以警告方式接受尾部空格等历史值。建索引属于 DDL,
// 严格模式会把同一警告升级为错误;仅在固定连接的索引构建期间保持旧转换行为。
return db.Connection(func(conn *gorm.DB) (result error) {
var previous string
if err := conn.Raw("SELECT @@session.sql_mode").Scan(&previous).Error; err != nil {
return err
}
modes := []string{}
for _, mode := range strings.Split(previous, ",") {
if mode != "STRICT_TRANS_TABLES" && mode != "STRICT_ALL_TABLES" {
modes = append(modes, mode)
}
}
if err := conn.Exec("SET SESSION sql_mode=?", strings.Join(modes, ",")).Error; err != nil {
return err
}
defer func() {
if err := conn.Exec("SET SESSION sql_mode=?", previous).Error; err != nil {
result = errors.Join(result, err)
}
}()
return conn.Exec("ALTER TABLE tab_user ADD INDEX idx_sw_user_registration(source_db,puid,sw_register_time_unix,id), ALGORITHM=INPLACE, LOCK=NONE").Error
})
}
+45 -9
View File
@@ -41,14 +41,12 @@ func (r *reportBuildRepo) Rebuild(ctx context.Context, kind business.ReportBuild
reportTable, keyTable = "tab_sw_report_server_day", "tab_sw_report_server_day_key"
rawTable = "tab_user_play_info"
}
for _, table := range []string{reportTable, keyTable, rawTable, "tab_user", "tab_spend", "tab_game"} {
available, err := businessTableAvailable(tx, table)
if err != nil {
return err
}
if !available {
return business.ErrReportBuildUnavailable
}
available, err := businessTablesAvailable(tx, reportTable, keyTable, rawTable, "tab_user", "tab_spend", "tab_game")
if err != nil {
return err
}
if !available {
return business.ErrReportBuildUnavailable
}
statDate := day.Format("2006-01-02")
lock := reportBuildLockPO{Kind: string(kind), StatDate: statDate}
@@ -133,7 +131,11 @@ func loadReportInput(db *gorm.DB, kind business.ReportBuildKind, day time.Time)
cast = "CAST(register_time AS UNSIGNED)"
}
var registers, actives []reportEventPO
if err := db.Table("tab_user").Select("id AS user_id,source_db,promote_id,fgame_id AS game_id").Where("puid=0").Where(cast+" BETWEEN ? AND ?", begin, end).Scan(&registers).Error; err != nil {
registerQuery, err := reportRegistrationQuery(db)
if err != nil {
return nil, err
}
if err := registerQuery.Select("id AS user_id,source_db,promote_id,fgame_id AS game_id").Where("puid=0").Where(cast+" BETWEEN ? AND ?", begin, end).Scan(&registers).Error; err != nil {
return nil, business.ErrStorage
}
if err := db.Table("tab_user_day_login").Select("user_id,source_db,promote_id,game_id").Where("login_time BETWEEN ? AND ?", begin, end).Scan(&actives).Error; err != nil {
@@ -204,3 +206,37 @@ func loadReportInput(db *gorm.DB, kind business.ReportBuildKind, day time.Time)
}
return input, nil
}
func reportRegistrationQuery(db *gorm.DB) (*gorm.DB, error) {
q := db.Table("tab_user")
if db.Dialector.Name() != "mysql" || !db.Migrator().HasIndex("tab_user", "idx_sw_user_registration") {
return q, nil
}
// 跨平台任务也必须提供组合索引的首列,避免又回到数百万主账号扫描。
// 平台值取同一事务中的实际数据,不写死平台名单,也保留历史 NULL。
var rows []struct {
SourceDB *string `gorm:"column:source_db"`
}
if err := db.Table("tab_user").Distinct("source_db").Scan(&rows).Error; err != nil {
return nil, business.ErrStorage
}
sources := []string{}
hasNull := false
for _, row := range rows {
if row.SourceDB == nil {
hasNull = true
} else {
sources = append(sources, *row.SourceDB)
}
}
if len(sources) == 0 {
if hasNull {
return q.Where("source_db IS NULL"), nil
}
return q.Where("1=0"), nil
}
if hasNull {
return q.Where("(source_db IN ? OR source_db IS NULL)", sources), nil
}
return q.Where("source_db IN ?", sources), nil
}
+26 -4
View File
@@ -19,14 +19,36 @@ func storageError(err error) error {
}
func businessTableAvailable(db *gorm.DB, name string) (bool, error) {
tables, err := db.Migrator().GetTables()
return businessTablesAvailable(db, name)
}
// 一次检查请求所需的表,不为每个依赖重复枚举整个业务库。
// 不缓存结果,保留迁移后新建表和运行时连接异常的可见性。
func businessTablesAvailable(db *gorm.DB, names ...string) (bool, error) {
if len(names) == 0 {
return true, nil
}
var tables []string
var err error
switch db.Dialector.Name() {
case "mysql":
err = db.Raw("SELECT table_name FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name IN ?", names).Scan(&tables).Error
case "sqlite":
err = db.Raw("SELECT name FROM sqlite_master WHERE type='table' AND name IN ?", names).Scan(&tables).Error
default:
tables, err = db.Migrator().GetTables()
}
if err != nil {
return false, business.ErrStorage
}
found := make(map[string]bool, len(tables))
for _, table := range tables {
if table == name {
return true, nil
found[table] = true
}
for _, name := range names {
if !found[name] {
return false, nil
}
}
return false, nil
return true, nil
}
+49
View File
@@ -0,0 +1,49 @@
package business
import (
"path/filepath"
"testing"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"kra/internal/biz/business"
)
func TestBusinessTablesAvailabilityIsFreshAndPropagatesFailures(t *testing.T) {
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "schema.db")), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
pool, err := db.DB()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = pool.Close() })
check := func(want bool, names ...string) {
t.Helper()
got, err := businessTablesAvailable(db, names...)
if err != nil || got != want {
t.Fatalf("tables %v: got %v, %v; want %v", names, got, err, want)
}
}
check(true)
check(false, "first", "second")
for _, table := range []string{"first", "second"} {
if err := db.Exec("CREATE TABLE " + table + " (id INTEGER)").Error; err != nil {
t.Fatal(err)
}
}
check(true, "first", "second", "first")
check(false, "first", "missing")
check(false, "first') OR 1=1 --")
if err := db.Exec("DROP TABLE second").Error; err != nil {
t.Fatal(err)
}
check(false, "first", "second")
if err := pool.Close(); err != nil {
t.Fatal(err)
}
if available, err := businessTablesAvailable(db, "first"); available || err != business.ErrStorage {
t.Fatalf("connection error must not become a missing-table result: %v, %v", available, err)
}
}
@@ -0,0 +1,18 @@
package business
import "gorm.io/gorm"
// 区服汇总按创角时间读取原始记录;旧表只有 play_time 索引,导致每日创角扫描全表。
func migrateServerReportIndex(db *gorm.DB) error {
if db.Dialector.Name() != "mysql" || !db.Migrator().HasTable("tab_user_play_info") {
return nil
}
var existing int64
if err := db.Raw("SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name='tab_user_play_info' AND column_name='create_time' AND seq_in_index=1").Scan(&existing).Error; err != nil {
return err
}
if existing > 0 {
return nil
}
return db.Exec("ALTER TABLE tab_user_play_info ADD INDEX idx_sw_play_create_time(create_time), ALGORITHM=INPLACE, LOCK=NONE").Error
}
+103 -59
View File
@@ -4,6 +4,8 @@ import (
"context"
"time"
"golang.org/x/sync/errgroup"
"gorm.io/gorm"
"kra/internal/biz/business"
)
@@ -23,13 +25,8 @@ func workbenchAvailable(db *gorm.DB, scope business.AnalyticsScope, sources []st
if kind == "" || kind == business.WorkbenchRegister {
tables = append(tables, "tab_sw_report_day_key")
}
for _, table := range tables {
available, err := businessTableAvailable(db, table)
if err != nil || !available {
return false, available, err
}
}
return false, true, nil
available, err = businessTablesAvailable(db, tables...)
return false, available, err
}
func (r *workbenchRepo) Summary(ctx context.Context, scope business.AnalyticsScope, sources []string, w business.WorkbenchWindow) (*business.WorkbenchTotals, error) {
db := r.data.DB().WithContext(ctx)
@@ -46,47 +43,83 @@ func (r *workbenchRepo) Summary(ctx context.Context, scope business.AnalyticsSco
if scope.IDCardDedup {
mode = 1
}
for _, period := range []struct {
day time.Time
target *business.WorkbenchDay
}{{w.Today, &out.Today}, {w.Yesterday, &out.Yesterday}} {
f := business.AnalyticsFilter{Sources: sources, Begin: period.day.Format("2006-01-02"), End: period.day.Format("2006-01-02")}
var counts []portraitCountPO
if err := analyticsQuery(db, scope, f, "tab_sw_report_day_key", mode).Select("r.metric,COUNT(DISTINCT " + portraitIdentity(db, scope.IDCardDedup) + ") AS count").Where("r.metric IN (1,2)").Group("r.metric").Scan(&counts).Error; err != nil {
return nil, business.ErrStorage
}
period.target.Counts = toPortraitCounts(counts)
if err := analyticsQuery(db, scope, f, "tab_sw_report_day", 0).Select("COALESCE(SUM(r.total_pay),0)").Scan(&period.target.Pay).Error; err != nil {
return nil, business.ErrStorage
}
// 活跃渠道按自身注册或正充值统计,不能归并一级渠道后再计数。
active := map[business.AnalyticsKey]bool{}
var registrations, pays []analyticsAggPO
if err := analyticsQuery(db, scope, f, "tab_sw_report_day_key", mode).Where("r.metric=1").Select("r.source_db,r.promote_id AS entity_id").Group("r.source_db,r.promote_id").Having("COUNT(DISTINCT r.identity_key)>0").Scan(&registrations).Error; err != nil {
return nil, business.ErrStorage
}
if err := analyticsQuery(db, scope, f, "tab_sw_report_day", 0).Select("r.source_db,r.promote_id AS entity_id").Group("r.source_db,r.promote_id").Having("SUM(r.total_pay)>0").Scan(&pays).Error; err != nil {
return nil, business.ErrStorage
}
for _, po := range append(registrations, pays...) {
if po.ID != nil {
active[business.AnalyticsKey{SourceDB: po.SourceDB, ID: *po.ID}] = true
}
}
period.target.ActiveChannels = int64(len(active))
if err := analyticsBasePromotes(db, scope, f, "p").Where("p.create_time BETWEEN ? AND ?", period.day.Unix(), period.day.AddDate(0, 0, 1).Unix()-1).Count(&period.target.NewChannels).Error; err != nil {
return nil, business.ErrStorage
date := func(value time.Time) string { return value.Format("2006-01-02") }
today, yesterday := date(w.Today), date(w.Yesterday)
monthStart, previousStart, previousEnd := date(w.MonthStart), date(w.PreviousMonthStart), date(w.PreviousMonthEnd)
dayFilter := business.AnalyticsFilter{Sources: sources, Begin: yesterday, End: today}
type datedCount struct {
StatDate string
Metric int
Count int64
}
type datedChannel struct {
StatDate string
SourceDB string `gorm:"column:source_db"`
ID *int64 `gorm:"column:entity_id"`
}
var counts []datedCount
var registrations, pays []datedChannel
var amounts struct{ TodayPay, YesterdayPay, MonthPay, PreviousMonthPay string }
var created struct{ TodayCount, YesterdayCount int64 }
group, queryContext := errgroup.WithContext(ctx)
group.SetLimit(2)
queryDB := db.WithContext(queryContext)
group.Go(func() error {
return storageError(analyticsQuery(queryDB, scope, dayFilter, "tab_sw_report_day_key", mode).
Select("CAST(r.stat_date AS CHAR) AS stat_date,r.metric,COUNT(DISTINCT " + portraitIdentity(queryDB, scope.IDCardDedup) + ") AS count").
Where("r.metric IN (1,2)").Group("r.stat_date,r.metric").Scan(&counts).Error)
})
group.Go(func() error {
f := business.AnalyticsFilter{Sources: sources, Begin: previousStart, End: today}
// 同时取得两日和两个自然月区间的金额;月初的昨日须额外纳入扫描范围。
return storageError(analyticsQuery(queryDB, scope, f, "tab_sw_report_day", 0).
Where("((r.stat_date BETWEEN ? AND ?) OR (r.stat_date BETWEEN ? AND ?) OR r.stat_date=?)", monthStart, today, previousStart, previousEnd, yesterday).
Select("COALESCE(SUM(CASE WHEN r.stat_date=? THEN r.total_pay ELSE 0 END),0) AS today_pay,"+
"COALESCE(SUM(CASE WHEN r.stat_date=? THEN r.total_pay ELSE 0 END),0) AS yesterday_pay,"+
"COALESCE(SUM(CASE WHEN r.stat_date BETWEEN ? AND ? THEN r.total_pay ELSE 0 END),0) AS month_pay,"+
"COALESCE(SUM(CASE WHEN r.stat_date BETWEEN ? AND ? THEN r.total_pay ELSE 0 END),0) AS previous_month_pay",
today, yesterday, monthStart, today, previousStart, previousEnd).Scan(&amounts).Error)
})
group.Go(func() error {
// COUNT(DISTINCT identity_key)>0 等价于存在非 NULL 的注册键;保留空字符串的旧语义。
return storageError(analyticsQuery(queryDB, scope, dayFilter, "tab_sw_report_day_key", mode).
Where("r.metric=1 AND r.identity_key IS NOT NULL").Distinct().
Select("CAST(r.stat_date AS CHAR) AS stat_date,r.source_db,r.promote_id AS entity_id").Scan(&registrations).Error)
})
group.Go(func() error {
return storageError(analyticsQuery(queryDB, scope, dayFilter, "tab_sw_report_day", 0).
Select("CAST(r.stat_date AS CHAR) AS stat_date,r.source_db,r.promote_id AS entity_id").
Group("r.stat_date,r.source_db,r.promote_id").Having("SUM(r.total_pay)>0").Scan(&pays).Error)
})
group.Go(func() error {
todayEnd := w.Today.AddDate(0, 0, 1).Unix() - 1
yesterdayEnd := w.Yesterday.AddDate(0, 0, 1).Unix() - 1
return storageError(analyticsBasePromotes(queryDB, scope, dayFilter, "p").
Where("p.create_time BETWEEN ? AND ?", w.Yesterday.Unix(), todayEnd).
Select("COALESCE(SUM(CASE WHEN p.create_time BETWEEN ? AND ? THEN 1 ELSE 0 END),0) AS today_count,"+
"COALESCE(SUM(CASE WHEN p.create_time BETWEEN ? AND ? THEN 1 ELSE 0 END),0) AS yesterday_count",
w.Today.Unix(), todayEnd, w.Yesterday.Unix(), yesterdayEnd).Scan(&created).Error)
})
if err := group.Wait(); err != nil {
return nil, err
}
days := map[string]*business.WorkbenchDay{today: &out.Today, yesterday: &out.Yesterday}
for _, row := range counts {
if day := days[row.StatDate]; day != nil {
putAnalyticsCount(&day.Counts, row.Metric, row.Count)
}
}
for _, period := range []struct {
begin, end time.Time
target *string
}{{w.MonthStart, w.Today, &out.MonthPay}, {w.PreviousMonthStart, w.PreviousMonthEnd, &out.PreviousMonthPay}} {
f := business.AnalyticsFilter{Sources: sources, Begin: period.begin.Format("2006-01-02"), End: period.end.Format("2006-01-02")}
if err := analyticsQuery(db, scope, f, "tab_sw_report_day", 0).Select("COALESCE(SUM(r.total_pay),0)").Scan(period.target).Error; err != nil {
return nil, business.ErrStorage
active := map[string]map[business.AnalyticsKey]bool{today: {}, yesterday: {}}
for _, row := range append(registrations, pays...) {
if keys := active[row.StatDate]; keys != nil && row.ID != nil {
keys[business.AnalyticsKey{SourceDB: row.SourceDB, ID: *row.ID}] = true
}
}
out.Today.ActiveChannels, out.Yesterday.ActiveChannels = int64(len(active[today])), int64(len(active[yesterday]))
out.Today.Pay, out.Yesterday.Pay = amounts.TodayPay, amounts.YesterdayPay
out.Today.NewChannels, out.Yesterday.NewChannels = created.TodayCount, created.YesterdayCount
out.MonthPay, out.PreviousMonthPay = amounts.MonthPay, amounts.PreviousMonthPay
return out, nil
}
@@ -118,12 +151,12 @@ func (r *workbenchRepo) Ranks(ctx context.Context, scope business.AnalyticsScope
}
if kind == business.WorkbenchPayLow || kind == business.WorkbenchUnlogin {
var channels []workbenchPromotePO
if err := analyticsBasePromotes(db, scope, f, "p").Select("p.id,p.source_db,p.account,p.create_time,p.last_login_time").Where("p.promote_level=1").Scan(&channels).Error; err != nil {
if err := analyticsBasePromotes(db, scope, f, "p").Select("p.id,p.source_db,p.create_time,p.last_login_time").Where("p.promote_level=1").Scan(&channels).Error; err != nil {
return nil, business.ErrStorage
}
for _, po := range channels {
row := rowFor(po.SourceDB, po.ID)
row.Name, row.CreatedUnix, row.LastLoginUnix = po.Account, po.CreateTime, po.LastLoginTime
row.CreatedUnix, row.LastLoginUnix = po.CreateTime, po.LastLoginTime
}
}
if kind != business.WorkbenchUnlogin {
@@ -144,6 +177,7 @@ func (r *workbenchRepo) Ranks(ctx context.Context, scope business.AnalyticsScope
id = "CASE WHEN root_p.promote_level=1 THEN root_p.id WHEN root_p.top_promote_id>0 THEN root_p.top_promote_id ELSE root_p.id END"
}
} else {
q = analyticsPayLeaves(db, q)
q = q.Joins("LEFT JOIN tab_promote root_p ON root_p.source_db=r.source_db AND root_p.id=r.promote_id")
id = "CASE WHEN root_p.promote_level=1 THEN root_p.id WHEN root_p.top_promote_id>0 THEN root_p.top_promote_id ELSE COALESCE(root_p.id,r.promote_id) END"
}
@@ -160,22 +194,32 @@ func (r *workbenchRepo) Ranks(ctx context.Context, scope business.AnalyticsScope
rowFor(po.SourceDB, *po.ID).Value = po.Amount
}
}
keys := make([]business.AnalyticsKey, 0, len(rows))
for key := range rows {
keys = append(keys, key)
}
for start := 0; start < len(keys); start += 300 {
var names []workbenchPromotePO
if err := db.Table("tab_promote").Select("id,source_db,account").Where("(source_db,id) IN ?", analyticsPairs(keys[start:min(start+300, len(keys))])).Scan(&names).Error; err != nil {
return nil, business.ErrStorage
}
for _, po := range names {
rowFor(po.SourceDB, po.ID).Name = po.Account
}
}
}
for _, row := range rows {
out.Rows = append(out.Rows, row)
}
return out, nil
}
func (r *workbenchRepo) FillRankNames(ctx context.Context, rows []*business.WorkbenchRankRow) error {
if len(rows) == 0 {
return nil
}
keys := make([]business.AnalyticsKey, 0, len(rows))
for _, row := range rows {
keys = append(keys, row.AnalyticsKey)
}
var names []workbenchPromotePO
if err := r.data.DB().WithContext(ctx).Table("tab_promote").Select("id,source_db,account").Where("(source_db,id) IN ?", analyticsPairs(keys)).Scan(&names).Error; err != nil {
return business.ErrStorage
}
byKey := make(map[business.AnalyticsKey]string, len(names))
for _, po := range names {
byKey[business.AnalyticsKey{SourceDB: po.SourceDB, ID: po.ID}] = po.Account
}
for _, row := range rows {
row.Name = byKey[row.AnalyticsKey]
}
return nil
}
+29
View File
@@ -85,3 +85,32 @@ func TestWorkbenchStorageSummaryAndRankRules(t *testing.T) {
t.Fatal("六卡缺表应明确未接入")
}
}
func TestWorkbenchSummaryMonthStartAndNullableRegistrationKeys(t *testing.T) {
db, _ := prepareChannels(t)
for _, statement := range []string{
"DELETE FROM tab_sw_report_day",
"DELETE FROM tab_sw_report_day_key",
"INSERT INTO tab_sw_report_day VALUES('lx',100,1,0,'2026-10-01',10),('lx',100,1,0,'2026-10-02',999),('lx',100,1,0,'2026-10-31',30),('lx',100,1,0,'2026-11-01',20),('lx',103,1,0,'2026-11-01',-1)",
"INSERT INTO tab_sw_report_day_key VALUES('lx',103,1,1,'2026-11-01',1,NULL),('lx',999,1,1,'2026-11-01',1,''),('lx',100,1,1,'2026-10-31',1,'A')",
} {
if err := db.Exec(statement).Error; err != nil {
t.Fatal(err)
}
}
day := time.Date(2026, 11, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600))
if err := db.Exec("UPDATE tab_promote SET create_time=CASE WHEN id=100 THEN ? ELSE ? END WHERE source_db='lx' AND id IN(100,101)", day.Unix(), day.AddDate(0, 0, -1).Unix()).Error; err != nil {
t.Fatal(err)
}
window := business.WorkbenchWindow{Today: day, Yesterday: day.AddDate(0, 0, -1), MonthStart: day, PreviousMonthStart: day.AddDate(0, -1, 0), PreviousMonthEnd: day.AddDate(0, -1, 0)}
out, err := NewWorkbenchRepo(testData{db}).Summary(context.Background(), business.AnalyticsScope{Viewer: business.UserScope{All: true}, IncludeSub: true, IDCardDedup: true}, []string{"lx"}, window)
if err != nil {
t.Fatal(err)
}
if portraitTestMoney(out.Today.Pay) != "19.00" || portraitTestMoney(out.Yesterday.Pay) != "30.00" || portraitTestMoney(out.MonthPay) != "19.00" || portraitTestMoney(out.PreviousMonthPay) != "10.00" {
t.Fatalf("month-first comparison mixed yesterday with the previous-month window: %+v", out)
}
if out.Today.ActiveChannels != 2 || out.Today.Counts.Register != 1 || out.Yesterday.ActiveChannels != 1 || out.Today.NewChannels != 1 || out.Yesterday.NewChannels != 1 {
t.Fatalf("daily groups or NULL/empty registration-key behavior changed: %+v", out)
}
}
+1
View File
@@ -47,6 +47,7 @@ var ProviderSet = wire.NewSet(
databusiness.NewWorkbenchRepo,
databusiness.NewRankingRepo,
databusiness.NewReportBuildRepo,
databusiness.NewAlertTriggerRepo,
NewIntegrationRuntime,
wire.Bind(new(datasystem.Provider), new(*Data)),
wire.Bind(new(datasystem.DatabaseProvider), new(*Data)),
+29 -7
View File
@@ -10,9 +10,11 @@ import (
)
const (
dataScopeAuditQueueSize = 1024
dataScopeAuditBatchSize = 100
dataScopeAuditInterval = 2 * time.Second
dataScopeAuditQueueSize = 1024
dataScopeAuditBatchSize = 100
dataScopeAuditInterval = 2 * time.Second
dataScopeAuditWriteTimeout = 2 * time.Second
dataScopeAuditCloseTimeout = 5 * time.Second
)
// dataScopeAuditWriter keeps row-scope auditing best-effort: callbacks only
@@ -26,6 +28,8 @@ type dataScopeAuditWriter struct {
stop chan struct{}
done chan struct{}
closeOnce sync.Once
enqueueMu sync.RWMutex
closed bool
}
func newDataScopeAuditWriter(data *Data, logger *slog.Logger) *dataScopeAuditWriter {
@@ -57,6 +61,11 @@ func (w *dataScopeAuditWriter) Enqueue(record dataAccessLogPO) {
if w == nil {
return
}
w.enqueueMu.RLock()
defer w.enqueueMu.RUnlock()
if w.closed {
return
}
select {
case w.queue <- record:
default:
@@ -68,8 +77,19 @@ func (w *dataScopeAuditWriter) Close() {
if w == nil {
return
}
w.closeOnce.Do(func() { close(w.stop) })
<-w.done
w.closeOnce.Do(func() {
w.enqueueMu.Lock()
w.closed = true
close(w.stop)
w.enqueueMu.Unlock()
})
timer := time.NewTimer(dataScopeAuditCloseTimeout)
defer timer.Stop()
select {
case <-w.done:
case <-timer.C:
w.log().Warn("等待数据权限审计写入超时", "mod", "data-scope")
}
}
func (w *dataScopeAuditWriter) run() {
@@ -78,6 +98,7 @@ func (w *dataScopeAuditWriter) run() {
defer ticker.Stop()
batch := make([]dataAccessLogPO, 0, w.batchSize)
flush := func() {
defer func() { batch = batch[:0] }()
if len(batch) == 0 || w.data == nil || w.data.gormDB == nil {
return
}
@@ -85,10 +106,11 @@ func (w *dataScopeAuditWriter) run() {
if db == nil {
return
}
if err := db.WithContext(context.Background()).Session(&gorm.Session{NewDB: true, SkipHooks: true}).Create(&batch).Error; err != nil {
ctx, cancel := context.WithTimeout(context.Background(), dataScopeAuditWriteTimeout)
defer cancel()
if err := db.WithContext(ctx).Session(&gorm.Session{NewDB: true, SkipHooks: true}).Create(&batch).Error; err != nil {
w.log().Warn("数据权限审计批量写入失败", "mod", "data-scope", "error", err)
}
batch = batch[:0]
}
for {
select {
+29
View File
@@ -76,3 +76,32 @@ func TestDataScopeAuditWriterUsesReloadedDatabase(t *testing.T) {
t.Fatalf("audit log was written to retired database: %d", firstCount)
}
}
func TestDataScopeAuditCloseCancelsBlockedWrite(t *testing.T) {
db := openDataScopeAuditTestDB(t, "data-scope-audit-blocked")
entered := make(chan struct{})
if err := db.Callback().Create().Before("gorm:create").Register("test:block", func(tx *gorm.DB) {
close(entered)
<-tx.Statement.Context.Done()
tx.AddError(tx.Statement.Context.Err())
}); err != nil {
t.Fatal(err)
}
w := newDataScopeAuditWriterWithOptions(&Data{gormDB: newReloadableDB(db)}, slog.New(slog.NewTextHandler(io.Discard, nil)), 8, 1, time.Hour)
w.Enqueue(dataAccessLogPO{EventType: "blocked_write"})
select {
case <-entered:
case <-time.After(time.Second):
t.Fatal("writer did not start")
}
w.Close()
select {
case <-w.done:
default:
t.Fatal("database write did not honor its deadline")
}
w.Enqueue(dataAccessLogPO{EventType: "after_close"})
if len(w.queue) != 0 {
t.Fatal("closed writer accepted a new event")
}
}
@@ -52,6 +52,26 @@ func TestIntegrationConfigSavePublishesUnmaskedRuntimeValues(t *testing.T) {
}
}
func TestCertificateAliasesMaskAndRoundTrip(t *testing.T) {
old := map[string]any{"app_cert_content": "cert", "key_content": "key", "pkcs12_file": "bundle", "template_content": "keep"}
values := map[string]any{}
for key, value := range old {
values[key] = value
}
maskIntegrationSecrets("payment", "alipay", values)
for _, key := range []string{"app_cert_content", "key_content", "pkcs12_file"} {
if values[key] == old[key] {
t.Fatalf("unmasked alias: %s", key)
}
}
mergeIntegrationSecrets("payment", "alipay", values, old)
for key, value := range old {
if values[key] != value {
t.Fatalf("masked edit changed %s", key)
}
}
}
func TestCommunicationDefaultsIncludeKafka(t *testing.T) {
db, err := openWithDriver("sqlite", "file:"+t.Name()+"?mode=memory&cache=shared")
if err != nil {
+2 -2
View File
@@ -34,8 +34,8 @@ func TestMigrateAllRunsModuleSchemasWithoutBootstrapSeed(t *testing.T) {
if err = db.Table(migration.TableName).Count(&versions).Error; err != nil {
t.Fatal(err)
}
if versions != 23 {
t.Fatalf("migration versions = %d, want 23", versions)
if versions != 27 {
t.Fatalf("migration versions = %d, want 27", versions)
}
var communicationRows []dataintegration.ConfigPO
if err = db.Where("kind IN ?", []string{"mq", "websocket"}).Order("kind, provider").Find(&communicationRows).Error; err != nil {
+4 -1
View File
@@ -125,6 +125,9 @@ func (r *mediaRepo) UpdateMediaName(ctx context.Context, id uint, name string) e
// not-found error instead of silently succeeding with RowsAffected=0.
var po mediaPO
if err := r.data.DB().WithContext(ctx).Where("id = ?", id).First(&po).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return system.ErrMediaNotFound
}
return err
}
return r.data.DB().WithContext(ctx).Model(&po).Update("name", name).Error
@@ -147,7 +150,7 @@ func (r *mediaRepo) CreateMediaBatch(ctx context.Context, items []*system.MediaF
func (r *mediaRepo) SaveCategory(ctx context.Context, v *system.AttachmentCategory) error {
db := r.data.DB().WithContext(ctx)
var count int64
query := db.Model(&categoryPO{}).Where("name = ? AND pid = ?", v.Name, v.ParentID)
query := db.Model(&categoryPO{}).Where("name = ? AND pid = ? AND id <> ?", v.Name, v.ParentID, v.ID)
if err := query.Count(&count).Error; err != nil {
return err
}
+40
View File
@@ -0,0 +1,40 @@
package system
import (
"context"
"kra/internal/biz/system"
"path/filepath"
"testing"
)
func TestMediaEditsKeepUnchangedCategoryNameAndMapMissingFile(t *testing.T) {
db, err := openWithDriver("sqlite", filepath.Join(t.TempDir(), "media.db"))
if err != nil {
t.Fatal(err)
}
pool, _ := db.DB()
t.Cleanup(func() { _ = pool.Close() })
if err := db.AutoMigrate(&mediaPO{}, &categoryPO{}); err != nil {
t.Fatal(err)
}
r := NewMediaRepo(&Data{gormDB: newReloadableDB(db, nil)})
ctx := context.Background()
category := &system.AttachmentCategory{Name: "screenshots"}
if err := r.SaveCategory(ctx, category); err != nil {
t.Fatal(err)
}
var row categoryPO
if err := db.First(&row).Error; err != nil {
t.Fatal(err)
}
category.ID = row.ID
if err := r.SaveCategory(ctx, category); err != nil {
t.Fatalf("unchanged category rejected: %v", err)
}
if err := r.SaveCategory(ctx, &system.AttachmentCategory{Name: "screenshots"}); err == nil {
t.Fatal("duplicate category accepted")
}
if err := r.UpdateMediaName(ctx, 999, "missing"); err != system.ErrMediaNotFound {
t.Fatalf("missing file: %v", err)
}
}