fix: validate explicit migrations and preserve large business ids

This commit is contained in:
Yvan
2026-11-06 07:00:05 +08:00
parent f6f2a4745e
commit 0f0ec2e6d7
53 changed files with 2653 additions and 1445 deletions
+10 -2
View File
@@ -119,7 +119,7 @@ func TestChannelHTTPListsMetricsOptionsAndAtomicCreation(t *testing.T) {
rootRow := func(data map[string]any) map[string]any {
for _, raw := range data["rows"].([]any) {
row := raw.(map[string]any)
if row["id"] == float64(100) && row["sourceDb"] == "lx" {
if row["id"] == "100" && row["sourceDb"] == "lx" {
return row
}
}
@@ -151,7 +151,7 @@ func TestChannelHTTPListsMetricsOptionsAndAtomicCreation(t *testing.T) {
list(8, "?lockStatus=-1", 1)
list(1, "?busierAccount=leaf", 2)
options := call(8, "GET", "/sw/promote/bindOptions?sourceDb=lx", "", 0).([]any)
if len(options) != 1 || options[0].(map[string]any)["platformBusierId"] != float64(14) {
if len(options) != 1 || options[0].(map[string]any)["platformBusierId"] != "14" {
t.Fatal("上级未获取下级正常绑定")
}
body := `{"sourceDb":"lx","platformBusierId":"14","account":"中文渠道","password":"demo123","registerType":1}`
@@ -193,6 +193,14 @@ func TestChannelHTTPListsMetricsOptionsAndAtomicCreation(t *testing.T) {
if game["gameName"] != "汇总游戏" || game["registerCount"] != float64(2) || game["payAmount"] != "70.00" {
t.Fatalf("游戏分布误用了原始注册/充值或错误金额口径: %+v", game)
}
if err := db.Exec("UPDATE tab_promote SET id=9007199254740993 WHERE source_db='lx' AND id=100").Error; err != nil {
t.Fatal(err)
}
large := list(8, "?keyword=main-channel", 1)["rows"].([]any)[0].(map[string]any)
if large["id"] != "9007199254740993" {
t.Fatalf("HTTP response rounded channel ID: %v", large["id"])
}
call(8, "GET", "/sw/promote/game-distribution?sourceDb=lx&promoteId="+large["id"].(string), "", 0)
if err := db.Exec("DROP TABLE tab_promote").Error; err != nil {
t.Fatal(err)
}
+2 -2
View File
@@ -180,7 +180,7 @@ func TestChannelPortraitHTTPTotalTiersAndScope(t *testing.T) {
}
data = call(7, "sourceDb=lx&promoteId=101&period=last7", 0)
basic = data["basic"].(map[string]any)
if basic["parentPromoteId"] != float64(100) || basic["parentPromoteName"] != "root-account" || basic["lastLoginTime"] != nil {
if basic["parentPromoteId"] != "100" || basic["parentPromoteName"] != "root-account" || basic["lastLoginTime"] != nil {
t.Fatal("上级应读同平台当前账号,不能沿用旧快照名")
}
call(99, "sourceDb=lx&promoteId=100", 7)
@@ -189,7 +189,7 @@ func TestChannelPortraitHTTPTotalTiersAndScope(t *testing.T) {
t.Fatal(err)
}
data = call(7, "sourceDb=lx&promoteId=100", 0)
if data["available"] != false || data["metrics"] != nil || data["basic"].(map[string]any)["promoteId"] != float64(100) {
if data["available"] != false || data["metrics"] != nil || data["basic"].(map[string]any)["promoteId"] != "100" {
t.Fatal("缺数据时应保留已授权基本资料并标明未知")
}
}
+1 -1
View File
@@ -110,7 +110,7 @@ func TestLedgerHTTPQueriesAndExcelExports(t *testing.T) {
data := call(7, "GET", "/sw/register/list?sourceDb=lx&sortField=totalPayAmount", "", 0).(map[string]any)
rows := data["rows"].([]any)
first := rows[0].(map[string]any)
if data["total"] != float64(2) || first["userId"] != float64(1) || first["totalPayAmount"] != "1012.34" || first["fgameName"] != "历史游戏" || first["busierName"] != "owner→leaf" {
if data["total"] != float64(2) || first["userId"] != "1" || first["totalPayAmount"] != "1012.34" || first["fgameName"] != "历史游戏" || first["busierName"] != "owner→leaf" {
t.Fatalf("注册明细口径或字段错误: %+v", data)
}
games := call(7, "GET", "/sw/serverAnalysis/games?sourceDb=lx&limit=50", "", 0).([]any)
+21 -1
View File
@@ -9,7 +9,9 @@ import (
"strings"
"kra/internal/config"
"kra/internal/data"
"kra/internal/logging"
"kra/internal/modules"
"kra/internal/server/router"
"kra/internal/service/dto"
systemservice "kra/internal/service/system"
@@ -32,7 +34,9 @@ var (
// Version is the version of the compiled software.
Version string
// flagconf is the config flag.
flagconf string
flagconf string
flagMigrate bool
flagCheckSchema bool
id, _ = os.Hostname()
)
@@ -47,6 +51,8 @@ const (
func init() {
flag.StringVar(&flagconf, "conf", "./configs", "config path, eg: -conf config.yaml")
flag.BoolVar(&flagMigrate, "migrate", false, "apply and verify all migrations, then exit without starting the service")
flag.BoolVar(&flagCheckSchema, "check-schema", false, "verify migration versions and required schema without writing, then exit")
}
// taskRegistry builds the process-wide registry from the dependency-free task
@@ -159,6 +165,20 @@ func main() {
loggerControl.Reload(root, options)
})
defer unsubscribeLogger()
if flagMigrate || flagCheckSchema {
if flagMigrate && flagCheckSchema {
panic("-migrate and -check-schema are mutually exclusive")
}
var database *config.Database
if bc.Data != nil {
database = bc.Data.Database
}
if err := data.MigrateDatabase(context.Background(), database, modules.Catalog(), logger, flagCheckSchema); err != nil {
panic(err)
}
logger.Info("database migration verification completed", "mod", "system", "check_only", flagCheckSchema)
return
}
application, cleanup, err := wireApp(bc.Server, runtime, logger, loggerControl, Version)
if err != nil {
+18 -3
View File
@@ -15,7 +15,22 @@
- 后端仅绑定 `127.0.0.1:9602`,前端仅绑定 `127.0.0.1:9601`。
- 宝塔 `sw2.fuhua91.com` 代理至 9601,前端容器将 `/api/` 转发至后端并去除前缀,支持 WebSocket、SSE 和文件上传。
数据库沿用已迁移并验证的 `sw10`,平台别名为 `lx/hw/yq/oa`,其中三个 SDK 别名连接 `game_lx/game_hw/game_yq` 测试库,Redis 沿用当前关闭状态。生产配置使用 `admin.app.env: production`,关闭开发 Swagger。七条常规任务启用(四条商务/区服日汇总、预警检测、两条有效端刷新),两个历史回填停用。**使用同一 sw10 时只运行一个调度实例,切换到服务器前停止本机的后端服务。**
数据库沿用已迁移并验证的 `sw10`,设置 `admin.system.disable_auto_migrate: true`,由部署前的独立迁移命令更新结构。平台别名为 `lx/hw/yq/oa`,其中三个 SDK 别名连接 `game_lx/game_hw/game_yq` 测试库,Redis 沿用当前关闭状态。生产配置使用 `admin.app.env: production`,关闭开发 Swagger。七条常规任务启用(四条商务/区服日汇总、预警检测、两条有效端刷新),两个历史回填停用。**使用同一 sw10 时只运行一个调度实例,切换到服务器前停止本机的后端服务。**
## 数据库迁移
新副本必须先完整复制旧商务数据库,再执行 [旧框架转换 SQL](../migrations/legacy-framework.sql),最后使用本次发布的后端执行全部增量迁移:
```bash
/app/kratos-admin -conf /data/conf/config.yaml -migrate
/app/kratos-admin -conf /data/conf/config.yaml -check-schema
```
框架转换 SQL 的 `USE` 默认指向 sw10,其他目标库需修改该行。`-migrate` 使用与服务相同的模块迁移目录,包含新表、字段、索引、权限及任务定义,完成检查后退出;不会启动 HTTP、定时任务或 SDK 连接。`-check-schema` 只读检查版本、必需表和字段、主键与索引。旧业务表缺失时应重新完成复制,不能用空表替代历史数据。
转换 SQL 和增量迁移必须全部执行。服务关闭自动迁移后,会在连接失败、版本未完成或结构缺失时明确报错,不进入初始化向导。新增迁移需同时更新模块结构检查与相邻回归测试。部署脚本在停止旧调度实例后自动运行 `-migrate`,全部检查通过才启动新服务。
旧库 `tab_promote.id` 的 INT 自增容量由增量迁移扩大为同符号 BIGINT,保留现存 ID、自增高水位、关联数据与注释。渠道、玩家、流水等可能变大的旧业务 ID 在 JSON 和 Excel 中使用文本,请求兼容十进制字符串;不降低自增值或重新编号。
## 构建与发布
@@ -35,9 +50,9 @@ bash /www/wwwroot/sw2.fuhua91.com/releases/<commit>/deploy.sh <commit> --build-o
bash /www/wwwroot/sw2.fuhua91.com/releases/<commit>/deploy.sh <commit>
```
脚本先构建和检查镜像,再停止当前受管容器,保留一份停止的 `-previous` 容器。健康检查失败时恢复前一组容器。配置、上传文件和数据库不随镜像替换;索引迁移由后端启动按版本执行。
脚本先构建和检查镜像,再停止当前受管容器,保留一份停止的 `-previous` 容器。迁移或健康检查失败时恢复前一组容器。配置、上传文件和数据库不随镜像替换;索引在独立迁移容器中按版本执行。
首次启动持续等待建表与索引迁移完成,以后端健康检查通过为准,不设置总等待时限。容器退出或重启时立即回退;迁移日志记录每一步的名称、结果和耗时。切换数据库前需备份运行配置;镜像回退不会自动还原数据库连接配置。
迁移不设置总等待时限,日志记录每一步的名称、结果和耗时;耗时索引完成前不会启动业务服务。业务容器退出或重启时立即回退。切换数据库前需备份运行配置;镜像回退不会自动还原数据库连接配置或撤销已执行的数据库迁移,回退版本需兼容当前结构和迁移版本。
首次接入域名时,备份宝塔原站点配置,再安装 `bt-nginx-sw2.fuhua91.com.conf` 至 `/www/server/panel/vhost/nginx/`,执行 `/www/server/nginx/sbin/nginx -t` 通过后重载。模板保留证书验证目录,当前监听 HTTP;后续由宝塔为 sw2 单独配置匹配域名的证书。
+1
View File
@@ -47,6 +47,7 @@ try {
foreach ($name in @('deploy.sh', 'wait-backend.sh', 'bt-nginx-sw2.fuhua91.com.conf')) {
Copy-Item -LiteralPath (Join-Path $PSScriptRoot $name) -Destination (Join-Path $stage $name)
}
Copy-Item -LiteralPath (Join-Path $PSScriptRoot '..\migrations') -Destination (Join-Path $stage 'migrations') -Recurse
& tar -czf $archive -C $stage .
if ($LASTEXITCODE -ne 0) { throw 'Release archive failed' }
$hash = (Get-FileHash -LiteralPath $archive -Algorithm SHA256).Hash.ToLowerInvariant()
+6
View File
@@ -91,6 +91,12 @@ for name in "${NAMES[@]}"; do
fi
done
echo 'Applying and verifying database migrations before starting the service.'
docker run --rm --name sw2-migrate --label com.sw2.application=sw2 \
--network "$NETWORK" --env-file "$BASE_DIR/runtime.env" \
-v "$BASE_DIR/config:/data/conf:ro" -v "$BASE_DIR/log:/app/logs" \
"$BACKEND_IMAGE" /app/kratos-admin -conf /data/conf/config.yaml -migrate
created+=(sw2-backend)
docker run -d --name sw2-backend --label com.sw2.application=sw2 \
--restart unless-stopped --network "$NETWORK" \
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-1
View File
@@ -1,6 +1,5 @@
# 文档入口
- [后台说明](ADMIN_BACKEND.md)、[支付接入](PAYMENT.md)、[短信接入](SMS_INTEGRATION.md)。
- [OA 迁移计划](OA_MIGRATION_PLAN.md):独立 OA 工作的设计材料,不是当前商务迁移的完成证明。
此目录仅保留持续维护的文档。临时迁移笔记、验证报告和脚本用完即删,不纳入版本控制。
+12 -10
View File
@@ -55,16 +55,7 @@ func migrateAlertV2(db *gorm.DB) error {
if err := migrateAlertV2UniqueKey(db); err != nil {
return err
}
return migrateQueryIndexes(db, []businessQueryIndex{
{"sys_alert_record", "idx_alert_owner_status_time", []string{"business_id", "status", "created_at", "id"}},
{"sys_alert_record", "idx_alert_channel_type_time", []string{"platform_code", "target_promote_id", "rule_type", "created_at", "id"}},
{"sys_alert_record", "idx_alert_owner_time", []string{"business_id", "created_at", "id"}},
{"sys_alert_record", "idx_alert_created", []string{"created_at", "id"}},
{"sys_alert_operation_log", "idx_alert_log_time", []string{"alert_id", "created_at", "id"}},
{"tab_spend", "idx_alert_spend_window", []string{"source_db", "promote_id", "pay_status", "pay_time", "pay_amount"}},
{"tab_user", "idx_alert_user_window", []string{"source_db", "promote_id", "sw_register_time_unix", "puid"}},
{"tab_promote", "idx_alert_promote_tree", []string{"source_db", "top_promote_id", "id"}},
})
return migrateQueryIndexes(db, alertV2QueryIndexes)
}
func migrateAlertV2UniqueKey(db *gorm.DB) error {
@@ -131,3 +122,14 @@ func migrateAlertV2UniqueKey(db *gorm.DB) error {
}
return db.Exec("ALTER TABLE sys_alert_record " + strings.Join(clauses, ", ") + ", ALGORITHM=INPLACE, LOCK=NONE").Error
}
var alertV2QueryIndexes = []businessQueryIndex{
{"sys_alert_record", "idx_alert_owner_status_time", []string{"business_id", "status", "created_at", "id"}},
{"sys_alert_record", "idx_alert_channel_type_time", []string{"platform_code", "target_promote_id", "rule_type", "created_at", "id"}},
{"sys_alert_record", "idx_alert_owner_time", []string{"business_id", "created_at", "id"}},
{"sys_alert_record", "idx_alert_created", []string{"created_at", "id"}},
{"sys_alert_operation_log", "idx_alert_log_time", []string{"alert_id", "created_at", "id"}},
{"tab_spend", "idx_alert_spend_window", []string{"source_db", "promote_id", "pay_status", "pay_time", "pay_amount"}},
{"tab_user", "idx_alert_user_window", []string{"source_db", "promote_id", "sw_register_time_unix", "puid"}},
{"tab_promote", "idx_alert_promote_tree", []string{"source_db", "top_promote_id", "id"}},
}
+1 -1
View File
@@ -14,7 +14,7 @@ func businessDisplayPaths(db *gorm.DB, owners map[uint]string) (map[uint][]strin
for owner, path := range owners {
seen := map[uint]bool{}
for _, part := range strings.Split(path, ",") {
id, err := strconv.ParseUint(strings.TrimSpace(part), 10, 32)
id, err := strconv.ParseUint(strings.TrimSpace(part), 10, strconv.IntSize)
if err == nil && id > 0 && uint(id) != owner && !seen[uint(id)] {
paths[owner] = append(paths[owner], uint(id))
ids[uint(id)], seen[uint(id)] = true, true
+62
View File
@@ -0,0 +1,62 @@
package business
import (
"fmt"
"strings"
"gorm.io/gorm"
)
func migrateBindingStatusIndex(db *gorm.DB) error {
const name = "idx_sys_user_bind_platform_status"
if !db.Migrator().HasTable(&bindingPO{}) {
return nil
}
if db.Migrator().HasIndex(&bindingPO{}, name) {
return nil
}
return db.Migrator().CreateIndex(&bindingPO{}, name)
}
// Preserve IDs and the high-water mark: lowering AUTO_INCREMENT could reuse
// a deleted ID that still appears in historical records or external systems.
func migratePromoteIDCapacity(db *gorm.DB) error {
if db.Dialector.Name() != "mysql" || !db.Migrator().HasTable("tab_promote") {
return nil
}
var column struct {
DataType, ColumnType, IsNullable, Extra, ColumnComment string
}
if err := db.Raw("SELECT data_type,column_type,is_nullable,extra,column_comment FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='tab_promote' AND column_name='id'").Scan(&column).Error; err != nil {
return err
}
if column.DataType == "bigint" {
return nil
}
if column.DataType != "int" || column.IsNullable != "NO" || column.Extra != "auto_increment" {
return fmt.Errorf("unexpected tab_promote.id definition; capacity migration stopped")
}
var next uint64
if err := db.Raw("SELECT auto_increment FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name='tab_promote'").Scan(&next).Error; err != nil {
return err
}
typeName := "BIGINT"
if strings.Contains(column.ColumnType, "unsigned") {
typeName += " UNSIGNED"
}
// MySQL 5.7 rebuilds the table when widening an integer. Execute before
// starting writers and retain the original next ID and column comment.
return db.Connection(func(conn *gorm.DB) error {
var mode string
if err := conn.Raw("SELECT @@session.sql_mode").Scan(&mode).Error; err != nil {
return err
}
comment := column.ColumnComment
if !strings.Contains(mode, "NO_BACKSLASH_ESCAPES") {
comment = strings.ReplaceAll(comment, `\`, `\\`)
}
comment = "'" + strings.ReplaceAll(comment, "'", "''") + "'"
// MySQL does not permit a prepared-statement placeholder in COMMENT.
return conn.Session(&gorm.Session{NewDB: true}).Exec(fmt.Sprintf("ALTER TABLE tab_promote MODIFY COLUMN id %s NOT NULL AUTO_INCREMENT COMMENT %s, AUTO_INCREMENT=%d", typeName, comment, next)).Error
})
}
@@ -0,0 +1,51 @@
package business
import (
"testing"
)
func TestMySQLPromoteCapacityPreservesIDsAndNextValue(t *testing.T) {
db := openRehearsalMySQL(t)
if err := db.Exec("CREATE TABLE tab_promote(id INT NOT NULL AUTO_INCREMENT COMMENT '渠道主键',source_db VARCHAR(10) NOT NULL,PRIMARY KEY(id,source_db)) AUTO_INCREMENT=2147483647").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("INSERT INTO tab_promote(id,source_db) VALUES(29701,'lx'),(29701,'hw')").Error; err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
if err := migratePromoteIDCapacity(db); err != nil {
t.Fatal(err)
}
}
var next uint64
if err := db.Raw("SELECT AUTO_INCREMENT FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name='tab_promote'").Scan(&next).Error; err != nil || next != 2147483647 {
t.Fatalf("high-water mark changed: %d, %v", next, err)
}
if err := db.Exec("INSERT INTO tab_promote(source_db) VALUES('lx'),('lx')").Error; err != nil {
t.Fatal(err)
}
var ids []uint64
if err := db.Table("tab_promote").Order("id").Pluck("id", &ids).Error; err != nil || len(ids) != 4 || ids[0] != 29701 || ids[1] != 29701 || ids[2] != 2147483647 || ids[3] != 2147483648 {
t.Fatalf("IDs changed or overflowed: %v, %v", ids, err)
}
var comment string
if err := db.Raw("SELECT column_comment FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='tab_promote' AND column_name='id'").Scan(&comment).Error; err != nil || comment != "渠道主键" {
t.Fatalf("comment changed: %q, %v", comment, err)
}
}
func TestBindingStatusIndexRepairsAnExistingLegacyTable(t *testing.T) {
db := openRehearsalMySQL(t)
const index = "idx_sys_user_bind_platform_status"
if err := db.Migrator().DropIndex(&bindingPO{}, index); err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
if err := migrateBindingStatusIndex(db); err != nil {
t.Fatal(err)
}
}
if !db.Migrator().HasIndex(&bindingPO{}, index) {
t.Fatal("legacy status index was not created")
}
}
+1 -1
View File
@@ -385,7 +385,7 @@ func fillLedgerNames(db *gorm.DB, scope business.LedgerScope, filter business.Le
paths := map[business.PlatformKey][]uint{}
for key, binding := range names {
for _, part := range strings.Split(binding.IDPath, ",") {
id, err := strconv.ParseUint(strings.TrimSpace(part), 10, 32)
id, err := strconv.ParseUint(strings.TrimSpace(part), 10, strconv.IntSize)
if err == nil && id > 0 && uint(id) != binding.UserID {
paths[key] = append(paths[key], uint(id))
ancestorIDs[uint(id)] = true
@@ -0,0 +1,91 @@
package business
import (
"fmt"
"strings"
"gorm.io/gorm"
"kra/pkg/database/migration"
)
// These source tables carry history. An incomplete copy must not be replaced
// with empty tables. SDK and OA schemas belong to their respective databases.
var importedTables = []string{
"tab_game", "tab_game_server", "tab_promote", "tab_promote_apply",
"tab_spend", "tab_user", "tab_user_day_login", "tab_user_play_info",
"tab_sw_report_day", "tab_sw_report_day_key", "tab_sw_report_server_day", "tab_sw_report_server_day_key",
}
func CheckPrerequisites(db *gorm.DB) error {
if db.Dialector.Name() != "mysql" {
return nil
}
if db.Migrator().HasTable("sys_user") && !db.Migrator().HasTable("sys_users") {
return fmt.Errorf("legacy framework conversion must run before incremental migrations")
}
var missing []string
for _, table := range importedTables {
if !db.Migrator().HasTable(table) {
missing = append(missing, table)
}
}
if len(missing) > 0 {
return fmt.Errorf("incomplete legacy copy, missing tables: %s", strings.Join(missing, ", "))
}
return nil
}
func CheckSchema(db *gorm.DB) error {
if err := CheckPrerequisites(db); err != nil {
return err
}
if err := migration.CheckModels(db,
&preferencePO{}, &bindingPO{}, &bindingLockPO{}, &resetCodePO{}, &resetTokenPO{},
&alertRulePO{}, &alertRecordPO{}, &alertLogPO{}, &alertGapPO{}, &alertExemptionPO{}, &alertChannelLockPO{},
&alertTriggerLockPO{}, &reportBuildLockPO{}, &businessGroupPO{}, &businessGroupMemberPO{},
&businessMemberProfilePO{}, &businessGroupStatePO{}, &performanceGoalPO{}, &effectivenessStatePO{}, &effectivenessPO{},
); err != nil {
return err
}
if db.Dialector.Name() != "mysql" {
return nil
}
indexes := append([]businessQueryIndex{}, businessQueryIndexes...)
var idType string
if err := db.Raw("SELECT data_type FROM information_schema.columns WHERE table_schema=DATABASE() AND table_name='tab_promote' AND column_name='id'").Scan(&idType).Error; err != nil {
return err
}
if idType != "bigint" {
return fmt.Errorf("tab_promote.id must be BIGINT after capacity migration")
}
indexes = append(indexes, reportBuildQueryIndexes...)
indexes = append(indexes, alertV2QueryIndexes...)
indexes = append(indexes,
businessQueryIndex{"tab_spend", "idx_sw_spend_effective_scan", []string{"promote_id", "source_db", "pay_status", "pay_time", "id", "pay_amount"}},
businessQueryIndex{"tab_promote_apply", "idx_sw_apply_game_scope", []string{"game_id", "source_db", "status", "promote_id"}},
businessQueryIndex{"tab_user", "idx_sw_user_registration", []string{"source_db", "puid", "sw_register_time_unix", "id"}},
businessQueryIndex{"tab_user_play_info", "idx_sw_play_create_time", []string{"create_time"}},
businessQueryIndex{"sys_alert_record", "uk_alert_record_v2_open", []string{"business_id", "platform_code", "target_promote_id", "rule_type", "v2_open_key"}},
)
for _, index := range indexes {
var existing []struct {
ColumnName string
NonUnique int
SubPart *int
}
if err := db.Raw("SELECT column_name,non_unique,sub_part FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name=? AND index_name=? ORDER BY seq_in_index", index.table, index.name).Scan(&existing).Error; err != nil {
return err
}
columns := []string{}
for _, field := range existing {
if field.SubPart != nil || index.name == "uk_alert_record_v2_open" && field.NonUnique != 0 {
return fmt.Errorf("invalid index %s.%s", index.table, index.name)
}
columns = append(columns, field.ColumnName)
}
if strings.Join(columns, ",") != strings.Join(index.columns, ",") {
return fmt.Errorf("missing or incompatible index %s.%s", index.table, index.name)
}
}
return nil
}
+1 -1
View File
@@ -6,7 +6,7 @@ import (
)
func Migrations() []migration.Step {
return []migration.Step{{ID: "202611050020_business_alert_v2", Migrate: migrateAlertV2}, {ID: "202611040017_business_groups_goals_effectiveness", Migrate: migrateBusinessManagement}, {ID: "202611040018_business_effectiveness_query_index", Migrate: migrateEffectivenessQueryIndex}, {ID: "202611040015_business_approved_game_index", Migrate: migrateApprovedGameIndexes}, {ID: "202611040014_business_report_build_indexes", Migrate: migrateReportBuildQueryIndexes}, {ID: "202611040013_business_query_indexes", Migrate: migrateBusinessQueryIndexes}, {ID: "202611040012_business_server_report_index", Migrate: migrateServerReportIndex}, {ID: "202611040011_business_registration_index", Migrate: migrateRegistrationIndex}, {ID: "202611040010_business_alert_trigger_locks", Migrate: func(db *gorm.DB) error { return migration.CreateMissingTables(db, &alertTriggerLockPO{}) }}, {ID: "202611040009_business_menu_icons", Migrate: migrateBusinessMenuIcons}, {ID: "202611040008_business_report_build_locks", Migrate: func(db *gorm.DB) error { return migration.CreateMissingTables(db, &reportBuildLockPO{}) }}, {ID: "202611030007_business_alert_attachments", Migrate: migrateAlertAttachments}, {ID: "202611030006_business_alert_schema", Migrate: migrateAlertSchema}, {ID: "202611030005_business_password_reset", Migrate: func(db *gorm.DB) error {
return []migration.Step{{ID: "202611060023_business_binding_status_index", Migrate: migrateBindingStatusIndex}, {ID: "202611060022_business_promote_id_capacity", Migrate: migratePromoteIDCapacity}, {ID: "202611050020_business_alert_v2", Migrate: migrateAlertV2}, {ID: "202611040017_business_groups_goals_effectiveness", Migrate: migrateBusinessManagement}, {ID: "202611040018_business_effectiveness_query_index", Migrate: migrateEffectivenessQueryIndex}, {ID: "202611040015_business_approved_game_index", Migrate: migrateApprovedGameIndexes}, {ID: "202611040014_business_report_build_indexes", Migrate: migrateReportBuildQueryIndexes}, {ID: "202611040013_business_query_indexes", Migrate: migrateBusinessQueryIndexes}, {ID: "202611040012_business_server_report_index", Migrate: migrateServerReportIndex}, {ID: "202611040011_business_registration_index", Migrate: migrateRegistrationIndex}, {ID: "202611040010_business_alert_trigger_locks", Migrate: func(db *gorm.DB) error { return migration.CreateMissingTables(db, &alertTriggerLockPO{}) }}, {ID: "202611040009_business_menu_icons", Migrate: migrateBusinessMenuIcons}, {ID: "202611040008_business_report_build_locks", Migrate: func(db *gorm.DB) error { return migration.CreateMissingTables(db, &reportBuildLockPO{}) }}, {ID: "202611030007_business_alert_attachments", Migrate: migrateAlertAttachments}, {ID: "202611030006_business_alert_schema", Migrate: migrateAlertSchema}, {ID: "202611030005_business_password_reset", Migrate: func(db *gorm.DB) error {
return migration.CreateMissingTables(db, &resetCodePO{}, &resetTokenPO{})
}}, {ID: "202611030004_business_binding_locks", Migrate: func(db *gorm.DB) error {
return migration.CreateMissingTables(db, &bindingLockPO{})
+8
View File
@@ -145,6 +145,9 @@ func (d *Data) reloadConfig(ctx context.Context) error {
var undo rollback
defer undo.run()
databaseReady := databaseConnectionConfigured(next.Data.Database)
if !databaseReady && next.Admin.System != nil && next.Admin.System.DisableAutoMigrate {
return fmt.Errorf("automatic migration is disabled; an existing migrated database is required")
}
var candidateDB *gorm.DB
if databaseReady {
candidateDB, err = openDatabase(next.Data.Database, false, "", d.logger())
@@ -170,6 +173,11 @@ func (d *Data) reloadConfig(ctx context.Context) error {
return fmt.Errorf("reload database migrations: %w", err)
}
}
if databaseReady && next.Admin.System != nil && next.Admin.System.DisableAutoMigrate {
if err = checkMigratedSchema(candidateDB.WithContext(ctx), d.catalog); err != nil {
return fmt.Errorf("reload database schema: %w", err)
}
}
storageConfig, err := dataintegration.ResolveStorageConfig(candidateDB.WithContext(ctx), next.Admin.Storage)
if err != nil {
return fmt.Errorf("reload storage configuration: %w", err)
+13 -5
View File
@@ -246,12 +246,20 @@ func NewData(runtime *config.Store, appLogger *slog.Logger, storageManager *stor
undo.add(cleanup)
defer undo.run()
usingFallback := !databaseConnectionConfigured(c.Database)
admin := runtime.Admin()
if admin == nil {
admin = &config.Admin{}
}
disableAutoMigrate := admin.System != nil && admin.System.DisableAutoMigrate
var db *gorm.DB
var err error
if !usingFallback {
db, err = openDatabase(c.Database, false, "", appLogger)
}
if usingFallback || err != nil {
if disableAutoMigrate {
return nil, nil, fmt.Errorf("automatic migration is disabled; an available migrated database is required")
}
// The initialization endpoint must remain available when the configured
// target database has not been created yet.
if err != nil {
@@ -274,16 +282,16 @@ func NewData(runtime *config.Store, appLogger *slog.Logger, storageManager *stor
for _, item := range d.dbList {
registerDataScopeCallbacks(item, d.enqueueDataScopeAudit)
}
admin := runtime.Admin()
if admin == nil {
admin = &config.Admin{}
}
disableAutoMigrate := admin.System != nil && admin.System.DisableAutoMigrate
if !usingFallback && !disableAutoMigrate {
if err = migrateAll(db, catalog); err != nil {
return nil, nil, fmt.Errorf("migrate tables: %w", err)
}
}
if !usingFallback && disableAutoMigrate {
if err = checkMigratedSchema(db, catalog); err != nil {
return nil, nil, err
}
}
if !usingFallback {
storageConfig, storageErr := dataintegration.ResolveStorageConfig(db, admin.Storage)
if storageErr != nil {
+2
View File
@@ -151,3 +151,5 @@ func ensurePaymentIntegrationConfigs(db *gorm.DB) error {
}
return nil
}
func CheckSchema(db *gorm.DB) error { return migration.CheckModels(db, &ConfigPO{}) }
+79
View File
@@ -0,0 +1,79 @@
package data
import (
"context"
"fmt"
"log/slog"
"time"
"gorm.io/gorm"
"kra/internal/config"
"kra/pkg/database/migration"
"kra/pkg/module"
)
// MigrateDatabase opens only the configured primary database. It does not
// create a database, start a scheduler, open SDK clients, or initialize users.
func MigrateDatabase(ctx context.Context, c *config.Database, catalog module.Catalog, logger *slog.Logger, checkOnly bool) error {
if !databaseConnectionConfigured(c) {
return fmt.Errorf("migration requires an existing configured database")
}
migrationConfig := *c
// One connection holds the advisory lock, a second executes migrations.
if !checkOnly && migrationConfig.MaxOpenConns > 0 && migrationConfig.MaxOpenConns < 2 {
migrationConfig.MaxOpenConns = 2
}
db, err := openDatabase(&migrationConfig, false, "", logger)
if err != nil {
return fmt.Errorf("open migration database: %w", err)
}
defer closeGormDB(db)
db = db.WithContext(ctx)
if checkOnly {
return checkMigratedSchema(db, catalog)
}
// A dedicated connection owns the lock even when a migration uses a
// transaction or temporarily changes connection-local SQL compatibility.
run := func() error {
if err := catalog.CheckPrerequisites(db); err != nil {
return err
}
if err := migrateAll(db, catalog); err != nil {
return err
}
return checkMigratedSchema(db, catalog)
}
if db.Dialector.Name() != "mysql" {
return run()
}
return db.Connection(func(conn *gorm.DB) error {
var name string
if err := conn.Raw("SELECT CONCAT('sw_to_kra:', MD5(DATABASE()))").Scan(&name).Error; err != nil {
return err
}
var locked int
if err := conn.Session(&gorm.Session{NewDB: true}).Raw("SELECT GET_LOCK(?, 0)", name).Scan(&locked).Error; err != nil {
return err
}
if locked != 1 {
return fmt.Errorf("another migration owns the database lock")
}
defer func() {
cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
conn.Session(&gorm.Session{NewDB: true}).WithContext(cleanup).Exec("DO RELEASE_LOCK(?)", name)
}()
return run()
})
}
func checkMigratedSchema(db *gorm.DB, catalog module.Catalog) error {
db = db.Session(&gorm.Session{NewDB: true})
if err := migration.CheckApplied(db, catalog.MigrationSteps()); err != nil {
return fmt.Errorf("run -migrate before starting the service: %w", err)
}
if err := catalog.CheckSchema(db); err != nil {
return fmt.Errorf("database schema is incomplete: %w", err)
}
return nil
}
+3
View File
@@ -25,6 +25,9 @@ func TestMigrateAllRunsModuleSchemasWithoutBootstrapSeed(t *testing.T) {
if err = migrateAll(db, testCatalog()); err != nil {
t.Fatalf("second migration run: %v", err)
}
if err := checkMigratedSchema(db, testCatalog()); err != nil {
t.Fatalf("complete migration must satisfy every module schema: %v", err)
}
for _, table := range []string{"sys_integration_configs", "sys_users", "sys_base_menus", "pay_orders", "pay_events"} {
if !db.Migrator().HasTable(table) {
t.Fatalf("migration did not create %s", table)
+4
View File
@@ -16,3 +16,7 @@ func Migrations() []migration.Step {
}},
}
}
func CheckSchema(db *gorm.DB) error {
return migration.CheckModels(db, &paymentOrderPO{}, &paymentEventPO{})
}
+1 -10
View File
@@ -16,16 +16,7 @@ func Migrations() []migration.Step {
{
ID: "202608200002_system_schema",
Migrate: func(db *gorm.DB) error {
return migration.CreateMissingTables(db,
&userPO{}, &authorityPO{}, &menuPO{}, &userAuthorityPO{}, &authorityMenuPO{}, &menuParameterPO{},
&apiPO{}, &ignoredAPIPO{}, &authorityAPIPO{}, &casbinRulePO{}, &menuButtonPO{}, &authorityButtonPO{},
&departmentPO{}, &positionPO{}, &userDepartmentPO{}, &userPositionPO{}, &authorityDepartmentPO{},
&dictionaryPO{}, &dictionaryDetailPO{}, &parameterPO{}, &apiTokenPO{}, &jwtBlacklistPO{}, &SecurityConfigPO{},
&versionPO{}, &exportTemplatePO{}, &exportConditionPO{}, &exportJoinPO{},
&operationPO{}, &loginLogPO{}, &DataAccessLogPO{}, &errorRecordPO{},
&mediaPO{}, &categoryPO{}, &uploadSessionPO{}, &uploadChunkPO{},
&announcementPO{},
)
return migration.CreateMissingTables(db, schemaModels()...)
},
},
{ID: "202608210002_communication_surface", Migrate: ensureCommunicationSurface},
+20
View File
@@ -0,0 +1,20 @@
package system
import (
"gorm.io/gorm"
"kra/pkg/database/migration"
)
func schemaModels() []any {
return []any{
&userPO{}, &authorityPO{}, &menuPO{}, &userAuthorityPO{}, &authorityMenuPO{}, &menuParameterPO{},
&apiPO{}, &ignoredAPIPO{}, &authorityAPIPO{}, &casbinRulePO{}, &menuButtonPO{}, &authorityButtonPO{},
&departmentPO{}, &positionPO{}, &userDepartmentPO{}, &userPositionPO{}, &authorityDepartmentPO{},
&dictionaryPO{}, &dictionaryDetailPO{}, &parameterPO{}, &apiTokenPO{}, &jwtBlacklistPO{}, &SecurityConfigPO{},
&versionPO{}, &exportTemplatePO{}, &exportConditionPO{}, &exportJoinPO{},
&operationPO{}, &loginLogPO{}, &DataAccessLogPO{}, &errorRecordPO{},
&mediaPO{}, &categoryPO{}, &uploadSessionPO{}, &uploadChunkPO{}, &announcementPO{},
}
}
func CheckSchema(db *gorm.DB) error { return migration.CheckModels(db, schemaModels()...) }
+2
View File
@@ -38,3 +38,5 @@ func SeedDefaults(ctx context.Context, db *gorm.DB, defaults []platformmodule.Ti
return nil
})
}
func CheckSchema(db *gorm.DB) error { return migration.CheckModels(db, &taskPO{}, &taskLogPO{}) }
+1 -1
View File
@@ -106,7 +106,7 @@ func Definition() module.Definition {
})
}})
migrations = append(migrations, migration.Step{ID: "202611050021_business_alert_surface", Migrate: migrateAlertSurface})
definition := module.Definition{Name: "business", Migrations: migrations, TimedTasks: append([]module.TimedTask{alertDetectionTask}, tasks...), Surface: module.Surface{
definition := module.Definition{Name: "business", Migrations: migrations, Prerequisites: databusiness.CheckPrerequisites, ValidateSchema: databusiness.CheckSchema, TimedTasks: append([]module.TimedTask{alertDetectionTask}, tasks...), Surface: module.Surface{
Menus: []module.Menu{
alertExemptionMenu,
performanceMenu,
+2 -2
View File
@@ -25,8 +25,8 @@ func TestCatalogContainsBuiltInModulesInDependencyOrder(t *testing.T) {
t.Fatalf("definition[%d] = %q, want %q", index, got, name)
}
}
if got := catalog.MigrationSteps(); len(got) != 35 {
t.Fatalf("module migrations = %d, want 35", len(got))
if got := catalog.MigrationSteps(); len(got) != 37 {
t.Fatalf("module migrations = %d, want 37", len(got))
}
var menus, apis int
for _, definition := range catalog.Definitions {
+3 -2
View File
@@ -20,8 +20,9 @@ func Definition() module.Definition {
}, 888)
}})
return module.Definition{
Name: "integration",
Migrations: migrations,
Name: "integration",
ValidateSchema: dataintegration.CheckSchema,
Migrations: migrations,
Surface: module.Surface{
Menus: []module.Menu{
{Name: "integrationConfig", Path: "integrationConfig", ParentName: "extensions", Component: "view/systemTools/integration/config.vue", Title: "通信集成", Icon: "connection", Sort: 8},
+4 -3
View File
@@ -56,8 +56,9 @@ func Definition() module.Definition {
return datasystem.GrantMenuAPIs(db, "paymentOrders", orderReadAPIs)
}})
return module.Definition{
Name: "payment",
Migrations: migrations,
Surface: adminSurface,
Name: "payment",
ValidateSchema: datapayment.CheckSchema,
Migrations: migrations,
Surface: adminSurface,
}
}
+3 -2
View File
@@ -11,7 +11,8 @@ import (
// catalog. Other business modules can expose the same shape independently.
func Definition() module.Definition {
return module.Definition{
Name: "system",
Migrations: datasystem.Migrations(),
Name: "system",
Migrations: datasystem.Migrations(),
ValidateSchema: datasystem.CheckSchema,
}
}
+4 -3
View File
@@ -8,9 +8,10 @@ import (
func Definition() module.Definition {
return module.Definition{
Name: "task",
Migrations: datatask.Migrations(),
TimedTasks: TimedTasks(),
Name: "task",
ValidateSchema: datatask.CheckSchema,
Migrations: datatask.Migrations(),
TimedTasks: TimedTasks(),
}
}
+1 -1
View File
@@ -117,5 +117,5 @@ func (s *BindingService) Unbind(ctx context.Context, req *dto.BusinessBindingReq
if err != nil {
return err
}
return s.uc.Unbind(ctx, business.PlatformKey{SourceDB: source, BusierID: req.PlatformBusierID})
return s.uc.Unbind(ctx, business.PlatformKey{SourceDB: source, BusierID: uint64(req.PlatformBusierID)})
}
+8 -2
View File
@@ -21,6 +21,12 @@ func optionalNumber[T ~int | ~int64](value *T) ledgerCell {
}
return numericCell(strconv.FormatInt(int64(*value), 10))
}
func optionalID(value *int64) ledgerCell {
if value == nil {
return textCell("")
}
return textCell(strconv.FormatInt(*value, 10))
}
func optionalMoney(value *string) ledgerCell {
if value == nil {
return textCell("")
@@ -83,7 +89,7 @@ func (s *LedgerService) ExportRegistrations(ctx context.Context, req *dto.Ledger
rows := make([][]ledgerCell, 0, len(items))
for _, item := range items {
r := registrationResponse(item)
rows = append(rows, []ledgerCell{textCell(r.SourceDB), textCell(r.SourceDBLabel), numericCell(strconv.FormatUint(r.ID, 10)), numericCell(strconv.FormatUint(r.UserID, 10)), optionalNumber(r.RegisterTime), optionalNumber(r.FirstGameID), textCell(r.FirstGameName), optionalMoney(r.TotalPayAmount), textCell(r.RegisterIP), textCell(r.DeviceName), textCell(r.PromoteAccount), optionalNumber(r.PromoteID), textCell(r.BusierName)})
rows = append(rows, []ledgerCell{textCell(r.SourceDB), textCell(r.SourceDBLabel), textCell(strconv.FormatUint(r.ID, 10)), textCell(strconv.FormatUint(r.UserID, 10)), optionalNumber(r.RegisterTime), optionalID(r.FirstGameID), textCell(r.FirstGameName), optionalMoney(r.TotalPayAmount), textCell(r.RegisterIP), textCell(r.DeviceName), textCell(r.PromoteAccount), optionalID(r.PromoteID), textCell(r.BusierName)})
}
return ledgerWorkbook("注册明细", []string{"sourceDb", "所属平台", "ID", "玩家ID", "注册时间", "fgameId", "游戏", "累计充值", "注册IP", "机型", "渠道", "promoteId", "所属商务"}, rows)
}
@@ -99,7 +105,7 @@ func (s *LedgerService) ExportSpends(ctx context.Context, req *dto.LedgerQuery)
rows := make([][]ledgerCell, 0, len(items))
for _, item := range items {
r := spendResponse(item)
rows = append(rows, []ledgerCell{textCell(r.SourceDB), textCell(r.SourceDBLabel), numericCell(strconv.FormatUint(r.ID, 10)), optionalNumber(r.UserID), textCell(r.OrderNumber), optionalNumber(r.GameID), textCell(r.GameName), textCell(r.ServerID), textCell(r.ServerName), textCell(r.PlayerName), optionalMoney(r.PayAmount), optionalNumber(r.PayTime), optionalNumber(r.PayWay), optionalNumber(r.PayStatus), textCell(r.IP), textCell(r.PromoteAccount), optionalNumber(r.PromoteID), textCell(r.BusierName)})
rows = append(rows, []ledgerCell{textCell(r.SourceDB), textCell(r.SourceDBLabel), textCell(strconv.FormatUint(r.ID, 10)), optionalID(r.UserID), textCell(r.OrderNumber), optionalID(r.GameID), textCell(r.GameName), textCell(r.ServerID), textCell(r.ServerName), textCell(r.PlayerName), optionalMoney(r.PayAmount), optionalNumber(r.PayTime), optionalNumber(r.PayWay), optionalNumber(r.PayStatus), textCell(r.IP), textCell(r.PromoteAccount), optionalID(r.PromoteID), textCell(r.BusierName)})
}
return ledgerWorkbook("充值明细", []string{"sourceDb", "所属平台", "ID", "玩家ID", "订单号", "gameId", "游戏", "serverId", "区服", "角色", "实付金额", "完成时间", "方式", "状态", "IP", "渠道", "promoteId", "所属商务"}, rows)
}
@@ -15,11 +15,11 @@ type ledgerExportFake struct{ LedgerUsecase }
func (ledgerExportFake) ExportRegistrations(context.Context, business.LedgerFilter) ([]*business.Registration, error) {
amount := "9007199254740993.10"
stamp := int64(1793577600)
return []*business.Registration{{ChannelKey: business.ChannelKey{SourceDB: "lx", ID: 7}, RegisterTime: &stamp, FirstGameName: "游戏", TotalPayAmount: &amount, LedgerNames: business.LedgerNames{PromoteAccount: "=1+1"}}}, nil
return []*business.Registration{{ChannelKey: business.ChannelKey{SourceDB: "lx", ID: 9007199254740993}, RegisterTime: &stamp, FirstGameName: "游戏", TotalPayAmount: &amount, LedgerNames: business.LedgerNames{PromoteAccount: "=1+1"}}}, nil
}
func (ledgerExportFake) ExportSpends(context.Context, business.LedgerFilter) ([]*business.Spend, error) {
amount := "123.45"
return []*business.Spend{{ChannelKey: business.ChannelKey{SourceDB: "lx", ID: 8}, OrderNumber: "000000000000000001", PayAmount: &amount}}, nil
return []*business.Spend{{ChannelKey: business.ChannelKey{SourceDB: "lx", ID: 9007199254740995}, OrderNumber: "000000000000000001", PayAmount: &amount}}, nil
}
func TestLedgerWorkbookPreservesNumbersAndText(t *testing.T) {
s := &LedgerService{uc: ledgerExportFake{}}
@@ -41,6 +41,13 @@ func TestLedgerWorkbookPreservesNumbersAndText(t *testing.T) {
if amount != "9007199254740993.10" || formula != "" || text != "=1+1" {
t.Fatalf("导出丢失金额精度或把账号当成公式: %s %s %s", amount, formula, text)
}
for _, cell := range []string{"C2", "D2"} {
value, _ := f.GetCellValue("注册明细", cell)
kind, _ := f.GetCellType("注册明细", cell)
if value != "9007199254740993" || kind != excelize.CellTypeSharedString {
t.Fatalf("ID must be exact text: %s %v", value, kind)
}
}
rows, err := f.GetRows("注册明细")
if err != nil || len(rows) != 2 || rows[0][0] != "sourceDb" || rows[0][12] != "所属商务" {
t.Fatal("原导出字段与顺序丢失")
+2 -2
View File
@@ -26,7 +26,7 @@ type AnalyticsMetrics struct {
type ChannelAnalyticsRow struct {
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
PromoteID *int64 `json:"promoteId"`
PromoteID *int64 `json:"promoteId,string"`
PromoteName string `json:"promoteName"`
PromoteType *int `json:"promoteType"`
BusierName string `json:"busierName"`
@@ -35,7 +35,7 @@ type ChannelAnalyticsRow struct {
type GameAnalyticsRow struct {
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
GameID *int64 `json:"gameId"`
GameID *int64 `json:"gameId,string"`
GameName string `json:"gameName"`
AnalyticsMetrics
ARPU string `json:"arpu"`
+2 -2
View File
@@ -11,13 +11,13 @@ type BusinessBindingRequest struct {
SourceDB string `json:"sourceDb"`
Account string `json:"account"`
Password string `json:"password"`
PlatformBusierID uint64 `json:"platformBusierId"`
PlatformBusierID ID `json:"platformBusierId"`
}
type BusinessBindingResponse struct {
UserID uint `json:"userId"`
SourceDB string `json:"sourceDb"`
PlatformName string `json:"platformName"`
PlatformBusierID uint64 `json:"platformBusierId"`
PlatformBusierID uint64 `json:"platformBusierId,string"`
Account string `json:"platformBusierAccount"`
UserName string `json:"bindUserName"`
PromoteCount *int64 `json:"promoteCount"`
+1 -1
View File
@@ -45,7 +45,7 @@ type PortraitRank struct {
Rank int `json:"rank"`
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
PromoteID uint64 `json:"promoteId"`
PromoteID uint64 `json:"promoteId,string"`
PromoteName string `json:"promoteName"`
Value string `json:"value"`
}
+2 -2
View File
@@ -7,11 +7,11 @@ type BusinessPromoteQuery struct {
PlatformBusierID uint64 `form:"platformBusierId"`
}
type BusinessPromoteResponse struct {
PromoteID uint64 `json:"promoteId"`
PromoteID uint64 `json:"promoteId,string"`
PromoteName string `json:"promoteName"`
SourceDB string `json:"sourceDb"`
PlatformName string `json:"platformName"`
PlatformBusierID uint64 `json:"platformBusierId"`
PlatformBusierID uint64 `json:"platformBusierId,string"`
}
type BusinessPromotePage struct {
Rows []*BusinessPromoteResponse `json:"rows"`
+6 -6
View File
@@ -30,17 +30,17 @@ type ChannelResponse struct {
LastLoginAt *int64 `json:"lastLoginAt"`
Phone string `json:"phone"`
SettlementDays *int `json:"settlementDays"`
ID uint64 `json:"id"`
ID uint64 `json:"id,string"`
SourceDB string `json:"sourceDb"`
PlatformName string `json:"platformName"`
Account string `json:"account"`
Nickname string `json:"nickname"`
Level *int `json:"promoteLevel"`
ParentID uint64 `json:"parentId"`
ParentID uint64 `json:"parentId,string"`
ParentName string `json:"parentName"`
RealName string `json:"realName"`
RegisterType *int `json:"registerType"`
BusierID uint64 `json:"busierId"`
BusierID uint64 `json:"busierId,string"`
BusierUserID *uint `json:"busierUserId"`
BusierDisplayName string `json:"busierDisplayName"`
Status *int `json:"status"`
@@ -62,7 +62,7 @@ type ChannelStatusCounts struct {
}
type ChannelBindOption struct {
SourceDB string `json:"sourceDb"`
PlatformBusierID uint64 `json:"platformBusierId"`
PlatformBusierID uint64 `json:"platformBusierId,string"`
Account string `json:"platformBusierAccount"`
UserID uint `json:"userId"`
}
@@ -74,7 +74,7 @@ type ChannelCreateRequest struct {
RegisterType *int `json:"registerType"`
}
type ChannelCreateResult struct {
ID uint64 `json:"id"`
ID uint64 `json:"id,string"`
SourceDB string `json:"sourceDb"`
}
type ChannelGameQuery struct {
@@ -82,7 +82,7 @@ type ChannelGameQuery struct {
PromoteID uint64 `form:"promoteId"`
}
type ChannelGameResponse struct {
GameID uint64 `json:"gameId"`
GameID uint64 `json:"gameId,string"`
GameName string `json:"gameName"`
RegisterCount int64 `json:"registerCount"`
PayAmount string `json:"payAmount"`
+2 -2
View File
@@ -6,7 +6,7 @@ type ChannelPortraitQuery struct {
Period string `form:"period"`
}
type ChannelPortraitBasic struct {
PromoteID uint64 `json:"promoteId"`
PromoteID uint64 `json:"promoteId,string"`
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
Account string `json:"account"`
@@ -26,7 +26,7 @@ type ChannelPortraitBasic struct {
LastLoginTime *string `json:"lastLoginTime"`
BusierUserID *uint `json:"busierUserId"`
BusierDisplayName string `json:"busierDisplayName"`
ParentPromoteID *uint64 `json:"parentPromoteId"`
ParentPromoteID *uint64 `json:"parentPromoteId,string"`
ParentPromoteName *string `json:"parentPromoteName"`
ParentPromoteAccount *string `json:"parentPromoteAccount"`
GameCount *int64 `json:"gameCount"`
+38
View File
@@ -0,0 +1,38 @@
package dto
import (
"encoding/json"
"fmt"
"strconv"
)
// ID accepts legacy JSON integers and decimal strings. Its wire representation
// is always a string so a browser can round-trip all uint64 values exactly.
type ID uint64
func (id ID) MarshalJSON() ([]byte, error) {
return json.Marshal(strconv.FormatUint(uint64(id), 10))
}
func (id *ID) UnmarshalJSON(raw []byte) error {
text := string(raw)
if len(raw) > 0 && raw[0] == '"' {
if err := json.Unmarshal(raw, &text); err != nil {
return err
}
}
if text == "" {
return fmt.Errorf("ID must be a decimal integer")
}
for _, digit := range text {
if digit < '0' || digit > '9' {
return fmt.Errorf("ID must be a decimal integer")
}
}
value, err := strconv.ParseUint(text, 10, 64)
if err != nil {
return fmt.Errorf("ID is outside the uint64 range")
}
*id = ID(value)
return nil
}
+68
View File
@@ -0,0 +1,68 @@
package dto
import (
"encoding/json"
"math"
"strconv"
"testing"
)
func TestIDRoundTripsDecimalStringsAndLegacyNumbers(t *testing.T) {
for _, value := range []uint64{1, math.MaxInt32, math.MaxInt32 + 1, 1<<53 - 1, 1 << 53, 1<<53 + 1, math.MaxInt64, math.MaxUint64} {
want := strconv.FormatUint(value, 10)
for _, raw := range []string{want, strconv.Quote(want)} {
var request BusinessBindingRequest
if err := json.Unmarshal([]byte(`{"platformBusierId":`+raw+`}`), &request); err != nil || uint64(request.PlatformBusierID) != value {
t.Fatalf("decode %s: %v, %d", raw, err, request.PlatformBusierID)
}
encoded, err := json.Marshal(request.PlatformBusierID)
if err != nil || string(encoded) != strconv.Quote(want) {
t.Fatalf("encode %s: %s, %v", raw, encoded, err)
}
}
}
for _, raw := range []string{`null`, `true`, `-1`, `1.5`, `1e3`, `18446744073709551616`, `""`, `"-1"`, `"1.0"`, `"1e3"`, `" 1"`, `"+1"`, `"18446744073709551616"`} {
var id ID
if err := json.Unmarshal([]byte(raw), &id); err == nil {
t.Fatalf("accepted invalid ID %s", raw)
}
}
}
func TestLegacyResponsesPreserveLargeIDsAndNumericMetrics(t *testing.T) {
const value = uint64(1<<53 + 1)
signed := int64(value)
for _, response := range []any{
ChannelResponse{ID: value, ParentID: value, BusierID: value},
ChannelCreateResult{ID: value}, ChannelBindOption{PlatformBusierID: value},
ChannelGameResponse{GameID: value, RegisterCount: 3},
BusinessBindingResponse{PlatformBusierID: value},
BusinessPromoteResponse{PromoteID: value, PlatformBusierID: value},
ChannelPortraitBasic{PromoteID: value, ParentPromoteID: func() *uint64 { v := value; return &v }()},
RegistrationResponse{ID: value, UserID: value, FirstGameID: &signed, PromoteID: &signed},
SpendResponse{ID: value, UserID: &signed, GameID: &signed, PromoteID: &signed},
ChannelAnalyticsRow{PromoteID: &signed}, GameAnalyticsRow{GameID: &signed},
PerformanceChannel{ID: value}, PortraitRank{PromoteID: value},
} {
raw, err := json.Marshal(response)
if err != nil {
t.Fatal(err)
}
var browser map[string]any
if err := json.Unmarshal(raw, &browser); err != nil {
t.Fatal(err)
}
for _, key := range []string{"id", "userId", "parentId", "busierId", "platformBusierId", "promoteId", "parentPromoteId", "gameId", "fgameId"} {
actual, exists := browser[key]
if !exists || actual == nil || actual == float64(0) || actual == "0" {
continue
}
if actual != "9007199254740993" {
t.Fatalf("%T.%s lost ID: %v", response, key, actual)
}
}
if count, ok := browser["registerCount"]; ok && count != float64(3) {
t.Fatalf("metric changed type: %v", count)
}
}
}
+9 -9
View File
@@ -22,25 +22,25 @@ type LedgerQuery struct {
type RegistrationResponse struct {
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
ID uint64 `json:"id"`
UserID uint64 `json:"userId"`
ID uint64 `json:"id,string"`
UserID uint64 `json:"userId,string"`
RegisterTime *int64 `json:"registerTime"`
FirstGameID *int64 `json:"fgameId"`
FirstGameID *int64 `json:"fgameId,string"`
FirstGameName string `json:"fgameName"`
TotalPayAmount *string `json:"totalPayAmount"`
RegisterIP string `json:"registerIp"`
DeviceName string `json:"deviceName"`
PromoteAccount string `json:"promoteAccount"`
PromoteID *int64 `json:"promoteId"`
PromoteID *int64 `json:"promoteId,string"`
BusierName string `json:"busierName"`
}
type SpendResponse struct {
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
ID uint64 `json:"id"`
UserID *int64 `json:"userId"`
ID uint64 `json:"id,string"`
UserID *int64 `json:"userId,string"`
OrderNumber string `json:"payOrderNumber"`
GameID *int64 `json:"gameId"`
GameID *int64 `json:"gameId,string"`
GameName string `json:"gameName"`
ServerID string `json:"serverId"`
ServerName string `json:"serverName"`
@@ -51,7 +51,7 @@ type SpendResponse struct {
PayStatus *int `json:"payStatus"`
IP string `json:"spendIp"`
PromoteAccount string `json:"promoteAccount"`
PromoteID *int64 `json:"promoteId"`
PromoteID *int64 `json:"promoteId,string"`
BusierName string `json:"busierName"`
}
type RegistrationPage struct {
@@ -73,6 +73,6 @@ type GameOptionQuery struct {
type GameOptionResponse struct {
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
GameID uint64 `json:"gameId"`
GameID uint64 `json:"gameId,string"`
GameName string `json:"gameName"`
}
+1 -1
View File
@@ -138,7 +138,7 @@ type PerformanceChannel struct {
ContributorName string `json:"contributorName"`
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
ID uint64 `json:"id"`
ID uint64 `json:"id,string"`
Account string `json:"account"`
Name string `json:"name"`
OwnerName string `json:"ownerName"`
+1 -1
View File
@@ -25,7 +25,7 @@ type ServerOptionResponse struct {
type ServerAnalysisRow struct {
SourceDB string `json:"sourceDb"`
SourceDBLabel string `json:"sourceDbLabel"`
GameID *int64 `json:"gameId"`
GameID *int64 `json:"gameId,string"`
GameName string `json:"gameName"`
ServerOptionResponse
Register int64 `json:"newRegisterCount"`
+5 -2
View File
@@ -5,10 +5,13 @@ import "strings"
// Normalize removes only a complete configured router prefix. Paths
// such as /administrator must not be shortened when the prefix is /admin.
func Normalize(path, routerPrefix string) string {
prefix := strings.TrimSuffix(strings.TrimSpace(routerPrefix), "/")
if prefix == "" || prefix == "/" {
prefix := strings.Trim(strings.TrimSpace(routerPrefix), "/")
if prefix == "" {
return path
}
// Gin also accepts a relative group name such as "api". Normalize it
// the same way before comparing a request path with Casbin policies.
prefix = "/" + prefix
if path == prefix {
return "/"
}
@@ -11,6 +11,8 @@ func TestNormalize(t *testing.T) {
}{
{name: "empty prefix", path: "/api/login", prefix: "", want: "/api/login"},
{name: "configured prefix", path: "/admin/api/login", prefix: "/admin", want: "/api/login"},
{name: "relative prefix accepted by Gin", path: "/api/sw/promote/list", prefix: "api", want: "/sw/promote/list"},
{name: "relative prefix boundary", path: "/apiary/list", prefix: "api", want: "/apiary/list"},
{name: "trailing slash prefix", path: "/admin/api/login", prefix: "/admin/", want: "/api/login"},
{name: "prefix root", path: "/admin", prefix: "/admin", want: "/"},
{name: "prefix boundary", path: "/administrator/api/login", prefix: "/admin", want: "/administrator/api/login"},
+159
View File
@@ -0,0 +1,159 @@
package migration
import (
"fmt"
"sort"
"strings"
"gorm.io/gorm"
)
// CheckApplied is read-only; a missing or corrupt ledger must not be repaired
// by a normal service startup with automatic migrations disabled.
func CheckApplied(db *gorm.DB, steps []Step) error {
db = db.Session(&gorm.Session{NewDB: true})
var rows []record
if err := db.Model(&record{}).Find(&rows).Error; err != nil {
return fmt.Errorf("read migration versions: %w", err)
}
known := map[string]bool{}
for _, step := range steps {
if step.ID == "" || known[step.ID] {
return fmt.Errorf("invalid or duplicate migration ID %q", step.ID)
}
known[step.ID] = true
}
for _, row := range rows {
if _, ok := known[row.ID]; !ok {
return fmt.Errorf("unknown applied migration %q", row.ID)
}
known[row.ID] = false
}
var pending []string
for id, missing := range known {
if missing {
pending = append(pending, id)
}
}
sort.Strings(pending)
if len(pending) > 0 {
return fmt.Errorf("pending migrations: %s", strings.Join(pending, ", "))
}
return nil
}
// CheckModels verifies persistent columns and declared indexes without
// changing data or schema. Extra legacy columns and indexes are retained.
func CheckModels(db *gorm.DB, models ...any) error {
db = db.Session(&gorm.Session{NewDB: true})
var problems []string
for _, model := range models {
stmt := &gorm.Statement{DB: db}
if err := stmt.Parse(model); err != nil {
return err
}
table := stmt.Schema.Table
if !db.Migrator().HasTable(model) {
problems = append(problems, "missing table "+table)
continue
}
columns, err := db.Migrator().ColumnTypes(model)
if err != nil {
return err
}
found := map[string]bool{}
for _, column := range columns {
found[column.Name()] = true
}
for _, name := range stmt.Schema.DBNames {
if !stmt.Schema.FieldsByDBName[name].IgnoreMigration && !found[name] {
problems = append(problems, "missing column "+table+"."+name)
}
}
indexes, err := readIndexes(db, model, table)
if err != nil {
return err
}
if db.Dialector.Name() == "mysql" && len(stmt.Schema.PrimaryFields) > 0 {
var wanted []string
for _, field := range stmt.Schema.PrimaryFields {
wanted = append(wanted, field.DBName)
}
matched := false
for _, actual := range indexes {
if actual.primary && strings.Join(actual.columns, ",") == strings.Join(wanted, ",") {
matched = true
}
}
if !matched {
problems = append(problems, "missing or incompatible primary key "+table)
}
}
for _, index := range stmt.Schema.ParseIndexes() {
wanted := make([]string, 0, len(index.Fields))
for _, field := range index.Fields {
wanted = append(wanted, field.DBName)
}
covered := false
for _, actual := range indexes {
if actual.prefix || len(actual.columns) < len(wanted) {
continue
}
if index.Class == "UNIQUE" && (!actual.unique || len(actual.columns) != len(wanted)) {
continue
}
if strings.Join(actual.columns[:len(wanted)], ",") == strings.Join(wanted, ",") {
covered = true
break
}
}
if !covered {
problems = append(problems, "missing index "+table+"."+index.Name)
}
}
}
if len(problems) > 0 {
return fmt.Errorf("%s", strings.Join(problems, "; "))
}
return nil
}
type schemaIndex struct {
columns []string
unique, prefix, primary bool
}
func readIndexes(db *gorm.DB, model any, table string) ([]schemaIndex, error) {
if db.Dialector.Name() == "mysql" {
var rows []struct {
IndexName, ColumnName string
NonUnique int
SubPart *int
}
if err := db.Raw("SELECT index_name,column_name,non_unique,sub_part FROM information_schema.statistics WHERE table_schema=DATABASE() AND table_name=? ORDER BY index_name,seq_in_index", table).Scan(&rows).Error; err != nil {
return nil, err
}
var indexes []schemaIndex
previous := ""
for _, row := range rows {
if row.IndexName != previous {
indexes = append(indexes, schemaIndex{unique: row.NonUnique == 0, primary: row.IndexName == "PRIMARY"})
previous = row.IndexName
}
index := &indexes[len(indexes)-1]
index.columns = append(index.columns, row.ColumnName)
index.prefix = index.prefix || row.SubPart != nil
}
return indexes, nil
}
rows, err := db.Migrator().GetIndexes(model)
if err != nil {
return nil, err
}
indexes := make([]schemaIndex, 0, len(rows))
for _, row := range rows {
unique, _ := row.Unique()
indexes = append(indexes, schemaIndex{columns: row.Columns(), unique: unique})
}
return indexes, nil
}
+109
View File
@@ -0,0 +1,109 @@
package migration
import (
"strings"
"testing"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
type checkedSchema struct {
ID uint `gorm:"primaryKey"`
Owner uint `gorm:"index:idx_checked_owner_state,priority:1"`
State int `gorm:"index:idx_checked_owner_state,priority:2"`
Code string `gorm:"uniqueIndex"`
}
func TestSchemaCheckRejectsMissingColumnsAndIncorrectIndexes(t *testing.T) {
db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
if err := CheckModels(db, &checkedSchema{}); err == nil {
t.Fatal("missing table passed")
}
if err := db.Migrator().CreateTable(&checkedSchema{}); err != nil {
t.Fatal(err)
}
if err := CheckModels(db, &checkedSchema{}); err != nil {
t.Fatal(err)
}
for _, statement := range []string{
"DROP INDEX idx_checked_owner_state",
"CREATE INDEX idx_checked_owner_state ON checked_schemas(state,owner)",
} {
if err := db.Exec(statement).Error; err != nil {
t.Fatal(err)
}
}
if err := CheckModels(db, &checkedSchema{}); err == nil || !strings.Contains(err.Error(), "idx_checked_owner_state") {
t.Fatalf("reversed index passed: %v", err)
}
if err := db.Exec("CREATE INDEX legacy_owner_state ON checked_schemas(owner,state,id)").Error; err != nil {
t.Fatal(err)
}
if err := CheckModels(db, &checkedSchema{}); err != nil {
t.Fatalf("equivalent covering index rejected: %v", err)
}
if err := db.Exec("DROP INDEX idx_checked_schemas_code").Error; err != nil {
t.Fatal(err)
}
if err := db.Exec("CREATE INDEX idx_checked_schemas_code ON checked_schemas(code)").Error; err != nil {
t.Fatal(err)
}
if err := CheckModels(db, &checkedSchema{}); err == nil {
t.Fatal("non-unique replacement passed")
}
if err := db.Migrator().DropColumn(&checkedSchema{}, "Owner"); err != nil {
t.Fatal(err)
}
if err := CheckModels(db, &checkedSchema{}); err == nil || !strings.Contains(err.Error(), "missing column") {
t.Fatalf("missing column passed: %v", err)
}
}
func TestCheckAppliedDoesNotCreateOrApplyMigrations(t *testing.T) {
db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
if err != nil {
t.Fatal(err)
}
called := false
steps := []Step{{ID: "required", Migrate: func(*gorm.DB) error { called = true; return nil }}}
if err := CheckApplied(db, steps); err == nil {
t.Fatal("missing ledger passed")
}
if db.Migrator().HasTable(TableName) || called {
t.Fatal("read-only check mutated the database")
}
if err := db.Migrator().CreateTable(&record{}); err != nil {
t.Fatal(err)
}
if err := CheckApplied(db, steps); err == nil || !strings.Contains(err.Error(), "required") {
t.Fatalf("pending migration passed: %v", err)
}
if err := db.Create(&record{ID: "required"}).Error; err != nil {
t.Fatal(err)
}
// openWithDriverConfig returns a settings-bearing handle for MySQL's
// table options. Read-only checks must not retain their Model on it.
configured := db.Set("gorm:table_options", "ENGINE=InnoDB")
if err := CheckApplied(configured, steps); err != nil {
t.Fatal(err)
}
if configured.Statement.Model != nil {
t.Fatal("schema check leaked the migration model into the shared handle")
}
if err := CheckApplied(db, steps); err != nil {
t.Fatal(err)
}
if err := db.Create(&record{ID: "unknown"}).Error; err != nil {
t.Fatal(err)
}
if err := CheckApplied(db, steps); err == nil {
t.Fatal("unknown version passed")
}
if called {
t.Fatal("read-only check ran a migration")
}
}
+30 -4
View File
@@ -4,7 +4,9 @@
package module
import (
"fmt"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"kra/pkg/database/migration"
)
@@ -28,16 +30,40 @@ type TimedTask struct {
}
type Definition struct {
Name string
Migrations []migration.Step
Surface Surface
TimedTasks []TimedTask
Name string
Migrations []migration.Step
Surface Surface
TimedTasks []TimedTask
Prerequisites func(*gorm.DB) error
ValidateSchema func(*gorm.DB) error
}
type Catalog struct {
Definitions []Definition
}
func (c Catalog) CheckPrerequisites(db *gorm.DB) error {
for _, item := range c.Definitions {
if item.Prerequisites != nil {
if err := item.Prerequisites(db); err != nil {
return fmt.Errorf("%s prerequisites: %w", item.Name, err)
}
}
}
return nil
}
func (c Catalog) CheckSchema(db *gorm.DB) error {
for _, item := range c.Definitions {
if item.ValidateSchema != nil {
if err := item.ValidateSchema(db); err != nil {
return fmt.Errorf("%s: %w", item.Name, err)
}
}
}
return nil
}
func (c Catalog) MigrationSteps() []migration.Step {
var steps []migration.Step
for _, item := range c.Definitions {
@@ -19,7 +19,7 @@ import { createLatestRequest } from '@/utils/latestRequest'
import { listBusinessChannels } from '@/api/business'
defineOptions({ name: 'SwBusinessChannels' })
const props = defineProps({ userId: { type: Number, required: true }, sourceDb: { type: String, default: '' }, platformBusierId: { type: Number, default: 0 } })
const props = defineProps({ userId: { type: Number, required: true }, sourceDb: { type: String, default: '' }, platformBusierId: { type: [String, Number], default: 0 } })
const page = ref(1)
const rows = ref([])
const total = ref(0)
@@ -62,7 +62,7 @@ import { channelCount, channelMoney } from '../channel-display'
import { alertStatuses } from '../alert-display'
import BusinessTrend from './BusinessTrend.vue'
import BusinessChannels from './BusinessChannels.vue'
const props = defineProps({ modelValue: Boolean, userId: { type: Number, default: 0 }, sourceDb: { type: String, default: '' }, platformBusierId: { type: Number, default: 0 }, name: { type: String, default: '' } })
const props = defineProps({ modelValue: Boolean, userId: { type: Number, default: 0 }, sourceDb: { type: String, default: '' }, platformBusierId: { type: [String, Number], default: 0 }, name: { type: String, default: '' } })
const emit = defineEmits(['update:modelValue'])
const visible = computed({ get: () => props.modelValue, set: value => emit('update:modelValue', value) })
const periods = [{ label: '今日', value: 'today' }, { label: '昨日', value: 'yesterday' }, { label: '近7天', value: 'last7' }, { label: '近30天', value: 'last30' }, { label: '本月', value: 'thismonth' }]