fix: 完成商务迁移复核与接口优化并清理历史资料
This commit is contained in:
Yvan
@@ -0,0 +1,119 @@
|
||||
"""备份/比较/恢复 sw8 与 sw9 指定自然日的四张报表，不备份或改写原始业务流水。"""
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from sw_database_rehearsal import client_options, connect, query, write_json
|
||||
|
||||
TABLES = ("tab_sw_report_day", "tab_sw_report_day_key", "tab_sw_report_server_day", "tab_sw_report_server_day_key")
|
||||
|
||||
|
||||
def snapshot(conn, database, day, logical=False):
|
||||
result = {}
|
||||
for table in TABLES:
|
||||
rows = query(conn, f"SELECT * FROM `{database}`.`{table}` WHERE stat_date=%s ORDER BY id", (day,))
|
||||
if logical:
|
||||
rows = [{key: value for key, value in row.items() if key not in ("id", "created_at", "updated_at")} for row in rows]
|
||||
serialized = [json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) for row in rows]
|
||||
if logical:
|
||||
serialized.sort()
|
||||
result[table] = {"rows": len(rows), "sha256": hashlib.sha256("\n".join(serialized).encode()).hexdigest()}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("action", choices=("backup", "verify", "restore"))
|
||||
parser.add_argument("--config", type=Path, default=Path("configs/config.yaml"))
|
||||
parser.add_argument("--date", default="2026-11-03")
|
||||
parser.add_argument("--state", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args()
|
||||
dt.date.fromisoformat(args.date)
|
||||
args.state = args.state.resolve()
|
||||
if args.state.is_relative_to(Path(__file__).resolve().parents[1]):
|
||||
raise ValueError("报表备份和身份键必须保存在仓库外")
|
||||
args.state.mkdir(parents=True, exist_ok=True)
|
||||
config = yaml.safe_load(args.config.read_text(encoding="utf-8"))["data"]["database"]
|
||||
manifest_path = args.state / "report-baseline.json"
|
||||
if args.action == "backup":
|
||||
if manifest_path.exists():
|
||||
raise ValueError("已有基线备份，禁止覆盖")
|
||||
manifest = {"date": args.date, "databases": {}}
|
||||
with connect(config) as conn:
|
||||
manifest["server"] = query(conn, "SELECT @@hostname hostname,@@version version")[0]
|
||||
for database in ("sw8", "sw9"):
|
||||
baseline = snapshot(conn, database, args.date)
|
||||
options = client_options(config, args.state)
|
||||
file = args.state / (database + "-report-partition.sql.gz")
|
||||
try:
|
||||
with (args.state / (database + "-dump.stderr.log")).open("wb") as log:
|
||||
process = subprocess.Popen([shutil.which("mysqldump"), "--defaults-extra-file=" + str(options), "--single-transaction", "--quick", "--hex-blob",
|
||||
"--no-create-info", "--skip-triggers", "--skip-add-locks", "--skip-disable-keys", "--complete-insert", "--set-gtid-purged=OFF",
|
||||
"--where=stat_date='" + args.date + "'", database, *TABLES], stdout=subprocess.PIPE, stderr=log)
|
||||
with gzip.open(file, "wb") as target:
|
||||
shutil.copyfileobj(process.stdout, target)
|
||||
if process.wait() != 0:
|
||||
raise RuntimeError("报表备份失败")
|
||||
with gzip.open(file, "rb") as stream:
|
||||
while stream.read(1024 * 1024):
|
||||
pass
|
||||
manifest["databases"][database] = {"tables": baseline, "backup": str(file), "sha256": hashlib.sha256(file.read_bytes()).hexdigest()}
|
||||
finally:
|
||||
options.unlink(missing_ok=True)
|
||||
write_json(manifest_path, manifest)
|
||||
print(json.dumps({"date": args.date, "tables": {db: value["tables"] for db, value in manifest["databases"].items()}}, ensure_ascii=False), flush=True)
|
||||
return
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if manifest["date"] != args.date:
|
||||
raise ValueError("日期与备份不符")
|
||||
with connect(config) as conn:
|
||||
if query(conn, "SELECT @@hostname hostname,@@version version")[0] != manifest["server"]:
|
||||
raise ValueError("数据库服务器与备份不符")
|
||||
if args.action == "restore":
|
||||
for database, value in manifest["databases"].items():
|
||||
if database not in ("sw8", "sw9"):
|
||||
raise ValueError("未知数据库")
|
||||
file = Path(value["backup"])
|
||||
if hashlib.sha256(file.read_bytes()).hexdigest() != value["sha256"]:
|
||||
raise ValueError("备份校验失败")
|
||||
content = gzip.decompress(file.read_bytes())
|
||||
if b"LOCK TABLES" in content or b"ALTER TABLE" in content or b"CREATE TABLE" in content:
|
||||
raise ValueError("分区恢复文件不允许包含 DDL 或隐式提交")
|
||||
deletes = "\n".join(f"DELETE FROM `{table}` WHERE stat_date='{args.date}';" for table in TABLES)
|
||||
script = ("START TRANSACTION;\n" + deletes + "\n").encode() + content + b"\nCOMMIT;\n"
|
||||
options = client_options(config, args.state)
|
||||
try:
|
||||
completed = subprocess.run([shutil.which("mysql"), "--defaults-extra-file=" + str(options), "--batch", database], input=script, capture_output=True)
|
||||
(args.state / (database + "-restore.stderr.log")).write_bytes(completed.stderr)
|
||||
if completed.returncode:
|
||||
raise RuntimeError("分区恢复失败")
|
||||
finally:
|
||||
options.unlink(missing_ok=True)
|
||||
with connect(config) as conn:
|
||||
checks = [{"database": db, "passed": snapshot(conn, db, args.date) == value["tables"]} for db, value in manifest["databases"].items()]
|
||||
else:
|
||||
with connect(config) as conn:
|
||||
source, target = [snapshot(conn, db, args.date, logical=True) for db in ("sw8", "sw9")]
|
||||
checks = [{"table": table, "source": source[table], "target": target[table], "passed": source[table] == target[table]} for table in TABLES]
|
||||
for db in ("sw8", "sw9"):
|
||||
for table in TABLES:
|
||||
rows = query(conn, f"SELECT * FROM `{db}`.`{table}` WHERE stat_date=%s ORDER BY id", (args.date,))
|
||||
write_json(args.state / (db + "-" + table + "-rebuilt.json"), rows)
|
||||
result = {"action": args.action, "date": args.date, "checks": checks, "passed": all(item["passed"] for item in checks)}
|
||||
write_json(args.output or args.state / (args.action + ".json"), result)
|
||||
print(json.dumps(result, ensure_ascii=False), flush=True)
|
||||
if not result["passed"]:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user