120 lines
6.6 KiB
Python
120 lines
6.6 KiB
Python
"""备份/比较/恢复 sw8 与 sw9 指定自然日的四张报表，不备份或改写原始业务流水。"""
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
from sw_database_rehearsal import client_options, connect, query, write_json
|
|
|
|
TABLES = ("tab_sw_report_day", "tab_sw_report_day_key", "tab_sw_report_server_day", "tab_sw_report_server_day_key")
|
|
|
|
|
|
def snapshot(conn, database, day, logical=False):
|
|
result = {}
|
|
for table in TABLES:
|
|
rows = query(conn, f"SELECT * FROM `{database}`.`{table}` WHERE stat_date=%s ORDER BY id", (day,))
|
|
if logical:
|
|
rows = [{key: value for key, value in row.items() if key not in ("id", "created_at", "updated_at")} for row in rows]
|
|
serialized = [json.dumps(row, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) for row in rows]
|
|
if logical:
|
|
serialized.sort()
|
|
result[table] = {"rows": len(rows), "sha256": hashlib.sha256("\n".join(serialized).encode()).hexdigest()}
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("action", choices=("backup", "verify", "restore"))
|
|
parser.add_argument("--config", type=Path, default=Path("configs/config.yaml"))
|
|
parser.add_argument("--date", default="2026-11-03")
|
|
parser.add_argument("--state", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path)
|
|
args = parser.parse_args()
|
|
dt.date.fromisoformat(args.date)
|
|
args.state = args.state.resolve()
|
|
if args.state.is_relative_to(Path(__file__).resolve().parents[1]):
|
|
raise ValueError("报表备份和身份键必须保存在仓库外")
|
|
args.state.mkdir(parents=True, exist_ok=True)
|
|
config = yaml.safe_load(args.config.read_text(encoding="utf-8"))["data"]["database"]
|
|
manifest_path = args.state / "report-baseline.json"
|
|
if args.action == "backup":
|
|
if manifest_path.exists():
|
|
raise ValueError("已有基线备份，禁止覆盖")
|
|
manifest = {"date": args.date, "databases": {}}
|
|
with connect(config) as conn:
|
|
manifest["server"] = query(conn, "SELECT @@hostname hostname,@@version version")[0]
|
|
for database in ("sw8", "sw9"):
|
|
baseline = snapshot(conn, database, args.date)
|
|
options = client_options(config, args.state)
|
|
file = args.state / (database + "-report-partition.sql.gz")
|
|
try:
|
|
with (args.state / (database + "-dump.stderr.log")).open("wb") as log:
|
|
process = subprocess.Popen([shutil.which("mysqldump"), "--defaults-extra-file=" + str(options), "--single-transaction", "--quick", "--hex-blob",
|
|
"--no-create-info", "--skip-triggers", "--skip-add-locks", "--skip-disable-keys", "--complete-insert", "--set-gtid-purged=OFF",
|
|
"--where=stat_date='" + args.date + "'", database, *TABLES], stdout=subprocess.PIPE, stderr=log)
|
|
with gzip.open(file, "wb") as target:
|
|
shutil.copyfileobj(process.stdout, target)
|
|
if process.wait() != 0:
|
|
raise RuntimeError("报表备份失败")
|
|
with gzip.open(file, "rb") as stream:
|
|
while stream.read(1024 * 1024):
|
|
pass
|
|
manifest["databases"][database] = {"tables": baseline, "backup": str(file), "sha256": hashlib.sha256(file.read_bytes()).hexdigest()}
|
|
finally:
|
|
options.unlink(missing_ok=True)
|
|
write_json(manifest_path, manifest)
|
|
print(json.dumps({"date": args.date, "tables": {db: value["tables"] for db, value in manifest["databases"].items()}}, ensure_ascii=False), flush=True)
|
|
return
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
if manifest["date"] != args.date:
|
|
raise ValueError("日期与备份不符")
|
|
with connect(config) as conn:
|
|
if query(conn, "SELECT @@hostname hostname,@@version version")[0] != manifest["server"]:
|
|
raise ValueError("数据库服务器与备份不符")
|
|
if args.action == "restore":
|
|
for database, value in manifest["databases"].items():
|
|
if database not in ("sw8", "sw9"):
|
|
raise ValueError("未知数据库")
|
|
file = Path(value["backup"])
|
|
if hashlib.sha256(file.read_bytes()).hexdigest() != value["sha256"]:
|
|
raise ValueError("备份校验失败")
|
|
content = gzip.decompress(file.read_bytes())
|
|
if b"LOCK TABLES" in content or b"ALTER TABLE" in content or b"CREATE TABLE" in content:
|
|
raise ValueError("分区恢复文件不允许包含 DDL 或隐式提交")
|
|
deletes = "\n".join(f"DELETE FROM `{table}` WHERE stat_date='{args.date}';" for table in TABLES)
|
|
script = ("START TRANSACTION;\n" + deletes + "\n").encode() + content + b"\nCOMMIT;\n"
|
|
options = client_options(config, args.state)
|
|
try:
|
|
completed = subprocess.run([shutil.which("mysql"), "--defaults-extra-file=" + str(options), "--batch", database], input=script, capture_output=True)
|
|
(args.state / (database + "-restore.stderr.log")).write_bytes(completed.stderr)
|
|
if completed.returncode:
|
|
raise RuntimeError("分区恢复失败")
|
|
finally:
|
|
options.unlink(missing_ok=True)
|
|
with connect(config) as conn:
|
|
checks = [{"database": db, "passed": snapshot(conn, db, args.date) == value["tables"]} for db, value in manifest["databases"].items()]
|
|
else:
|
|
with connect(config) as conn:
|
|
source, target = [snapshot(conn, db, args.date, logical=True) for db in ("sw8", "sw9")]
|
|
checks = [{"table": table, "source": source[table], "target": target[table], "passed": source[table] == target[table]} for table in TABLES]
|
|
for db in ("sw8", "sw9"):
|
|
for table in TABLES:
|
|
rows = query(conn, f"SELECT * FROM `{db}`.`{table}` WHERE stat_date=%s ORDER BY id", (args.date,))
|
|
write_json(args.state / (db + "-" + table + "-rebuilt.json"), rows)
|
|
result = {"action": args.action, "date": args.date, "checks": checks, "passed": all(item["passed"] for item in checks)}
|
|
write_json(args.output or args.state / (args.action + ".json"), result)
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
if not result["passed"]:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|