189 lines
10 KiB
Python
189 lines
10 KiB
Python
"""在同一测试服务器核对 SW 源副本、备份目标库并执行迁移；凭据只读取本地配置。"""
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pymysql
|
|
import yaml
|
|
|
|
|
|
def identifier(value):
|
|
if not re.fullmatch(r"[A-Za-z0-9_]+", value):
|
|
raise ValueError("数据库和表名只能包含字母、数字、下划线")
|
|
return "`" + value + "`"
|
|
|
|
|
|
def write_json(path, data):
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2, default=str) + "\n", encoding="utf-8")
|
|
|
|
|
|
def connect(config):
|
|
return pymysql.connect(host=config["host"], port=int(config["port"]), user=config["user"],
|
|
password=config["password"], charset="utf8mb4", connect_timeout=10,
|
|
read_timeout=180, autocommit=True, cursorclass=pymysql.cursors.DictCursor)
|
|
|
|
|
|
def query(conn, sql, args=None):
|
|
with conn.cursor() as cursor:
|
|
cursor.execute(sql, args)
|
|
return cursor.fetchall()
|
|
|
|
|
|
def source_tables(sql):
|
|
tables = re.findall(r"CREATE TABLE `bak_sw2kra_20261103_([^`]+)` LIKE", sql)
|
|
if len(tables) != 26 or len(set(tables)) != 26:
|
|
raise ValueError("迁移源表清单发生变化，请先复核备份范围")
|
|
return tables
|
|
|
|
|
|
def fingerprint(conn, database, table):
|
|
columns = query(conn, "SELECT column_name,data_type FROM information_schema.columns WHERE table_schema=%s AND table_name=%s ORDER BY ordinal_position", (database, table))
|
|
keys = query(conn, "SELECT column_name FROM information_schema.statistics WHERE table_schema=%s AND table_name=%s AND index_name='PRIMARY' ORDER BY seq_in_index", (database, table))
|
|
if not keys:
|
|
raise ValueError("备份核对要求稳定主键：" + table)
|
|
order = ",".join(identifier(item["column_name"]) for item in keys)
|
|
digest = hashlib.sha256()
|
|
rows = query(conn, f"SELECT * FROM {identifier(database)}.{identifier(table)} ORDER BY {order}")
|
|
json_columns = [item["column_name"] for item in columns if item["data_type"] == "json"]
|
|
for row in rows:
|
|
# JSON 的内部二进制布局在导入后可能不同，比较实际业务值而非 CHECKSUM 的存储字节。
|
|
for name in json_columns:
|
|
if row[name] is not None:
|
|
row[name] = json.loads(row[name])
|
|
digest.update(json.dumps(row, ensure_ascii=False, sort_keys=True, default=str, separators=(",", ":")).encode())
|
|
digest.update(b"\n")
|
|
return {"rows": len(rows), "sha256": digest.hexdigest()}
|
|
|
|
|
|
def client_options(config, folder):
|
|
# 使用独立选项文件，避免密码出现在进程命令行或输出中。
|
|
def escape(value):
|
|
return str(value).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n").replace("\r", "\\r")
|
|
handle = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", suffix=".cnf", dir=folder, delete=False)
|
|
with handle:
|
|
handle.write("[client]\n")
|
|
for key in ("host", "port", "user", "password"):
|
|
handle.write(f'{key}="{escape(config[key])}"\n')
|
|
handle.write("default-character-set=utf8mb4\n")
|
|
return Path(handle.name)
|
|
|
|
|
|
def prepare(args, config, sql):
|
|
if (args.state / (args.target + "-migration-source.sql.gz")).exists():
|
|
raise ValueError("目标备份已存在；请使用新的状态目录，保留原始备份")
|
|
tables = source_tables(sql)
|
|
manifest = {"source": args.source, "target": args.target, "created_at": dt.datetime.now(dt.timezone.utc),
|
|
"sql_sha256": hashlib.sha256(sql.encode()).hexdigest(), "checks": []}
|
|
with connect(config) as conn:
|
|
manifest["server"] = query(conn, "SELECT @@hostname hostname,@@version version")[0]
|
|
if query(conn, "SELECT 1 FROM information_schema.tables WHERE table_schema=%s AND table_name='sys_users'", (args.target,)):
|
|
raise ValueError("目标已迁移，不能覆盖原始基线")
|
|
for table in tables:
|
|
checksums = query(conn, f"CHECKSUM TABLE {identifier(args.source)}.{identifier(table)}, {identifier(args.target)}.{identifier(table)} EXTENDED")
|
|
values = [row["Checksum"] for row in checksums]
|
|
fingerprints = [fingerprint(conn, db, table) for db in (args.source, args.target)]
|
|
item = {"table": table, "rows": [item["rows"] for item in fingerprints], "checksums": values,
|
|
"fingerprints": fingerprints, "passed": fingerprints[0] == fingerprints[1]}
|
|
manifest["checks"].append(item)
|
|
print(json.dumps(item, ensure_ascii=False), flush=True)
|
|
if not all(item["passed"] for item in manifest["checks"]):
|
|
write_json(args.state / "baseline.json", manifest)
|
|
raise RuntimeError("源库与目标库框架数据不一致，未执行迁移")
|
|
manifest["tables_before"] = query(conn, "SELECT table_name,engine FROM information_schema.tables WHERE table_schema=%s ORDER BY table_name", (args.target,))
|
|
schemas = {}
|
|
for row in manifest["tables_before"]:
|
|
table = row["table_name"]
|
|
schemas[table] = query(conn, f"SHOW CREATE TABLE {identifier(args.target)}.{identifier(table)}")[0]["Create Table"]
|
|
write_json(args.state / "schema-before.json", schemas)
|
|
dump = Path(shutil.which("mysqldump") or Path(shutil.which("mysql")).with_name("mysqldump.exe"))
|
|
options = client_options(config, args.state)
|
|
backup = args.state / (args.target + "-migration-source.sql.gz")
|
|
try:
|
|
command = [str(dump), "--defaults-extra-file=" + str(options), "--single-transaction", "--quick", "--hex-blob",
|
|
"--set-gtid-purged=OFF", "--skip-lock-tables", args.target, *tables]
|
|
with (args.state / "backup.stderr.log").open("wb") as errors:
|
|
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=errors)
|
|
with gzip.open(backup, "wb") as output:
|
|
shutil.copyfileobj(process.stdout, output)
|
|
if process.wait() != 0:
|
|
raise RuntimeError("备份失败，详情仅保存在本地日志")
|
|
# 完整解压检查，避免把未写完的压缩文件当作成功备份。
|
|
with gzip.open(backup, "rb") as stream:
|
|
while stream.read(1024 * 1024):
|
|
pass
|
|
manifest["backup"] = {"path": str(backup.resolve()), "bytes": backup.stat().st_size,
|
|
"sha256": hashlib.sha256(backup.read_bytes()).hexdigest()}
|
|
write_json(args.state / "baseline.json", manifest)
|
|
print(json.dumps({"backup": manifest["backup"], "passed": True}), flush=True)
|
|
finally:
|
|
options.unlink(missing_ok=True)
|
|
|
|
|
|
def migrate(args, config, sql):
|
|
manifest = json.loads((args.state / "baseline.json").read_text(encoding="utf-8"))
|
|
if (manifest["source"], manifest["target"]) != (args.source, args.target):
|
|
raise ValueError("备份对应的数据库与本次选择不符")
|
|
backup = Path(manifest["backup"]["path"])
|
|
if hashlib.sha256(backup.read_bytes()).hexdigest() != manifest["backup"]["sha256"]:
|
|
raise ValueError("备份文件校验失败")
|
|
statements = re.findall(r"(?m)^USE `[^`]+`;", sql)
|
|
if len(statements) != 1:
|
|
raise ValueError("SQL 必须只有一个显式 USE 入口")
|
|
prepared = sql.replace(statements[0], "USE " + identifier(args.target) + ";")
|
|
(args.state / "executed-migration.sql").write_text(prepared, encoding="utf-8")
|
|
with connect(config) as conn:
|
|
actual_server = query(conn, "SELECT @@hostname hostname,@@version version")[0]
|
|
if actual_server != manifest["server"]:
|
|
raise ValueError("连接服务器与备份时不同")
|
|
# 仅首次执行需要核对原始基线；重复执行由迁移 SQL 的完成账本保护。
|
|
migrated = query(conn, "SELECT 1 FROM information_schema.tables WHERE table_schema=%s AND table_name='sys_users'", (args.target,))
|
|
if not migrated:
|
|
for item in manifest["checks"]:
|
|
if fingerprint(conn, args.target, item["table"]) != item["fingerprints"][1]:
|
|
raise ValueError("目标数据在备份后变化：" + item["table"])
|
|
options = client_options(config, args.state)
|
|
try:
|
|
with (args.state / "migration.stdout.log").open("wb") as output, (args.state / "migration.stderr.log").open("wb") as errors:
|
|
result = subprocess.run([shutil.which("mysql"), "--defaults-extra-file=" + str(options), "--batch", "--binary-mode"],
|
|
input=prepared.encode("utf-8"), stdout=output, stderr=errors)
|
|
print(json.dumps({"target": args.target, "exit_code": result.returncode,
|
|
"sql_sha256": hashlib.sha256(prepared.encode()).hexdigest()}), flush=True)
|
|
if result.returncode:
|
|
raise RuntimeError("迁移 SQL 失败，请检查本地日志；不会自动覆盖部分迁移状态")
|
|
finally:
|
|
options.unlink(missing_ok=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("action", choices=("prepare", "migrate"))
|
|
parser.add_argument("--config", type=Path, default=Path("configs/config.yaml"))
|
|
parser.add_argument("--sql", type=Path, default=Path("docs/sw6-to-kra/sw_to_kra_migrate.sql"))
|
|
parser.add_argument("--source", default="sw8")
|
|
parser.add_argument("--target", default="sw9")
|
|
parser.add_argument("--state", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
identifier(args.source)
|
|
identifier(args.target)
|
|
if args.source == args.target or args.target in ("sw", "sw6"):
|
|
raise ValueError("演练目标必须为独立测试副本")
|
|
args.state = args.state.resolve()
|
|
if args.state.is_relative_to(Path(__file__).resolve().parents[1]):
|
|
raise ValueError("含账号数据的备份必须保存在仓库外")
|
|
args.state.mkdir(parents=True, exist_ok=True)
|
|
config = yaml.safe_load(args.config.read_text(encoding="utf-8"))["data"]["database"]
|
|
sql = args.sql.read_text(encoding="utf-8")
|
|
{"prepare": prepare, "migrate": migrate}[args.action](args, config, sql)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|