"""请求隔离的新旧商务实例；原始响应和令牌留在仓库外，报告只记录校验结果。""" import argparse import concurrent.futures import decimal import hashlib import json import re import time import urllib.error import urllib.parse import urllib.request from pathlib import Path def fetch(base, token, path, params, legacy, client_id="", timeout=120): url = base.rstrip("/") + path if params: url += "?" + urllib.parse.urlencode(params, doseq=True) headers = {"Authorization": "Bearer " + token} if legacy else {"x-token": token} if legacy: headers["clientid"] = client_id request = urllib.request.Request(url, headers=headers) started = time.monotonic() try: with urllib.request.urlopen(request, timeout=timeout) as response: body = json.load(response) return {"http": response.status, "elapsed_ms": round((time.monotonic() - started) * 1000), "body": body} except (urllib.error.URLError, TimeoutError, ValueError) as error: return {"http": None, "elapsed_ms": round((time.monotonic() - started) * 1000), "error": type(error).__name__} def data_of(reply): body = reply.get("body", {}) if "rows" in body: return {key: value for key, value in body.items() if key not in ("code", "msg")} return body.get("data") def shape(value): if isinstance(value, dict): return {key: (len(item) if isinstance(item, list) else type(item).__name__) for key, item in value.items()} return {"type": type(value).__name__, "length": len(value) if isinstance(value, list) else None} def expected_relative_pay_rate(row): """用户确认的新口径：由旧的本期付费率及百分点差推导相对环比。""" if row.get("momValue") is None: return row current = decimal.Decimal(str(row["metricValue"])) difference = decimal.Decimal(str(row["momValue"])) previous = current - difference result = dict(row) if previous <= 0: result.update(momValue=None, momStatus=None) else: change = (difference / previous * 100).quantize(decimal.Decimal("0.01"), rounding=decimal.ROUND_HALF_UP) result.update(momValue=format(abs(change) if change == 0 else change, ".2f"), momStatus=0 if change == 0 else 1 if change > 0 else -1) return result def compare_data(source, target, case, users): differences = [] omitted = {"source": case.get("source_omit_fields", []), "target": case.get("target_omit_fields", [])} # 两边通过 HTTP 分别创建的专用测试账号，其创建时刻天然不同。 # 只忽略显式列出的夹具账号时间，真实迁移账号的所有时间字段仍须相等。 generated = {str(value) for value in case.get("generated_user_ids", [])} generated_ids = {"source": generated, "target": {users.get(value, value) for value in generated}} user_fields = set(case.get("user_fields", ["userId", "parentId", "creatorId", "businessId", "operatorId", "parentUserId"])) def normalize(value, side, field="", path=""): if isinstance(value, dict): if side == "source" and field == "payRateRankList" and case.get("relative_pay_rate"): value = expected_relative_pay_rate(value) if side == "target" and value.get("available") is False: differences.append({"path": path + ".available", "kind": "unavailable"}) if side == "source" and case.get("normalize_root_path") and value.get("level") in (0, 1, "0", "1") and value.get("idPath") is None: value = {**value, "idPath": "0"} excluded = set(omitted[side]) if str(value.get("userId")) in generated_ids[side]: excluded.add("createTime") return {key: normalize(item, side, key, path + "." + key) for key, item in value.items() if key not in excluded} if isinstance(value, list): rows = [normalize(item, side, field, path) for item in value] keys = case.get("row_key") if keys and rows and all(isinstance(row, dict) and all(key in row for key in keys) for row in rows): rows.sort(key=lambda row: tuple(str(row[key]) for key in keys)) return rows if value is None and field in case.get("null_empty_fields", []): return "" if side == "source" and case.get("map_users", True) and value is not None: if field in user_fields: value = users.get(str(value), str(value)) elif field == "idPath": value = ",".join(users.get(item, item) for item in str(value).split(",")) return value def walk(left, right, path="data"): # 只兼容数值与 JSON 数字字符串；账号、手机号、区服编号等字符串保留前导零。 numeric_field = path.rsplit(".", 1)[-1] in case.get("numeric_fields", []) if not isinstance(left, bool) and not isinstance(right, bool) and (numeric_field or isinstance(left, (int, float)) or isinstance(right, (int, float))): try: if decimal.Decimal(str(left)) == decimal.Decimal(str(right)): return except decimal.InvalidOperation: pass if type(left) is not type(right): differences.append({"path": path, "kind": "type", "source": type(left).__name__, "target": type(right).__name__}) elif isinstance(left, dict): for key in sorted(left.keys() | right.keys()): if key not in left or key not in right: differences.append({"path": path + "." + key, "kind": "missing_field", "side": "source" if key not in left else "target"}) else: walk(left[key], right[key], path + "." + key) elif isinstance(left, list): if len(left) != len(right): differences.append({"path": path, "kind": "length", "source": len(left), "target": len(right)}) for i, (a, b) in enumerate(zip(left, right)): walk(a, b, path + f"[{i}]") elif left != right: # 数据值只留在仓库外；公开报告保留字段路径和摘要供复核。 differences.append({"path": path, "kind": "value", "source_sha256": hashlib.sha256(str(left).encode()).hexdigest(), "target_sha256": hashlib.sha256(str(right).encode()).hexdigest()}) walk(normalize(source, "source"), normalize(target, "target")) return {"semantic_equal": not differences, "difference_count": len(differences), "differences": differences[:30], "normalization": {"user_ids_mapped": case.get("map_users", True), "numeric_representation": True, "user_fields": sorted(user_fields), "numeric_string_fields": case.get("numeric_fields", []), "normalize_root_path": case.get("normalize_root_path", False), "expected_relative_pay_rate": case.get("relative_pay_rate", False), "generated_user_creation_times": len(generated), "row_key": case.get("row_key"), "omitted_fields": omitted, "null_empty_fields": case.get("null_empty_fields", [])}} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--runtime", type=Path, required=True) parser.add_argument("--cases", type=Path, required=True) parser.add_argument("--identity", default="admin") parser.add_argument("--old-url", default="http://127.0.0.1:18080") parser.add_argument("--new-url", default="http://127.0.0.1:19000") parser.add_argument("--output", type=Path, required=True) mode = parser.add_mutually_exclusive_group() mode.add_argument("--recompare", action="store_true", help="使用已保存的同参数 HTTP 响应重新核对，不重复访问数据库") mode.add_argument("--refresh-target", action="store_true", help="保留已采集的同参数旧响应，重新请求优化后的新实现；新增用例仍请求两边") args = parser.parse_args() root = Path(__file__).resolve().parents[1] if args.runtime.resolve().is_relative_to(root): raise ValueError("原始业务响应必须保存在仓库外") tokens = [(args.runtime / (prefix + "-" + args.identity + ".token")).read_text(encoding="utf-8").strip() for prefix in ("legacy", "new")] client_id = json.loads((args.runtime / "legacy-client.json").read_text(encoding="utf-8"))["client_id"] users = {str(row["old_id"]): str(row["new_id"]) for row in json.loads((args.runtime / "id-maps.json").read_text(encoding="utf-8")) if row["entity"] == "user"} cases = json.loads(args.cases.read_text(encoding="utf-8")) reports = [] folder = args.runtime / ("responses-" + args.identity) folder.mkdir(exist_ok=True) with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: for case in cases: capture = folder / (case["name"] + ".json") reused_source = args.refresh_target and capture.exists() if args.recompare or reused_source: saved = json.loads(capture.read_text(encoding="utf-8")) if any(saved["case"].get(key) != case.get(key) for key in ("source", "target", "params", "source_params", "target_params")): raise ValueError("已有 HTTP 响应的请求条件不符：" + case["name"]) replies = [saved["source"], saved["target"]] if args.refresh_target: replies[1] = fetch(args.new_url, tokens[1], case["target"], case.get("target_params", case.get("params", {})), False, timeout=case.get("timeout_seconds", 120)) capture.write_text(json.dumps({"case": case, "source": replies[0], "target": replies[1]}, ensure_ascii=False, indent=2), encoding="utf-8") else: pairs = [pool.submit(fetch, base, tokens[i], case[side], case.get(side + "_params", case.get("params", {})), i == 0, client_id, case.get("timeout_seconds", 120)) for i, (base, side) in enumerate(((args.old_url, "source"), (args.new_url, "target")))] replies = [job.result() for job in pairs] capture.write_text(json.dumps({"case": case, "source": replies[0], "target": replies[1]}, ensure_ascii=False, indent=2), encoding="utf-8") values = [data_of(reply) for reply in replies] for i, side in enumerate(("source", "target")): for key in case.get(side + "_data_path", []): values[i] = values[i][key] codes = [reply.get("body", {}).get("code") for reply in replies] report = {"case": case["name"], "source": case["source"], "target": case["target"], "codes": codes, "source_capture_reused": reused_source, "http_passed": codes == [200, 0], "elapsed_ms": [reply["elapsed_ms"] for reply in replies], "shapes": [shape(value) for value in values], "exact_data_equal": values[0] == values[1]} if all(isinstance(value, dict) and "total" in value for value in values): report["totals"] = [value["total"] for value in values] report["totals_equal"] = str(values[0]["total"]) == str(values[1]["total"]) reports.append(report) report.update(compare_data(values[0], values[1], case, users)) report["passed"] = report["http_passed"] and report["semantic_equal"] if case.get("expected_error_codes"): messages = [reply.get("body", {}).get("msg") for reply in replies] report["rejection_equal"] = codes == case["expected_error_codes"] and messages[0] == messages[1] report["expected_error_codes"] = case["expected_error_codes"] report["passed"] = report["rejection_equal"] print(json.dumps(report, ensure_ascii=False), flush=True) args.output.write_text(json.dumps({"identity": args.identity, "checks": reports}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") if not all(report["passed"] for report in reports): raise SystemExit(1) if __name__ == "__main__": main()