202 lines
12 KiB
Python
202 lines
12 KiB
Python
"""请求隔离的新旧商务实例；原始响应和令牌留在仓库外，报告只记录校验结果。"""
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import decimal
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
|
|
def fetch(base, token, path, params, legacy, client_id="", timeout=120):
|
|
url = base.rstrip("/") + path
|
|
if params:
|
|
url += "?" + urllib.parse.urlencode(params, doseq=True)
|
|
headers = {"Authorization": "Bearer " + token} if legacy else {"x-token": token}
|
|
if legacy:
|
|
headers["clientid"] = client_id
|
|
request = urllib.request.Request(url, headers=headers)
|
|
started = time.monotonic()
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
body = json.load(response)
|
|
return {"http": response.status, "elapsed_ms": round((time.monotonic() - started) * 1000), "body": body}
|
|
except (urllib.error.URLError, TimeoutError, ValueError) as error:
|
|
return {"http": None, "elapsed_ms": round((time.monotonic() - started) * 1000), "error": type(error).__name__}
|
|
|
|
|
|
def data_of(reply):
|
|
body = reply.get("body", {})
|
|
if "rows" in body:
|
|
return {key: value for key, value in body.items() if key not in ("code", "msg")}
|
|
return body.get("data")
|
|
|
|
|
|
def shape(value):
|
|
if isinstance(value, dict):
|
|
return {key: (len(item) if isinstance(item, list) else type(item).__name__) for key, item in value.items()}
|
|
return {"type": type(value).__name__, "length": len(value) if isinstance(value, list) else None}
|
|
|
|
|
|
def expected_relative_pay_rate(row):
|
|
"""用户确认的新口径：由旧的本期付费率及百分点差推导相对环比。"""
|
|
if row.get("momValue") is None:
|
|
return row
|
|
current = decimal.Decimal(str(row["metricValue"]))
|
|
difference = decimal.Decimal(str(row["momValue"]))
|
|
previous = current - difference
|
|
result = dict(row)
|
|
if previous <= 0:
|
|
result.update(momValue=None, momStatus=None)
|
|
else:
|
|
change = (difference / previous * 100).quantize(decimal.Decimal("0.01"), rounding=decimal.ROUND_HALF_UP)
|
|
result.update(momValue=format(abs(change) if change == 0 else change, ".2f"), momStatus=0 if change == 0 else 1 if change > 0 else -1)
|
|
return result
|
|
|
|
|
|
def compare_data(source, target, case, users):
|
|
differences = []
|
|
omitted = {"source": case.get("source_omit_fields", []), "target": case.get("target_omit_fields", [])}
|
|
# 两边通过 HTTP 分别创建的专用测试账号，其创建时刻天然不同。
|
|
# 只忽略显式列出的夹具账号时间，真实迁移账号的所有时间字段仍须相等。
|
|
generated = {str(value) for value in case.get("generated_user_ids", [])}
|
|
generated_ids = {"source": generated, "target": {users.get(value, value) for value in generated}}
|
|
user_fields = set(case.get("user_fields", ["userId", "parentId", "creatorId", "businessId", "operatorId", "parentUserId"]))
|
|
|
|
def normalize(value, side, field="", path=""):
|
|
if isinstance(value, dict):
|
|
if side == "source" and field == "payRateRankList" and case.get("relative_pay_rate"):
|
|
value = expected_relative_pay_rate(value)
|
|
if side == "target" and value.get("available") is False:
|
|
differences.append({"path": path + ".available", "kind": "unavailable"})
|
|
if side == "source" and case.get("normalize_root_path") and value.get("level") in (0, 1, "0", "1") and value.get("idPath") is None:
|
|
value = {**value, "idPath": "0"}
|
|
excluded = set(omitted[side])
|
|
if str(value.get("userId")) in generated_ids[side]:
|
|
excluded.add("createTime")
|
|
return {key: normalize(item, side, key, path + "." + key) for key, item in value.items() if key not in excluded}
|
|
if isinstance(value, list):
|
|
rows = [normalize(item, side, field, path) for item in value]
|
|
keys = case.get("row_key")
|
|
if keys and rows and all(isinstance(row, dict) and all(key in row for key in keys) for row in rows):
|
|
rows.sort(key=lambda row: tuple(str(row[key]) for key in keys))
|
|
return rows
|
|
if value is None and field in case.get("null_empty_fields", []):
|
|
return ""
|
|
if side == "source" and case.get("map_users", True) and value is not None:
|
|
if field in user_fields:
|
|
value = users.get(str(value), str(value))
|
|
elif field == "idPath":
|
|
value = ",".join(users.get(item, item) for item in str(value).split(","))
|
|
return value
|
|
|
|
def walk(left, right, path="data"):
|
|
# 只兼容数值与 JSON 数字字符串；账号、手机号、区服编号等字符串保留前导零。
|
|
numeric_field = path.rsplit(".", 1)[-1] in case.get("numeric_fields", [])
|
|
if not isinstance(left, bool) and not isinstance(right, bool) and (numeric_field or isinstance(left, (int, float)) or isinstance(right, (int, float))):
|
|
try:
|
|
if decimal.Decimal(str(left)) == decimal.Decimal(str(right)):
|
|
return
|
|
except decimal.InvalidOperation:
|
|
pass
|
|
if type(left) is not type(right):
|
|
differences.append({"path": path, "kind": "type", "source": type(left).__name__, "target": type(right).__name__})
|
|
elif isinstance(left, dict):
|
|
for key in sorted(left.keys() | right.keys()):
|
|
if key not in left or key not in right:
|
|
differences.append({"path": path + "." + key, "kind": "missing_field", "side": "source" if key not in left else "target"})
|
|
else:
|
|
walk(left[key], right[key], path + "." + key)
|
|
elif isinstance(left, list):
|
|
if len(left) != len(right):
|
|
differences.append({"path": path, "kind": "length", "source": len(left), "target": len(right)})
|
|
for i, (a, b) in enumerate(zip(left, right)):
|
|
walk(a, b, path + f"[{i}]")
|
|
elif left != right:
|
|
# 数据值只留在仓库外；公开报告保留字段路径和摘要供复核。
|
|
differences.append({"path": path, "kind": "value", "source_sha256": hashlib.sha256(str(left).encode()).hexdigest(), "target_sha256": hashlib.sha256(str(right).encode()).hexdigest()})
|
|
|
|
walk(normalize(source, "source"), normalize(target, "target"))
|
|
return {"semantic_equal": not differences, "difference_count": len(differences), "differences": differences[:30],
|
|
"normalization": {"user_ids_mapped": case.get("map_users", True), "numeric_representation": True,
|
|
"user_fields": sorted(user_fields), "numeric_string_fields": case.get("numeric_fields", []),
|
|
"normalize_root_path": case.get("normalize_root_path", False),
|
|
"expected_relative_pay_rate": case.get("relative_pay_rate", False),
|
|
"generated_user_creation_times": len(generated),
|
|
"row_key": case.get("row_key"), "omitted_fields": omitted, "null_empty_fields": case.get("null_empty_fields", [])}}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--runtime", type=Path, required=True)
|
|
parser.add_argument("--cases", type=Path, required=True)
|
|
parser.add_argument("--identity", default="admin")
|
|
parser.add_argument("--old-url", default="http://127.0.0.1:18080")
|
|
parser.add_argument("--new-url", default="http://127.0.0.1:19000")
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
mode = parser.add_mutually_exclusive_group()
|
|
mode.add_argument("--recompare", action="store_true", help="使用已保存的同参数 HTTP 响应重新核对，不重复访问数据库")
|
|
mode.add_argument("--refresh-target", action="store_true", help="保留已采集的同参数旧响应，重新请求优化后的新实现；新增用例仍请求两边")
|
|
args = parser.parse_args()
|
|
root = Path(__file__).resolve().parents[1]
|
|
if args.runtime.resolve().is_relative_to(root):
|
|
raise ValueError("原始业务响应必须保存在仓库外")
|
|
tokens = [(args.runtime / (prefix + "-" + args.identity + ".token")).read_text(encoding="utf-8").strip() for prefix in ("legacy", "new")]
|
|
client_id = json.loads((args.runtime / "legacy-client.json").read_text(encoding="utf-8"))["client_id"]
|
|
users = {str(row["old_id"]): str(row["new_id"]) for row in json.loads((args.runtime / "id-maps.json").read_text(encoding="utf-8")) if row["entity"] == "user"}
|
|
cases = json.loads(args.cases.read_text(encoding="utf-8"))
|
|
reports = []
|
|
folder = args.runtime / ("responses-" + args.identity)
|
|
folder.mkdir(exist_ok=True)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
|
for case in cases:
|
|
capture = folder / (case["name"] + ".json")
|
|
reused_source = args.refresh_target and capture.exists()
|
|
if args.recompare or reused_source:
|
|
saved = json.loads(capture.read_text(encoding="utf-8"))
|
|
if any(saved["case"].get(key) != case.get(key) for key in ("source", "target", "params", "source_params", "target_params")):
|
|
raise ValueError("已有 HTTP 响应的请求条件不符：" + case["name"])
|
|
replies = [saved["source"], saved["target"]]
|
|
if args.refresh_target:
|
|
replies[1] = fetch(args.new_url, tokens[1], case["target"], case.get("target_params", case.get("params", {})), False, timeout=case.get("timeout_seconds", 120))
|
|
capture.write_text(json.dumps({"case": case, "source": replies[0], "target": replies[1]}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
else:
|
|
pairs = [pool.submit(fetch, base, tokens[i], case[side], case.get(side + "_params", case.get("params", {})), i == 0, client_id, case.get("timeout_seconds", 120))
|
|
for i, (base, side) in enumerate(((args.old_url, "source"), (args.new_url, "target")))]
|
|
replies = [job.result() for job in pairs]
|
|
capture.write_text(json.dumps({"case": case, "source": replies[0], "target": replies[1]}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
values = [data_of(reply) for reply in replies]
|
|
for i, side in enumerate(("source", "target")):
|
|
for key in case.get(side + "_data_path", []):
|
|
values[i] = values[i][key]
|
|
codes = [reply.get("body", {}).get("code") for reply in replies]
|
|
report = {"case": case["name"], "source": case["source"], "target": case["target"], "codes": codes,
|
|
"source_capture_reused": reused_source,
|
|
"http_passed": codes == [200, 0], "elapsed_ms": [reply["elapsed_ms"] for reply in replies],
|
|
"shapes": [shape(value) for value in values],
|
|
"exact_data_equal": values[0] == values[1]}
|
|
if all(isinstance(value, dict) and "total" in value for value in values):
|
|
report["totals"] = [value["total"] for value in values]
|
|
report["totals_equal"] = str(values[0]["total"]) == str(values[1]["total"])
|
|
reports.append(report)
|
|
report.update(compare_data(values[0], values[1], case, users))
|
|
report["passed"] = report["http_passed"] and report["semantic_equal"]
|
|
if case.get("expected_error_codes"):
|
|
messages = [reply.get("body", {}).get("msg") for reply in replies]
|
|
report["rejection_equal"] = codes == case["expected_error_codes"] and messages[0] == messages[1]
|
|
report["expected_error_codes"] = case["expected_error_codes"]
|
|
report["passed"] = report["rejection_equal"]
|
|
print(json.dumps(report, ensure_ascii=False), flush=True)
|
|
args.output.write_text(json.dumps({"identity": args.identity, "checks": reports}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
if not all(report["passed"] for report in reports):
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|