#!/usr/bin/env python3
"""把两轮采集到的原始来源文件整理成 data_provenance.json，并逐条回到原文里核一遍。

规则（与预注册一致）：
- 每个数字必须能在随包交付的某个来源文件里原文找到，找不到就不进面板。
- 不插值、不用增长率反推、不用估算值冒充实测值。
- 单位换算只做一种：亿人次 → 万人次（×10000），且在记录里写明原文单位与精度。
"""
from __future__ import annotations

import json
import re
import sys
from pathlib import Path

CODE = Path(__file__).resolve().parent
WORK = CODE.parent
SRC = WORK / "data" / "sources"

FIELD_MAP = {
    "tourism_visitors": "tourism_visitors",
    "tourism_revenue": "tourism_revenue",
    "resident_population": "resident_pop",
    "secondary_industry_gdp": "secondary_gdp",
}


def text_of(path: Path) -> str:
    raw = path.read_bytes()
    for enc in ("utf-8", "gbk", "gb18030"):
        try:
            t = raw.decode(enc)
            break
        except UnicodeDecodeError:
            continue
    else:
        t = raw.decode("utf-8", "ignore")
    t = re.sub(r"<[^>]+>", "", t)
    return re.sub(r"[\s　]+", "", t)


def number_present(text: str, value: float) -> bool:
    """原文里可能写成 4403.93 / 4 403.93 / 99 / 99.0 / 1.3；空白已在 text_of 里去掉。"""
    cands = {f"{value:g}", f"{value}"}
    if float(value).is_integer():
        cands.add(str(int(value)))
        cands.add(f"{int(value)}.0")
    if abs(value - round(value, 1)) < 1e-9:
        cands.add(f"{value:.1f}")
    if abs(value - round(value, 2)) < 1e-9:
        cands.add(f"{value:.2f}")
    return any(c in text for c in cands)


def main() -> int:
    records, unverified = [], []
    incoming = json.loads((WORK / "data" / "sources_collected.json").read_text(encoding="utf-8"))
    for rec in incoming["records"]:
        local = SRC / rec["source_file"]
        if not local.is_file():
            unverified.append({**rec, "why": "来源文件不在包里"})
            continue
        text = text_of(local)
        probe = rec.get("raw_value", rec["value"])
        if not number_present(text, float(probe)):
            unverified.append({**rec, "why": "在来源文件里没找到这个数字的原文"})
            continue
        snippet = rec.get("snippet", "")
        if snippet and re.sub(r"[\s　]+", "", snippet)[:20] not in text:
            unverified.append({**rec, "why": "原文片段与来源文件对不上"})
            continue
        records.append({
            "city": rec["city"], "year": int(rec["year"]), "field": rec["field"],
            "value": float(rec["value"]), "unit": rec["unit"],
            "raw_value": rec.get("raw_value"), "raw_unit": rec.get("raw_unit"),
            "indicator_text": rec["indicator_text"], "caliber": rec.get("caliber"),
            "source_file": f"work/data/sources/{rec['source_file']}",
            "url": rec["url"], "source_type": rec.get("source_type", ""),
            "snippet": rec.get("snippet", ""),
        })
    out = {
        "collected_at": incoming["collected_at"],
        "method": incoming["method"],
        "verification": "每条记录都在随包交付的来源文件里重新 grep 过数字原文与片段；对不上的不进面板，列在 unverified 里。",
        "records": records,
        "unverified": unverified,
        "missing": incoming.get("missing", []),
    }
    (WORK / "data" / "data_provenance.json").write_text(
        json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")
    print(f"通过核对 {len(records)} 条，未通过 {len(unverified)} 条，"
          f"已知缺失 {len(out['missing'])} 条")
    for u in unverified:
        print("  未通过:", u["city"], u["year"], u["field"], u["why"])
    return 1 if unverified else 0


if __name__ == "__main__":
    raise SystemExit(main())