#!/usr/bin/env python3
"""Build a metadata-only inventory for a password-protected Baidu Pan share.

The script never downloads file bodies. It verifies the share password, walks the
directory tree through Baidu's list endpoint, and records names, paths and sizes.
"""

from __future__ import annotations

import argparse
import collections
import datetime as dt
import http.cookiejar
import json
import http.client
import os
import re
import time
import urllib.parse
import urllib.error
import urllib.request
from pathlib import Path


USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36"


def request(opener, url: str, *, data: bytes | None = None, referer: str | None = None):
    headers = {"User-Agent": USER_AGENT, "Accept": "application/json,text/html,*/*"}
    if referer:
        headers["Referer"] = referer
    last_error = None
    for attempt in range(6):
        req = urllib.request.Request(url, data=data, headers=headers)
        try:
            with opener.open(req, timeout=30) as response:
                return response.read()
        except (urllib.error.URLError, TimeoutError, http.client.RemoteDisconnected, ConnectionError) as error:
            last_error = error
            time.sleep(min(12, 0.8 * (2**attempt)))
    raise last_error


def parse_share_identity(html: str) -> tuple[str, str]:
    share_uk = re.search(r'"share_uk"\s*:\s*"?(\d+)', html)
    share_id = re.search(r'"shareid"\s*:\s*"?(\d+)', html)
    if not share_uk or not share_id:
        raise RuntimeError("BAIDU_SHARE_IDENTITY_NOT_FOUND")
    return share_uk.group(1), share_id.group(1)


def get_json(opener, url: str, referer: str):
    return json.loads(request(opener, url, referer=referer).decode("utf-8"))


def list_page(opener, referer: str, share_uk: str, share_id: str, *, page: int, directory: str | None):
    params = {
        "app_id": "250528",
        "web": "1",
        "channel": "chunlei",
        "clienttype": "0",
        "desc": "0",
        "num": "100",
        "order": "name",
        "page": str(page),
        "shareid": share_id,
        "showempty": "0",
        "uk": share_uk,
    }
    if directory is None:
        params["root"] = "1"
    else:
        params["dir"] = directory
    url = "https://pan.baidu.com/share/list?" + urllib.parse.urlencode(params)
    payload = get_json(opener, url, referer)
    if payload.get("errno") != 0:
        raise RuntimeError(f"BAIDU_LIST_FAILED:{payload.get('errno')}:{directory or '/'}")
    return payload.get("list") or []


def human_bytes(value: int) -> str:
    size = float(value)
    units = ["B", "KB", "MB", "GB", "TB"]
    for unit in units:
        if size < 1024 or unit == units[-1]:
            return f"{size:.2f} {unit}"
        size /= 1024
    return f"{value} B"


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--url", required=True)
    parser.add_argument("--password", default=os.environ.get("BAIDU_SHARE_PASSWORD", ""))
    parser.add_argument("--result", required=True)
    parser.add_argument("--report", required=True)
    args = parser.parse_args()
    if not args.password:
        raise SystemExit("BAIDU_SHARE_PASSWORD_REQUIRED")

    cookies = http.cookiejar.CookieJar()
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookies))
    parsed = urllib.parse.urlparse(args.url)
    surl = parsed.path.rsplit("/", 1)[-1]
    if surl.startswith("1"):
        surl = surl[1:]
    clean_url = urllib.parse.urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
    initial_html = request(opener, clean_url).decode("utf-8", errors="replace")
    share_uk, share_id = parse_share_identity(initial_html)

    verify_url = "https://pan.baidu.com/share/verify?" + urllib.parse.urlencode({
        "surl": surl,
        "t": str(int(time.time() * 1000)),
        "channel": "chunlei",
        "web": "1",
        "app_id": "250528",
    })
    verify_body = urllib.parse.urlencode({"pwd": args.password, "vcode": "", "vcode_str": ""}).encode()
    verify = json.loads(request(opener, verify_url, data=verify_body, referer=clean_url).decode("utf-8"))
    if verify.get("errno") != 0:
        raise RuntimeError(f"BAIDU_PASSWORD_VERIFY_FAILED:{verify.get('errno')}")

    root_html = request(opener, clean_url, referer="https://pan.baidu.com/").decode("utf-8", errors="replace")
    share_uk, share_id = parse_share_identity(root_html)
    queue = collections.deque([None])
    visited = set()
    records = []
    while queue:
        directory = queue.popleft()
        key = directory or "/"
        if key in visited:
            continue
        visited.add(key)
        page = 1
        while True:
            items = list_page(opener, clean_url, share_uk, share_id, page=page, directory=directory)
            for item in items:
                record = {
                    "path": item.get("path") or item.get("server_filename"),
                    "name": item.get("server_filename"),
                    "isDirectory": bool(int(item.get("isdir", 0))),
                    "size": int(item.get("size", 0) or 0),
                    "fsId": str(item.get("fs_id", "")),
                    "md5": item.get("md5") or "",
                    "serverModified": int(item.get("server_mtime", 0) or 0),
                }
                records.append(record)
                if record["isDirectory"]:
                    queue.append(record["path"])
            if len(items) < 100:
                break
            page += 1
        if len(visited) % 25 == 0:
            print(f"indexed {len(visited):,} directories, {len(records):,} entries", flush=True)
        time.sleep(0.25)

    files = [item for item in records if not item["isDirectory"]]
    directories = [item for item in records if item["isDirectory"]]
    by_extension = collections.Counter()
    by_year = collections.Counter()
    total_bytes = 0
    for item in files:
        total_bytes += item["size"]
        suffix = Path(item["name"] or "").suffix.lower() or "[no extension]"
        by_extension[suffix] += 1
        for year in set(re.findall(r"(?<!\d)((?:199\d|20[01]\d|202[0-6]))(?:[01]\d[0-3]\d)?(?!\d)", item["name"] or "")):
            by_year[year] += 1

    generated_at = dt.datetime.now(dt.timezone.utc).isoformat()
    result = {
        "version": "baidu-share-metadata-inventory-v1",
        "generatedAt": generated_at,
        "source": clean_url,
        "mode": "METADATA_ONLY_NO_FILE_DOWNLOAD",
        "directories": len(directories),
        "files": len(files),
        "totalBytes": total_bytes,
        "totalHuman": human_bytes(total_bytes),
        "byExtension": dict(by_extension.most_common()),
        "byYear": dict(sorted(by_year.items())),
        "records": records,
    }
    result_path = Path(args.result)
    result_path.parent.mkdir(parents=True, exist_ok=True)
    # Keep the complete record set but write it compactly so the auditable
    # artifact remains practical to publish even for tens of thousands of files.
    result_path.write_text(json.dumps(result, ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8")

    largest = sorted(files, key=lambda item: item["size"], reverse=True)[:30]
    report = [
        "# 百度网盘 A 股历史数据远程清单",
        "",
        f"- 生成时间：{generated_at}",
        f"- 模式：只读取目录元数据，不下载文件正文。",
        f"- 目录：{len(directories):,}；文件：{len(files):,}；标称总量：{human_bytes(total_bytes)}。",
        "- 用途：与本地数据按名称、年份、大小和校验值去重，按回测需要选择分片。",
        "",
        "## 最大文件",
        "",
        "| 文件 | 大小 |",
        "|---|---:|",
        *[f"| {item['path']} | {human_bytes(item['size'])} |" for item in largest],
        "",
        "## 约束",
        "",
        "目录清单可以远程读取；策略计算不能直接在百度网盘网页中执行。回测所需的压缩分片仍需按需暂存到本机或云端计算节点，但无需一次下载整套数据。",
        "",
    ]
    report_path = Path(args.report)
    report_path.parent.mkdir(parents=True, exist_ok=True)
    report_path.write_text("\n".join(report), encoding="utf-8")
    print(json.dumps({key: result[key] for key in ["version", "generatedAt", "directories", "files", "totalBytes", "totalHuman", "byExtension", "byYear"]}, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()