#!/usr/bin/env python3
"""Join detector results to public manifest metadata and emit the aggregate
summary + the flagged-document list for human verification.

Aggregate-only. For flagged docs we record public metadata (court, docket,
date, URL, page numbers) and counts — never the recovered text.
"""
import json, os, sys, collections

HERE = os.path.dirname(os.path.abspath(__file__))
res = json.load(open(os.path.join(HERE, "results3.json")))
man = {d["filepath"].replace("/", "__"): d for d in json.load(open(os.path.join(HERE, "manifest.json")))}

analyzed = ok = errors = no_text = box = failed = 0
leak_pages = leak_words = leak_chars = 0
courts_denom, dockets_denom = set(), set()
flagged = []

for fn, r in res.items():
    analyzed += 1
    if "error" in r:
        errors += 1
        continue
    ok += 1
    if not r.get("text_pages"):
        no_text += 1
        continue
    if not r.get("boxes"):
        continue
    box += 1
    m = man.get(fn, {})
    courts_denom.add(m.get("court"))
    dockets_denom.add(m.get("docket_id"))
    if r.get("leak_pages"):
        failed += 1
        leak_pages += r["leak_pages"]
        leak_words += r["leak_words"]
        leak_chars += r["leak_chars"]
        flagged.append({
            "file": fn,
            "court": m.get("court"),
            "docket_id": m.get("docket_id"),
            "date_filed": m.get("date_filed"),
            "pages": r.get("pages"),
            "url": m.get("url"),
            "leak_pages": r["leak_pages"],
            "leak_words": r["leak_words"],
            "leak_chars": r["leak_chars"],
        })

flagged.sort(key=lambda x: -x["leak_chars"])
summary = {
    "detector": "v3 render-based: a word counts as leaked only if extractable from the text layer and invisible in the rendered page under an opaque uniform near-black region. v3 = v2 rule plus (a) page-rotation normalisation and (b) 120-page cap matching the collector.",
    "corpus_analyzed": analyzed,
    "parse_errors": errors,
    "docs_with_text_layer": ok - no_text,
    "used_box_redaction": box,
    "box_redaction_failed": failed,
    "headline_failure_rate_pct": round(100 * failed / box, 2) if box else None,
    "leak_pages_total": leak_pages,
    "leak_words_total": leak_words,
    "leak_chars_total": leak_chars,
    "distinct_courts_in_denominator": len(courts_denom - {None}),
    "distinct_dockets_in_denominator": len(dockets_denom - {None}),
}
json.dump(summary, open(os.path.join(HERE, "summary3.json"), "w"), indent=1)
json.dump(flagged, open(os.path.join(HERE, "flagged3.json"), "w"), indent=1)
print(json.dumps(summary, indent=1))
print(f"\n{len(flagged)} FLAGGED:")
for f in flagged:
    print(f"  {f['court']:6} {f['date_filed']} pages={f['pages']:3} leak_pages={f['leak_pages']:2} "
          f"words={f['leak_words']:4} chars={f['leak_chars']:5}  {f['url']}")