#!/usr/bin/env python3
"""Verification: confirm no source page's content was dropped.

For each of the 267 scanned pages, take several word-windows from its OCR text
and check they survive into the final document. Catches whole pages, columns,
or paragraphs lost anywhere in the pipeline.
"""

import os as _os
# resolve paths relative to this archive rather than the machine it was built on
_ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))
ARCHIVE = _ROOT
_DATA = _os.path.join(_ROOT, 'data')
import json, re, random, sys

random.seed(7)

def norm(s):
    s = s.replace('“', '"').replace('”', '"').replace('’', "'")
    return re.sub(r'[^a-z0-9 ]', ' ', s.lower())

_raw = open('/tmp/final.txt').read()
# drop the running header and page-break marks, which otherwise interrupt
# phrases that straddle a page boundary in the rendered output
_raw = _raw.replace('\f', '\n')
_raw = re.sub(r'Walton / The China Affair / \d+', ' ', _raw)
final = re.sub(r'\s+', ' ', norm(_raw))

LINES = json.load(open('/tmp/lines.json'))
OV = {int(k) for k in json.load(open(_os.path.join(_DATA, 'overrides.json'))) if k.isdigit()}
FRONT = {1, 2, 3}

W = 6              # window length in words
bad, checked = [], 0
for p in LINES:
    pno = p['page']
    if pno in FRONT:
        continue
    # use only confidently-OCR'd, substantial body lines as probes
    probes = [l['text'] for l in p['lines']
              if l['conf'] >= 92 and l['nwords'] >= 8 and l['top'] > 250]
    if not probes:
        continue
    wins = []
    for text in probes:
        ws = norm(text).split()
        if len(ws) >= W:
            wins.append(' '.join(ws[:W]))
    if not wins:
        continue
    sample = random.sample(wins, min(4, len(wins)))
    hits = sum(1 for w in sample if w in final)
    checked += 1
    if hits < len(sample):
        bad.append((pno, hits, len(sample),
                    [w for w in sample if w not in final]))

print(f"pages probed: {checked}")
print(f"pages with every probe found: {checked - len(bad)}")
print(f"pages with a missing probe: {len(bad)}")
for pno, h, n, miss in bad[:30]:
    tag = ' (manually transcribed)' if pno in OV else ''
    print(f"  page {pno:3d}: {h}/{n}{tag}")
    for m in miss[:2]:
        print(f"        missing: {m!r}")
