#!/usr/bin/env python3
"""Stage 1: parse tesseract TSV into structured lines with geometry, per page."""

import os as _os
# resolve paths relative to this archive rather than the machine it was built on
_ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))
ARCHIVE = _ROOT
_DATA = _os.path.join(_ROOT, 'data')
import csv, glob, json, os, re, statistics
from collections import OrderedDict

OCR_DIR = "/tmp/ocr"

def page_lines(path):
    with open(path, newline='') as fh:
        rows = list(csv.DictReader(fh, delimiter='\t', quoting=csv.QUOTE_NONE))
    groups = OrderedDict()
    for r in rows:
        if r.get('level') != '5':
            continue
        txt = (r.get('text') or '')
        if not txt.strip():
            continue
        key = (int(r['block_num']), int(r['par_num']), int(r['line_num']))
        groups.setdefault(key, []).append(r)
    out = []
    for key, ws in groups.items():
        left = min(int(w['left']) for w in ws)
        right = max(int(w['left']) + int(w['width']) for w in ws)
        top = min(int(w['top']) for w in ws)
        confs = [float(w['conf']) for w in ws if float(w['conf']) >= 0]
        text = ' '.join(w['text'] for w in ws)
        out.append({
            'block': key[0], 'par': key[1], 'line': key[2],
            'left': left, 'right': right, 'top': top,
            'conf': round(statistics.mean(confs), 1) if confs else 0.0,
            'nwords': len(ws),
            'text': text,
        })
    out.sort(key=lambda l: (l['top'], l['left']))
    return out

pages = []
for path in sorted(glob.glob(os.path.join(OCR_DIR, '*.tsv'))):
    pno = int(re.search(r'p-(\d+)', path).group(1))
    pages.append({'page': pno, 'lines': page_lines(path)})

with open('/tmp/lines.json', 'w') as fh:
    json.dump(pages, fh)

# ---- corpus geometry report, to calibrate thresholds ----
lefts = [l['left'] for p in pages for l in p['lines'] if l['nwords'] >= 8]
print(f"pages: {len(pages)}  total lines: {sum(len(p['lines']) for p in pages)}")
print(f"body-left (>=8 words): median={statistics.median(lefts)} "
      f"p10={sorted(lefts)[len(lefts)//10]} p90={sorted(lefts)[9*len(lefts)//10]}")

allconf = [l['conf'] for p in pages for l in p['lines']]
print(f"line confidence: median={statistics.median(allconf):.1f} "
      f"min={min(allconf):.1f}  lines<70={sum(1 for c in allconf if c < 70)}")

# distribution of left positions, bucketed
from collections import Counter
buckets = Counter((l['left'] // 100) * 100 for p in pages for l in p['lines'])
print("\nleft-position histogram (bucket:count):")
for b in sorted(buckets):
    if buckets[b] > 20:
        print(f"  {b:5d}: {buckets[b]}")

print("\n--- first 3 lines of pages 1,2,5,50,150,267 ---")
for p in pages:
    if p['page'] in (1, 2, 5, 50, 150, 267):
        print(f"[page {p['page']}]")
        for l in p['lines'][:3]:
            print(f"   left={l['left']:5d} top={l['top']:5d} conf={l['conf']:5.1f} {l['text'][:66]}")
