#!/usr/bin/env python3
"""Find every OCR line containing a non-dictionary token and crop it from the
page image, so the garbled lines can be re-read visually in batches."""

import os as _os
# resolve paths relative to this archive rather than the machine it was built on
_ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))
ARCHIVE = _ROOT
_DATA = _os.path.join(_ROOT, 'data')
import csv, glob, json, os, re, sys
from collections import OrderedDict
from PIL import Image
from spellchecker import SpellChecker

sp = SpellChecker()
OV = {int(k) for k in json.load(open(_os.path.join(_DATA, 'overrides.json'))) if k.isdigit()}

# tokens we already know are fine: German, names, author's own spellings
ALLOW = set("""von der die das und im mit ist des den dem zu auf aus flieg
abgebrannt vertretbare elan askaris askari duraluminum altimetric zeppelin
zeppelins Zeppelin nach uns Krieg Hilf England wird uber hoert hoert
occured socalled anonimity darkeness embarassment longitudenal accomodated
accomodations aeronatical appearence assesment disasterous dilemna
reconaissance selfassurance clounds alkso battallions combattants brakish
caried boyancy capablities condtions consitute machinegun machineguns
groundcrew bombload airforce airforces rudderman ocurred degress
envisionaged Penninsula propellors Portugese sailmaker Sailmaker
""".split())

rows = []
for path in sorted(glob.glob('/tmp/ocr/p-*.tsv')):
    pno = int(re.search(r'p-(\d+)', path).group(1))
    if pno in OV:
        continue
    with open(path, newline='') as fh:
        rr = [r for r in csv.DictReader(fh, delimiter='\t', quoting=csv.QUOTE_NONE)
              if r.get('level') == '5' and (r.get('text') or '').strip()]
    groups = OrderedDict()
    for r in rr:
        groups.setdefault((r['block_num'], r['par_num'], r['line_num']), []).append(r)
    for key, ws in groups.items():
        text = ' '.join(w['text'] for w in ws)
        toks = re.findall(r"[A-Za-z][A-Za-z']*", text)
        bad = [t for t in toks
               if len(t) >= 3 and t not in ALLOW and t.lower() not in ALLOW
               and t.lower() not in sp]
        # also flag lines with umlaut-corruption signatures
        weird = re.findall(r"[A-Za-z]*[&À-ÿ][A-Za-z]*|[a-z][A-Z]", text)
        if not bad and not weird:
            continue
        top = min(int(w['top']) for w in ws)
        bot = max(int(w['top']) + int(w['height']) for w in ws)
        rows.append({'page': pno, 'top': top, 'bot': bot,
                     'text': text, 'bad': bad + weird})

print(f"lines needing review: {len(rows)}", file=sys.stderr)
json.dump(rows, open('/tmp/suspects.json', 'w'))

# ---- render montages of the offending lines ----
os.makedirs('/tmp/montage', exist_ok=True)
PER = 22
W = 2100
for bi in range(0, len(rows), PER):
    chunk = rows[bi:bi+PER]
    crops = []
    for r in chunk:
        im = Image.open('/tmp/img/p-%03d.pgm' % r['page'])
        y0, y1 = max(0, r['top'] - 12), min(im.height, r['bot'] + 12)
        c = im.crop((150, y0, min(im.width, 150 + W), y1))
        crops.append((r, c))
    H = sum(c.height + 8 for _, c in crops)
    sheet = Image.new('L', (W + 120, H), 255)
    y = 0
    for r, c in crops:
        sheet.paste(c, (120, y))
        y += c.height + 8
    sheet = sheet.resize((int(sheet.width * 0.78), int(sheet.height * 0.78)))
    sheet.save(_os.path.join(_ROOT, 'm%02d.png') % (bi // PER))
    print(f"m{bi//PER:02d}.png  lines {bi}-{bi+len(chunk)-1}", file=sys.stderr)
