#!/usr/bin/env python3
"""Crop the source scan line for each remaining suspect token, so the last of
the OCR residue can be checked against the page rather than guessed at."""

import os as _os
# resolve paths relative to this archive rather than the machine it was built on
_ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))
ARCHIVE = _ROOT
_DATA = _os.path.join(_ROOT, 'data')
import csv, glob, json, os, re, sys, unicodedata
from collections import OrderedDict, Counter
from PIL import Image
from spellchecker import SpellChecker

sp = SpellChecker()

# The author's own spellings, German/French words, and house compounds. These
# are deliberately preserved, so they are not flagged for review.
PRESERVE = set("""
von der des die das und im mit ist den dem zu auf aus nach uns
flieg abgebrannt vertretbare elan dictu opera attaches
occured ocurred socalled askaris askari longitudenal anonimity darkeness
embarassment dilemna disasterous reconaissance selfassurance accomodated
accomodations aeronatical aggrevated appearence assesment assidiously
battallions boyancy capablities caried clounds alkso combattants concesssion
condtions consitute degress criticised emphasise cyphers envisionaged
machinegun machineguns groundcrew bombload airforce airforces rudderman
duraluminum altimetric airdefense airwar airmasses airlayers airlegs
airstrength airshipmen wingracks teaparty brakish penninsula propellors
portugese bulgartian zagged lle nigsberg tte terbog
""".split())


def sa(s):
    return ''.join(c for c in unicodedata.normalize('NFD', s)
                   if not unicodedata.combining(c))


def compound(t):
    t = t.lower()
    return any(t[:i] in sp and t[i:] in sp and len(t[:i]) >= 4 and len(t[i:]) >= 4
               for i in range(4, len(t) - 3))


text = open('/tmp/final.txt').read()
counts = Counter(re.findall(r"[A-Za-zÀ-ÿ][A-Za-zÀ-ÿ']*", text))
suspect = set()
for k in counts:
    if len(k) < 3 or not k[0].islower():
        continue
    if k.lower() in sp or sa(k.lower()) in sp:
        continue
    if k.lower() in PRESERVE or sa(k.lower()) in PRESERVE or compound(k):
        continue
    suspect.add(k)

print(f"tokens to review: {len(suspect)}", file=sys.stderr)

# locate each suspect in the OCR line data
OV = {int(k) for k in json.load(open(_os.path.join(_DATA, 'overrides.json'))) if k.isdigit()}
found, seen = [], set()
for path in sorted(glob.glob('/tmp/ocr/p-*.tsv')):
    pno = int(re.search(r'p-(\d+)', path).group(1))
    if pno in OV:
        continue
    with open(path, newline='') as fh:
        rr = [r for r in csv.DictReader(fh, delimiter='\t', quoting=csv.QUOTE_NONE)
              if r.get('level') == '5' and (r.get('text') or '').strip()]
    g = OrderedDict()
    for r in rr:
        g.setdefault((r['block_num'], r['par_num'], r['line_num']), []).append(r)
    for key, ws in g.items():
        line = ' '.join(w['text'] for w in ws)
        hits = [t for t in re.findall(r"[A-Za-zÀ-ÿ'][A-Za-zÀ-ÿ']*", line)
                if t in suspect and t not in seen]
        if not hits:
            continue
        seen.update(hits)
        found.append({'page': pno, 'hits': hits, 'text': line,
                      'top': min(int(w['top']) for w in ws),
                      'bot': max(int(w['top']) + int(w['height']) for w in ws)})

print(f"lines to review: {len(found)}", file=sys.stderr)
json.dump(found, open('/tmp/residue.json', 'w'))

PER = 20
for bi in range(0, len(found), PER):
    chunk = found[bi:bi+PER]
    crops = []
    for r in chunk:
        im = Image.open('/tmp/img/p-%03d.pgm' % r['page'])
        crops.append(im.crop((150, max(0, r['top'] - 14),
                              min(im.width, 2380), min(im.height, r['bot'] + 14))))
    H = sum(c.height + 12 for c in crops)
    W = max(c.width for c in crops)
    sheet = Image.new('L', (W, H), 255)
    y = 0
    for c in crops:
        sheet.paste(c, (0, y))
        y += c.height + 12
    sheet.resize((int(W * 0.72), int(H * 0.72))).save(
        _os.path.join(_ROOT, 'rv%02d.png') % (bi // PER))
    print(f"rv{bi//PER:02d}.png " + ' | '.join(','.join(r['hits']) for r in chunk),
          file=sys.stderr)
