#!/usr/bin/env python3
"""Generate the written reports for the archive: the corrections log, the
statistics summary, and the verification report."""

import os as _os
# resolve paths relative to this archive rather than the machine it was built on
_ROOT = _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__)))
ARCHIVE = _ROOT
_DATA = _os.path.join(_ROOT, 'data')
import csv, glob, json, os, re, statistics, unicodedata
from collections import Counter, defaultdict

ARCHIVE = _ROOT
R = f'{ARCHIVE}/reports'

doc = json.load(open('/tmp/doc.json'))
blocks = doc['blocks']
fixlog = json.load(open('/tmp/fixlog.json'))
lines = json.load(open('/tmp/lines.json'))
rules = {int(k): v for k, v in json.load(open('/tmp/rules.json')).items()}
overrides = {k: v for k, v in json.load(
    open(_os.path.join(_DATA, 'overrides.json'))).items()
    if k.isdigit()}
fixes = json.load(open(_os.path.join(_DATA, 'fixes.json')))


def pg(pages):
    if not pages:
        return '—'
    return str(pages[0]) if len(pages) == 1 else f'{pages[0]}–{pages[-1]}'


# ══════════════════════════════════════════════════ 1. corrections log
UMLAUT = {'Königsberg', 'Kölle', 'Göring', 'Göbel', 'Jüterbog', 'Königin',
          'Spähkorb', 'Göppingen', 'Bröcker', 'Bödecker', 'Schütte', 'Füssen',
          'Kapitänleutnant', 'Deuxième', 'Düsseldorf', 'hört'}

tok_by_pair = defaultdict(list)
for x in fixlog['tokens']:
    tok_by_pair[(x['from'], x['to'])].append(pg(x['pages']))

umlaut_rows, letter_rows = [], []
for (a, b), pages in sorted(tok_by_pair.items(), key=lambda kv: kv[0][1]):
    row = (a, b, len(pages), ', '.join(sorted(set(pages), key=lambda s: int(
        re.match(r'\d+', s).group(0)))))
    (umlaut_rows if b in UMLAUT else letter_rows).append(row)

L = []
L += ['# Corrections log', '',
      'Every change made to the raw OCR output, with the scan page it was applied',
      'on. Each was checked against the page image before being added; nothing here',
      'was inferred from context alone.', '',
      'Page numbers are positions in the scanned PDF. The typist\'s own folio',
      'numbers, hand-written at the foot of each page, run three lower.', '',
      f'**Totals:** {len(fixlog["phrases"])} phrase corrections, '
      f'{len(fixlog["tokens"])} token corrections, '
      f'{len(fixlog["capitalisations"])} capitalisations, '
      f'{len(overrides)} pages re-transcribed by hand.', '',
      'Corrections are listed as `scanned reading → corrected reading`. For what',
      'was deliberately *not* changed, see `preserved-spellings.md`.', '',
      '---', '']

L += ['## 1. German names and umlauts', '',
      'The scanner rendered ö/ü/ä inconsistently as `&`, `é`, `S`, `8` or `d`,',
      'often differently on different pages for the same name. Corrected against',
      'the page image in each case.', '',
      '| Scanned | Corrected | Times | Pages |', '|---|---|---|---|']
for a, b, n, pages in umlaut_rows:
    L.append(f'| `{a}` | **{b}** | {n} | {pages} |')

L += ['', '## 2. Letter-confusion and single-word errors', '',
      'Classic OCR substitutions on a typewritten original — n/h, l/1, i/l, v/y,',
      'c/e, rn/m — plus a few digits read into words.', '',
      '| Scanned | Corrected | Times | Pages |', '|---|---|---|---|']
for a, b, n, pages in letter_rows:
    L.append(f'| `{a}` | **{b}** | {n} | {pages} |')

L += ['', '## 3. Phrase and passage corrections', '',
      'Longer spans where the scan was faint enough that several words were',
      'garbled together. The corrected reading was taken from the page image.', '',
      '| Page | Scanned | Corrected |', '|---|---|---|']
seen = set()
for x in sorted(fixlog['phrases'], key=lambda x: (x['pages'] or [0])[0]):
    key = (x['from'], x['to'])
    if key in seen:
        continue
    seen.add(key)
    a = x['from'].replace('|', '\\|')
    b = x['to'].replace('|', '\\|')
    L.append(f'| {pg(x["pages"])} | `{a}` | **{b}** |')

L += ['', '## 4. Capitalisation', '',
      'A paragraph always opens a sentence. Two paragraph-initial capitals were',
      'lost by the OCR; both were confirmed capitalised on the scan.', '',
      '| Page | Word as scanned |', '|---|---|']
for c in fixlog['capitalisations']:
    L.append(f'| {pg(c["pages"])} | `{c["word"]}` |')

L += ['', '## 5. Pages re-transcribed by hand', '',
      'Pages 150–156 are faint carbon copies. Tesseract dropped whole lines and',
      'garbled others past the point where targeted correction was safe, so these',
      'were read off the page images and typed out in full. The transcriptions live',
      'in `data/overrides.json`; the page scans are in `evidence/faint-pages/`.', '',
      '| Page | Mean OCR confidence | Words re-typed | Has footnote |',
      '|---|---|---|---|']
conf_by_page = {}
for p in lines:
    body = [l for l in p['lines'] if l['nwords'] >= 6 and l['top'] > 200]
    if body:
        conf_by_page[p['page']] = statistics.mean(l['conf'] for l in body)
for k in sorted(overrides, key=int):
    ov = overrides[k]
    words = sum(len(t.split()) for t in ov['paras'])
    L.append(f'| {k} | {conf_by_page.get(int(k), 0):.1f}% | {words} | '
             f'{"yes" if ov.get("footnote") else "—"} |')

L += ['', '## 6. Corrections that were considered and rejected', '',
      'Cases where the scan was checked and the odd reading turned out to be the',
      "author's own, so the text was left alone.", '',
      '| Page | Reading | Why it was kept |', '|---|---|---|',
      '| 253 | "seems to have been into circulation" | The word *put* is genuinely '
      'absent from the typescript — an authorial slip, not a scanning loss. |',
      '| 150 | "43 degress F." | Typed that way; the next line spells *degrees* '
      'correctly, so the inconsistency is the author\'s. |',
      '| 155 | "broke through the clounds" | Typed that way. |',
      '| 155 | "and alkso from the altimetric record" | Typed that way. |',
      '| 187 | "the Bulgartian recruits" | Typed that way, though *Bulgarian* '
      'appears correctly in the same sentence. |',
      '| 25 | "mililtary airships" | Typed that way. |',
      '| various | "occured", "socalled", "longitudenal", "embarassment" | '
      'Consistent authorial spellings — see `preserved-spellings.md`. |', '']

L += ['## 7. One editorial judgment call', '',
      'On scan page 230 the typed word *doing* is struck through by hand in',
      '"...before he ~~doing~~ anything further", with a handwritten correction above',
      'it too faint to read. **"did"** was set, as the only reading the sentence',
      'admits. This is the single place in the document where a word was chosen',
      'rather than read. See `evidence/decisions/p230-struck-through-word.png`.', '']

open(f'{R}/corrections.md', 'w').write('\n'.join(L) + '\n')

# ══════════════════════════════════════════════════ 2. statistics
allconf = [l['conf'] for p in lines for l in p['lines']]
bodyconf = sorted(conf_by_page.values())
counts = Counter(b['type'] for b in blocks)
words = sum(len(b.get('text', '').split()) for b in blocks)

S = ['# Pipeline statistics', '',
     '## Source', '',
     '| | |', '|---|---|',
     '| File | `The China Affair manuscript (scanned 7-26).pdf` |',
     '| Pages | 267 |', '| Size | 52.6 MB |',
     '| Text layer | none — pure raster scan |',
     '| Page size | 792 × 612 pt (US Letter, landscape-flagged) |',
     '| Producer | EFI Cyclone |', '',
     '## OCR', '',
     '| | |', '|---|---|',
     '| Engine | Tesseract 4.1.1, `--psm 3`, English |',
     '| Render | `pdftoppm -gray -r 300` (2550 × 3301 px/page) |',
     '| Output | TSV with per-word bounding boxes and confidence |',
     f'| Lines recognised | {sum(len(p["lines"]) for p in lines):,} |',
     f'| Median line confidence | {statistics.median(allconf):.1f}% |',
     f'| Pages ≥ 90% mean body confidence | '
     f'{sum(1 for c in bodyconf if c >= 90)} / {len(bodyconf)} |',
     f'| Pages 80–90% | {sum(1 for c in bodyconf if 80 <= c < 90)} |',
     f'| Pages < 80% | {sum(1 for c in bodyconf if c < 80)} |', '',
     'PSM 3 was chosen over 4 and 6 after comparing all three on sample pages: it',
     'preserved the blank lines between paragraphs and avoided a spurious leading',
     'character that PSM 6 introduced.', '',
     '## Reconstructed structure', '',
     '| Block type | Count |', '|---|---|',
     f'| Body paragraphs | {counts["para"]} |',
     f'| Chapter headings | {counts["heading"]} |',
     f'| Footnotes | {counts["footnote"]} |',
     f'| Block quotations | {counts["quote"]} |',
     f'| Verse / telegram blocks | {counts["verse"]} |',
     f'| **Total words** | **{words:,}** |', '',
     f'Footnote separator rules detected from pixel data on '
     f'{sum(1 for v in rules.values() if v["rule_y"])} pages.', '',
     '## Output', '',
     '| | |', '|---|---|',
     '| Format | .docx, US Letter, 1"/1.25" margins |',
     '| Type | 12 pt Times New Roman, double-spaced, 0.5" first-line indent |',
     '| Length | 227 pages |',
     '| Footnotes | 31, as real Word footnotes anchored at the author\'s asterisks |',
     '| Validation | passed structural XSD checks |', '']
open(f'{R}/statistics.md', 'w').write('\n'.join(S) + '\n')

print('wrote corrections.md and statistics.md')
print('  umlaut corrections:', len(umlaut_rows))
print('  letter corrections:', len(letter_rows))
print('  distinct phrase corrections:', len(seen))
