"""Score an OCR result of scan-letter (the 02-libreoffice-arial.pdf letter, scanned).
Usage: python3 -I check-ocr.py <result.pdf|.docx|.txt> [...]
Reads the text (pdftotext for PDF, document.xml for Word, as-is for .txt) and counts:
words of the letter found exactly as written, lines found as written, lines in
reversed word order, and presentation-form ("letter shape") code points."""
import sys, zipfile, re, json, os, subprocess
here = os.path.dirname(os.path.abspath(__file__))
TRUE = next(f['lines'] for f in json.load(open(os.path.join(here, 'cases.json')))['files'] if f['file'] == '02-libreoffice-arial.pdf')
def text_of(p):
    if p.endswith('.pdf'):
        return subprocess.run(['pdftotext', p, '-'], capture_output=True, text=True).stdout
    if p.endswith('.docx'):
        x = zipfile.ZipFile(p).read('word/document.xml').decode('utf-8')
        return '\n'.join(''.join(re.findall(r'<w:t[^>]*>([^<]*)</w:t>', para)) for para in re.findall(r'<w:p[ >].*?</w:p>', x, re.S))
    return open(p, encoding='utf-8', errors='replace').read()
clean = lambda s: re.sub(r'[‎‏‪-‮⁦-⁩]', '', s)
def tokens(s):
    s = re.sub(r'[0-9٠-٩.,:;،؛!؟?()«»"\'\-]', ' ', clean(s))
    return [w for w in s.split() if re.search(r'[؀-ۿ]', w)]
for f in sys.argv[1:]:
    raw = text_of(f)
    out = tokens(raw)
    outset = set(out)
    truth_lines = [tokens(l) for l in TRUE]
    words = [w for l in truth_lines for w in l]
    found = sum(1 for w in words if w in outset)
    pairs = [(a, b) for l in truth_lines for a, b in zip(l, l[1:])]
    seq = ' '.join(out)
    in_order = sum(1 for a, b in pairs if f' {a} {b} ' in f' {seq} ')
    reversed_pairs = sum(1 for a, b in pairs if f' {b} {a} ' in f' {seq} ')
    shapes = len(re.findall(r'[ﭐ-﷿ﹰ-﻿]', raw))
    print(json.dumps({"file": os.path.basename(f), "words_found": f"{found}/{len(words)}", "pairs_in_order": f"{in_order}/{len(pairs)}", "reversed_pairs": reversed_pairs, "letter_shape_codes": shapes, "sample": ' '.join(out[:14])}, ensure_ascii=False))
