"""Text-cleanup decorators for `plugins.normalize_spaced_text`. Pure module: no FastAPI imports, no HTTPException, and no module-level state. Each helper carries its own lookup tables and thresholds, so a caller can tune one call site without moving a shared constant that every other caller reads. `normalize_unicode` and `despace_line` are stacked onto the formatter and run outermost-first, so the text arrives already folded and already rebuilt: raw -> normalize_unicode -> despace_line -> normalize_spaced_text """ from __future__ import annotations import re import unicodedata from functools import wraps def is_letter_spaced(line, *, min_tokens=4, single_char_ratio=0.6) -> bool: """True when the line looks glyph-padded rather than normally typed. Both gates have to pass: a short line like "Next.js 3 A B" clears the ratio on its own, so the token count is what keeps it out. """ tokens = line.split() if len(tokens) < min_tokens: return False singles = sum(1 for token in tokens if len(token) == 1) return singles / len(tokens) >= single_char_ratio def normalize_unicode(func): """Fold ligatures, drop invisibles, flatten every space variant to U+0020. Runs before the spacing heuristics so they only ever see one kind of gap. """ @wraps(func) def wrapper(text, *args, **kwargs): text = text or "" ligatures = {"ff": "ff", "fi": "fi", "fl": "fl", "ffi": "ffi", "ffl": "ffl"} # Zero-width and soft-hyphen glyphs pypdf emits; they break word matching. invisible = dict.fromkeys(map(ord, "​‌‍­"), None) for ligature, plain in ligatures.items(): text = text.replace(ligature, plain) text = text.translate(invisible) folded = "".join( " " if char == "\t" or unicodedata.category(char) == "Zs" else char for char in text ) return func(folded, *args, **kwargs) return wrapper def despace_line(func): """Rebuild every glyph-padded line: 2+ spaces are word gaps, single spaces are noise. Lines that fail `is_letter_spaced` are passed through untouched, because the same rule applied to normally typed text would glue its words together. """ @wraps(func) def wrapper(text, *args, **kwargs): lines = [] for line in (text or "").splitlines(): if is_letter_spaced(line): words = re.split(r" {2,}", line.strip()) line = " ".join(word.replace(" ", "") for word in words if word.strip()) lines.append(line) return func("\n".join(lines), *args, **kwargs) return wrapper