75 lines
2.6 KiB
Python
75 lines
2.6 KiB
Python
"""Text-cleanup decorators for `plugins.normalize_spaced_text`.
|
||
|
||
Pure module: no FastAPI imports, no HTTPException, and no module-level state.
|
||
Each helper carries its own lookup tables and thresholds, so a caller can tune
|
||
one call site without moving a shared constant that every other caller reads.
|
||
|
||
`normalize_unicode` and `despace_line` are stacked onto the formatter and run
|
||
outermost-first, so the text arrives already folded and already rebuilt:
|
||
|
||
raw -> normalize_unicode -> despace_line -> normalize_spaced_text
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import unicodedata
|
||
from functools import wraps
|
||
|
||
|
||
def is_letter_spaced(line, *, min_tokens=4, single_char_ratio=0.6) -> bool:
|
||
"""True when the line looks glyph-padded rather than normally typed.
|
||
|
||
Both gates have to pass: a short line like "Next.js 3 A B" clears the ratio
|
||
on its own, so the token count is what keeps it out.
|
||
"""
|
||
tokens = line.split()
|
||
if len(tokens) < min_tokens:
|
||
return False
|
||
singles = sum(1 for token in tokens if len(token) == 1)
|
||
return singles / len(tokens) >= single_char_ratio
|
||
|
||
|
||
def normalize_unicode(func):
|
||
"""Fold ligatures, drop invisibles, flatten every space variant to U+0020.
|
||
|
||
Runs before the spacing heuristics so they only ever see one kind of gap.
|
||
"""
|
||
|
||
@wraps(func)
|
||
def wrapper(text, *args, **kwargs):
|
||
text = text or ""
|
||
ligatures = {"ff": "ff", "fi": "fi", "fl": "fl", "ffi": "ffi", "ffl": "ffl"}
|
||
# Zero-width and soft-hyphen glyphs pypdf emits; they break word matching.
|
||
invisible = dict.fromkeys(map(ord, ""), None)
|
||
for ligature, plain in ligatures.items():
|
||
text = text.replace(ligature, plain)
|
||
text = text.translate(invisible)
|
||
folded = "".join(
|
||
" " if char == "\t" or unicodedata.category(char) == "Zs" else char
|
||
for char in text
|
||
)
|
||
return func(folded, *args, **kwargs)
|
||
|
||
return wrapper
|
||
|
||
|
||
def despace_line(func):
|
||
"""Rebuild every glyph-padded line: 2+ spaces are word gaps, single spaces are noise.
|
||
|
||
Lines that fail `is_letter_spaced` are passed through untouched, because the
|
||
same rule applied to normally typed text would glue its words together.
|
||
"""
|
||
|
||
@wraps(func)
|
||
def wrapper(text, *args, **kwargs):
|
||
lines = []
|
||
for line in (text or "").splitlines():
|
||
if is_letter_spaced(line):
|
||
words = re.split(r" {2,}", line.strip())
|
||
line = " ".join(word.replace(" ", "") for word in words if word.strip())
|
||
lines.append(line)
|
||
return func("\n".join(lines), *args, **kwargs)
|
||
|
||
return wrapper
|