223 lines
7.9 KiB
Python
223 lines
7.9 KiB
Python
"""PDF validation and text extraction.
|
|
|
|
Uploads are read once into memory and parsed from ``io.BytesIO``. Nothing is written
|
|
to disk, so no shared predictable path exists to race on.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import logging
|
|
import re
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from pathlib import PurePosixPath, PureWindowsPath
|
|
from typing import Literal
|
|
|
|
from pypdf import PdfReader
|
|
|
|
from app.core.errors import (
|
|
EncryptedPDFError,
|
|
InvalidPDFError,
|
|
PDFTextUnavailableError,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
PDF_SIGNATURE = b"%PDF-"
|
|
# Some real-world PDFs carry a few junk bytes before the header.
|
|
_SIGNATURE_SEARCH_WINDOW = 1024
|
|
|
|
# Below this, extraction produced nothing a reviewer could act on -- almost always a
|
|
# scanned/image-only PDF.
|
|
_MIN_USABLE_CHARS = 30
|
|
|
|
_UNSAFE_FILENAME_CHARS = re.compile(r'[<>:"|?*\x00-\x1f]')
|
|
_CONTROL_CHARS = re.compile(r"[\x01-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
|
_HORIZONTAL_RUNS = re.compile(r"[ \t]{2,}")
|
|
_TRAILING_SPACE = re.compile(r"[ \t]+\n")
|
|
_BLANK_RUNS = re.compile(r"\n{3,}")
|
|
_ALPHANUMERIC = re.compile(r"[A-Za-z0-9]")
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ExtractedResume:
|
|
"""One resume that survived extraction and is ready to score."""
|
|
|
|
filename: str
|
|
candidate_id: str
|
|
text: str
|
|
page_count: int
|
|
truncated: bool
|
|
|
|
|
|
def sanitize_filename(raw: str | None) -> str:
|
|
"""Reduce an uploaded filename to a bare, safe basename.
|
|
|
|
``PurePosixPath(...).name`` alone is not enough: on POSIX it leaves a
|
|
Windows-style ``..\\..\\evil.pdf`` fully intact. ``PureWindowsPath`` treats both
|
|
``/`` and ``\\`` as separators, so it is applied first.
|
|
"""
|
|
if not raw:
|
|
return "resume.pdf"
|
|
|
|
# Strip control characters before path parsing so pathlib never sees a NUL.
|
|
cleaned = _UNSAFE_FILENAME_CHARS.sub("_", raw)
|
|
name = PureWindowsPath(cleaned).name
|
|
name = PurePosixPath(name).name
|
|
name = name.strip().strip(".")
|
|
|
|
if not name:
|
|
return "resume.pdf"
|
|
return name[:255]
|
|
|
|
|
|
def _normalize_text(text: str) -> str:
|
|
"""Strip NULs and control characters, collapse runs, keep meaningful line breaks."""
|
|
text = text.replace("\x00", "")
|
|
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
text = _CONTROL_CHARS.sub("", text)
|
|
text = _HORIZONTAL_RUNS.sub(" ", text)
|
|
text = _TRAILING_SPACE.sub("\n", text)
|
|
text = _BLANK_RUNS.sub("\n\n", text)
|
|
return text.strip()
|
|
|
|
|
|
def is_glyph_fragmented(text: str | None, *, min_lines: int = 20, ratio: float = 0.4) -> bool:
|
|
"""True when pypdf emitted one character per line instead of words.
|
|
|
|
Design tools that position every glyph separately (Canva, InDesign and
|
|
friends) make pypdf's default mode break after each one, so a CV reading
|
|
"LinkedIn: linkedin.com/in/jane" arrives as thirty single-character lines.
|
|
A model reads that fine, which is why it hides: what breaks is every
|
|
substring check downstream. ``verify_matched_keywords`` drops every keyword,
|
|
and on the recruiting side the LinkedIn scan and the skills, company and
|
|
education clamps all return nothing, silently.
|
|
|
|
``min_lines`` stops a two-line PDF or a near-empty page from tripping the
|
|
check on a handful of legitimately short lines.
|
|
"""
|
|
lines = [ln.strip() for ln in (text or "").splitlines() if ln.strip()]
|
|
if len(lines) < min_lines:
|
|
return False
|
|
singles = sum(1 for ln in lines if len(ln) == 1)
|
|
return singles / len(lines) >= ratio
|
|
|
|
|
|
def extract_pdf_text(reader: PdfReader) -> str:
|
|
"""Page text from a reader, repaired when the default mode shatters it.
|
|
|
|
Default mode first: it is faster and already correct for ordinary CVs.
|
|
Layout mode is the fallback, never the default -- it rebuilds the page from
|
|
glyph coordinates, which recovers word and line structure on a fragmented
|
|
file but is slower and pads ordinary documents with alignment whitespace.
|
|
Reaching for it only when the default output is measurably broken means a
|
|
CV that extracts cleanly today keeps extracting exactly as it does now.
|
|
|
|
The fallback is checked before it is trusted: if layout mode comes back
|
|
fragmented too, or empty, the default text is kept. Fragmented text still
|
|
scores a candidate; empty text fails them outright.
|
|
"""
|
|
default = "\n".join((page.extract_text() or "") for page in reader.pages)
|
|
if not is_glyph_fragmented(default):
|
|
return default
|
|
try:
|
|
layout = "\n".join(
|
|
(page.extract_text(extraction_mode="layout") or "") for page in reader.pages
|
|
)
|
|
except Exception: # older pypdf, or a page layout mode chokes on
|
|
return default
|
|
if not layout.strip() or is_glyph_fragmented(layout):
|
|
return default
|
|
return layout
|
|
|
|
|
|
def _truncate(text: str, max_chars: int) -> tuple[str, bool]:
|
|
"""Cut at a line boundary near the limit rather than mid-word."""
|
|
if len(text) <= max_chars:
|
|
return text, False
|
|
|
|
window = text[:max_chars]
|
|
boundary = window.rfind("\n")
|
|
if boundary >= int(max_chars * 0.8):
|
|
window = window[:boundary]
|
|
return window.rstrip(), True
|
|
|
|
|
|
def extract_resume(data: bytes, filename: str, max_chars: int) -> ExtractedResume:
|
|
"""Validate and extract one PDF.
|
|
|
|
Raises :class:`~app.core.errors.ATSError` subclasses; callers turn those into
|
|
per-candidate failures so one bad file never aborts a batch.
|
|
"""
|
|
if data[:_SIGNATURE_SEARCH_WINDOW].find(PDF_SIGNATURE) == -1:
|
|
raise InvalidPDFError("missing %PDF- signature")
|
|
|
|
try:
|
|
reader = PdfReader(io.BytesIO(data))
|
|
except Exception as exc: # pypdf raises a wide family of parse errors
|
|
raise InvalidPDFError("pypdf failed to open the document") from exc
|
|
|
|
if reader.is_encrypted:
|
|
# Password handling is deliberately out of scope.
|
|
raise EncryptedPDFError("document is encrypted")
|
|
|
|
try:
|
|
pages = list(reader.pages)
|
|
except Exception as exc:
|
|
raise InvalidPDFError("pypdf failed to enumerate pages") from exc
|
|
|
|
if not pages:
|
|
raise InvalidPDFError("document has no pages")
|
|
|
|
def _pages_in_mode(mode: Literal["plain", "layout"]) -> list[str]:
|
|
out: list[str] = []
|
|
for page in pages:
|
|
try:
|
|
raw = page.extract_text(extraction_mode=mode) or ""
|
|
except Exception: # a single bad page must not sink the whole document
|
|
raw = ""
|
|
out.append(_normalize_text(raw))
|
|
return out
|
|
|
|
page_texts = _pages_in_mode("plain")
|
|
# The same repair extract_pdf_text performs, but page by page, because the
|
|
# page markers below need the split preserved. The whole document is judged
|
|
# together and then every page is re-extracted in one mode, so a document
|
|
# cannot end up half in each.
|
|
if is_glyph_fragmented("\n".join(page_texts)):
|
|
repaired = _pages_in_mode("layout")
|
|
joined = "\n".join(repaired)
|
|
if joined.strip() and not is_glyph_fragmented(joined):
|
|
page_texts = repaired
|
|
|
|
body = "\n".join(chunk for chunk in page_texts if chunk)
|
|
if len(body) < _MIN_USABLE_CHARS or not _ALPHANUMERIC.search(body):
|
|
raise PDFTextUnavailableError("extracted text was empty or unusable")
|
|
|
|
# Page separators are added only after the usability check, so the markers can
|
|
# never make an image-only PDF look like it contained text.
|
|
marked = "\n\n".join(
|
|
f"[Page {index}]\n{chunk}" for index, chunk in enumerate(page_texts, start=1) if chunk
|
|
)
|
|
text, truncated = _truncate(marked, max_chars)
|
|
|
|
resume = ExtractedResume(
|
|
filename=filename,
|
|
candidate_id=uuid.uuid4().hex,
|
|
text=text,
|
|
page_count=len(pages),
|
|
truncated=truncated,
|
|
)
|
|
logger.info(
|
|
"pdf_extracted",
|
|
extra={
|
|
"file_name": resume.filename,
|
|
"candidate_id": resume.candidate_id,
|
|
"page_count": resume.page_count,
|
|
"extracted_chars": len(resume.text),
|
|
"truncated": resume.truncated,
|
|
},
|
|
)
|
|
return resume
|