diff --git a/.gitignore b/.gitignore index 27df84e..bffe937 100644 --- a/.gitignore +++ b/.gitignore @@ -71,7 +71,6 @@ Utopia-ai-hr-ats-portal 1.pem # Local-only Compose overrides (never deployed) docker.local.env -tests/** **/.env** # Paper form source documents (Annexure A/E/J) — reference material, not code. @@ -81,8 +80,12 @@ frontend/dist/** */ docker.local.frontend/dist/** */ frontend/dist/index.html frontend/dist/index.html -tests/** -/backend/tests/** +# `tests/**` and `/backend/tests/**` used to sit here. Both test suites are +# tracked and both are run by scripts/ci-checks.sh, so the rules were inert for +# the files that already existed and did nothing but silently swallow NEW ones: +# a test added to either suite never showed up in `git status`, and CI ran a +# suite that did not include it. Removed rather than negated, because there is +# nothing under either path that should be ignored. frontend/dist/** nginx.conf smoke.test.mjs \ No newline at end of file diff --git a/app/services/pdf.py b/app/services/pdf.py index 822f3de..e3afdf4 100644 --- a/app/services/pdf.py +++ b/app/services/pdf.py @@ -12,6 +12,7 @@ import re import uuid from dataclasses import dataclass from pathlib import PurePosixPath, PureWindowsPath +from typing import Literal from pypdf import PdfReader @@ -82,6 +83,55 @@ def _normalize_text(text: str) -> str: return text.strip() +def is_glyph_fragmented(text: str | None, *, min_lines: int = 20, ratio: float = 0.4) -> bool: + """True when pypdf emitted one character per line instead of words. + + Design tools that position every glyph separately (Canva, InDesign and + friends) make pypdf's default mode break after each one, so a CV reading + "LinkedIn: linkedin.com/in/jane" arrives as thirty single-character lines. + A model reads that fine, which is why it hides: what breaks is every + substring check downstream. ``verify_matched_keywords`` drops every keyword, + and on the recruiting side the LinkedIn scan and the skills, company and + education clamps all return nothing, silently. + + ``min_lines`` stops a two-line PDF or a near-empty page from tripping the + check on a handful of legitimately short lines. + """ + lines = [ln.strip() for ln in (text or "").splitlines() if ln.strip()] + if len(lines) < min_lines: + return False + singles = sum(1 for ln in lines if len(ln) == 1) + return singles / len(lines) >= ratio + + +def extract_pdf_text(reader: PdfReader) -> str: + """Page text from a reader, repaired when the default mode shatters it. + + Default mode first: it is faster and already correct for ordinary CVs. + Layout mode is the fallback, never the default -- it rebuilds the page from + glyph coordinates, which recovers word and line structure on a fragmented + file but is slower and pads ordinary documents with alignment whitespace. + Reaching for it only when the default output is measurably broken means a + CV that extracts cleanly today keeps extracting exactly as it does now. + + The fallback is checked before it is trusted: if layout mode comes back + fragmented too, or empty, the default text is kept. Fragmented text still + scores a candidate; empty text fails them outright. + """ + default = "\n".join((page.extract_text() or "") for page in reader.pages) + if not is_glyph_fragmented(default): + return default + try: + layout = "\n".join( + (page.extract_text(extraction_mode="layout") or "") for page in reader.pages + ) + except Exception: # older pypdf, or a page layout mode chokes on + return default + if not layout.strip() or is_glyph_fragmented(layout): + return default + return layout + + def _truncate(text: str, max_chars: int) -> tuple[str, bool]: """Cut at a line boundary near the limit rather than mid-word.""" if len(text) <= max_chars: @@ -120,13 +170,26 @@ def extract_resume(data: bytes, filename: str, max_chars: int) -> ExtractedResum if not pages: raise InvalidPDFError("document has no pages") - page_texts: list[str] = [] - for page in pages: - try: - raw_text = page.extract_text() or "" - except Exception: # a single bad page must not sink the whole document - raw_text = "" - page_texts.append(_normalize_text(raw_text)) + def _pages_in_mode(mode: Literal["plain", "layout"]) -> list[str]: + out: list[str] = [] + for page in pages: + try: + raw = page.extract_text(extraction_mode=mode) or "" + except Exception: # a single bad page must not sink the whole document + raw = "" + out.append(_normalize_text(raw)) + return out + + page_texts = _pages_in_mode("plain") + # The same repair extract_pdf_text performs, but page by page, because the + # page markers below need the split preserved. The whole document is judged + # together and then every page is re-extracted in one mode, so a document + # cannot end up half in each. + if is_glyph_fragmented("\n".join(page_texts)): + repaired = _pages_in_mode("layout") + joined = "\n".join(repaired) + if joined.strip() and not is_glyph_fragmented(joined): + page_texts = repaired body = "\n".join(chunk for chunk in page_texts if chunk) if len(body) < _MIN_USABLE_CHARS or not _ALPHANUMERIC.search(body): diff --git a/backend/job/candidate/plugins.py b/backend/job/candidate/plugins.py index 606984b..cdcf3ec 100644 --- a/backend/job/candidate/plugins.py +++ b/backend/job/candidate/plugins.py @@ -19,6 +19,11 @@ from pathlib import Path from app.core.config import Settings, get_settings from app.services.llm import OpenAIScorer +# One definition, two extractors. The bulk-ATS engine and this recruiting path +# both read CVs with pypdf and both broke the same way on glyph-fragmented +# files, so the repair lives in the package that owns PDF handling and is +# re-exported here for the callers that import it from this module. +from app.services.pdf import extract_pdf_text, is_glyph_fragmented # noqa: F401 from dotenv import load_dotenv from job.candidate.decorators import despace_line, normalize_unicode diff --git a/backend/job/candidate/views.py b/backend/job/candidate/views.py index 5a079d7..1e96a75 100644 --- a/backend/job/candidate/views.py +++ b/backend/job/candidate/views.py @@ -19,6 +19,7 @@ from job.candidate.plugins import ( contained_download_path, documents_from_message, extract_pdf_link_uris, + extract_pdf_text, get_scorer, get_scoring_settings, normalize_spaced_text, @@ -125,8 +126,10 @@ class FileRead: reader = PdfReader(io.BytesIO(self.file)) if reader.is_encrypted: raise HTTPException(400, "PDF is password protected") - pages = [(page.extract_text() or "") for page in reader.pages] - text = normalize_spaced_text("\n".join(pages)) + # extract_pdf_text, not page.extract_text() directly: some CVs come + # out of pypdf one character per line, which reads fine to the LLM + # but defeats every substring check downstream. See its docstring. + text = normalize_spaced_text(extract_pdf_text(reader)) # Icon-only LinkedIn buttons never appear in extract_text(); the # URL is on the annotation. Append so the employment agent can # return linkedin_url as its own parsed key. diff --git a/tests/unit/test_pdf_fragmentation.py b/tests/unit/test_pdf_fragmentation.py new file mode 100644 index 0000000..0b573ec --- /dev/null +++ b/tests/unit/test_pdf_fragmentation.py @@ -0,0 +1,97 @@ +"""Glyph-fragmented CVs must not silently lose every substring check. + +The defect these cover, found on a live application: a CV whose PDF positions +each glyph separately came out of pypdf's default mode as one character per +line. The text was all there, so the LLM read it and scored the candidate +fine — but `linkedin.com/in/...` was spelled across forty lines, so the +LinkedIn scan found nothing, and so did the skills, company and education +clamps, which all ask whether a string appears in the resume. + +No real PDF and no pypdf call: a stub reader returns canned page text, which +is the only input the function under test actually reads. +""" + +from __future__ import annotations + +import pytest + +from app.services.pdf import extract_pdf_text, is_glyph_fragmented + +# The real shape of the failure, taken from the CV that exposed it. +FRAGMENTED = "\n".join("LinkedIn:linkedin.com/in/mohammad-raza-digital-marketer") +REPAIRED = ( + "MOHAMMAD RAZA\n" + "Performance Marketing Specialist\n" + "Karachi, Pakistan | +923362837939\n" + "LinkedIn:linkedin.com/in/mohammad-raza-digital-marketer\n" +) +ORDINARY = "\n".join(f"Line {i} of an ordinary resume with real words on it." for i in range(30)) + + +class _Page: + """Minimal pypdf page: default text, and optionally a layout-mode variant.""" + + def __init__(self, default, layout=None, layout_raises=False): + self._default = default + self._layout = layout + self._layout_raises = layout_raises + + def extract_text(self, extraction_mode="plain"): + if extraction_mode == "layout": + if self._layout_raises: + raise ValueError("layout mode unsupported") + return self._layout + return self._default + + +class _Reader: + def __init__(self, *pages): + self.pages = list(pages) + + +class TestIsGlyphFragmented: + def test_one_character_per_line_is_fragmented(self): + assert is_glyph_fragmented(FRAGMENTED) is True + + def test_ordinary_text_is_not(self): + assert is_glyph_fragmented(ORDINARY) is False + + def test_short_input_never_trips_it(self): + # A two-line PDF of initials is not evidence of a broken extractor, and + # treating it as such would send every tiny document through layout mode. + assert is_glyph_fragmented("A\nB\nC") is False + + @pytest.mark.parametrize("text", ["", None]) + def test_empty_is_not_fragmented(self, text): + assert is_glyph_fragmented(text) is False + + +class TestExtractPdfText: + def test_ordinary_pdf_is_returned_untouched(self): + # The guarantee that matters most: a CV that extracts cleanly today must + # keep extracting byte-identically, never routed through layout mode. + reader = _Reader(_Page(ORDINARY, layout="LAYOUT SHOULD NOT BE USED")) + assert extract_pdf_text(reader) == ORDINARY + + def test_fragmented_pdf_falls_back_to_layout(self): + reader = _Reader(_Page(FRAGMENTED, layout=REPAIRED)) + out = extract_pdf_text(reader) + assert out == REPAIRED + assert "linkedin.com/in/mohammad-raza-digital-marketer" in out.lower() + + def test_layout_that_is_also_fragmented_is_rejected(self): + reader = _Reader(_Page(FRAGMENTED, layout=FRAGMENTED)) + assert extract_pdf_text(reader) == FRAGMENTED + + def test_empty_layout_is_rejected(self): + # Fragmented text still scores a candidate. Empty text fails them outright. + reader = _Reader(_Page(FRAGMENTED, layout=" ")) + assert extract_pdf_text(reader) == FRAGMENTED + + def test_layout_mode_unsupported_falls_back_to_default(self): + reader = _Reader(_Page(FRAGMENTED, layout_raises=True)) + assert extract_pdf_text(reader) == FRAGMENTED + + def test_pages_are_joined(self): + reader = _Reader(_Page("page one text here"), _Page("page two text here")) + assert extract_pdf_text(reader) == "page one text here\npage two text here"