239 lines
8.5 KiB
Python
239 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
|
|
import httpx
|
|
import openai
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from app.core.errors import (
|
|
ErrorCode,
|
|
InvalidPDFError,
|
|
ModelRefusedError,
|
|
ModelResponseInvalidError,
|
|
classify_error,
|
|
)
|
|
from app.models.scoring import ATSScore, CompletedCandidate, FailedCandidate
|
|
from app.services.pdf import ExtractedResume
|
|
from app.services.scoring import score_batch, sort_results
|
|
from tests.conftest import FakeScorer, default_score
|
|
|
|
|
|
def resume(name: str, score: int | None = None) -> ExtractedResume:
|
|
text = f"Resume for {name}."
|
|
if score is not None:
|
|
text += f" SCORE {score}"
|
|
return ExtractedResume(
|
|
filename=f"{name}.pdf",
|
|
candidate_id=name,
|
|
text=text,
|
|
page_count=1,
|
|
truncated=False,
|
|
)
|
|
|
|
|
|
class TestConcurrency:
|
|
async def test_never_exceeds_the_configured_bound(self) -> None:
|
|
scorer = FakeScorer(delay=0.02)
|
|
items = [resume(f"c{i}") for i in range(12)]
|
|
|
|
await score_batch(items, job_description="JD", scorer=scorer, concurrency=3)
|
|
|
|
assert scorer.max_concurrent <= 3
|
|
|
|
async def test_a_bound_of_one_serialises_everything(self) -> None:
|
|
scorer = FakeScorer(delay=0.01)
|
|
items = [resume(f"c{i}") for i in range(5)]
|
|
|
|
await score_batch(items, job_description="JD", scorer=scorer, concurrency=1)
|
|
|
|
assert scorer.max_concurrent == 1
|
|
|
|
|
|
class TestCachePriming:
|
|
async def test_first_candidate_completes_before_any_other_starts(self) -> None:
|
|
"""Without this, all N candidates race and none can read the cached prefix."""
|
|
scorer = FakeScorer(delay=0.02)
|
|
items = [resume(f"c{i}") for i in range(4)]
|
|
|
|
await score_batch(items, job_description="JD", scorer=scorer, concurrency=4)
|
|
|
|
first_end = next(ts for kind, text, ts in scorer.events if kind == "end")
|
|
later_starts = [
|
|
ts for kind, text, ts in scorer.events if kind == "start" and text != items[0].text
|
|
]
|
|
assert later_starts
|
|
assert all(start >= first_end for start in later_starts)
|
|
|
|
async def test_single_candidate_batch_still_works(self) -> None:
|
|
scorer = FakeScorer()
|
|
results = await score_batch(
|
|
[resume("solo", 70)], job_description="JD", scorer=scorer, concurrency=5
|
|
)
|
|
|
|
assert len(results) == 1
|
|
assert isinstance(results[0], CompletedCandidate)
|
|
|
|
async def test_empty_batch_short_circuits(self) -> None:
|
|
scorer = FakeScorer()
|
|
assert await score_batch([], job_description="JD", scorer=scorer, concurrency=5) == []
|
|
assert scorer.calls == []
|
|
|
|
|
|
class TestFailureIsolation:
|
|
async def test_one_failure_does_not_abort_the_others(self) -> None:
|
|
def handler(text: str) -> ATSScore:
|
|
if "boom" in text:
|
|
raise RuntimeError("provider exploded")
|
|
return default_score(text)
|
|
|
|
scorer = FakeScorer(handler)
|
|
items = [resume("ok1", 60), resume("boom"), resume("ok2", 80)]
|
|
|
|
results = await score_batch(items, job_description="JD", scorer=scorer, concurrency=3)
|
|
|
|
assert [type(item).__name__ for item in results] == [
|
|
"CompletedCandidate",
|
|
"FailedCandidate",
|
|
"CompletedCandidate",
|
|
]
|
|
failed = results[1]
|
|
assert isinstance(failed, FailedCandidate)
|
|
assert failed.error_code == ErrorCode.INTERNAL_ERROR
|
|
# Provider text never reaches the client.
|
|
assert "exploded" not in failed.error_message
|
|
|
|
async def test_a_failure_on_the_priming_candidate_still_runs_the_rest(self) -> None:
|
|
def handler(text: str) -> ATSScore:
|
|
if "c0" in text:
|
|
raise RuntimeError("first one failed")
|
|
return default_score(text)
|
|
|
|
scorer = FakeScorer(handler)
|
|
items = [resume("c0"), resume("c1", 55), resume("c2", 65)]
|
|
|
|
results = await score_batch(items, job_description="JD", scorer=scorer, concurrency=3)
|
|
|
|
assert isinstance(results[0], FailedCandidate)
|
|
assert sum(isinstance(item, CompletedCandidate) for item in results) == 2
|
|
|
|
async def test_cancellation_is_not_swallowed(self) -> None:
|
|
def handler(text: str) -> ATSScore:
|
|
raise asyncio.CancelledError
|
|
|
|
scorer = FakeScorer(handler)
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await score_batch([resume("c0")], job_description="JD", scorer=scorer, concurrency=1)
|
|
|
|
async def test_results_are_returned_in_input_order(self) -> None:
|
|
scorer = FakeScorer(delay=0.01)
|
|
items = [resume("a", 10), resume("b", 90), resume("c", 50)]
|
|
|
|
results = await score_batch(items, job_description="JD", scorer=scorer, concurrency=3)
|
|
|
|
assert [item.filename for item in results] == ["a.pdf", "b.pdf", "c.pdf"]
|
|
|
|
|
|
class TestSorting:
|
|
def completed(self, name: str, score: int) -> CompletedCandidate:
|
|
return CompletedCandidate(
|
|
filename=name,
|
|
match_score=score,
|
|
matched_keywords=[],
|
|
missing_keywords=[],
|
|
summary_critique="ok",
|
|
)
|
|
|
|
def failed(self, name: str) -> FailedCandidate:
|
|
return FailedCandidate(filename=name, error_code=ErrorCode.INVALID_PDF, error_message="bad")
|
|
|
|
def test_completed_sorted_descending_failures_last(self) -> None:
|
|
results = sort_results(
|
|
[
|
|
self.completed("low.pdf", 10),
|
|
self.failed("bad.pdf"),
|
|
self.completed("high.pdf", 95),
|
|
self.completed("mid.pdf", 50),
|
|
]
|
|
)
|
|
|
|
assert [item.filename for item in results] == [
|
|
"high.pdf",
|
|
"mid.pdf",
|
|
"low.pdf",
|
|
"bad.pdf",
|
|
]
|
|
|
|
def test_ties_keep_upload_order(self) -> None:
|
|
results = sort_results(
|
|
[self.completed("a.pdf", 70), self.completed("b.pdf", 70), self.completed("c.pdf", 70)]
|
|
)
|
|
|
|
assert [item.filename for item in results] == ["a.pdf", "b.pdf", "c.pdf"]
|
|
|
|
def test_failures_keep_upload_order(self) -> None:
|
|
results = sort_results(
|
|
[self.failed("x.pdf"), self.completed("ok.pdf", 40), self.failed("y.pdf")]
|
|
)
|
|
|
|
assert [item.filename for item in results] == ["ok.pdf", "x.pdf", "y.pdf"]
|
|
|
|
def test_sorting_is_deterministic_across_runs(self) -> None:
|
|
batch = [
|
|
self.completed("a.pdf", 80),
|
|
self.failed("f1.pdf"),
|
|
self.completed("b.pdf", 80),
|
|
self.failed("f2.pdf"),
|
|
]
|
|
assert [i.filename for i in sort_results(list(batch))] == [
|
|
i.filename for i in sort_results(list(batch))
|
|
]
|
|
|
|
|
|
class TestErrorClassification:
|
|
def _request(self) -> httpx.Request:
|
|
return httpx.Request("POST", "https://api.openai.com/v1/responses")
|
|
|
|
def test_timeout(self) -> None:
|
|
code, _ = classify_error(openai.APITimeoutError(request=self._request()))
|
|
assert code == ErrorCode.MODEL_TIMEOUT
|
|
|
|
def test_asyncio_timeout(self) -> None:
|
|
code, _ = classify_error(TimeoutError())
|
|
assert code == ErrorCode.MODEL_TIMEOUT
|
|
|
|
def test_rate_limit(self) -> None:
|
|
response = httpx.Response(429, request=self._request())
|
|
exc = openai.RateLimitError("slow down", response=response, body=None)
|
|
assert classify_error(exc)[0] == ErrorCode.MODEL_RATE_LIMITED
|
|
|
|
def test_connection_error(self) -> None:
|
|
exc = openai.APIConnectionError(request=self._request())
|
|
assert classify_error(exc)[0] == ErrorCode.MODEL_UNAVAILABLE
|
|
|
|
def test_server_error(self) -> None:
|
|
response = httpx.Response(503, request=self._request())
|
|
exc = openai.InternalServerError("down", response=response, body=None)
|
|
assert classify_error(exc)[0] == ErrorCode.MODEL_UNAVAILABLE
|
|
|
|
def test_refusal(self) -> None:
|
|
assert classify_error(ModelRefusedError())[0] == ErrorCode.MODEL_REFUSED
|
|
|
|
def test_invalid_response(self) -> None:
|
|
assert classify_error(ModelResponseInvalidError())[0] == ErrorCode.MODEL_RESPONSE_INVALID
|
|
|
|
def test_pydantic_validation_error(self) -> None:
|
|
with pytest.raises(ValidationError) as caught:
|
|
ATSScore(match_score=150, summary_critique="x")
|
|
assert classify_error(caught.value)[0] == ErrorCode.MODEL_RESPONSE_INVALID
|
|
|
|
def test_pdf_error_passes_through(self) -> None:
|
|
assert classify_error(InvalidPDFError())[0] == ErrorCode.INVALID_PDF
|
|
|
|
def test_unknown_exception_collapses_to_internal(self) -> None:
|
|
code, message = classify_error(RuntimeError("secret detail about a resume"))
|
|
assert code == ErrorCode.INTERNAL_ERROR
|
|
assert "secret" not in message
|