284 lines
9.4 KiB
Python
284 lines
9.4 KiB
Python
"""Adapter tests.
|
|
|
|
These exercise :class:`OpenAIScorer` against a fake ``responses`` resource. The fake
|
|
records the exact prompt prefix each call sends, so a regression that leaks a filename,
|
|
timestamp, or candidate id into the job-description block shows up here as a changed
|
|
prefix rather than as a silent cache-miss cost increase in production.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from app.core.errors import (
|
|
ModelRefusedError,
|
|
ModelResponseInvalidError,
|
|
ModelUnavailableError,
|
|
)
|
|
from app.models.scoring import ATSScore
|
|
from app.services.llm import OpenAIScorer
|
|
|
|
SCORE = ATSScore(
|
|
match_score=77,
|
|
matched_keywords=["Python"],
|
|
missing_keywords=["AWS"],
|
|
summary_critique="Strong backend fit with a cloud gap.",
|
|
)
|
|
|
|
|
|
class FakeInputDetails:
|
|
def __init__(self, cached: int) -> None:
|
|
self.cached_tokens = cached
|
|
|
|
|
|
class FakeOutputDetails:
|
|
def __init__(self) -> None:
|
|
self.reasoning_tokens = 64
|
|
|
|
|
|
class FakeUsage:
|
|
def __init__(self, cached: int) -> None:
|
|
self.input_tokens = 1200
|
|
self.output_tokens = 180
|
|
self.input_tokens_details = FakeInputDetails(cached)
|
|
self.output_tokens_details = FakeOutputDetails()
|
|
|
|
|
|
class FakeIncomplete:
|
|
def __init__(self, reason: str) -> None:
|
|
self.reason = reason
|
|
|
|
|
|
class FakePart:
|
|
def __init__(self, type_: str, refusal: str | None = None) -> None:
|
|
self.type = type_
|
|
self.refusal = refusal
|
|
|
|
|
|
class FakeItem:
|
|
def __init__(self, content: list[FakePart]) -> None:
|
|
self.type = "message"
|
|
self.content = content
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(
|
|
self,
|
|
parsed: Any,
|
|
status: str,
|
|
cached: int,
|
|
*,
|
|
incomplete_reason: str | None = None,
|
|
refusal: str | None = None,
|
|
) -> None:
|
|
self.id = "resp_fake"
|
|
self.status = status
|
|
self.output_parsed = parsed
|
|
self.usage = FakeUsage(cached)
|
|
self.incomplete_details = FakeIncomplete(incomplete_reason) if incomplete_reason else None
|
|
self.output = [FakeItem([FakePart("refusal", refusal)])] if refusal else []
|
|
|
|
|
|
class FakeResponses:
|
|
"""Records every request and reports a cache read on a repeated stable prefix."""
|
|
|
|
def __init__(self, script: list[dict[str, Any]] | None = None) -> None:
|
|
self.calls: list[dict[str, Any]] = []
|
|
self._seen_prefixes: set[str] = set()
|
|
self._script = list(script or [])
|
|
|
|
async def parse(self, **kwargs: Any) -> FakeResponse:
|
|
self.calls.append(kwargs)
|
|
|
|
blocks = kwargs["input"][0]["content"]
|
|
prefix = json.dumps([kwargs["instructions"], blocks[0]], sort_keys=True)
|
|
cached = 4096 if prefix in self._seen_prefixes else 0
|
|
self._seen_prefixes.add(prefix)
|
|
|
|
spec = self._script.pop(0) if self._script else {}
|
|
return FakeResponse(
|
|
spec.get("parsed", SCORE),
|
|
spec.get("status", "completed"),
|
|
cached,
|
|
incomplete_reason=spec.get("incomplete_reason"),
|
|
refusal=spec.get("refusal"),
|
|
)
|
|
|
|
|
|
class FakeClient:
|
|
def __init__(self, responses: FakeResponses) -> None:
|
|
self.responses = responses
|
|
|
|
|
|
def build_scorer(responses: FakeResponses, **overrides: Any) -> OpenAIScorer:
|
|
kwargs: dict[str, Any] = {
|
|
"model": "gpt-5.4-mini",
|
|
"max_output_tokens": 4000,
|
|
"effort": "low",
|
|
"enable_cache": True,
|
|
}
|
|
kwargs.update(overrides)
|
|
return OpenAIScorer(FakeClient(responses), **kwargs) # type: ignore[arg-type]
|
|
|
|
|
|
class TestRequestShape:
|
|
async def test_sends_expected_parameters(self) -> None:
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses)
|
|
|
|
await scorer.score("Backend engineer JD", "Resume text")
|
|
|
|
call = responses.calls[0]
|
|
assert call["model"] == "gpt-5.4-mini"
|
|
assert call["max_output_tokens"] == 4000
|
|
assert call["reasoning"] == {"effort": "low"}
|
|
assert call["text_format"] is ATSScore
|
|
assert "You are a strict Applicant Tracking System evaluator." in call["instructions"]
|
|
|
|
async def test_never_sends_sampling_parameters(self) -> None:
|
|
"""Reasoning models reject temperature / top_p."""
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses)
|
|
|
|
await scorer.score("JD", "Resume")
|
|
|
|
assert not {"temperature", "top_p"} & set(responses.calls[0])
|
|
|
|
async def test_reasoning_is_omitted_for_a_non_reasoning_model(self) -> None:
|
|
"""gpt-4.1 is an allowed model but 400s if sent a reasoning parameter."""
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses, model="gpt-4.1")
|
|
|
|
await scorer.score("JD", "Resume")
|
|
|
|
assert "reasoning" not in responses.calls[0]
|
|
|
|
async def test_stable_block_precedes_the_volatile_one(self) -> None:
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses)
|
|
|
|
await scorer.score("JD", "Resume")
|
|
|
|
blocks = responses.calls[0]["input"][0]["content"]
|
|
assert "<job_description>" in blocks[0]["text"]
|
|
assert "<resume>" in blocks[1]["text"]
|
|
|
|
|
|
class TestPromptCaching:
|
|
async def test_cache_key_is_stable_across_a_batch(self) -> None:
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses)
|
|
|
|
for index in range(4):
|
|
await scorer.score("Shared JD", f"Resume {index}")
|
|
|
|
keys = {call["prompt_cache_key"] for call in responses.calls}
|
|
assert len(keys) == 1
|
|
|
|
async def test_cache_key_is_low_cardinality_not_per_candidate(self) -> None:
|
|
"""A per-candidate key would defeat the routing hint entirely."""
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses)
|
|
|
|
await scorer.score("Shared JD", "Resume A")
|
|
await scorer.score("Shared JD", "Resume B")
|
|
|
|
assert responses.calls[0]["prompt_cache_key"] == responses.calls[1]["prompt_cache_key"]
|
|
|
|
async def test_a_different_job_description_uses_a_different_key(self) -> None:
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses)
|
|
|
|
await scorer.score("JD one", "Resume")
|
|
await scorer.score("JD two", "Resume")
|
|
|
|
assert responses.calls[0]["prompt_cache_key"] != responses.calls[1]["prompt_cache_key"]
|
|
|
|
async def test_cache_key_omitted_when_disabled(self) -> None:
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses, enable_cache=False)
|
|
|
|
await scorer.score("JD", "Resume")
|
|
|
|
assert "prompt_cache_key" not in responses.calls[0]
|
|
|
|
async def test_whole_batch_shares_one_prompt_prefix(self) -> None:
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses)
|
|
|
|
for index in range(5):
|
|
await scorer.score("Shared JD", f"Resume {index}")
|
|
|
|
prefixes = {
|
|
json.dumps(call["input"][0]["content"][0], sort_keys=True) for call in responses.calls
|
|
}
|
|
assert len(prefixes) == 1
|
|
|
|
async def test_second_candidate_reuses_the_prefix(self) -> None:
|
|
responses = FakeResponses()
|
|
scorer = build_scorer(responses)
|
|
|
|
await scorer.score("Shared JD", "Resume A")
|
|
await scorer.score("Shared JD", "Resume B")
|
|
|
|
first, second = responses.calls
|
|
assert first["input"][0]["content"][0] == second["input"][0]["content"][0]
|
|
assert first["input"][0]["content"][1] != second["input"][0]["content"][1]
|
|
|
|
|
|
class TestStatusHandling:
|
|
async def test_returns_the_parsed_score(self) -> None:
|
|
scorer = build_scorer(FakeResponses())
|
|
assert await scorer.score("JD", "Resume") == SCORE
|
|
|
|
async def test_truncation_raises_response_invalid(self) -> None:
|
|
responses = FakeResponses(
|
|
script=[{"status": "incomplete", "incomplete_reason": "max_output_tokens"}]
|
|
)
|
|
scorer = build_scorer(responses)
|
|
|
|
with pytest.raises(ModelResponseInvalidError):
|
|
await scorer.score("JD", "Resume")
|
|
|
|
async def test_content_filter_raises_refused(self) -> None:
|
|
responses = FakeResponses(
|
|
script=[{"status": "incomplete", "incomplete_reason": "content_filter"}]
|
|
)
|
|
scorer = build_scorer(responses)
|
|
|
|
with pytest.raises(ModelRefusedError):
|
|
await scorer.score("JD", "Resume")
|
|
|
|
async def test_refusal_part_raises_refused(self) -> None:
|
|
responses = FakeResponses(
|
|
script=[{"status": "completed", "parsed": None, "refusal": "I can't help"}]
|
|
)
|
|
scorer = build_scorer(responses)
|
|
|
|
with pytest.raises(ModelRefusedError):
|
|
await scorer.score("JD", "Resume")
|
|
|
|
async def test_failed_status_raises_unavailable(self) -> None:
|
|
responses = FakeResponses(script=[{"status": "failed", "parsed": None}])
|
|
scorer = build_scorer(responses)
|
|
|
|
with pytest.raises(ModelUnavailableError):
|
|
await scorer.score("JD", "Resume")
|
|
|
|
async def test_missing_parsed_output_raises_response_invalid(self) -> None:
|
|
responses = FakeResponses(script=[{"status": "completed", "parsed": None}])
|
|
scorer = build_scorer(responses)
|
|
|
|
with pytest.raises(ModelResponseInvalidError):
|
|
await scorer.score("JD", "Resume")
|
|
|
|
async def test_unexpected_parsed_type_raises_response_invalid(self) -> None:
|
|
responses = FakeResponses(script=[{"status": "completed", "parsed": {"match_score": 50}}])
|
|
scorer = build_scorer(responses)
|
|
|
|
with pytest.raises(ModelResponseInvalidError):
|
|
await scorer.score("JD", "Resume")
|