"""Adapter tests. These exercise :class:`OpenAIScorer` against a fake ``responses`` resource. The fake records the exact prompt prefix each call sends, so a regression that leaks a filename, timestamp, or candidate id into the job-description block shows up here as a changed prefix rather than as a silent cache-miss cost increase in production. """ from __future__ import annotations import json from typing import Any import pytest from app.core.errors import ( ModelRefusedError, ModelResponseInvalidError, ModelUnavailableError, ) from app.models.scoring import ATSScore from app.services.llm import OpenAIScorer SCORE = ATSScore( match_score=77, matched_keywords=["Python"], missing_keywords=["AWS"], summary_critique="Strong backend fit with a cloud gap.", ) class FakeInputDetails: def __init__(self, cached: int) -> None: self.cached_tokens = cached class FakeOutputDetails: def __init__(self) -> None: self.reasoning_tokens = 64 class FakeUsage: def __init__(self, cached: int) -> None: self.input_tokens = 1200 self.output_tokens = 180 self.input_tokens_details = FakeInputDetails(cached) self.output_tokens_details = FakeOutputDetails() class FakeIncomplete: def __init__(self, reason: str) -> None: self.reason = reason class FakePart: def __init__(self, type_: str, refusal: str | None = None) -> None: self.type = type_ self.refusal = refusal class FakeItem: def __init__(self, content: list[FakePart]) -> None: self.type = "message" self.content = content class FakeResponse: def __init__( self, parsed: Any, status: str, cached: int, *, incomplete_reason: str | None = None, refusal: str | None = None, ) -> None: self.id = "resp_fake" self.status = status self.output_parsed = parsed self.usage = FakeUsage(cached) self.incomplete_details = FakeIncomplete(incomplete_reason) if incomplete_reason else None self.output = [FakeItem([FakePart("refusal", refusal)])] if refusal else [] class FakeResponses: """Records every request and reports a cache read on a repeated stable prefix.""" def __init__(self, script: list[dict[str, Any]] | None = None) -> None: self.calls: list[dict[str, Any]] = [] self._seen_prefixes: set[str] = set() self._script = list(script or []) async def parse(self, **kwargs: Any) -> FakeResponse: self.calls.append(kwargs) blocks = kwargs["input"][0]["content"] prefix = json.dumps([kwargs["instructions"], blocks[0]], sort_keys=True) cached = 4096 if prefix in self._seen_prefixes else 0 self._seen_prefixes.add(prefix) spec = self._script.pop(0) if self._script else {} return FakeResponse( spec.get("parsed", SCORE), spec.get("status", "completed"), cached, incomplete_reason=spec.get("incomplete_reason"), refusal=spec.get("refusal"), ) class FakeClient: def __init__(self, responses: FakeResponses) -> None: self.responses = responses def build_scorer(responses: FakeResponses, **overrides: Any) -> OpenAIScorer: kwargs: dict[str, Any] = { "model": "gpt-5.4-mini", "max_output_tokens": 4000, "effort": "low", "enable_cache": True, } kwargs.update(overrides) return OpenAIScorer(FakeClient(responses), **kwargs) # type: ignore[arg-type] class TestRequestShape: async def test_sends_expected_parameters(self) -> None: responses = FakeResponses() scorer = build_scorer(responses) await scorer.score("Backend engineer JD", "Resume text") call = responses.calls[0] assert call["model"] == "gpt-5.4-mini" assert call["max_output_tokens"] == 4000 assert call["reasoning"] == {"effort": "low"} assert call["text_format"] is ATSScore assert "You are a strict Applicant Tracking System evaluator." in call["instructions"] async def test_never_sends_sampling_parameters(self) -> None: """Reasoning models reject temperature / top_p.""" responses = FakeResponses() scorer = build_scorer(responses) await scorer.score("JD", "Resume") assert not {"temperature", "top_p"} & set(responses.calls[0]) async def test_reasoning_is_omitted_for_a_non_reasoning_model(self) -> None: """gpt-4.1 is an allowed model but 400s if sent a reasoning parameter.""" responses = FakeResponses() scorer = build_scorer(responses, model="gpt-4.1") await scorer.score("JD", "Resume") assert "reasoning" not in responses.calls[0] async def test_stable_block_precedes_the_volatile_one(self) -> None: responses = FakeResponses() scorer = build_scorer(responses) await scorer.score("JD", "Resume") blocks = responses.calls[0]["input"][0]["content"] assert "" in blocks[0]["text"] assert "" in blocks[1]["text"] class TestPromptCaching: async def test_cache_key_is_stable_across_a_batch(self) -> None: responses = FakeResponses() scorer = build_scorer(responses) for index in range(4): await scorer.score("Shared JD", f"Resume {index}") keys = {call["prompt_cache_key"] for call in responses.calls} assert len(keys) == 1 async def test_cache_key_is_low_cardinality_not_per_candidate(self) -> None: """A per-candidate key would defeat the routing hint entirely.""" responses = FakeResponses() scorer = build_scorer(responses) await scorer.score("Shared JD", "Resume A") await scorer.score("Shared JD", "Resume B") assert responses.calls[0]["prompt_cache_key"] == responses.calls[1]["prompt_cache_key"] async def test_a_different_job_description_uses_a_different_key(self) -> None: responses = FakeResponses() scorer = build_scorer(responses) await scorer.score("JD one", "Resume") await scorer.score("JD two", "Resume") assert responses.calls[0]["prompt_cache_key"] != responses.calls[1]["prompt_cache_key"] async def test_cache_key_omitted_when_disabled(self) -> None: responses = FakeResponses() scorer = build_scorer(responses, enable_cache=False) await scorer.score("JD", "Resume") assert "prompt_cache_key" not in responses.calls[0] async def test_whole_batch_shares_one_prompt_prefix(self) -> None: responses = FakeResponses() scorer = build_scorer(responses) for index in range(5): await scorer.score("Shared JD", f"Resume {index}") prefixes = { json.dumps(call["input"][0]["content"][0], sort_keys=True) for call in responses.calls } assert len(prefixes) == 1 async def test_second_candidate_reuses_the_prefix(self) -> None: responses = FakeResponses() scorer = build_scorer(responses) await scorer.score("Shared JD", "Resume A") await scorer.score("Shared JD", "Resume B") first, second = responses.calls assert first["input"][0]["content"][0] == second["input"][0]["content"][0] assert first["input"][0]["content"][1] != second["input"][0]["content"][1] class TestStatusHandling: async def test_returns_the_parsed_score(self) -> None: scorer = build_scorer(FakeResponses()) assert await scorer.score("JD", "Resume") == SCORE async def test_truncation_raises_response_invalid(self) -> None: responses = FakeResponses( script=[{"status": "incomplete", "incomplete_reason": "max_output_tokens"}] ) scorer = build_scorer(responses) with pytest.raises(ModelResponseInvalidError): await scorer.score("JD", "Resume") async def test_content_filter_raises_refused(self) -> None: responses = FakeResponses( script=[{"status": "incomplete", "incomplete_reason": "content_filter"}] ) scorer = build_scorer(responses) with pytest.raises(ModelRefusedError): await scorer.score("JD", "Resume") async def test_refusal_part_raises_refused(self) -> None: responses = FakeResponses( script=[{"status": "completed", "parsed": None, "refusal": "I can't help"}] ) scorer = build_scorer(responses) with pytest.raises(ModelRefusedError): await scorer.score("JD", "Resume") async def test_failed_status_raises_unavailable(self) -> None: responses = FakeResponses(script=[{"status": "failed", "parsed": None}]) scorer = build_scorer(responses) with pytest.raises(ModelUnavailableError): await scorer.score("JD", "Resume") async def test_missing_parsed_output_raises_response_invalid(self) -> None: responses = FakeResponses(script=[{"status": "completed", "parsed": None}]) scorer = build_scorer(responses) with pytest.raises(ModelResponseInvalidError): await scorer.score("JD", "Resume") async def test_unexpected_parsed_type_raises_response_invalid(self) -> None: responses = FakeResponses(script=[{"status": "completed", "parsed": {"match_score": 50}}]) scorer = build_scorer(responses) with pytest.raises(ModelResponseInvalidError): await scorer.score("JD", "Resume")