"""Live smoke test for the request shape. Makes two real API calls. Run this once before trusting the service against a new model or SDK version: python scripts/smoke_structured_output.py It proves the three things unit tests cannot: 1. The schema derived from ``ATSScore`` is accepted by structured outputs, and the configured model supports both it and the requested reasoning effort. 2. ``output_parsed`` comes back as a valid ``ATSScore``. 3. The shared job-description prefix is actually cached -- the second call reports ``usage.input_tokens_details.cached_tokens > 0``. Needs OPENAI_API_KEY in the environment or .env, and spends a few cents. """ from __future__ import annotations import asyncio import sys from pathlib import Path # Running a script directly puts scripts/ on sys.path[0], not the repo root. This # environment has another project on the path via an editable-install .pth file, and # it also ships a top-level `app` package -- without this line `import app` silently # resolves to that one instead. sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from openai import AsyncOpenAI from app.core.config import get_settings, supports_reasoning from app.core.logging import configure_logging from app.services.llm import OpenAIScorer # OpenAI only caches prompts at or above 1024 tokens, so a short job description will # report zero cached tokens no matter how stable the prefix is. This one clears it. JOB_DESCRIPTION = ( "Senior backend engineer.\n\n" "Required: Python 3.12, FastAPI, asyncio, Docker, PostgreSQL, REST API design, " "and demonstrated ownership of production services.\n" "Preferred: AWS, Kubernetes, Terraform, observability tooling.\n\n" ) + ("Responsibilities include designing, shipping, and operating backend services. " * 200) RESUME_A = ( "Ada Lovelace\nBackend engineer, 6 years.\n" "Built FastAPI services on Python 3.12 with asyncio and PostgreSQL. " "Owned Docker-based deploys and on-call for a payments API." ) RESUME_B = ( "Grace Hopper\nData engineer, 3 years.\n" "Primarily ETL in Python with pandas and Airflow. Familiar with REST APIs. " "No production service ownership listed." ) async def main() -> int: settings = get_settings() configure_logging(level=settings.log_level, fmt=settings.log_format) print( f"model={settings.openai_model} " f"effort={settings.openai_effort if supports_reasoning(settings.openai_model) else 'n/a'} " f"max_output_tokens={settings.openai_max_output_tokens}" ) client = AsyncOpenAI( api_key=settings.openai_api_key or None, timeout=settings.openai_timeout_seconds, max_retries=settings.openai_max_retries, ) scorer = OpenAIScorer( client, model=settings.openai_model, max_output_tokens=settings.openai_max_output_tokens, effort=settings.openai_effort, enable_cache=settings.openai_enable_prompt_cache, ) try: # Sequential on purpose: a cache entry is only readable once the first # response exists, which is exactly what score_batch's priming step does. first = await scorer.score(JOB_DESCRIPTION, RESUME_A) print( f"call 1 ok: score={first.match_score} name={first.candidate_name!r} " f"title={first.job_title!r} years={first.years_experience} " f"critique={first.summary_critique!r}" ) second = await scorer.score(JOB_DESCRIPTION, RESUME_B) print( f"call 2 ok: score={second.match_score} name={second.candidate_name!r} " f"title={second.job_title!r} years={second.years_experience} " f"critique={second.summary_critique!r}" ) finally: await client.close() print( "\nSchema accepted and both responses parsed. " "Check the 'candidate_scored_upstream' log lines above: call 2 should show a " "non-zero cached_tokens. If it is zero, either the prompt is under the 1024-token " "caching minimum or the job-description prefix is not byte-stable across calls." ) return 0 if __name__ == "__main__": sys.exit(asyncio.run(main()))