109 lines
4.1 KiB
Python
109 lines
4.1 KiB
Python
"""Live smoke test for the request shape. Makes two real API calls.
|
|
|
|
Run this once before trusting the service against a new model or SDK version:
|
|
|
|
python scripts/smoke_structured_output.py
|
|
|
|
It proves the three things unit tests cannot:
|
|
|
|
1. The schema derived from ``ATSScore`` is accepted by structured outputs, and the
|
|
configured model supports both it and the requested reasoning effort.
|
|
2. ``output_parsed`` comes back as a valid ``ATSScore``.
|
|
3. The shared job-description prefix is actually cached -- the second call reports
|
|
``usage.input_tokens_details.cached_tokens > 0``.
|
|
|
|
Needs OPENAI_API_KEY in the environment or .env, and spends a few cents.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Running a script directly puts scripts/ on sys.path[0], not the repo root. This
|
|
# environment has another project on the path via an editable-install .pth file, and
|
|
# it also ships a top-level `app` package -- without this line `import app` silently
|
|
# resolves to that one instead.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from openai import AsyncOpenAI
|
|
|
|
from app.core.config import get_settings, supports_reasoning
|
|
from app.core.logging import configure_logging
|
|
from app.services.llm import OpenAIScorer
|
|
|
|
# OpenAI only caches prompts at or above 1024 tokens, so a short job description will
|
|
# report zero cached tokens no matter how stable the prefix is. This one clears it.
|
|
JOB_DESCRIPTION = (
|
|
"Senior backend engineer.\n\n"
|
|
"Required: Python 3.12, FastAPI, asyncio, Docker, PostgreSQL, REST API design, "
|
|
"and demonstrated ownership of production services.\n"
|
|
"Preferred: AWS, Kubernetes, Terraform, observability tooling.\n\n"
|
|
) + ("Responsibilities include designing, shipping, and operating backend services. " * 200)
|
|
|
|
RESUME_A = (
|
|
"Ada Lovelace\nBackend engineer, 6 years.\n"
|
|
"Built FastAPI services on Python 3.12 with asyncio and PostgreSQL. "
|
|
"Owned Docker-based deploys and on-call for a payments API."
|
|
)
|
|
RESUME_B = (
|
|
"Grace Hopper\nData engineer, 3 years.\n"
|
|
"Primarily ETL in Python with pandas and Airflow. Familiar with REST APIs. "
|
|
"No production service ownership listed."
|
|
)
|
|
|
|
|
|
async def main() -> int:
|
|
settings = get_settings()
|
|
configure_logging(level=settings.log_level, fmt=settings.log_format)
|
|
print(
|
|
f"model={settings.openai_model} "
|
|
f"effort={settings.openai_effort if supports_reasoning(settings.openai_model) else 'n/a'} "
|
|
f"max_output_tokens={settings.openai_max_output_tokens}"
|
|
)
|
|
|
|
client = AsyncOpenAI(
|
|
api_key=settings.openai_api_key or None,
|
|
timeout=settings.openai_timeout_seconds,
|
|
max_retries=settings.openai_max_retries,
|
|
)
|
|
scorer = OpenAIScorer(
|
|
client,
|
|
model=settings.openai_model,
|
|
max_output_tokens=settings.openai_max_output_tokens,
|
|
effort=settings.openai_effort,
|
|
enable_cache=settings.openai_enable_prompt_cache,
|
|
)
|
|
|
|
try:
|
|
# Sequential on purpose: a cache entry is only readable once the first
|
|
# response exists, which is exactly what score_batch's priming step does.
|
|
first = await scorer.score(JOB_DESCRIPTION, RESUME_A)
|
|
print(
|
|
f"call 1 ok: score={first.match_score} name={first.candidate_name!r} "
|
|
f"title={first.job_title!r} years={first.years_experience} "
|
|
f"critique={first.summary_critique!r}"
|
|
)
|
|
|
|
second = await scorer.score(JOB_DESCRIPTION, RESUME_B)
|
|
print(
|
|
f"call 2 ok: score={second.match_score} name={second.candidate_name!r} "
|
|
f"title={second.job_title!r} years={second.years_experience} "
|
|
f"critique={second.summary_critique!r}"
|
|
)
|
|
finally:
|
|
await client.close()
|
|
|
|
print(
|
|
"\nSchema accepted and both responses parsed. "
|
|
"Check the 'candidate_scored_upstream' log lines above: call 2 should show a "
|
|
"non-zero cached_tokens. If it is zero, either the prompt is under the 1024-token "
|
|
"caching minimum or the job-description prefix is not byte-stable across calls."
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main()))
|