From 867fea495ab5105f1dd9d04efb6a6134c2358cc1 Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Thu, 20 Aug 2026 18:22:33 +0500 Subject: [PATCH 1/7] m. --- backend/inbox/models.py | 8 +++++++- backend/job/app.py | 20 +++++++++++--------- backend/job/candidate/models.py | 32 +++++++++++++++++++++++++------- backend/job/candidate/views.py | 8 +++++--- backend/job/pipeline/views.py | 8 +++++--- frontend/src/api/candidates.js | 23 +++++++++++++++-------- 6 files changed, 68 insertions(+), 31 deletions(-) diff --git a/backend/inbox/models.py b/backend/inbox/models.py index ea1d579..2ba6092 100644 --- a/backend/inbox/models.py +++ b/backend/inbox/models.py @@ -105,9 +105,12 @@ class Inbox(SQLModel, table=True): .outerjoin(AtsResults,cls.ats_id==AtsResults.id) .where(Inbox_Messages.assigned_job_post_id.is_not(None)) .where(Roles.role_name==EnumRoles.CANDIDATE.value) + # Newest-first is the list contract; score is only a tiebreak + # within the same instant. id keeps paging stable. .order_by( - AtsResults.overall_score.desc().nulls_last(), cls.created_at.desc(), + AtsResults.overall_score.desc().nulls_last(), + cls.id.desc(), ) ) if job_post_id: @@ -202,6 +205,9 @@ class Inbox(SQLModel, table=True): qry = qry.where(cls.user_id == user_id) if search: qry = qry.where(cls._candidate_search_filter(search)) + # Most-recent-first is the list contract; id breaks ties so a page + # boundary can't drop or repeat a row when created_at collides. + qry = qry.order_by(cls.created_at.desc(), cls.id.desc()) qry = qry.limit(limit).offset(offset) result = await session.execute(qry) rows = result.scalars().all() diff --git a/backend/job/app.py b/backend/job/app.py index 4e2b225..8502194 100644 --- a/backend/job/app.py +++ b/backend/job/app.py @@ -391,15 +391,17 @@ async def score_inbox_candidates( @router.get("/candidate/scored/fetch") async def fetch_scored_candidates( job_id: str = Query(None), + limit: int = Query(10, ge=1, le=100), + offset: int = Query(0, ge=0), current_user: dict = Depends(require_permission(PermissionTag.CANDIDATES_VIEW)), session: AsyncSession = Depends(get_session), ): - """Persisted leaderboard: completed by score desc, failures last. Without job_id - returns the whole pool across jobs.""" + """Persisted scored candidates, newest first. Without job_id returns the whole + pool across jobs. `total` is the full result-set size, not the page length.""" try: service=CandidateScoring(session=session) - data=await service.fetch_candidates(job_id) - return JSONResponse(content={"data":data,"total":len(data),"status_code":200}) + data,total=await service.fetch_candidates(job_id,limit=limit,offset=offset) + return JSONResponse(content={"data":data,"total":total,"status_code":200}) except HTTPException: raise except Exception as e: @@ -409,7 +411,7 @@ async def fetch_scored_candidates( @router.get("/job/fetch") async def fetch_job_posts( search: str | None = Query(None), - top: int | None = Query(None), + top: int | None = Query(10, ge=1, le=100), skip: int = Query(0, ge=0), ids: str | None = Query(None), active_only: bool = Query(True), @@ -445,7 +447,7 @@ async def fetch_jobs( department: str | None = Query(None), requisition_status: str | None = Query(None), employment_type: str | None = Query(None), - top: int | None = Query(None), + top: int | None = Query(10, ge=1, le=100), skip: int = Query(0, ge=0), # Defaults False, unlike /job/fetch: a requisition list must show CLOSED # requisitions, and those carry is_active = false. Soft-deleted rows are still @@ -486,8 +488,8 @@ async def fetch_candidate_by_id( @router.get("/candidate/fetch") async def fetch_candidate( user_id:str=Query(None), - limit:int=Query(10), - offset:int=Query(0), + limit:int=Query(10,ge=1,le=100), + offset:int=Query(0,ge=0), search:str=Query(None), current_user: dict = Depends(require_permission(PermissionTag.CANDIDATES_VIEW)), session: AsyncSession = Depends(get_session), @@ -772,7 +774,7 @@ async def change_candidate_stage( @router.get("/pipeline/candidates/fetch") async def fetch_pipeline_candidates( job_post_id:Optional[uuid.UUID]=Query(None), - limit:int=Query(200,ge=1,le=1000), + limit:int=Query(10,ge=1,le=1000), offset:int=Query(0,ge=0), current_user: dict = Depends(require_permission(PermissionTag.PIPELINE_VIEW)), session: AsyncSession = Depends(get_session), diff --git a/backend/job/candidate/models.py b/backend/job/candidate/models.py index 6ec9676..92e9107 100644 --- a/backend/job/candidate/models.py +++ b/backend/job/candidate/models.py @@ -90,9 +90,12 @@ class Manual_UPLOAD_CANDIDATE(SQLModel, table=True): &(AtsResults.job_post_id==cls.job_post_id) &(AtsResults.is_current==True), # noqa: E712 ) + # Newest-first is the list contract; score is only a tiebreak + # within the same instant. id keeps paging stable. .order_by( - AtsResults.overall_score.desc().nulls_last(), cls.created_at.desc(), + AtsResults.overall_score.desc().nulls_last(), + cls.id.desc(), ) ) if job_post_id: @@ -278,25 +281,40 @@ class Candidates(SQLModel, table=True): return result.scalars().first() @classmethod - async def get_candidates_by_job(cls, session: AsyncSession, job_id: str | None = None): - """Leaderboard order: completed by score desc, failures last, ties stable. + async def get_candidates_by_job( + cls, + session: AsyncSession, + job_id: str | None = None, + limit: int | None = None, + offset: int = 0, + ): + """Most-recent-first list, paged. Score is only a tiebreak within an instant. job_id=None returns the whole pool across jobs (same ordering) for the - frontend's unscoped Candidates/Talent Pool views. + frontend's unscoped Candidates/Talent Pool views. Returns (rows, total) so + the caller can page without a second count query of its own. """ statement = select(cls) if job_id is not None: uid = cls._as_uuid(job_id) if uid is None: - return [] + return [], 0 statement = statement.where(cls.job_id == uid) + total = ( + await session.execute(select(func.count()).select_from(statement.subquery())) + ).scalar_one() statement = statement.order_by( + cls.created_at.desc(), cls.status.asc(), # "completed" < "failed" cls.match_score.desc().nulls_last(), - cls.created_at.asc(), + cls.id.desc(), ) + if offset: + statement = statement.offset(offset) + if limit is not None: + statement = statement.limit(limit) result = await session.execute(statement) - return result.scalars().all() + return list(result.scalars().all()), total @classmethod async def get_completed_by_email_job(cls, session: AsyncSession, email, job_id): diff --git a/backend/job/candidate/views.py b/backend/job/candidate/views.py index 3c0fa40..eeb29d5 100644 --- a/backend/job/candidate/views.py +++ b/backend/job/candidate/views.py @@ -350,14 +350,16 @@ class CandidateScoring: raise HTTPException(status_code=400,detail="No attachments found for the given message(s)") return await self._score_and_persist(job_id,sources,"inbox",current_user) - async def fetch_candidates(self,job_id=None): + async def fetch_candidates(self,job_id=None,limit=10,offset=0): # job_id omitted -> the whole pool across jobs (frontend Candidates/TalentPool). if job_id is not None: job=await JobPosts.get_job_post_by_id(self.session,job_id) if job is None or job.is_deleted: raise HTTPException(status_code=404,detail="Job post not found") - rows=await Candidates.get_candidates_by_job(self.session,job_id) - return [serialize_candidate(row) for row in rows] + rows,total=await Candidates.get_candidates_by_job( + self.session,job_id,limit=limit,offset=offset, + ) + return [serialize_candidate(row) for row in rows],total async def fetch_candidate_by_id(self,candidate_id): row=await Candidates.get_candidate_by_id(self.session,candidate_id) diff --git a/backend/job/pipeline/views.py b/backend/job/pipeline/views.py index 48ef30d..1db7ed4 100644 --- a/backend/job/pipeline/views.py +++ b/backend/job/pipeline/views.py @@ -13,9 +13,11 @@ class Pipeline: def __init__(self,session:AsyncSession): self.session=session - async def get_all(self,job_post_id=None,limit=None,offset=0): - # limit/offset are per-source, not a merged page: two tables, no common - # order key. limit=200 returns up to 200 inbox AND up to 200 manual rows. + async def get_all(self,job_post_id=None,limit=10,offset=0): + # limit/offset are per-source, not a merged page: two tables that cannot be + # paged as one. limit=10 returns up to 10 inbox AND up to 10 manual rows, + # each newest-first by created_at. `counts`/`total` stay full-set sizes so + # the caller can drive paging off them. try: inbox_data=await Inbox.get_all(self.session,job_post_id=job_post_id,limit=limit,offset=offset) manual_upload_data=await Manual_UPLOAD_CANDIDATE.get_all(self.session,job_post_id=job_post_id,limit=limit,offset=offset) diff --git a/frontend/src/api/candidates.js b/frontend/src/api/candidates.js index 72d4c1b..b2a6ed0 100644 --- a/frontend/src/api/candidates.js +++ b/frontend/src/api/candidates.js @@ -14,18 +14,25 @@ import { downloadFile, request } from '../lib/apiClient' -/** Active job posts for pickers. Needs job_board.view OR candidates.view. */ -export function listJobs() { - return request('/job/fetch') +/** Active job posts for pickers. Needs job_board.view OR candidates.view. + * + * `top` is explicit because /job/fetch now defaults to 10 — a picker dropdown + * that silently showed only the 10 newest jobs would hide the rest. + */ +export function listJobs({ top = 100 } = {}) { + return request('/job/fetch', { params: { top } }) } /** - * Persisted scoring leaderboard. Needs candidates.view. - * Omit jobId for the whole pool across jobs; rows are ordered completed-by- - * score-desc, then failed rows. + * Persisted scored candidates. Needs candidates.view. + * Omit jobId for the whole pool across jobs. Rows come back newest-first by + * created_at and PAGED (limit defaults to 10 server-side); `total` in the + * envelope is the full result-set size, not the page length. */ -export function listCandidates({ jobId } = {}) { - return request('/candidate/scored/fetch', { params: { job_id: jobId } }) +export function listCandidates({ jobId, limit, offset } = {}) { + return request('/candidate/scored/fetch', { + params: { job_id: jobId, limit, offset }, + }) } /** One scored candidate row by id. Needs candidates.view. 404s on unknown ids. */ -- 2.40.1 From 40edc092a38eef8b582c5b20d916d91cdf5946b6 Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Thu, 20 Aug 2026 18:40:24 +0500 Subject: [PATCH 2/7] top limit maitnained --- frontend/src/screens/Jobs.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/screens/Jobs.jsx b/frontend/src/screens/Jobs.jsx index 17f189d..a37c6da 100644 --- a/frontend/src/screens/Jobs.jsx +++ b/frontend/src/screens/Jobs.jsx @@ -28,7 +28,8 @@ import * as tasksApi from '../api/tasks' import { JOB_STATUSES } from '../api/jobs' import { empTypes, fmtShort } from '../data/seed' -const JOB_LIMIT = 200 +// Must stay within GET /jobs/fetch `top` max (le=100); 200 returns 422 and an empty board. +const JOB_LIMIT = 100 async function fetchJobs() { const res = await jobsApi.list({ top: JOB_LIMIT }) -- 2.40.1 From c536def5858659d4ead89d781a98586e5a0443e4 Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Thu, 20 Aug 2026 18:55:49 +0500 Subject: [PATCH 3/7] Update Docker and API configurations for improved job fetching and API proxying - Adjusted `VITE_API_BASE` in Dockerfile and docker-compose.yml to allow same-origin requests, enhancing compatibility with nginx proxy settings. - Increased the `top` query limit in `app.py` to 500 to accommodate frontend requirements while ensuring consistency across job fetching in `Jobs.jsx` and `Managers.jsx`. - Updated nginx configuration to properly proxy API requests, preventing incorrect responses for job-related endpoints. These changes streamline the interaction between the frontend and backend, ensuring a smoother user experience when fetching job data. --- backend/job/app.py | 4 +++- docker-compose.yml | 9 +++++++-- frontend/Dockerfile | 12 +++++++----- frontend/nginx.conf | 14 ++++++++++++++ frontend/src/screens/Jobs.jsx | 2 +- frontend/src/screens/Managers.jsx | 4 +++- 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/backend/job/app.py b/backend/job/app.py index 8502194..a16758b 100644 --- a/backend/job/app.py +++ b/backend/job/app.py @@ -447,7 +447,9 @@ async def fetch_jobs( department: str | None = Query(None), requisition_status: str | None = Query(None), employment_type: str | None = Query(None), - top: int | None = Query(10, ge=1, le=100), + # le=500 (not 100): the Jobs board loads a full client-side page for facets; + # a 200 ceiling used to 422 the SPA and render an empty requisition list. + top: int | None = Query(10, ge=1, le=500), skip: int = Query(0, ge=0), # Defaults False, unlike /job/fetch: a requisition list must show CLOSED # requisitions, and those carry is_active = false. Soft-deleted rows are still diff --git a/docker-compose.yml b/docker-compose.yml index 745e7f2..3d33432 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -151,10 +151,15 @@ services: build: context: ./frontend args: - # Baked into the bundle at build time — change it and rebuild, not restart. - VITE_API_BASE: ${VITE_API_BASE:-http://localhost:8000} + # Empty = same-origin; nginx proxies API paths to backend-api (see nginx.conf). + # A baked http://localhost:8000 / LAN IP sends the browser to a *different* + # listener than the SPA (Cursor steals 127.0.0.1:8000) and empties /jobs. + VITE_API_BASE: ${VITE_API_BASE:-} image: hrms-frontend:local container_name: hrms-frontend + depends_on: + backend-api: + condition: service_healthy ports: - "${FRONTEND_PORT:-5173}:80" healthcheck: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index f2a2e3a..2be9bd2 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -14,11 +14,13 @@ RUN npm ci COPY . . -# Vite inlines VITE_* at BUILD time, so the API origin is fixed when the image is -# built, not when the container starts — rebuild the image to point it elsewhere. -# `.env.production.local` outranks every other env file, so this wins over the empty -# VITE_API_BASE in .env.production (which means "same origin, behind a proxy"). -ARG VITE_API_BASE=http://172.16.204.191:8000 +# Empty VITE_API_BASE = same-origin requests. nginx.conf proxies API paths to +# backend-api:8000, so a LAN IP baked into the bundle can no longer send the +# browser to a different listener than the one serving the SPA (the localhost +# vs 127.0.0.1 vs Docker split that emptied /jobs). +# Override with --build-arg VITE_API_BASE=https://api.example.com only when the +# API is intentionally on another origin. +ARG VITE_API_BASE= RUN printf 'VITE_API_BASE=%s\n' "$VITE_API_BASE" > .env.production.local \ && npm run build diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 147d1dd..7a9def0 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -5,6 +5,20 @@ server { root /usr/share/nginx/html; index index.html; + # Same-origin API proxy. The SPA is built with an empty VITE_API_BASE so + # fetch('/jobs/fetch') stays on this host; without this block nginx would + # serve index.html for those paths (200 HTML) and the Jobs board would + # parse an empty payload. backend-api is the compose service name. + location ~ ^/(users|roles|permissions|permission-tags|managers|inbox|email|job|jobs|candidate|notes|interview|feedback|activity|pipeline|notifications|analytics|offers|tasks|assessments|org-settings|saved-searches|search|docs|openapi\.json|redoc)(/|$) { + proxy_pass http://backend-api:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Authorization $http_authorization; + } + # One SPA at `/`. `vite dev` and `vite preview` serve index.html for every # unmatched path; vite.config.js notes that a static deploy needs the equivalent # rewrite rule. This is it — without it /auth/confirm-email (a router path, not a diff --git a/frontend/src/screens/Jobs.jsx b/frontend/src/screens/Jobs.jsx index a37c6da..2793868 100644 --- a/frontend/src/screens/Jobs.jsx +++ b/frontend/src/screens/Jobs.jsx @@ -28,7 +28,7 @@ import * as tasksApi from '../api/tasks' import { JOB_STATUSES } from '../api/jobs' import { empTypes, fmtShort } from '../data/seed' -// Must stay within GET /jobs/fetch `top` max (le=100); 200 returns 422 and an empty board. +// Backend allows up to 500; stay at 100 so we match Managers.jsx (shared qk.jobs.list). const JOB_LIMIT = 100 async function fetchJobs() { diff --git a/frontend/src/screens/Managers.jsx b/frontend/src/screens/Managers.jsx index c62da51..01ef2c7 100644 --- a/frontend/src/screens/Managers.jsx +++ b/frontend/src/screens/Managers.jsx @@ -19,7 +19,9 @@ async function fetchManagers() { } async function fetchJobs() { - const res = await jobsApi.list({ top: 200 }) + // Keep within GET /jobs/fetch `top` ceiling (and match Jobs.jsx) so a shared + // qk.jobs.list() cache entry is never poisoned by a 422 from top=200. + const res = await jobsApi.list({ top: 100 }) const rows = Array.isArray(res?.data) ? res.data : [] return rows.map(jobsApi.toJobView) } -- 2.40.1 From a8b213be39083f28c5bb5fdd341a743ea043cb04 Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Fri, 21 Aug 2026 13:00:22 +0500 Subject: [PATCH 4/7] commited --- .dockerignore | 5 + .env.example | 37 +++- .gitignore | 1 - DOCKER.md | 116 ++++++++++++ README.md | 78 +------- app/Dockerfile | 17 +- backend/Dockerfile | 23 +-- backend/README.md | 18 +- backend/alembic_setup.py | 117 +++++++++++- .../application_default_credentials.json | 8 + backend/credentials/client_secret.json | 1 + backend/g_sheet/export_json.py | 0 backend/g_sheet/read_sheet.py | 0 backend/main.py | 8 + docker-compose.dev.yml | 98 ++++++++-- docker-compose.yml | 177 ++++++++---------- docker/postgres/Dockerfile | 13 +- frontend/nginx.conf | 26 ++- 18 files changed, 511 insertions(+), 232 deletions(-) create mode 100644 DOCKER.md create mode 100644 backend/credentials/application_default_credentials.json create mode 100644 backend/credentials/client_secret.json create mode 100644 backend/g_sheet/export_json.py create mode 100644 backend/g_sheet/read_sheet.py diff --git a/.dockerignore b/.dockerignore index fa8125f..bd2cfe6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -31,6 +31,11 @@ frontend/ # Candidate CVs live on the bind mount, not inside an image. backend/inbox/decoded_attachments/ +# Alembic revision scripts stay out of images (gitignored; never ship to prod). +# Schema drift is applied filelessly at API boot when DB_AUTOGENERATE=true. +backend/migrations/versions/*.py +!backend/migrations/versions/.gitkeep + docs/ tests/ scripts/ diff --git a/.env.example b/.env.example index 8950763..afcfa78 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,38 @@ -# Copy to .env and fill in. Never commit .env. +# Copy to `.env` at the repo root for Compose variable substitution. +# Secrets for the app itself still live in `backend/.env` (see backend/.env.example). +# Never commit a filled `.env`. + +# --- Compose / production stack ---------------------------------------------------- +# Used by docker-compose.yml for the container Postgres and frontend publish port. + +DB_USERNAME=postgres +DB_PASSWORD=change-me-strong-password +DB_NAME=hrms + +# Public SPA port only (production). Dev overlay defaults FRONTEND_PORT to 5173. +FRONTEND_PORT=80 + +# API workers behind nginx (production backend-api command). +UVICORN_WORKERS=2 + +# Schema sync on API startup (docker compose up -d --build). +# true = upgrade (if any on-disk revisions) + apply model drift in-memory. +# Revision .py files stay gitignored and are not written on the server. +DB_AUTO_MIGRATE=true +DB_AUTOGENERATE=true + +# Empty = same-origin fetches; frontend nginx proxies to backend-api. +# Set only when the API is intentionally on another origin. +VITE_API_BASE= + +# Optional: override mail service URL for containers (else host.docker.internal:5000). +# EMAIL_URL=http://host.docker.internal:5000 + +# Optional: point containers at host Postgres instead of Compose postgres +# (also set by docker-compose.dev.yml). +# DB_HOST=host.docker.internal + +# --- Bulk ATS scoring engine (also read by ats-engine / backend) -------------------- OPENAI_API_KEY= @@ -15,7 +49,6 @@ OPENAI_MODEL=gpt-5.4-mini OPENAI_MAX_OUTPUT_TOKENS=4000 # none | minimal | low | medium | high | xhigh -# Per-model support varies; the API rejects a level the model does not implement. OPENAI_EFFORT=low OPENAI_MAX_RETRIES=3 diff --git a/.gitignore b/.gitignore index c30fcdb..8700273 100644 --- a/.gitignore +++ b/.gitignore @@ -57,4 +57,3 @@ frontend/dist/ **.pdf **_**_**.py Utopia-ai-hr-ats-portal 1.pem -db_setup.py \ No newline at end of file diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 0000000..8c19b33 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,116 @@ +# Docker + +Production stack is self-contained: Postgres, Redis, API, ATS engine, Taskiq +workers, and the React SPA. Only the frontend is published on the host. + +## Production + +```bash +# 1. Root compose env (DB password, port, workers) — see .env.example +cp .env.example .env +# Edit DB_PASSWORD (and JWT_SECRET_KEY inside backend/.env) + +# 2. App secrets (JWT, OpenAI, email, Buffer, …) +cp backend/.env.example backend/.env +# Fill JWT_SECRET_KEY, OPENAI_API_KEY, DB_* matching the Compose postgres, … + +# 3. Build and start +docker compose up -d --build +docker compose ps +``` + +| Service | Image | Host port | +|---|---|---| +| `frontend` | `hrms-frontend:local` | `${FRONTEND_PORT:-80}` | +| `backend-api` | `hrms-backend:local` | (internal) | +| `ats-engine` | `hrms-ats-engine:local` | (internal) | +| `redis` | `redis:7-alpine` | (internal) | +| `postgres` | `hrms-postgres:local` | (internal) | +| Taskiq workers / schedulers | `hrms-backend:local` | (internal) | + +Browser → `http:///` → nginx (same-origin) → `backend-api:8000`. +CV files live in the `attachments-data` named volume (shared by API + workers). + +### Schema / migrations (automatic) + +On every `backend-api` start (`docker compose up -d --build`): + +1. Fresh empty Postgres → create all tables from models and stamp a marker. +2. Otherwise → `alembic upgrade head` if any revision files exist in the image + (they normally do not — versions stay gitignored and are excluded from builds). +3. If `DB_AUTOGENERATE=true` (Compose default) → detect ORM drift (new/changed/removed + columns or tables) and apply DDL **in-memory** — no `versions/*.py` is written + on the server. +4. Apply any pending `backend/migrations/manual/*.sql` (seed/RBAC batches only). + +Revision scripts under `backend/migrations/versions/` remain gitignored and are +listed in `.dockerignore` so they never ship in the production image. + +Toggle with root `.env`: `DB_AUTO_MIGRATE` / `DB_AUTOGENERATE` (default `true`). + +### Verify + +```bash +docker compose config +curl -sf http://127.0.0.1/health # nginx → backend /health +curl -sf -o /dev/null -w "%{http_code}\n" http://127.0.0.1/ +docker compose logs -f backend-api +``` + +Open the SPA, sign in, confirm `/jobs` lists requisitions (Network: `/jobs/fetch` +same-origin JSON, not HTML). + +### Secrets + +- Never bake `.env` into images (`.dockerignore` already excludes them). +- Require a strong `DB_PASSWORD` and `JWT_SECRET_KEY` before any real deploy. +- Root `.env` is for Compose substitution; `backend/.env` is for the app process. + +### TLS + +This stack serves HTTP on the frontend port. Terminate TLS at a reverse proxy or +cloud load balancer in front of port 80. + +## Local development (host Postgres) + +Restores published ports, bind-mounted attachments, and `--reload`: + +```bash +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build +``` + +| Override | Value | +|---|---| +| `DB_HOST` | `host.docker.internal` | +| Frontend | `${FRONTEND_PORT:-5173}` | +| API | `${BACKEND_PORT:-8000}` | +| ATS | `${ATS_PORT:-8100}` | +| Redis | `${REDIS_PORT:-6379}` | +| Attachments | `./backend/inbox/decoded_attachments` | + +Container Postgres still starts (production default) but is unused while +`DB_HOST=host.docker.internal`. Host Postgres must accept Docker-bridge clients +(`listen_addresses = '*'`, `pg_hba` for the bridge subnet). + +For the Vite HMR loop, run `npm run dev` on the host against +`frontend/.env.development` (`VITE_API_BASE=http://127.0.0.1:8000`). + +## Data migration + +The Compose Postgres volume starts empty. To move an existing host database: + +```bash +pg_dump -Fc hrms > hrms.dump +# with postgres published via the dev overlay, or `docker compose exec -T postgres …` +pg_restore -h 127.0.0.1 -p 5433 -U postgres -d hrms --clean --if-exists hrms.dump +``` + +## Useful commands + +```bash +docker compose logs -f backend-api +docker compose logs -f taskiq-worker +docker compose restart backend-api +docker compose down # keep volumes +docker compose down -v # wipe postgres + attachments + redis data +``` diff --git a/README.md b/README.md index 67d821b..781e108 100644 --- a/README.md +++ b/README.md @@ -115,82 +115,24 @@ docker compose up redis taskiq-worker taskiq-scheduler ## Docker -Every service has its own image and its own container, all in one -[docker-compose.yml](docker-compose.yml). **Postgres is in that file but does not run** -— it sits behind a compose profile, and the stack talks to the PostgreSQL server -already running on the host. +Self-contained production stack (Postgres in Compose; only the SPA is published). +See **[DOCKER.md](DOCKER.md)** for env checklist, verification, TLS notes, and the +local host-Postgres overlay. ```bash -docker compose build -docker compose up -d -docker compose ps +cp .env.example .env # set DB_PASSWORD +cp backend/.env.example backend/.env # set JWT_SECRET_KEY, OPENAI_API_KEY, … +docker compose up -d --build +# SPA: http://localhost/ health: http://localhost/health ``` -| Service | Image | Host port | Built from | -|---|---|---|---| -| `backend-api` | `hrms-backend:local` | 8000 | [backend/Dockerfile](backend/Dockerfile) | -| `taskiq-worker` · `taskiq-scheduler` · `taskiq-cv-worker` · `taskiq-cv-scheduler` | `hrms-backend:local` (same image, different `command`) | — | same | -| `ats-engine` | `hrms-ats-engine:local` | 8100 | [app/Dockerfile](app/Dockerfile) | -| `frontend` | `hrms-frontend:local` | 5173 | [frontend/Dockerfile](frontend/Dockerfile) | -| `redis` | `redis:7-alpine` | 6379 | — | -| `postgres` *(profile `postgres` — never starts by default)* | `hrms-postgres:local` | 5433 | [docker/postgres/Dockerfile](docker/postgres/Dockerfile) | - -The backend image builds from the **repo root**, not `./backend`: `job/candidate` -imports the scoring engine from `app/`, and `inbox.plugins` pulls that in transitively, -so a `./backend` context produces workers that die on `No module named 'app'`. - -### The shared file mount - -`backend/inbox/decoded_attachments/` on the host is bind-mounted into every container -that touches a CV — `backend-api`, `taskiq-worker`, `taskiq-cv-worker` — at the -identical path `/app/inbox/decoded_attachments`. A PDF written by the API is the same -file the worker opens, and absolute paths stored in the database resolve in either -direction (`inbox.plugins.resolve_attachment_path` also falls back to -basename-under-that-folder for rows written by a host process). Point it elsewhere with -`ATTACHMENTS_DIR=/some/host/path`. - -### Talking to the host - -`backend/.env` is written for host processes, so compose overrides the three values a -container needs: `DB_HOST=host.docker.internal` (the local Postgres), -`EMAIL_URL=http://host.docker.internal:5000` (the email service on the host), and -`REDIS_URL=redis://redis:6379/0`. The host Postgres must accept connections from the -Docker bridge — `listen_addresses = '*'` plus a `pg_hba.conf` entry for `172.16.0.0/12`. - -> **Stop the host `uvicorn` and `npm run dev` first.** Windows lets a host process bind -> `127.0.0.1:8000` while Docker binds `0.0.0.0:8000`, and `localhost` resolves to `::1` -> first — so both listen and requests silently reach whichever won. Same for 5173. Use -> `BACKEND_PORT` / `FRONTEND_PORT` / `ATS_PORT` if both must run. - -`VITE_API_BASE` is inlined into the bundle at **build** time (default -`http://localhost:8000`), so changing the API origin means rebuilding the frontend -image, not restarting the container. - -### The Postgres profile - -The image is defined alongside everything else, but the `postgres` profile keeps it out -of `docker compose build` and `docker compose up` — bringing it up is always explicit: +Local day-to-day (host Postgres, exposed API ports, `--reload`): ```bash -docker compose --profile postgres build postgres -docker compose --profile postgres up -d postgres # host port 5433; 5432 is the host server's +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build ``` -Pointing the app at it is a second, deliberate step: set `DB_HOST=postgres` (the only -value that changes — services reach it on 5432 over the compose network) and recreate -the services. Its volume starts empty, so Alembic rebuilds the schema on first boot; it -does not share the host server's data. - -### Live-code overlay - -[docker-compose.dev.yml](docker-compose.dev.yml) is not a second stack — it defines no -services or images, it only adds source bind mounts and `--reload` to the ones above: - -```bash -docker compose -f docker-compose.yml -f docker-compose.dev.yml up -``` - -### 4. First run +### First run 1. Sign up / log in (`/auth/login`) — the user needs a role carrying `candidates.create` + `candidates.view` (RBAC screen or seed a role). diff --git a/app/Dockerfile b/app/Dockerfile index 99613fb..58b7cc0 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -3,10 +3,6 @@ # Bulk ATS scoring engine — the standalone FastAPI service (CLAUDE.md is its spec). # Serves POST /api/v1/score, GET /api/v1/health and the card-grid test UI at /. # -# The backend imports this same package as a library; this image is the separate -# service form of it, so it can be scaled, restarted or pointed at a different model -# independently of the portal API. -# # THE BUILD CONTEXT IS THE REPO ROOT (pyproject.toml lives there): # # docker build -f app/Dockerfile -t hrms-ats-engine:local . @@ -20,16 +16,17 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /srv +RUN groupadd --system app && useradd --system --gid app --home-dir /srv --shell /usr/sbin/nologin app + COPY pyproject.toml ./ COPY app/ ./app/ -# Installs the pinned dependencies from pyproject.toml along with the package. The -# copy at /srv/app stays on sys.path ahead of the installed one, so the dev overlay's -# source bind mount is what actually executes. -RUN pip install --no-cache-dir . +# Installs the pinned dependencies from pyproject.toml along with the package. +RUN pip install --no-cache-dir . \ + && chown -R app:app /srv + +USER app EXPOSE 8100 -# No module-level `app` object exists on purpose (app/main.py), so the factory form -# is mandatory here. CMD ["uvicorn", "app.main:create_app", "--factory", "--host", "0.0.0.0", "--port", "8100"] diff --git a/backend/Dockerfile b/backend/Dockerfile index 1da523b..58ec160 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,8 +1,7 @@ # syntax=docker/dockerfile:1 # -# Backend image. The FastAPI API and all four Taskiq processes (inbox worker and -# scheduler, CV worker and scheduler) run from this one image; docker-compose picks -# the process with `command:`. +# Backend image. The FastAPI API and all Taskiq processes run from this one image; +# docker-compose picks the process with `command:`. # # THE BUILD CONTEXT IS THE REPO ROOT, not ./backend: # @@ -22,21 +21,23 @@ ENV PYTHONUNBUFFERED=1 \ WORKDIR /app +RUN groupadd --system app && useradd --system --gid app --home-dir /app --shell /usr/sbin/nologin app + COPY backend/requirements.txt ./requirements.txt RUN pip install --no-cache-dir -r requirements.txt # Backend tree at /app; the scoring engine at /app/app so `import app.services.pdf` -# resolves under PYTHONPATH=/app. requirements.txt says to `pip install -e ..` for -# this in a host environment — copying it in is the container equivalent, and its -# dependencies (openai, pypdf, pydantic-settings, python-multipart) are already pinned -# above. +# resolves under PYTHONPATH=/app. COPY backend/ /app/ COPY app/ /app/app/ -# Decoded CV attachments are read and written here. docker-compose bind-mounts the -# host folder over this path so every container shares one set of files; creating it -# in the image keeps an un-mounted container from failing on first write. -RUN mkdir -p /app/inbox/decoded_attachments +# Decoded CV attachments are read and written here. Compose mounts a named volume +# (prod) or a host bind (dev) over this path; creating it in the image keeps an +# un-mounted container from failing on first write. +RUN mkdir -p /app/inbox/decoded_attachments \ + && chown -R app:app /app + +USER app EXPOSE 8000 diff --git a/backend/README.md b/backend/README.md index bc4307f..6807e88 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1046,19 +1046,23 @@ and a module called `alembic.py` would shadow the installed package. ## Docker -The repo-root `docker-compose.yml` runs Redis plus the four Taskiq processes (inbox -worker/scheduler and CV-upload worker/scheduler); the API itself is expected to run on the -host (the compose file points the containers at `host.docker.internal` for the database). +Production Compose is self-contained (Postgres in Docker; only the SPA is published). +Local host-Postgres + reload uses the dev overlay. See repo-root **[DOCKER.md](../DOCKER.md)**. ```bash -docker compose up -d # from the repo root +# Production +docker compose up -d --build + +# Local (host DB, published ports, --reload) +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build docker compose logs -f taskiq-worker docker compose logs -f taskiq-cv-worker ``` -`backend/Dockerfile` builds a `python:3.12-slim` image whose default command is the Taskiq -worker. `backend/inbox/decoded_attachments` is bind-mounted so the worker can read the -attachments the API wrote. +`backend/Dockerfile` builds a `python:3.12-slim` image (non-root `app` user) used by +the API and every Taskiq process. In production, CV attachments live in the +`attachments-data` named volume; the dev overlay bind-mounts +`backend/inbox/decoded_attachments`. --- diff --git a/backend/alembic_setup.py b/backend/alembic_setup.py index 8823770..4063f48 100644 --- a/backend/alembic_setup.py +++ b/backend/alembic_setup.py @@ -23,8 +23,9 @@ from pathlib import Path from typing import Any, AsyncIterator, Callable, Sequence from alembic import command -from alembic.autogenerate import compare_metadata +from alembic.autogenerate import compare_metadata, produce_migrations from alembic.config import Config +from alembic.operations import Operations from alembic.runtime.migration import MigrationContext from alembic.script import ScriptDirectory from sqlalchemy import MetaData, text @@ -197,7 +198,10 @@ def context_options() -> dict[str, Any]: s = get_settings() return { "compare_type": True, - "compare_server_default": True, + # Server-default string forms differ between reflection and models + # (e.g. now() vs CURRENT_TIMESTAMP); comparing them re-applies the same + # ALTER on every boot when we sync filelessly. + "compare_server_default": False, "include_schemas": bool(s.db_schemas), "version_table_schema": s.db_default_schema or None, "include_object": _include_object, @@ -233,18 +237,110 @@ async def current() -> str | None: return await _run(lambda c: MigrationContext.configure(c, opts=opts).get_current_revision()) +MODELS_STAMP = "models" # alembic_version marker when no revision files ship in the image + + async def upgrade(revision: str = "head") -> None: + if revision == "head" and not head(): + logger.info("no alembic revisions on disk; skipping upgrade") + return await _run(lambda c: command.upgrade(config(c), revision)) logger.info("upgraded to %s", revision) +async def stamp(revision: str = "head") -> None: + target = revision + if target == "head" and not head(): + target = MODELS_STAMP + await _run(lambda c: command.stamp(config(c), target)) + logger.info("stamped database at %s", target) + + +async def _schema_is_empty() -> bool: + """True when the app schema has never been populated (fresh Compose volume).""" + schema = get_settings().db_default_schema or "public" + async with get_engine().connect() as conn: + row = ( + await conn.execute( + text( + "SELECT 1 FROM information_schema.tables " + "WHERE table_schema = :schema AND table_name = 'users' LIMIT 1" + ), + {"schema": schema}, + ) + ).first() + return row is None + + +async def bootstrap_empty() -> None: + """Create every model table and stamp a revision marker. + + Several historical revisions assume tables (e.g. job_posts) that were never + given a create_table in the chain — they only exist on DBs that grew via + autogenerate. A brand-new Compose Postgres volume therefore cannot + `upgrade head`. Creating from metadata then stamping is the production + bootstrap for that case; existing databases keep the normal upgrade path. + + Revision `.py` files are gitignored and excluded from images; stamp uses + `models` when the versions directory is empty. + """ + metadata = target_metadata() + async with get_engine().begin() as conn: + await conn.run_sync(metadata.create_all) + await stamp("head") + logger.info("bootstrapped empty database from models") + + async def downgrade(revision: str = "-1") -> None: await _run(lambda c: command.downgrade(config(c), revision)) logger.info("downgraded to %s", revision) +def _apply_upgrade_ops(connection: Connection) -> int: + """Apply ORM→DB diffs in-process without writing a revision file.""" + opts = {k: v for k, v in context_options().items() if k != "process_revision_directives"} + ctx = MigrationContext.configure(connection, opts=opts) + script = produce_migrations(ctx, target_metadata()) + if script.upgrade_ops.is_empty(): + return 0 + operations = Operations(ctx) + applied = 0 + stack = [script.upgrade_ops] + while stack: + elem = stack.pop(0) + if hasattr(elem, "ops"): + stack.extend(elem.ops) + else: + operations.invoke(elem) + applied += 1 + return applied + + +async def apply_model_drift() -> bool: + """Sync the live schema to the ORM without creating migration files. + + Used by Docker/prod boots so `versions/*.py` can stay gitignored and out of + the image. Returns True when at least one DDL op was applied. + """ + opts = {k: v for k, v in context_options().items() if k != "process_revision_directives"} + diffs = await _run( + lambda c: compare_metadata(MigrationContext.configure(c, opts=opts), target_metadata()) + ) + if not diffs: + logger.info("schema matches the models") + return False + logger.info("%s schema difference(s) detected; applying without revision files", len(diffs)) + applied = await _run(_apply_upgrade_ops) + logger.info("applied %s schema operation(s)", applied) + return applied > 0 + + async def autogenerate(message: str = "auto") -> str | None: - """Write a revision if the models have drifted; return its id, or None.""" + """Write a revision if the models have drifted; return its id, or None. + + Local/CLI only. Docker boots use `apply_model_drift` instead so revision + files are never written on the server. + """ opts = {k: v for k, v in context_options().items() if k != "process_revision_directives"} diffs = await _run( lambda c: compare_metadata(MigrationContext.configure(c, opts=opts), target_metadata()) @@ -308,12 +404,21 @@ async def _lock() -> AsyncIterator[None]: async def migrate(*, autogen: bool | None = None, message: str = "auto") -> None: - """Apply pending revisions, fresh model drift, then manual SQL, under the lock.""" + """Bring the DB in line with models, then run manual SQL, under the lock. + + Empty Compose volumes bootstrap from models (create_all + stamp). Every boot + upgrades any on-disk revisions (skipped when versions are absent from the + image), then — when DB_AUTOGENERATE is on — applies ORM drift in-memory so + no `versions/*.py` files are written on the server. + """ should_autogen = get_settings().db_autogenerate if autogen is None else autogen async with _lock(): - await upgrade() - if should_autogen and await autogenerate(message): + if await _schema_is_empty(): + await bootstrap_empty() + else: await upgrade() + if should_autogen: + await apply_model_drift() await run_manual_sql() logger.info("database at revision %s", await current()) diff --git a/backend/credentials/application_default_credentials.json b/backend/credentials/application_default_credentials.json new file mode 100644 index 0000000..7c33d3a --- /dev/null +++ b/backend/credentials/application_default_credentials.json @@ -0,0 +1,8 @@ +{ + "account": "", + "client_id": "679334897177-ufal3rogbg8cgm3qcren6pv20phqd7nl.apps.googleusercontent.com", + "client_secret": "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", + "refresh_token": "1//038IfgCu3D42fCgYIARAAGAMSNwF-L9IrYAZ_DJUqwC9ETwLtH23D46j61gWMwFQjRWPklFZIiLmv7Q3-TOgcNxTrjO30jgkeYOo", + "type": "authorized_user", + "universe_domain": "googleapis.com" +} \ No newline at end of file diff --git a/backend/credentials/client_secret.json b/backend/credentials/client_secret.json new file mode 100644 index 0000000..ffd8c9c --- /dev/null +++ b/backend/credentials/client_secret.json @@ -0,0 +1 @@ +{"installed":{"client_id":"679334897177-ufal3rogbg8cgm3qcren6pv20phqd7nl.apps.googleusercontent.com","project_id":"hrms-ats-portal","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"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","redirect_uris":["http://localhost"]}} \ No newline at end of file diff --git a/backend/g_sheet/export_json.py b/backend/g_sheet/export_json.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/g_sheet/read_sheet.py b/backend/g_sheet/read_sheet.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/main.py b/backend/main.py index b171a05..ead81b9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -3,6 +3,7 @@ from contextlib import asynccontextmanager from fastapi.middleware.cors import CORSMiddleware from fastapi import FastAPI +from fastapi.responses import JSONResponse from db_setup import lifespan as db_lifespan from inbox.app import router as inbox_router from users.app import router as users_router @@ -88,6 +89,13 @@ app.add_middleware( allow_headers=["*"], ) + +@app.get("/health") +async def health(): + # Liveness only — no DB. Compose healthchecks and load balancers hit this. + return JSONResponse(content={"status":"ok","status_code":200}) + + app.include_router(inbox_router) app.include_router(users_router) app.include_router(role_router) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 64730d3..63eb9f1 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,28 +1,49 @@ -# Live-code overlay. Mounts the source folders into the running containers, so what -# executes is what is on disk on the host — edit, save, uvicorn reloads. +# Local / host-Postgres overlay. Restores the previous day-to-day workflow on top +# of the production docker-compose.yml: # -# docker compose -f docker-compose.yml -f docker-compose.dev.yml up +# docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build # -# The CV attachments mount from the base file still applies: compose merges volumes -# by target path, and /app/inbox/decoded_attachments is nested under the /app mount, -# so the daemon mounts the parent first and the attachments folder on top. +# - DB_HOST=host.docker.internal (host Postgres; container postgres still starts +# but is unused unless you flip DB_HOST back) +# - Publishes API / ATS / Redis / frontend ports for direct host access +# - Bind-mounts CV attachments to ./backend/inbox/decoded_attachments +# - Live source mounts + --reload on API / ATS / workers # -# Two mounts per Python service, not one: /app is the backend tree and /app/app is the -# bulk-ats engine that backend/job/candidate imports. Mounting only ./backend over -# /app would hide the engine baked into the image and every worker would fail on -# import. -# -# The frontend stays as the built nginx image — a Vite dev server wants node_modules -# on the mount, which is slow and fragile across a Windows bind mount. Run -# `npm run dev` on the host for the frontend loop. +# Frontend stays as the built nginx image — run `npm run dev` on the host for the +# Vite loop if you need HMR. services: + redis: + ports: + - "${REDIS_PORT:-6379}:6379" + + postgres: + ports: + - "127.0.0.1:${POSTGRES_PORT:-5433}:5432" + backend-api: + environment: + PYTHONPATH: /app + DB_HOST: host.docker.internal + REDIS_URL: redis://redis:6379/0 + EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} + BACKEND_URL: http://backend-api:8000 command: - ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] + [ + "uvicorn", + "main:app", + "--host", + "0.0.0.0", + "--port", + "8000", + "--reload", + ] + ports: + - "${BACKEND_PORT:-8000}:8000" volumes: - ./backend:/app - ./app:/app/app + - ${ATTACHMENTS_DIR:-./backend/inbox/decoded_attachments}:/app/inbox/decoded_attachments ats-engine: command: @@ -36,30 +57,77 @@ services: "8100", "--reload", ] + ports: + - "${ATS_PORT:-8100}:8100" volumes: - ./app:/srv/app + frontend: + ports: + - "${FRONTEND_PORT:-5173}:80" + taskiq-worker: + environment: + PYTHONPATH: /app + DB_HOST: host.docker.internal + REDIS_URL: redis://redis:6379/0 + EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} + BACKEND_URL: http://backend-api:8000 + TASKIQ_QUEUE_NAME: inbox + TASKIQ_WORKER_NAME: worker-01 volumes: - ./backend:/app - ./app:/app/app + - ${ATTACHMENTS_DIR:-./backend/inbox/decoded_attachments}:/app/inbox/decoded_attachments taskiq-scheduler: + environment: + PYTHONPATH: /app + DB_HOST: host.docker.internal + REDIS_URL: redis://redis:6379/0 + EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} + BACKEND_URL: http://backend-api:8000 + TASKIQ_QUEUE_NAME: inbox volumes: - ./backend:/app - ./app:/app/app taskiq-cv-worker: + environment: + PYTHONPATH: /app + DB_HOST: host.docker.internal + REDIS_URL: redis://redis:6379/0 + EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} + BACKEND_URL: http://backend-api:8000 + TASKIQ_CV_QUEUE_NAME: cv_upload + TASKIQ_WORKER_NAME: cv-worker-01 volumes: - ./backend:/app - ./app:/app/app + - ${ATTACHMENTS_DIR:-./backend/inbox/decoded_attachments}:/app/inbox/decoded_attachments taskiq-cv-scheduler: + environment: + PYTHONPATH: /app + DB_HOST: host.docker.internal + REDIS_URL: redis://redis:6379/0 + EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} + BACKEND_URL: http://backend-api:8000 + TASKIQ_CV_QUEUE_NAME: cv_upload volumes: - ./backend:/app - ./app:/app/app taskiq-mailbox-sync-worker: + environment: + PYTHONPATH: /app + DB_HOST: host.docker.internal + REDIS_URL: redis://redis:6379/0 + EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} + BACKEND_URL: http://backend-api:8000 + TASKIQ_MAILBOX_SYNC_QUEUE_NAME: mailbox_sync + TASKIQ_WORKER_NAME: mailbox-sync-worker-01 volumes: - ./backend:/app - ./app:/app/app + - ${ATTACHMENTS_DIR:-./backend/inbox/decoded_attachments}:/app/inbox/decoded_attachments diff --git a/docker-compose.yml b/docker-compose.yml index 3d33432..884bc7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,32 +1,23 @@ -# HR-ATS-Portal — every service and every image, in one file. +# HR-ATS-Portal — production Compose stack (self-contained). # -# docker compose build # hrms-backend / hrms-ats-engine / hrms-frontend -# docker compose up -d +# docker compose up -d --build # docker compose ps # docker compose logs -f backend-api # -# ── Postgres ────────────────────────────────────────────────────────────────────── -# The `postgres` service at the bottom is behind a compose PROFILE, so it is defined -# and buildable here but never starts with a plain `docker compose up`. The stack -# talks to the PostgreSQL server already running on the host, via -# DB_HOST=host.docker.internal (backend/.env says localhost — correct for a host -# process, wrong inside a container). +# Public surface: only the frontend (default host port 80). The browser talks +# same-origin to nginx, which proxies API paths to backend-api. Postgres, Redis, +# the API, ATS engine and Taskiq workers stay on the Compose network. # -# Host Postgres must accept connections from the Docker bridge: listen_addresses = '*' -# in postgresql.conf and a pg_hba.conf line for 172.16.0.0/12 (or the specific subnet). +# Local / current host-Postgres workflow (exposed ports, --reload, bind mounts): +# docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build # -# To build/run the containerised database instead, see the comments on that service. -# -# ── Ports ───────────────────────────────────────────────────────────────────────── -# Stop a host `uvicorn` (8000) and `npm run dev` (5173) before starting these, or -# override with BACKEND_PORT / FRONTEND_PORT / ATS_PORT. Windows lets a host process -# bind 127.0.0.1:8000 while Docker binds 0.0.0.0:8000, and `localhost` resolves to ::1 -# first — both listen, and requests reach whichever won. -# -# ── Overlay ─────────────────────────────────────────────────────────────────────── -# docker-compose.dev.yml adds live source mounts and --reload on top of this file. It -# defines no services or images of its own; it only overrides the ones here: -# docker compose -f docker-compose.yml -f docker-compose.dev.yml up +# See DOCKER.md for env checklist and verification. + +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" x-backend-build: &backend-build # Root context, not ./backend: backend/job/candidate imports the bulk-ats engine @@ -36,26 +27,28 @@ x-backend-build: &backend-build x-backend-env: &backend-env PYTHONPATH: /app - # backend/.env is written for host processes; these are the values a container needs. - # - # DB_HOST defaults to the LOCAL Postgres server on the host. Set DB_HOST=postgres in - # the shell (or a root .env) to point the whole stack at the container below instead - # — that is the only value that has to change, since services reach it over the - # compose network on 5432, not the published host port. - DB_HOST: ${DB_HOST:-host.docker.internal} + # Production default: container Postgres on the Compose network. The dev overlay + # overrides DB_HOST to host.docker.internal. Credentials come from root .env so + # they stay in lockstep with the postgres service (backend/.env is for host runs). + DB_HOST: ${DB_HOST:-postgres} + DB_PORT: ${DB_PORT:-5432} + DB_USERNAME: ${DB_USERNAME:-postgres} + DB_PASSWORD: ${DB_PASSWORD:-postgres} + DB_NAME: ${DB_NAME:-hrms} + # On every API boot: upgrade any on-disk revisions (usually none in images), + # then apply ORM drift in-memory (no revision files written — versions stay + # gitignored and out of the image). + DB_AUTO_MIGRATE: ${DB_AUTO_MIGRATE:-true} + DB_AUTOGENERATE: ${DB_AUTOGENERATE:-true} REDIS_URL: redis://redis:6379/0 - # Prefer root/.env EMAIL_URL (e.g. http://3.140.173.13:5000). Fall back to the - # host-gateway alias when the mail service runs on this machine's :5000. + # Prefer root/.env EMAIL_URL. Fall back to host-gateway when mail is on this machine. EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} BACKEND_URL: http://backend-api:8000 -# The one shared folder. Every process that decodes, scores or serves a CV reads and -# writes the same host directory, so a file written by the API is the same file the -# worker opens. Absolute paths stored in the DB match across services because the -# mount target is identical everywhere; inbox.plugins.resolve_attachment_path also -# falls back to basename-under-this-directory for rows written by a host process. +# Shared CV storage. Named volume in production so API + workers see the same +# files without a host path. Dev overlay remounts ./backend/inbox/decoded_attachments. x-attachments: &attachments - - ${ATTACHMENTS_DIR:-./backend/inbox/decoded_attachments}:/app/inbox/decoded_attachments + - attachments-data:/app/inbox/decoded_attachments x-backend-service: &backend-service build: *backend-build @@ -69,13 +62,10 @@ x-backend-service: &backend-service depends_on: redis: condition: service_healthy - # required:false — the default stack uses the host's Postgres and never starts - # this one, and that must not be an error. When the profile IS active, startup - # waits for it to pass pg_isready. postgres: condition: service_healthy - required: false restart: unless-stopped + logging: *default-logging services: # --- Redis (broker + result backend for taskiq) ---------------------------------- @@ -83,8 +73,7 @@ services: image: redis:7-alpine container_name: hrms-redis command: ["redis-server", "--appendonly", "yes"] - ports: - - "${REDIS_PORT:-6379}:6379" + # Not published in production. Dev overlay binds ${REDIS_PORT:-6379}:6379. volumes: - redis-data:/data healthcheck: @@ -93,24 +82,59 @@ services: timeout: 5s retries: 5 restart: unless-stopped + logging: *default-logging + + # --- Postgres (always on in production) ------------------------------------------ + postgres: + build: + context: ./docker/postgres + image: hrms-postgres:local + container_name: hrms-postgres + environment: + # Interpolated from the shell or root .env, NOT from backend/.env. + POSTGRES_USER: ${DB_USERNAME:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres} + POSTGRES_DB: ${DB_NAME:-hrms} + POSTGRES_INITDB_ARGS: "--encoding=UTF8" + # Not published in production. Dev overlay can bind 127.0.0.1:${POSTGRES_PORT:-5433}:5432. + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: + ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-postgres} -d ${DB_NAME:-hrms}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 20s + restart: unless-stopped + logging: *default-logging # --- portal API ------------------------------------------------------------------ backend-api: <<: *backend-service container_name: hrms-backend-api - command: ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] - ports: - - "${BACKEND_PORT:-8000}:8000" + command: + [ + "uvicorn", + "main:app", + "--host", + "0.0.0.0", + "--port", + "8000", + "--workers", + "${UVICORN_WORKERS:-2}", + "--proxy-headers", + "--forwarded-allow-ips=*", + ] + # Not published in production — browsers reach the API via frontend nginx. volumes: *attachments healthcheck: - # python:slim has no curl and the API exposes no /health route, so this is a - # plain TCP check against the uvicorn socket. test: [ "CMD", "python", "-c", - "import socket;socket.create_connection(('127.0.0.1',8000),3).close()", + "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=3)", ] interval: 15s timeout: 5s @@ -125,13 +149,10 @@ services: image: hrms-ats-engine:local container_name: hrms-ats-engine env_file: - # OPENAI_API_KEY currently lives in backend/.env; a root .env (see .env.example) - # overrides it when present, and the stack still comes up when it is not. - ./backend/.env - path: ./.env required: false - ports: - - "${ATS_PORT:-8100}:8100" + # Not published in production. healthcheck: test: [ @@ -145,15 +166,14 @@ services: retries: 5 start_period: 20s restart: unless-stopped + logging: *default-logging - # --- React portal ----------------------------------------------------------------- + # --- React portal (only published host port) ------------------------------------- frontend: build: context: ./frontend args: # Empty = same-origin; nginx proxies API paths to backend-api (see nginx.conf). - # A baked http://localhost:8000 / LAN IP sends the browser to a *different* - # listener than the SPA (Cursor steals 127.0.0.1:8000) and empties /jobs. VITE_API_BASE: ${VITE_API_BASE:-} image: hrms-frontend:local container_name: hrms-frontend @@ -161,13 +181,14 @@ services: backend-api: condition: service_healthy ports: - - "${FRONTEND_PORT:-5173}:80" + - "${FRONTEND_PORT:-80}:80" healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1/"] interval: 15s timeout: 5s retries: 5 restart: unless-stopped + logging: *default-logging # --- background processing (same image as backend-api, different command) --------- taskiq-worker: @@ -255,45 +276,7 @@ services: TASKIQ_WORKER_NAME: mailbox-sync-worker-01 volumes: *attachments - # --- Postgres: DEFINED HERE, NOT STARTED BY DEFAULT ------------------------------- - # The profile is what keeps it out of `docker compose build` and `docker compose up`. - # Nothing about the default stack changes by its presence in this file. - # - # docker compose --profile postgres build postgres # build the image - # docker compose --profile postgres up -d postgres # run it, host port 5433 - # - # Pointing the app at it is a separate, deliberate step — set DB_HOST=postgres (see - # x-backend-env) and recreate the services. The volume starts empty, so Alembic - # rebuilds the schema on first boot; it does not share the host server's data. - postgres: - profiles: ["postgres"] - build: - context: ./docker/postgres - image: hrms-postgres:local - container_name: hrms-postgres - environment: - # Interpolated from the shell or a root .env, NOT from backend/.env — compose - # variable substitution and container environment are different things. - # Defaults match backend/.env.example. - POSTGRES_USER: ${DB_USERNAME:-postgres} - POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres} - POSTGRES_DB: ${DB_NAME:-hrms} - POSTGRES_INITDB_ARGS: "--encoding=UTF8" - ports: - # 5433: the host's own Postgres server owns 5432. Only for host-side tools — - # containers reach this one on 5432 over the compose network. - - "${POSTGRES_PORT:-5433}:5432" - volumes: - - postgres-data:/var/lib/postgresql/data - healthcheck: - test: - ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-postgres} -d ${DB_NAME:-hrms}"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 20s - restart: unless-stopped - volumes: redis-data: postgres-data: + attachments-data: diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile index ac681d2..473bf50 100644 --- a/docker/postgres/Dockerfile +++ b/docker/postgres/Dockerfile @@ -1,15 +1,8 @@ # syntax=docker/dockerfile:1 # -# Postgres image — BUILT, BUT NOT USED BY THE DEFAULT STACK. -# -# Every service in docker-compose.yml points at the Postgres already running on the -# host (DB_HOST=host.docker.internal). This image exists so the database can be -# containerised on demand — a clean machine, a throwaway test run, a second dev — and -# it is kept in its own file (docker-compose.postgres.yml) so bringing it up is always -# a deliberate act: -# -# docker compose -f docker-compose.yml -f docker-compose.postgres.yml build postgres -# docker compose -f docker-compose.yml -f docker-compose.postgres.yml up -d postgres +# Postgres image used by the production Compose stack (DB_HOST=postgres). +# The volume starts empty on first boot; Alembic rebuilds the schema. It does +# not share the host server's data — migrate with pg_dump/pg_restore if needed. # # Context is ./docker/postgres. diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 7a9def0..9ebb5f2 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -5,11 +5,20 @@ server { root /usr/share/nginx/html; index index.html; + # CV / multipart uploads (MAX_PDF_SIZE_MB is 10; leave headroom for form fields). + client_max_body_size 25m; + + # Security headers on every response. + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + # Same-origin API proxy. The SPA is built with an empty VITE_API_BASE so # fetch('/jobs/fetch') stays on this host; without this block nginx would # serve index.html for those paths (200 HTML) and the Jobs board would - # parse an empty payload. backend-api is the compose service name. - location ~ ^/(users|roles|permissions|permission-tags|managers|inbox|email|job|jobs|candidate|notes|interview|feedback|activity|pipeline|notifications|analytics|offers|tasks|assessments|org-settings|saved-searches|search|docs|openapi\.json|redoc)(/|$) { + # parse an empty payload. OpenAPI (/docs, /redoc, /openapi.json) is + # intentionally NOT proxied — keep the schema off the public edge. + location ~ ^/(health|users|roles|permissions|permission-tags|managers|inbox|email|job|jobs|candidate|notes|interview|feedback|activity|pipeline|notifications|analytics|offers|tasks|assessments|org-settings|saved-searches|search)(/|$) { proxy_pass http://backend-api:8000; proxy_http_version 1.1; proxy_set_header Host $host; @@ -17,11 +26,12 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_send_timeout 120s; + proxy_read_timeout 120s; } - # One SPA at `/`. `vite dev` and `vite preview` serve index.html for every - # unmatched path; vite.config.js notes that a static deploy needs the equivalent - # rewrite rule. This is it — without it /auth/confirm-email (a router path, not a + # One SPA at `/`. Without this, /auth/confirm-email (a router path, not a # file) 404s when a confirmation email link is opened cold. location / { try_files $uri $uri/ /index.html; @@ -31,11 +41,17 @@ server { location /assets/ { expires 1y; add_header Cache-Control "public, immutable"; + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy strict-origin-when-cross-origin always; } # index.html must never be cached, or a redeploy keeps serving the old asset hashes. location = /index.html { add_header Cache-Control "no-store"; + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy strict-origin-when-cross-origin always; } gzip on; -- 2.40.1 From ea6f67786c515f6c5751094d3398ba9e6343e60d Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Mon, 24 Aug 2026 15:04:45 +0500 Subject: [PATCH 5/7] . --- backend/g_sheet/app.py | 264 ++++++++++++++++ backend/g_sheet/enums.py | 279 +++++++++++++++++ backend/g_sheet/models.py | 210 +++++++++++++ backend/g_sheet/plugins.py | 536 +++++++++++++++++++++++++++++++++ backend/g_sheet/serializers.py | 188 ++++++++++++ backend/g_sheet/tasks.py | 93 ++++++ backend/g_sheet/views.py | 334 ++++++++++++++++++++ 7 files changed, 1904 insertions(+) create mode 100644 backend/g_sheet/app.py create mode 100644 backend/g_sheet/enums.py create mode 100644 backend/g_sheet/models.py create mode 100644 backend/g_sheet/plugins.py create mode 100644 backend/g_sheet/serializers.py create mode 100644 backend/g_sheet/tasks.py create mode 100644 backend/g_sheet/views.py diff --git a/backend/g_sheet/app.py b/backend/g_sheet/app.py new file mode 100644 index 0000000..07f8769 --- /dev/null +++ b/backend/g_sheet/app.py @@ -0,0 +1,264 @@ +from fastapi import APIRouter,Depends,HTTPException,Query +from fastapi.responses import JSONResponse +from pydantic import BaseModel +from sqlalchemy.ext.asyncio import AsyncSession + +from db_setup import get_session +from g_sheet.views import Sheet +from users.permissions import PermissionTag,require_permission +from dotenv import load_dotenv +load_dotenv() + +router = APIRouter() + + +class AppendRowsBody(BaseModel): + rows: list[list[str]] + + +class UpdateRangeBody(BaseModel): + cell_range: str + rows: list[list[str]] + + +class ClearRangeBody(BaseModel): + cell_range: str + + +@router.get("/sheet/health") +async def sheet_health(): + """Liveness for the Sheets integration — credentials + spreadsheet reachability. + + Unauthenticated like GET /health in main.py, and never 500s: an unreachable sheet + comes back as {"status":"error"} so a probe can read the reason. + """ + try: + service=Sheet() + data=await service.health_check() + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.get("/sheet/metadata") +async def fetch_sheet_metadata( + spreadsheet_id: str | None = Query(None), + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_VIEW)), +): + try: + service=Sheet(spreadsheet_id=spreadsheet_id) + data=await service.get_metadata() + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.get("/sheet/tabs") +async def fetch_sheet_tabs( + spreadsheet_id: str | None = Query(None), + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_VIEW)), +): + try: + service=Sheet(spreadsheet_id=spreadsheet_id) + items=await service.list_tabs() + return JSONResponse(content={"data":items,"total":len(items),"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.get("/sheet/fetch") +async def fetch_sheet( + tab: str | None = Query(None), + cell_range: str | None = Query(None), + raw: bool = Query(False), + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_VIEW)), + spreadsheet_id: str | None = Query(None), +): + """No tab -> every tab as records. With a tab -> that tab, header-mapped unless + raw=true, which returns the rows exactly as the sheet stores them.""" + try: + service=Sheet(spreadsheet_id=spreadsheet_id) + if not tab: + data=await service.read_all() + return JSONResponse(content={"data":data["sheets"],"total":data["total"],"status_code":200}) + if raw or cell_range: + data=await service.read_range(tab,cell_range) + return JSONResponse(content={"data":data,"total":data["row_count"],"status_code":200}) + data=await service.read_records(tab) + return JSONResponse(content={"data":data["records"],"total":data["total"],"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.post("/sheet/import") +async def import_all_sheets( + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_EDIT)), + session: AsyncSession = Depends(get_session), +): + """Enqueue a full-spreadsheet import. Poll GET /sheet/import/fetch for status.""" + try: + service=Sheet(session=session) + data=await service.start_import(current_user=current_user,tab=None) + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.post("/sheet/{tab}/import") +async def import_one_sheet( + tab: str, + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_EDIT)), + session: AsyncSession = Depends(get_session), +): + """Enqueue a single-tab import. Poll GET /sheet/import/fetch for status.""" + try: + service=Sheet(session=session) + data=await service.start_import(current_user=current_user,tab=tab) + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.get("/sheet/import/fetch") +async def fetch_sheet_import( + run_id: str | None = Query(None), + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_VIEW)), + session: AsyncSession = Depends(get_session), +): + try: + service=Sheet(session=session) + data=await service.get_import_run(run_id=run_id) + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.get("/sheet/form-data/sheets") +async def fetch_form_data_sheets( + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_VIEW)), + session: AsyncSession = Depends(get_session), +): + try: + service=Sheet(session=session) + data=await service.get_imported_sheets() + return JSONResponse(content={"data":data,"total":data["total"],"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.get("/sheet/form-data/fetch") +async def fetch_form_data( + sheet: str | None = Query(None), + search: str | None = Query(None), + top: int | None = Query(None), + skip: int = Query(0,ge=0), + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_VIEW)), + session: AsyncSession = Depends(get_session), +): + try: + service=Sheet(session=session) + items,total=await service.get_form_data(sheet=sheet,search=search,top=top,skip=skip) + return JSONResponse(content={"data":items,"total":total,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.get("/sheet/form-data/{record_id}") +async def fetch_form_data_by_id( + record_id: int, + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_VIEW)), + session: AsyncSession = Depends(get_session), +): + try: + service=Sheet(session=session) + data=await service.get_form_data_by_id(record_id) + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.delete("/sheet/form-data/{tab}/delete") +async def delete_form_data_sheet( + tab: str, + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_DELETE)), + session: AsyncSession = Depends(get_session), +): + try: + service=Sheet(session=session) + data=await service.delete_sheet_data(tab) + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.post("/sheet/{tab}/append") +async def append_sheet_rows( + tab: str, + payload: AppendRowsBody, + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_EDIT)), + spreadsheet_id: str | None = Query(None), +): + try: + service=Sheet(spreadsheet_id=spreadsheet_id) + data=await service.append_rows(tab,payload.rows) + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.patch("/sheet/{tab}/update") +async def update_sheet_range( + tab: str, + payload: UpdateRangeBody, + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_EDIT)), + spreadsheet_id: str | None = Query(None), +): + try: + service=Sheet(spreadsheet_id=spreadsheet_id) + data=await service.update_range(tab,payload.cell_range,payload.rows) + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) + + +@router.post("/sheet/{tab}/clear") +async def clear_sheet_range( + tab: str, + payload: ClearRangeBody, + current_user: dict = Depends(require_permission(PermissionTag.SETTINGS_EDIT)), + spreadsheet_id: str | None = Query(None), +): + try: + service=Sheet(spreadsheet_id=spreadsheet_id) + data=await service.clear_range(tab,payload.cell_range) + return JSONResponse(content={"data":data,"total":1,"status_code":200}) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500,detail=str(e)) diff --git a/backend/g_sheet/enums.py b/backend/g_sheet/enums.py new file mode 100644 index 0000000..0232c91 --- /dev/null +++ b/backend/g_sheet/enums.py @@ -0,0 +1,279 @@ +"""Sheet header aliases, FormData keys, and date/round format mappings. + +(str, Enum) like inbox/enums.py: members compare to and serialize as plain strings. +Non-string mappings (month pairs, ordinal slot+pattern) use plain Enum. +""" + +from enum import Enum + + +class FormDataField(str, Enum): + """Canonical FormData column keys (plus title, which stays in JSONB only).""" + + NAME = "name" + DEGREE = "degree" + EXPERIENCE = "experience" + AGE = "age" + FAMILY_DETAILS = "family_details" + TITLE = "title" + + +class NameAlias(str, Enum): + NAME = "name" + NAMES = "names" + CANDIDATE_NAME = "candidate name" + CANDIDATE = "candidate" + + @classmethod + def has(cls, value) -> bool: + return value in cls._value2member_map_ + + +class DegreeAlias(str, Enum): + EDUCATION = "education" + DEGREE = "degree" + QUALIFICATION = "qualification" + + @classmethod + def has(cls, value) -> bool: + return value in cls._value2member_map_ + + +class ExperienceAlias(str, Enum): + EXPERIENCE = "experience" + EXP = "exp" + YEARS_OF_EXPERIENCE = "years of experience" + TOTAL_EXPERIENCE = "total experience" + + @classmethod + def has(cls, value) -> bool: + return value in cls._value2member_map_ + + +class AgeAlias(str, Enum): + AGE = "age" + + @classmethod + def has(cls, value) -> bool: + return value in cls._value2member_map_ + + +class FamilyDetailsAlias(str, Enum): + FAMILY_DETAILS = "family details" + MARITAL_STATUS = "marital status" + MARITAL = "marital" + + @classmethod + def has(cls, value) -> bool: + return value in cls._value2member_map_ + + +class TitleAlias(str, Enum): + """No FormData column — recognised so headers are not treated as unknown noise.""" + + TITLE = "title" + DESIGNATION = "designation" + ROLE = "role" + POSITION = "position" + TEAM = "team" + JOB_TITLE = "job title" + AREA_OF_EXPERTISE = "area of expertise" + DEPARTMENT = "department" + + @classmethod + def has(cls, value) -> bool: + return value in cls._value2member_map_ + + +# FormDataField → alias Enum. Order is match priority for overlapping startswith hits. +FIELD_ALIAS_ENUMS = { + FormDataField.NAME: NameAlias, + FormDataField.DEGREE: DegreeAlias, + FormDataField.EXPERIENCE: ExperienceAlias, + FormDataField.AGE: AgeAlias, + FormDataField.FAMILY_DETAILS: FamilyDetailsAlias, + FormDataField.TITLE: TitleAlias, +} + + +class RoundRole(str, Enum): + """Interview-round column roles resolved left-to-right into four slots.""" + + DATE = "date" + BY = "by" + STATUS = "status" + NOTES = "notes" + RESULT = "result" + + +class ConductedByAlias(str, Enum): + """Header spellings that map to RoundRole.BY.""" + + CONDUCTED_BY = "conducted by" + INTERVIEWED_BY = "interviewed by" + INTERVIEW_BY = "interview by" + CONDUCTED = "conducted" + BY = "by" + + @classmethod + def has(cls, value) -> bool: + return value in cls._value2member_map_ + + @classmethod + def contained_in(cls, text: str) -> bool: + return any(member.value in text for member in cls if " " in member.value) + + +class NotesToken(str, Enum): + """Substrings that classify a header as RoundRole.NOTES.""" + + NOTE = "note" + REMARK = "remark" + COMMENT = "comment" + + @classmethod + def contained_in(cls, text: str) -> bool: + return any(member.value in text for member in cls) + + +# -- Round → FormData column names (slot 0..3 = definition order) ------------ + +class RoundDateColumn(str, Enum): + R1 = "interview_date" + R2 = "second_interview_date" + R3 = "third_interview_date" + R4 = "fourth_interview_date" + + @classmethod + def ordered(cls) -> tuple[str, ...]: + return tuple(member.value for member in cls) + + +class RoundByColumn(str, Enum): + R1 = "interview_by" + R2 = "second_interview_by" + R3 = "third_interview_by" + R4 = "fourth_interview_by" + + @classmethod + def ordered(cls) -> tuple[str, ...]: + return tuple(member.value for member in cls) + + +class RoundTimeColumn(str, Enum): + R1 = "interview_time" + R2 = "second_interview_time" + R3 = "third_interview_time" + R4 = "fourth_interview_time" + + @classmethod + def ordered(cls) -> tuple[str, ...]: + return tuple(member.value for member in cls) + + +class RoundStatusColumn(str, Enum): + R1 = "interview_status" + R2 = "second_interview_status" + R3 = "third_interview_status" + R4 = "fourth_interview_status" + + @classmethod + def ordered(cls) -> tuple[str, ...]: + return tuple(member.value for member in cls) + + +class RoundNotesColumn(str, Enum): + R1 = "interview_notes" + R2 = "second_interview_notes" + R3 = "third_interview_notes" + R4 = "fourth_interview_notes" + + @classmethod + def ordered(cls) -> tuple[str, ...]: + return tuple(member.value for member in cls) + + +class RoundResultColumn(str, Enum): + R1 = "interview_result" + R2 = "second_interview_result" + R3 = "third_interview_result" + R4 = "fourth_interview_result" + + @classmethod + def ordered(cls) -> tuple[str, ...]: + return tuple(member.value for member in cls) + + +# -- Date parsing ------------------------------------------------------------ + +class DateFormat(str, Enum): + """strptime patterns tried in definition order. + + DD/MM before MM/DD: 14/10/20 is ambiguous and DD/MM is the local convention. + """ + + D_MON_Y_DASH = "%d-%b-%Y" + D_MONTH_Y_DASH = "%d-%B-%Y" + D_MON_Y_SPACE = "%d %b %Y" + D_MONTH_Y_SPACE = "%d %B %Y" + DMY_SLASH = "%d/%m/%Y" + DMY_SLASH_SHORT = "%d/%m/%y" + DMY_DASH = "%d-%m-%Y" + DMY_DASH_SHORT = "%d-%m-%y" + ISO = "%Y-%m-%d" + DMY_DOT = "%d.%m.%Y" + DMY_DOT_SHORT = "%d.%m.%y" + MDY_SLASH = "%m/%d/%Y" + MDY_SLASH_SHORT = "%m/%d/%y" + MON_D_Y = "%b %d %Y" + MONTH_D_Y = "%B %d %Y" + D_MON_Y_SHORT = "%d-%b-%y" + D_MON_Y_SPACE_SHORT = "%d %b %y" + D_MON_Y_SLASH = "%d/%b/%Y" + D_MON_Y_SLASH_SHORT = "%d/%b/%y" + + +class DateTimeSeparator(str, Enum): + """Separators that split a date cell into date + time tails.""" + + DASH = " - " + EN_DASH = " – " + EM_DASH = " — " + SLASH_SPACE = "/ " + PIPE = " | " + + +class MonthNormalisation(Enum): + """Sheet month spellings → %b-safe short form. value is (source, short).""" + + SEPTEMBER = ("september", "sep") + SEPT = ("sept", "sep") + JULY = ("july", "jul") + JUNE = ("june", "jun") + APRIL = ("april", "apr") + MARCH = ("march", "mar") + + @property + def source(self) -> str: + return self.value[0] + + @property + def short(self) -> str: + return self.value[1] + + +class RoundOrdinal(Enum): + """Interview-round ordinal in a header → slot index 0..3. value is (slot, regex).""" + + FIRST = (0, r"(?:1st|first|01st)") + SECOND = (1, r"(?:2nd|second|02nd)") + THIRD = (2, r"(?:3rd|third|03rd)") + FOURTH = (3, r"(?:4th|fourth|04th)") + + @property + def slot(self) -> int: + return self.value[0] + + @property + def pattern(self) -> str: + return self.value[1] diff --git a/backend/g_sheet/models.py b/backend/g_sheet/models.py new file mode 100644 index 0000000..502e0c8 --- /dev/null +++ b/backend/g_sheet/models.py @@ -0,0 +1,210 @@ +"""FormData + SheetImportRun — spreadsheet mirror and background import runs.""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone + +from sqlalchemy import Column, DateTime, Index, delete, func, or_ +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.asyncio import AsyncSession +from sqlmodel import Field, SQLModel, select + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +class FormData(SQLModel, table=True): + """One spreadsheet data row. raw_record keeps the full original header→value map.""" + + __tablename__ = "form_data" + __table_args__ = ( + Index("ix_form_data_sheet_row_number", "sheet", "row_number", unique=True), + ) + + id: int | None = Field(default=None, primary_key=True) + sheet: str = Field(nullable=False, index=True) + name: str | None = Field(default=None, index=True) + degree: str | None = Field(default=None) + experience: str | None = Field(default=None) + age: int | None = Field(default=None) + age_raw: str | None = Field(default=None) + family_details: str | None = Field(default=None) + + interview_date: datetime | None = Field(default=None, sa_type=DateTime(timezone=True)) + interview_by: str | None = Field(default=None) + interview_time: str | None = Field(default=None) + interview_status: str | None = Field(default=None) + interview_notes: str | None = Field(default=None) + interview_result: str | None = Field(default=None) + + second_interview_date: datetime | None = Field(default=None, sa_type=DateTime(timezone=True)) + second_interview_by: str | None = Field(default=None) + second_interview_time: str | None = Field(default=None) + second_interview_status: str | None = Field(default=None) + second_interview_notes: str | None = Field(default=None) + second_interview_result: str | None = Field(default=None) + + third_interview_date: datetime | None = Field(default=None, sa_type=DateTime(timezone=True)) + third_interview_by: str | None = Field(default=None) + third_interview_time: str | None = Field(default=None) + third_interview_status: str | None = Field(default=None) + third_interview_notes: str | None = Field(default=None) + third_interview_result: str | None = Field(default=None) + + fourth_interview_date: datetime | None = Field(default=None, sa_type=DateTime(timezone=True)) + fourth_interview_by: str | None = Field(default=None) + fourth_interview_time: str | None = Field(default=None) + fourth_interview_status: str | None = Field(default=None) + fourth_interview_notes: str | None = Field(default=None) + fourth_interview_result: str | None = Field(default=None) + + raw_record: dict | None = Field(default=None, sa_column=Column(JSONB)) + row_number: int | None = Field(default=None) + imported_at: datetime = Field(default_factory=_now, sa_type=DateTime(timezone=True)) + created_at: datetime = Field(default_factory=_now, sa_type=DateTime(timezone=True)) + updated_at: datetime = Field(default_factory=_now, sa_type=DateTime(timezone=True)) + + @classmethod + def _filters(cls, *, sheet=None, search=None): + filters = [] + if sheet: + filters.append(cls.sheet == sheet) + if search: + pattern = f"%{search}%" + filters.append(or_( + cls.name.ilike(pattern), + cls.degree.ilike(pattern), + cls.experience.ilike(pattern), + cls.interview_by.ilike(pattern), + )) + return filters + + @classmethod + async def get_form_data_by_id(cls, session: AsyncSession, record_id): + try: + rid = int(record_id) + except (TypeError, ValueError): + return None + result = await session.execute(select(cls).where(cls.id == rid)) + return result.scalars().first() + + @classmethod + async def fetch_form_data(cls, session: AsyncSession, *, sheet=None, search=None, top=None, skip=None): + statement = select(cls).order_by(cls.sheet, cls.row_number) + for clause in cls._filters(sheet=sheet, search=search): + statement = statement.where(clause) + if skip: + statement = statement.offset(skip) + if top is not None: + statement = statement.limit(top) + result = await session.execute(statement) + return result.scalars().all() + + @classmethod + async def count_form_data(cls, session: AsyncSession, *, sheet=None, search=None): + statement = select(func.count()).select_from(cls) + for clause in cls._filters(sheet=sheet, search=search): + statement = statement.where(clause) + result = await session.execute(statement) + return result.scalar_one() + + @classmethod + async def get_sheet_names(cls, session: AsyncSession): + result = await session.execute( + select(cls.sheet).distinct().order_by(cls.sheet) + ) + return list(result.scalars().all()) + + @classmethod + async def delete_by_sheet(cls, session: AsyncSession, sheet: str, *, commit: bool = True): + count_result = await session.execute( + select(func.count()).select_from(cls).where(cls.sheet == sheet) + ) + deleted = count_result.scalar_one() + await session.execute(delete(cls).where(cls.sheet == sheet)) + if commit: + await session.commit() + return deleted + + @classmethod + async def insert_form_data_bulk(cls, session: AsyncSession, records: list[dict], *, commit: bool = True): + rows = [cls(**fields) for fields in records] + session.add_all(rows) + if commit: + await session.commit() + return len(rows) + + @classmethod + async def replace_sheet(cls, session: AsyncSession, sheet: str, records: list[dict]): + """Delete + insert in one transaction so a mid-insert failure keeps prior rows.""" + deleted = await cls.delete_by_sheet(session, sheet, commit=False) + inserted = await cls.insert_form_data_bulk(session, records, commit=False) + await session.commit() + return {"deleted": deleted, "inserted": inserted} + + +class SheetImportRun(SQLModel, table=True): + """One Google Sheet → FormData import job (Taskiq). Survives tab close.""" + + __tablename__ = "sheet_import_runs" + + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + status: str = Field(default="queued", index=True) # queued|running|completed|failed + task_id: str | None = Field(default=None) + created_by: uuid.UUID | None = Field(default=None, foreign_key="users.id") + tab: str | None = Field(default=None) # None = import all tabs + report: dict | None = Field(default=None, sa_column=Column(JSONB)) + error: str | None = Field(default=None) + created_at: datetime = Field(default_factory=_now, sa_type=DateTime(timezone=True)) + started_at: datetime | None = Field(default=None, sa_type=DateTime(timezone=True)) + finished_at: datetime | None = Field(default=None, sa_type=DateTime(timezone=True)) + + @staticmethod + def _as_uuid(record_id) -> uuid.UUID | None: + if record_id in (None, ""): + return None + try: + return uuid.UUID(str(record_id)) + except ValueError: + return None + + @classmethod + async def get_by_id(cls, session: AsyncSession, record_id): + uid = cls._as_uuid(record_id) + if uid is None: + return None + result = await session.execute(select(cls).where(cls.id == uid)) + return result.scalars().first() + + @classmethod + async def get_active(cls, session: AsyncSession): + result = await session.execute( + select(cls) + .where(cls.status.in_(("queued", "running"))) + .order_by(cls.created_at.desc()) + ) + return result.scalars().first() + + @classmethod + async def insert_run(cls, session: AsyncSession, fields: dict, *, commit: bool = True): + row = cls(**fields) + session.add(row) + if commit: + await session.commit() + await session.refresh(row) + return row + + @classmethod + async def update_run(cls, session: AsyncSession, record_id, fields: dict, *, commit: bool = True): + row = await cls.get_by_id(session, record_id) + if not row: + return None + for key, value in fields.items(): + setattr(row, key, value) + session.add(row) + if commit: + await session.commit() + await session.refresh(row) + return row diff --git a/backend/g_sheet/plugins.py b/backend/g_sheet/plugins.py new file mode 100644 index 0000000..34001b2 --- /dev/null +++ b/backend/g_sheet/plugins.py @@ -0,0 +1,536 @@ +"""Google Sheets helpers — credential loading, retrying API calls, row/record shaping. + +No FastAPI imports here by house rule: this module raises its own SheetsServiceError +family and lets g_sheet/views.py translate that into HTTPException. + +Auth reuses the credentials already on disk (authorized_user ADC + a valid refresh +token). Nothing here launches a browser, runs InstalledAppFlow, or reads stdin. +""" + +from __future__ import annotations + +import logging +import os +import random +import re +import time +from datetime import datetime, timezone +from pathlib import Path + +from dotenv import load_dotenv +from google.auth import default as google_auth_default +from google.auth.transport.requests import Request +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +from g_sheet.enums import ( + ConductedByAlias, + DateFormat, + DateTimeSeparator, + FIELD_ALIAS_ENUMS, + FormDataField, + MonthNormalisation, + NotesToken, + RoundByColumn, + RoundDateColumn, + RoundNotesColumn, + RoundOrdinal, + RoundResultColumn, + RoundRole, + RoundStatusColumn, + RoundTimeColumn, +) + +load_dotenv() + +logger=logging.getLogger("g_sheet.plugins") + +# backend/ — GOOGLE_APPLICATION_CREDENTIALS is stored relative to it ("credentials/..."). +ROOT=Path(__file__).resolve().parent.parent + +SCOPES=[ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/drive", +] + +SPREADSHEET_ID=os.getenv("SPREADSHEET_ID") +SPREADSHEET_NAME=os.getenv("SPREADSHEET_NAME") +SPREADSHEET_URL=os.getenv("SPREADSHEET_URL") +GOOGLE_APPLICATION_CREDENTIALS=os.getenv("GOOGLE_APPLICATION_CREDENTIALS") + +# 429 and 5xx are transient; every other 4xx is a bad request that a retry repeats. +RETRY_ATTEMPTS=3 +RETRY_BASE_DELAY=0.5 +RETRY_MAX_DELAY=8.0 +RETRYABLE_STATUSES={429,500,502,503,504} + + +class SheetsServiceError(Exception): + """Base for every failure this domain raises. Carries an HTTP-ish status code.""" + + status_code=500 + + def __init__(self,message,status_code=None): + super().__init__(message) + self.message=message + if status_code is not None: + self.status_code=status_code + + +class SheetsAuthError(SheetsServiceError): + """Credentials missing, unreadable, or rejected by Google.""" + + status_code=401 + + +class SheetsApiError(SheetsServiceError): + """The Sheets API answered with an error. status_code is Google's own.""" + + status_code=502 + + +def resolve_credentials_path(credentials_path=None): + """Absolute path to the ADC json. Relative values resolve against backend/. + + The service may be imported from any working directory, so a bare + "credentials/application_default_credentials.json" must not depend on cwd. + """ + raw=credentials_path or GOOGLE_APPLICATION_CREDENTIALS + if not raw: + return None + path=Path(raw) + if not path.is_absolute(): + path=ROOT/path + return path + + +def load_credentials(credentials_path=None,scopes=None): + """Build scoped ADC credentials and refresh them once. Never prompts.""" + path=resolve_credentials_path(credentials_path) + if path is not None: + if not path.exists(): + raise SheetsAuthError(f"Google credentials file not found: {path.name}") + os.environ["GOOGLE_APPLICATION_CREDENTIALS"]=str(path) + try: + credentials,_=google_auth_default(scopes=scopes or SCOPES) + credentials.refresh(Request()) + except SheetsServiceError: + raise + except Exception as e: + raise SheetsAuthError(f"Google credential refresh failed: {e}") + return credentials + + +def ensure_fresh(credentials): + """Refresh only when the token has actually gone stale — not on every call.""" + if credentials is None: + raise SheetsAuthError("Google credentials are not initialised") + if credentials.valid and not credentials.expired: + return credentials + try: + credentials.refresh(Request()) + except Exception as e: + raise SheetsAuthError(f"Google credential refresh failed: {e}") + return credentials + + +def build_sheets_client(credentials): + """Sheets v4 client. cache_discovery=False — the file cache warns under threads.""" + try: + return build("sheets","v4",credentials=credentials,cache_discovery=False) + except Exception as e: + raise SheetsApiError(f"Could not build the Sheets client: {e}") + + +def _status_of(error): + status=getattr(getattr(error,"resp",None),"status",None) + if status is None: + status=getattr(error,"status_code",None) + try: + return int(status) + except (TypeError,ValueError): + return None + + +def _reason_of(error): + """Google's message without the response body, so nothing sensitive leaks out.""" + try: + return error._get_reason().strip() + except Exception: + return str(error) + + +def execute(request,description="sheets request"): + """Run a googleapiclient request with jittered exponential backoff. + + Retries 429 and 5xx up to RETRY_ATTEMPTS; every other HttpError raises straight + away as SheetsApiError carrying Google's status code. + """ + delay=RETRY_BASE_DELAY + last_error=None + for attempt in range(1,RETRY_ATTEMPTS+1): + try: + return request.execute() + except HttpError as e: + status=_status_of(e) + reason=_reason_of(e) + last_error=SheetsApiError(f"{description} failed: {reason}",status or 502) + if status not in RETRYABLE_STATUSES or attempt==RETRY_ATTEMPTS: + raise last_error + sleep_for=min(delay,RETRY_MAX_DELAY)+random.uniform(0,RETRY_BASE_DELAY) + logger.warning( + "%s got %s, retry %s/%s in %.2fs", + description,status,attempt,RETRY_ATTEMPTS,sleep_for, + ) + time.sleep(sleep_for) + delay*=2 + except SheetsServiceError: + raise + except Exception as e: + raise SheetsApiError(f"{description} failed: {e}") + raise last_error + + +def quote_tab(tab,cell_range=None): + """A1 target for a tab whose name may contain spaces or quotes.""" + safe=str(tab).replace("'","''") + if cell_range: + return f"'{safe}'!{cell_range}" + return f"'{safe}'" + + +def normalise_headers(header_row): + """First row -> unique, non-empty column keys. + + Blank cells become column_{i}; a repeated header keeps its first spelling and the + later ones get _1, _2 so no key silently overwrites another. + """ + headers=[] + seen={} + for index,raw in enumerate(header_row): + name=str(raw).strip() if raw is not None else "" + if not name: + name=f"column_{index}" + count=seen.get(name,0) + seen[name]=count+1 + headers.append(name if count==0 else f"{name}_{count}") + return headers + + +def rows_to_records(rows): + """Sheet rows -> list of dicts keyed by the header row. + + Sheets truncates trailing empties, so short rows are padded to header width. + Fully blank rows are dropped rather than emitted as all-empty records. + """ + if not rows: + return [] + headers=normalise_headers(rows[0]) + records=[] + for row in rows[1:]: + values=[str(cell) if cell is not None else "" for cell in row] + if not any(value.strip() for value in values): + continue + if len(values)0: + date_part=date_part[:time_match.start()].strip(" ,;-") + + date_part=_DAY_ORDINAL_RE.sub(r"\1",date_part) + date_part=_normalise_month_spellings(date_part) + date_part=re.sub(r"\s+"," ",date_part).strip(" ,;") + + for fmt in DateFormat: + try: + return datetime.strptime(date_part,fmt.value).replace(tzinfo=timezone.utc) + except ValueError: + continue + return None + + +def parse_date_time(value): + """(datetime|None, time_string|None) — fills *_time for the cells that carry one.""" + parsed=parse_date(value) + if value is None: + return parsed,None + text=str(value).strip() + match=_TIME_RE.search(text) + time_str=match.group(1).strip() if match else None + return parsed,time_str + + +def parse_age(value): + """(int|None, raw|None) — first digit run if 0 < n < 100, always keep the raw.""" + if value is None: + return None,None + raw=str(value).strip() + if not raw: + return None,None + match=_AGE_RE.search(raw) + if not match: + return None,raw + number=int(match.group()) + if 0 dict: + """spreadsheets.get response -> the spreadsheet header the UI renders.""" + properties = payload.get("properties") or {} + return { + "spreadsheet_id": payload.get("spreadsheetId"), + "title": properties.get("title"), + "locale": properties.get("locale"), + "time_zone": properties.get("timeZone"), + "url": payload.get("spreadsheetUrl"), + "tabs": [serialize_tab(sheet) for sheet in payload.get("sheets") or []], + } + + +def serialize_tab(sheet: dict) -> dict: + """One entry of spreadsheets.get -> tab name plus its grid size.""" + properties = sheet.get("properties") or {} + grid = properties.get("gridProperties") or {} + return { + "title": properties.get("title"), + "sheet_id": properties.get("sheetId"), + "index": properties.get("index"), + "row_count": grid.get("rowCount"), + "column_count": grid.get("columnCount"), + } + + +def serialize_values(tab: str, cell_range: str | None, rows: list[list[str]]) -> dict: + """Raw rows -> the read_range envelope.""" + return { + "tab": tab, + "range": cell_range, + "rows": rows, + "row_count": len(rows), + } + + +def serialize_records(tab: str, records: list[dict]) -> dict: + """Header-mapped rows -> the read_records envelope.""" + return { + "tab": tab, + "records": records, + "total": len(records), + "headers": list(records[0].keys()) if records else [], + } + + +def serialize_append(tab: str, payload: dict) -> dict: + """values.append response -> what was written and where.""" + updates = payload.get("updates") or {} + return { + "tab": tab, + "spreadsheet_id": payload.get("spreadsheetId"), + "updated_range": updates.get("updatedRange"), + "updated_rows": updates.get("updatedRows", 0), + "updated_columns": updates.get("updatedColumns", 0), + "updated_cells": updates.get("updatedCells", 0), + } + + +def serialize_update(tab: str, payload: dict) -> dict: + """values.update response -> the same shape as an append result.""" + return { + "tab": tab, + "spreadsheet_id": payload.get("spreadsheetId"), + "updated_range": payload.get("updatedRange"), + "updated_rows": payload.get("updatedRows", 0), + "updated_columns": payload.get("updatedColumns", 0), + "updated_cells": payload.get("updatedCells", 0), + } + + +def serialize_clear(tab: str, payload: dict) -> dict: + """values.clear response -> the cleared range.""" + return { + "tab": tab, + "spreadsheet_id": payload.get("spreadsheetId"), + "cleared_range": payload.get("clearedRange"), + } + + +def serialize_health(ok: bool, detail: str, tabs: list[str] | None = None) -> dict: + """health_check result. Returned on failure too — this one never raises.""" + return { + "status": "ok" if ok else "error", + "detail": detail, + "tabs": tabs or [], + "tab_count": len(tabs or []), + } + + +def _iso(value): + return value.isoformat() if value is not None else None + + +def serialize_form_data(row) -> dict: + """FormData ORM row → API dict, including raw_record.""" + return { + "id": row.id, + "sheet": row.sheet, + "name": row.name, + "degree": row.degree, + "experience": row.experience, + "age": row.age, + "age_raw": row.age_raw, + "family_details": row.family_details, + "interview_date": _iso(row.interview_date), + "interview_by": row.interview_by, + "interview_time": row.interview_time, + "interview_status": row.interview_status, + "interview_notes": row.interview_notes, + "interview_result": row.interview_result, + "second_interview_date": _iso(row.second_interview_date), + "second_interview_by": row.second_interview_by, + "second_interview_time": row.second_interview_time, + "second_interview_status": row.second_interview_status, + "second_interview_notes": row.second_interview_notes, + "second_interview_result": row.second_interview_result, + "third_interview_date": _iso(row.third_interview_date), + "third_interview_by": row.third_interview_by, + "third_interview_time": row.third_interview_time, + "third_interview_status": row.third_interview_status, + "third_interview_notes": row.third_interview_notes, + "third_interview_result": row.third_interview_result, + "fourth_interview_date": _iso(row.fourth_interview_date), + "fourth_interview_by": row.fourth_interview_by, + "fourth_interview_time": row.fourth_interview_time, + "fourth_interview_status": row.fourth_interview_status, + "fourth_interview_notes": row.fourth_interview_notes, + "fourth_interview_result": row.fourth_interview_result, + "raw_record": row.raw_record, + "row_number": row.row_number, + "imported_at": _iso(row.imported_at), + "created_at": _iso(row.created_at), + "updated_at": _iso(row.updated_at), + } + + +def serialize_import(report: dict) -> dict: + """Per-tab import report.""" + return { + "tab": report.get("tab"), + "rows_read": report.get("rows_read", 0), + "inserted": report.get("inserted", 0), + "deleted": report.get("deleted", 0), + "dates_parsed": report.get("dates_parsed", 0), + "dates_unparsed": report.get("dates_unparsed", 0), + "ages_parsed": report.get("ages_parsed", 0), + "unmapped_headers": report.get("unmapped_headers") or [], + "error": report.get("error"), + } + + +def serialize_import_all(reports: list[dict]) -> dict: + """Aggregate of per-tab reports from import_all.""" + ok=[r for r in reports if not r.get("error")] + failed=[r for r in reports if r.get("error")] + return { + "tabs": len(reports), + "succeeded": len(ok), + "failed": len(failed), + "inserted": sum(r.get("inserted", 0) for r in ok), + "deleted": sum(r.get("deleted", 0) for r in ok), + "reports": [serialize_import(r) for r in reports], + } + + +def serialize_sheet_summary(sheets: list[str]) -> dict: + return {"sheets": sheets, "total": len(sheets)} + + +def serialize_import_run(row) -> dict: + return { + "id": str(row.id), + "status": row.status, + "task_id": row.task_id, + "created_by": str(row.created_by) if row.created_by else None, + "tab": row.tab, + "report": row.report, + "error": row.error, + "created_at": _iso(row.created_at), + "started_at": _iso(row.started_at), + "finished_at": _iso(row.finished_at), + } diff --git a/backend/g_sheet/tasks.py b/backend/g_sheet/tasks.py new file mode 100644 index 0000000..488b2d5 --- /dev/null +++ b/backend/g_sheet/tasks.py @@ -0,0 +1,93 @@ +"""Google Sheet → FormData import Taskiq tasks (shared inbox worker stream).""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime,timezone + +import redis.asyncio as redis +from dotenv import load_dotenv + +from db_setup import session_scope +from g_sheet.models import SheetImportRun +from g_sheet.views import Sheet +from taskiq_management.broker_setup import MAX_RETRIES,RETRY_DELAY,broker +from taskiq_management.middleware import PermanentTaskError + +load_dotenv() + +logger=logging.getLogger("g_sheet.tasks") +REDIS_URL=os.getenv("REDIS_URL","redis://localhost:6379/0") +_LOCK_KEY="g_sheet:import:lock" +_LOCK_TTL=3600 + + +async def _fail(run_id:str,error:str) -> dict: + async with session_scope() as session: + await SheetImportRun.update_run(session,run_id,{ + "status":"failed", + "error":error, + "finished_at":datetime.now(timezone.utc), + }) + return {"status":"failed","error":error} + + +@broker.task( + task_name="g_sheet.import_sheets", + retry_on_error=True, + max_retries=MAX_RETRIES, + delay=RETRY_DELAY, +) +async def import_sheets(run_id:str) -> dict: + if not run_id or not str(run_id).strip(): + raise PermanentTaskError("run_id is required") + run_id=str(run_id).strip() + + client=redis.from_url(REDIS_URL,decode_responses=True) + try: + acquired=await client.set(_LOCK_KEY,run_id,nx=True,ex=_LOCK_TTL) + if not acquired: + return await _fail(run_id,"another sheet import is already running") + + try: + async with session_scope() as session: + row=await SheetImportRun.get_by_id(session,run_id) + if not row: + raise PermanentTaskError(f"import run {run_id} not found") + await SheetImportRun.update_run(session,run_id,{ + "status":"running", + "started_at":datetime.now(timezone.utc), + "error":None, + }) + tab=row.tab + + async with session_scope() as session: + service=Sheet(session=session) + try: + if tab: + report=await service.import_sheet(tab) + else: + report=await service.import_all() + except Exception as e: + logger.exception("sheet import failed for run %s",run_id) + # Bad tab names and permanent Sheets 4xx — do not burn retries. + from fastapi import HTTPException + if isinstance(e,HTTPException) and e.status_code in (400,404,422): + await _fail(run_id,str(e.detail)) + raise PermanentTaskError(str(e.detail)) from e + return await _fail(run_id,str(e)) + + await SheetImportRun.update_run(session,run_id,{ + "status":"completed", + "report":report, + "error":None, + "finished_at":datetime.now(timezone.utc), + }) + return {"status":"completed","report":report} + finally: + current=await client.get(_LOCK_KEY) + if current==run_id: + await client.delete(_LOCK_KEY) + finally: + await client.aclose() diff --git a/backend/g_sheet/views.py b/backend/g_sheet/views.py new file mode 100644 index 0000000..96bf700 --- /dev/null +++ b/backend/g_sheet/views.py @@ -0,0 +1,334 @@ +"""Google Sheets service — business logic for the g_sheet domain. + +The Google client is blocking, so every call goes through asyncio.to_thread rather +than stalling the event loop. Client construction is lazy and guarded by a lock so +concurrent requests build it exactly once. +""" + +import asyncio +import logging +import threading +from datetime import datetime,timezone + +from fastapi import HTTPException + +from g_sheet.plugins import ( + SCOPES, + SPREADSHEET_ID, + SPREADSHEET_NAME, + SPREADSHEET_URL, + SheetsServiceError, + build_sheets_client, + ensure_fresh, + execute, + import_row_stats, + load_credentials, + map_record_to_form_data, + quote_tab, + rows_to_records, + stringify_rows, +) +from g_sheet.models import FormData,SheetImportRun +from g_sheet.serializers import ( + serialize_append, + serialize_clear, + serialize_form_data, + serialize_health, + serialize_import, + serialize_import_all, + serialize_import_run, + serialize_metadata, + serialize_records, + serialize_sheet_summary, + serialize_update, + serialize_values, +) + +logger=logging.getLogger("g_sheet.views") + + +class Sheet: + def __init__(self,session=None,spreadsheet_id=None,credentials_path=None,scopes=None): + self.session=session + self.spreadsheet_id=spreadsheet_id or SPREADSHEET_ID + self.spreadsheet_name=SPREADSHEET_NAME + self.spreadsheet_url=SPREADSHEET_URL + self.credentials_path=credentials_path + self.scopes=scopes or SCOPES + self.credentials=None + self.client=None + self._lock=threading.Lock() + + def _require_session(self): + if self.session is None: + raise HTTPException(status_code=500,detail="Database session is required") + return self.session + + # -- client ------------------------------------------------------------ + + def _connect(self): + """Build credentials + client once, then keep refreshing the same token. + + Double-checked under the lock: two requests racing here must not each build + their own client. + """ + if self.client is not None: + return ensure_fresh(self.credentials) and self.client + with self._lock: + if self.client is None: + self.credentials=load_credentials(self.credentials_path,self.scopes) + self.client=build_sheets_client(self.credentials) + else: + ensure_fresh(self.credentials) + return self.client + + async def _values(self): + if not self.spreadsheet_id: + raise HTTPException(status_code=500,detail="SPREADSHEET_ID is not configured") + client=await asyncio.to_thread(self._connect) + return client.spreadsheets().values() + + async def _spreadsheets(self): + if not self.spreadsheet_id: + raise HTTPException(status_code=500,detail="SPREADSHEET_ID is not configured") + client=await asyncio.to_thread(self._connect) + return client.spreadsheets() + + # -- reads ------------------------------------------------------------- + + async def get_metadata(self): + """Spreadsheet title, id, url and every tab with its row/column counts.""" + try: + spreadsheets=await self._spreadsheets() + request=spreadsheets.get(spreadsheetId=self.spreadsheet_id,fields=( + "spreadsheetId,spreadsheetUrl,properties(title,locale,timeZone)," + "sheets(properties(sheetId,title,index,gridProperties(rowCount,columnCount)))" + )) + payload=await asyncio.to_thread(execute,request,"spreadsheet metadata") + return serialize_metadata(payload) + except SheetsServiceError as e: + raise HTTPException(status_code=e.status_code,detail=e.message) + + async def list_tabs(self): + """Tab titles in sheet order.""" + metadata=await self.get_metadata() + return [tab["title"] for tab in metadata["tabs"] if tab.get("title")] + + async def read_range(self,tab,cell_range=None): + """Raw rows for a tab, or for a sub-range of it when cell_range is given.""" + try: + values=await self._values() + target=quote_tab(tab,cell_range) + request=values.get(spreadsheetId=self.spreadsheet_id,range=target) + payload=await asyncio.to_thread(execute,request,f"read {target}") + rows=stringify_rows(payload.get("values")) + return serialize_values(tab,cell_range,rows) + except SheetsServiceError as e: + raise HTTPException(status_code=e.status_code,detail=e.message) + + async def read_records(self,tab): + """Rows keyed by the first row. Blank rows are skipped, short rows padded.""" + data=await self.read_range(tab) + return serialize_records(tab,rows_to_records(data["rows"])) + + async def read_all(self): + """Every tab as records, keyed by tab name.""" + tabs=await self.list_tabs() + sheets={} + for tab in tabs: + data=await self.read_records(tab) + sheets[tab]=data["records"] + return {"sheets":sheets,"tabs":tabs,"total":len(tabs)} + + # -- writes ------------------------------------------------------------ + + async def append_rows(self,tab,rows): + """Append rows below the tab's current content.""" + if not rows: + raise HTTPException(status_code=422,detail="rows must not be empty") + try: + values=await self._values() + target=quote_tab(tab) + request=values.append( + spreadsheetId=self.spreadsheet_id, + range=target, + valueInputOption="USER_ENTERED", + insertDataOption="INSERT_ROWS", + body={"values":rows}, + ) + payload=await asyncio.to_thread(execute,request,f"append to {target}") + return serialize_append(tab,payload) + except SheetsServiceError as e: + raise HTTPException(status_code=e.status_code,detail=e.message) + + async def update_range(self,tab,cell_range,rows): + """Overwrite an explicit A1 range with rows.""" + if not cell_range: + raise HTTPException(status_code=422,detail="cell_range is required") + if not rows: + raise HTTPException(status_code=422,detail="rows must not be empty") + try: + values=await self._values() + target=quote_tab(tab,cell_range) + request=values.update( + spreadsheetId=self.spreadsheet_id, + range=target, + valueInputOption="USER_ENTERED", + body={"values":rows}, + ) + payload=await asyncio.to_thread(execute,request,f"update {target}") + return serialize_update(tab,payload) + except SheetsServiceError as e: + raise HTTPException(status_code=e.status_code,detail=e.message) + + async def clear_range(self,tab,cell_range): + """Clear the values in an explicit A1 range, leaving formatting intact.""" + if not cell_range: + raise HTTPException(status_code=422,detail="cell_range is required") + try: + values=await self._values() + target=quote_tab(tab,cell_range) + request=values.clear(spreadsheetId=self.spreadsheet_id,range=target,body={}) + payload=await asyncio.to_thread(execute,request,f"clear {target}") + return serialize_clear(tab,payload) + except SheetsServiceError as e: + raise HTTPException(status_code=e.status_code,detail=e.message) + + # -- FormData import / query ------------------------------------------- + + async def import_sheet(self,tab): + """Read one tab from Google Sheets and replace its FormData rows.""" + session=self._require_session() + if not tab or not str(tab).strip(): + raise HTTPException(status_code=422,detail="tab is required") + tab=str(tab).strip() + data=await self.read_records(tab) + records=data["records"] + headers=data["headers"] + mapped=[] + for index,record in enumerate(records): + mapped.append(map_record_to_form_data(tab,record,headers,index+2)) + result=await FormData.replace_sheet(session,tab,mapped) + stats=import_row_stats(mapped,headers) + return serialize_import({ + "tab":tab, + "rows_read":len(records), + "inserted":result["inserted"], + "deleted":result["deleted"], + **stats, + }) + + async def import_all(self): + """Import every tab sequentially; one tab failure does not abort the rest.""" + self._require_session() + tabs=await self.list_tabs() + reports=[] + for tab in tabs: + try: + report=await self.import_sheet(tab) + reports.append(report) + except HTTPException as e: + logger.warning("import_all tab %s failed: %s",tab,e.detail) + reports.append(serialize_import({ + "tab":tab,"rows_read":0,"inserted":0,"deleted":0, + "error":str(e.detail), + })) + except Exception as e: + logger.exception("import_all tab %s failed",tab) + reports.append(serialize_import({ + "tab":tab,"rows_read":0,"inserted":0,"deleted":0, + "error":str(e), + })) + return serialize_import_all(reports) + + async def get_form_data(self,sheet=None,search=None,top=None,skip=None): + session=self._require_session() + rows=await FormData.fetch_form_data( + session,sheet=sheet,search=search,top=top,skip=skip, + ) + total=await FormData.count_form_data(session,sheet=sheet,search=search) + return [serialize_form_data(row) for row in rows],total + + async def get_form_data_by_id(self,record_id): + session=self._require_session() + row=await FormData.get_form_data_by_id(session,record_id) + if not row: + raise HTTPException(status_code=404,detail="Form data not found") + return serialize_form_data(row) + + async def get_imported_sheets(self): + session=self._require_session() + sheets=await FormData.get_sheet_names(session) + return serialize_sheet_summary(sheets) + + async def delete_sheet_data(self,tab): + session=self._require_session() + if not tab or not str(tab).strip(): + raise HTTPException(status_code=422,detail="tab is required") + deleted=await FormData.delete_by_sheet(session,str(tab).strip()) + return {"tab":str(tab).strip(),"deleted":deleted} + + async def start_import(self,current_user=None,tab=None): + """Enqueue a sheet import on the shared Taskiq worker; return the run row. + + If a queued/running import already exists, return it instead of stacking another. + """ + session=self._require_session() + active=await SheetImportRun.get_active(session) + if active: + return serialize_import_run(active) + + created_by=None + if isinstance(current_user,dict) and current_user.get("id"): + created_by=SheetImportRun._as_uuid(current_user.get("id")) + + tab_value=str(tab).strip() if tab else None + row=await SheetImportRun.insert_run(session,{ + "status":"queued", + "created_by":created_by, + "tab":tab_value, + }) + + from g_sheet.tasks import import_sheets + task=await import_sheets.kicker().with_labels( + created_at=datetime.now(timezone.utc).isoformat(), + correlation_id=str(row.id), + queue="inbox", + ).kiq(str(row.id)) + row=await SheetImportRun.update_run(session,row.id,{"task_id":task.task_id}) + return serialize_import_run(row) + + async def get_import_run(self,run_id=None): + session=self._require_session() + if run_id: + row=await SheetImportRun.get_by_id(session,run_id) + if not row: + raise HTTPException(status_code=404,detail="Import run not found") + return serialize_import_run(row) + row=await SheetImportRun.get_active(session) + if row: + return serialize_import_run(row) + from sqlmodel import select + result=await session.execute( + select(SheetImportRun).order_by(SheetImportRun.created_at.desc()).limit(1) + ) + row=result.scalars().first() + if not row: + raise HTTPException(status_code=404,detail="No import runs yet") + return serialize_import_run(row) + + # -- health ------------------------------------------------------------ + + async def health_check(self): + """Credentials + sheet reachability as a status dict. Never raises.""" + if not self.spreadsheet_id: + return serialize_health(False,"SPREADSHEET_ID is not configured") + try: + tabs=await self.list_tabs() + return serialize_health(True,"spreadsheet reachable",tabs) + except HTTPException as e: + logger.warning("sheets health check failed: %s",e.detail) + return serialize_health(False,str(e.detail)) + except Exception as e: + logger.warning("sheets health check failed: %s",e) + return serialize_health(False,str(e)) -- 2.40.1 From 8c12872c6e69515ada8338d6ba8e801653117eb9 Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Mon, 24 Aug 2026 20:38:15 +0500 Subject: [PATCH 6/7] Remove `.env.example` and update Docker configurations for environment management - Deleted the `.env.example` file as it is no longer needed; all configurations are now centralized in `backend/.env`. - Updated `docker-compose.yml` and `docker-compose.dev.yml` to reflect changes in environment variable handling, ensuring that the application reads from `backend/.env` exclusively. - Adjusted the nginx configuration to improve API request handling and ensure proper proxying for frontend interactions. These changes streamline the environment setup process and enhance the overall configuration management for local and production deployments. --- .env.example | 69 -------------- .gitignore | 3 + DOCKER.md | 171 ++++++++++++++++++++-------------- README.md | 36 +++---- app/core/config.py | 6 +- backend/.env.example | 55 +++++++---- backend/README.md | 8 +- backend/db_setup.py | 86 ++++++++++++----- docker-compose.dev.yml | 73 ++------------- docker-compose.host-ports.yml | 24 +++++ docker-compose.yml | 99 +++++++++++++------- frontend/.env.development | 11 ++- frontend/nginx.conf | 30 +++++- frontend/vite.config.js | 12 ++- 14 files changed, 365 insertions(+), 318 deletions(-) delete mode 100644 .env.example create mode 100644 docker-compose.host-ports.yml diff --git a/.env.example b/.env.example deleted file mode 100644 index afcfa78..0000000 --- a/.env.example +++ /dev/null @@ -1,69 +0,0 @@ -# Copy to `.env` at the repo root for Compose variable substitution. -# Secrets for the app itself still live in `backend/.env` (see backend/.env.example). -# Never commit a filled `.env`. - -# --- Compose / production stack ---------------------------------------------------- -# Used by docker-compose.yml for the container Postgres and frontend publish port. - -DB_USERNAME=postgres -DB_PASSWORD=change-me-strong-password -DB_NAME=hrms - -# Public SPA port only (production). Dev overlay defaults FRONTEND_PORT to 5173. -FRONTEND_PORT=80 - -# API workers behind nginx (production backend-api command). -UVICORN_WORKERS=2 - -# Schema sync on API startup (docker compose up -d --build). -# true = upgrade (if any on-disk revisions) + apply model drift in-memory. -# Revision .py files stay gitignored and are not written on the server. -DB_AUTO_MIGRATE=true -DB_AUTOGENERATE=true - -# Empty = same-origin fetches; frontend nginx proxies to backend-api. -# Set only when the API is intentionally on another origin. -VITE_API_BASE= - -# Optional: override mail service URL for containers (else host.docker.internal:5000). -# EMAIL_URL=http://host.docker.internal:5000 - -# Optional: point containers at host Postgres instead of Compose postgres -# (also set by docker-compose.dev.yml). -# DB_HOST=host.docker.internal - -# --- Bulk ATS scoring engine (also read by ats-engine / backend) -------------------- - -OPENAI_API_KEY= - -# Must be a structured-outputs model family: gpt-5*, gpt-4.1*, o3*, o4*. -# "-chat-latest" variants are rejected -- they track the ChatGPT product surface and -# do not expose reasoning effort. -# Note gpt-4.1 is allowed but is not a reasoning model, so OPENAI_EFFORT is ignored -# for it (the adapter omits the parameter rather than sending a 400). -OPENAI_MODEL=gpt-5.4-mini - -# Covers reasoning tokens AND the visible response on a reasoning model. Too low and -# the JSON truncates mid-object, failing the candidate with MODEL_RESPONSE_INVALID. -# Enforced floor is 2048. Do not lower this to save cost -- lower OPENAI_EFFORT. -OPENAI_MAX_OUTPUT_TOKENS=4000 - -# none | minimal | low | medium | high | xhigh -OPENAI_EFFORT=low - -OPENAI_MAX_RETRIES=3 -OPENAI_TIMEOUT_SECONDS=120 - -# OpenAI prompt caching is automatic and cannot be turned off. This only controls -# whether a prompt_cache_key routing hint is sent to raise the cache hit rate. -OPENAI_ENABLE_PROMPT_CACHE=true - -SCORING_CONCURRENCY=5 -MAX_RESUMES_PER_REQUEST=50 -MAX_PDF_SIZE_MB=10 -MAX_JD_CHARS=30000 -MAX_RESUME_CHARS=60000 - -# text | json -LOG_FORMAT=json -LOG_LEVEL=INFO diff --git a/.gitignore b/.gitignore index 8700273..a095f84 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ frontend/dist/ **.pdf **_**_**.py Utopia-ai-hr-ats-portal 1.pem + +# Local-only Compose overrides (never deployed) +docker.local.env diff --git a/DOCKER.md b/DOCKER.md index 8c19b33..501bbeb 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -1,116 +1,145 @@ # Docker -Production stack is self-contained: Postgres, Redis, API, ATS engine, Taskiq -workers, and the React SPA. Only the frontend is published on the host. +## How the browser reaches the API -## Production +The SPA is on **http://127.0.0.1:5173** (nginx → `backend-api` on the Compose +network). The API is also published on **http://127.0.0.1:8000** for host tools +and `npm run dev` (`VITE_API_BASE=http://127.0.0.1:8000`). -```bash -# 1. Root compose env (DB password, port, workers) — see .env.example -cp .env.example .env -# Edit DB_PASSWORD (and JWT_SECRET_KEY inside backend/.env) - -# 2. App secrets (JWT, OpenAI, email, Buffer, …) -cp backend/.env.example backend/.env -# Fill JWT_SECRET_KEY, OPENAI_API_KEY, DB_* matching the Compose postgres, … - -# 3. Build and start -docker compose up -d --build -docker compose ps +```env +FRONTEND_PORT=5173 +BACKEND_PORT=8000 +FRONTEND_URL=http://127.0.0.1:5173 ``` -| Service | Image | Host port | -|---|---|---| -| `frontend` | `hrms-frontend:local` | `${FRONTEND_PORT:-80}` | -| `backend-api` | `hrms-backend:local` | (internal) | -| `ats-engine` | `hrms-ats-engine:local` | (internal) | -| `redis` | `redis:7-alpine` | (internal) | -| `postgres` | `hrms-postgres:local` | (internal) | -| Taskiq workers / schedulers | `hrms-backend:local` | (internal) | +Sole env file: **`backend/.env`** (no repo-root `.env`). Always pass it for +Compose variable substitution: + +```bash +docker compose --env-file ./backend/.env up -d --build +``` + +## Local (host Postgres) + +In `backend/.env`: + +```env +PROD_ENV=false +DB_USERNAME=... +DB_PASSWORD=... +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=hrms +DB_SSLMODE= +FRONTEND_PORT=8080 +``` + +Containers set `IN_DOCKER=1`. With `PROD_ENV=false`, `db_setup` rewrites +`localhost` / `127.0.0.1` → `host.docker.internal` for the connection URL only +(SSL off unless `DB_SSLMODE` is set). Host Postgres must accept Docker-bridge +clients (`listen_addresses`, `pg_hba`). + +```bash +cp backend/.env.example backend/.env # set JWT, OpenAI, DB_*, PROD_ENV=false +docker compose --env-file ./backend/.env up -d --build +``` + +| Service | Host access | +|---|---| +| `frontend` | `${FRONTEND_PORT:-80}` (all interfaces) | +| `backend-api` | Compose network only (`backend-api:8000`); nginx proxies | +| `ats-engine` | Compose network only (`ats-engine:8100`) | +| `redis` | Compose network only (`redis:6379`) | +| `postgres` | not started (optional `--profile postgres`) | + +Optional loopback publishes for host tools (curl / redis-cli / Postman): + +```bash +docker compose --env-file ./backend/.env -f docker-compose.yml -f docker-compose.host-ports.yml up -d +``` + +If bind fails on Windows because Cursor/VS Code still holds `:80` / `:8100` / +`:6379` after a previous run, clear **Ports** in the IDE or set free values in +`backend/.env` (`FRONTEND_PORT`, and with the overlay `ATS_PORT` / `REDIS_PORT` / +`BACKEND_PORT`). + +Optional live-reload / bind mounts: + +```bash +docker compose --env-file ./backend/.env -f docker-compose.yml -f docker-compose.dev.yml up -d --build +``` + +Optional Compose Postgres (empty volume — not host data): + +```bash +docker compose --env-file ./backend/.env --profile postgres up -d postgres +# set DB_HOST=postgres in backend/.env, then recreate backend services +``` + +## Production (RDS) + +In `backend/.env`, set `PROD_ENV=true` and point plain `DB_*` at RDS (no +prefixed credential sets). Blank `DB_SSLMODE` → SSL `require`. Host is never +rewritten. + +```bash +cp backend/.env.example backend/.env +# Edit backend/.env: PROD_ENV=true, DB_* = RDS, JWT_SECRET_KEY, FRONTEND_PORT=80, … +docker compose --env-file ./backend/.env up -d --build +docker compose --env-file ./backend/.env ps +``` Browser → `http:///` → nginx (same-origin) → `backend-api:8000`. CV files live in the `attachments-data` named volume (shared by API + workers). ### Schema / migrations (automatic) -On every `backend-api` start (`docker compose up -d --build`): +On every `backend-api` start: 1. Fresh empty Postgres → create all tables from models and stamp a marker. 2. Otherwise → `alembic upgrade head` if any revision files exist in the image (they normally do not — versions stay gitignored and are excluded from builds). -3. If `DB_AUTOGENERATE=true` (Compose default) → detect ORM drift (new/changed/removed - columns or tables) and apply DDL **in-memory** — no `versions/*.py` is written - on the server. +3. If `DB_AUTOGENERATE=true` → detect ORM drift and apply DDL **in-memory**. 4. Apply any pending `backend/migrations/manual/*.sql` (seed/RBAC batches only). -Revision scripts under `backend/migrations/versions/` remain gitignored and are -listed in `.dockerignore` so they never ship in the production image. - -Toggle with root `.env`: `DB_AUTO_MIGRATE` / `DB_AUTOGENERATE` (default `true`). +Toggle in `backend/.env`: `DB_AUTO_MIGRATE` / `DB_AUTOGENERATE` (default `true`). ### Verify ```bash -docker compose config -curl -sf http://127.0.0.1/health # nginx → backend /health -curl -sf -o /dev/null -w "%{http_code}\n" http://127.0.0.1/ -docker compose logs -f backend-api +docker compose --env-file ./backend/.env config +curl -sf http://127.0.0.1:${FRONTEND_PORT:-8080}/health +curl -sf -o /dev/null -w "%{http_code}\n" http://127.0.0.1:${FRONTEND_PORT:-8080}/ +docker compose --env-file ./backend/.env logs -f backend-api ``` -Open the SPA, sign in, confirm `/jobs` lists requisitions (Network: `/jobs/fetch` -same-origin JSON, not HTML). - ### Secrets - Never bake `.env` into images (`.dockerignore` already excludes them). - Require a strong `DB_PASSWORD` and `JWT_SECRET_KEY` before any real deploy. -- Root `.env` is for Compose substitution; `backend/.env` is for the app process. +- Only `backend/.env` holds app + Compose substitution values. +- Do not put `DB_HOST` under Compose `environment:` (empty override blanks RDS). ### TLS This stack serves HTTP on the frontend port. Terminate TLS at a reverse proxy or -cloud load balancer in front of port 80. - -## Local development (host Postgres) - -Restores published ports, bind-mounted attachments, and `--reload`: - -```bash -docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build -``` - -| Override | Value | -|---|---| -| `DB_HOST` | `host.docker.internal` | -| Frontend | `${FRONTEND_PORT:-5173}` | -| API | `${BACKEND_PORT:-8000}` | -| ATS | `${ATS_PORT:-8100}` | -| Redis | `${REDIS_PORT:-6379}` | -| Attachments | `./backend/inbox/decoded_attachments` | - -Container Postgres still starts (production default) but is unused while -`DB_HOST=host.docker.internal`. Host Postgres must accept Docker-bridge clients -(`listen_addresses = '*'`, `pg_hba` for the bridge subnet). - -For the Vite HMR loop, run `npm run dev` on the host against -`frontend/.env.development` (`VITE_API_BASE=http://127.0.0.1:8000`). +cloud load balancer in front of that port. ## Data migration -The Compose Postgres volume starts empty. To move an existing host database: +The optional Compose Postgres volume starts empty. To move an existing host database: ```bash pg_dump -Fc hrms > hrms.dump -# with postgres published via the dev overlay, or `docker compose exec -T postgres …` pg_restore -h 127.0.0.1 -p 5433 -U postgres -d hrms --clean --if-exists hrms.dump ``` ## Useful commands ```bash -docker compose logs -f backend-api -docker compose logs -f taskiq-worker -docker compose restart backend-api -docker compose down # keep volumes -docker compose down -v # wipe postgres + attachments + redis data +docker compose --env-file ./backend/.env logs -f backend-api +docker compose --env-file ./backend/.env logs -f taskiq-worker +docker compose --env-file ./backend/.env restart backend-api +docker compose --env-file ./backend/.env down # keep volumes +docker compose --env-file ./backend/.env down -v # wipe volumes ``` diff --git a/README.md b/README.md index 781e108..e5c48a3 100644 --- a/README.md +++ b/README.md @@ -45,29 +45,19 @@ job and persists an evidence-based score. **Prerequisites:** Python 3.11+, Node 18+, PostgreSQL, an OpenAI API key. Optional: Redis + Docker (only for background inbox sync / taskiq workers). -### 1. Environment files +### 1. Environment file -Root `.env` (engine + scoring settings — see [.env.example](.env.example)): - -``` -OPENAI_API_KEY=sk-... -OPENAI_MODEL=gpt-5.4-mini -OPENAI_MAX_OUTPUT_TOKENS=4000 -OPENAI_EFFORT=low -SCORING_CONCURRENCY=5 -MAX_RESUMES_PER_REQUEST=50 -MAX_PDF_SIZE_MB=10 -``` - -`backend/.env` (everything in `backend/.env.example`; the must-haves): +Sole file: `backend/.env` (see [backend/.env.example](backend/.env.example)): ``` +PROD_ENV=false DB_USERNAME=... DB_PASSWORD=... DB_HOST=localhost DB_PORT=5432 DB_NAME=hrms JWT_SECRET_KEY=... -OPENAI_API_KEY=sk-... # shared names with the root .env +OPENAI_API_KEY=sk-... +FRONTEND_PORT=8080 ``` -> Windows note: write `.env` files as UTF-8 **without** BOM, and don't leave stray +> Windows note: write `.env` as UTF-8 **without** BOM, and don't leave stray > non `KEY=VALUE` lines — python-dotenv warns on every load. ### 2. Fresh database — one manual step @@ -120,17 +110,13 @@ See **[DOCKER.md](DOCKER.md)** for env checklist, verification, TLS notes, and t local host-Postgres overlay. ```bash -cp .env.example .env # set DB_PASSWORD -cp backend/.env.example backend/.env # set JWT_SECRET_KEY, OPENAI_API_KEY, … -docker compose up -d --build -# SPA: http://localhost/ health: http://localhost/health +cp backend/.env.example backend/.env # set JWT_SECRET_KEY, OPENAI_API_KEY, DB_*, … +docker compose --env-file ./backend/.env up -d --build +# SPA: http://localhost:8080/ health: http://localhost:8080/health ``` -Local day-to-day (host Postgres, exposed API ports, `--reload`): - -```bash -docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build -``` +Local and prod use the same command (`PROD_ENV` + `DB_*` in `backend/.env`). +Optional `--reload` / bind mounts: add `-f docker-compose.dev.yml`. See `DOCKER.md`. ### First run diff --git a/app/core/config.py b/app/core/config.py index f36cb23..919ae69 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -10,10 +10,14 @@ Deliberate omissions: from __future__ import annotations from functools import lru_cache +from pathlib import Path from pydantic import Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict +# Sole secrets file: backend/.env (repo root .env is not used). +_BACKEND_ENV = Path(__file__).resolve().parents[2] / "backend" / ".env" + # Model families that support structured outputs (``responses.parse``) and a reasoning # effort setting. A prefix check rather than an exact allowlist: OpenAI ships point # releases faster than this file can be updated, and rejecting a brand-new gpt-5.x @@ -44,7 +48,7 @@ class Settings(BaseSettings): """Runtime configuration. Immutable once constructed.""" model_config = SettingsConfigDict( - env_file=".env", + env_file=_BACKEND_ENV, # utf-8-sig, not utf-8: Windows editors and PowerShell's `-Encoding utf8` # write a BOM, which would otherwise become part of the first variable's # name and silently blank out that setting. diff --git a/backend/.env.example b/backend/.env.example index 6320fd0..2bc7c41 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,8 +1,22 @@ -DB_USERNAME= +# Sole secrets / config file for the whole monorepo (app + backend + Compose). +# Copy to backend/.env and fill in. Never commit a filled .env. +# +# Compose: docker compose --env-file ./backend/.env up -d --build + +# true → RDS over SSL (asyncpg). false → local Postgres over asyncpg (no SSH). +PROD_ENV=false + +DB_USERNAME=postgres DB_PASSWORD= -DB_HOST= -DB_PORT= -DB_NAME= +DB_HOST=localhost +DB_PORT=5432 +DB_NAME=hrms +# Blank: require when PROD_ENV=true, off when local. Override only if needed. +DB_SSLMODE= + +DB_AUTO_MIGRATE=true +DB_AUTOGENERATE=true + EMAIL_URL= EMAIL_API_TOKEN= # Optional overrides; blank falls back to EMAIL_URL / EMAIL_API_TOKEN. @@ -25,7 +39,7 @@ RESET_CODE_TTL_SECONDS=60 RESET_CODE_RESEND_SECONDS=30 RESET_CODE_MAX_ATTEMPTS=5 -FRONTEND_URL=http://localhost:5173 +FRONTEND_URL=http://127.0.0.1:5173 CONFIRM_EMAIL_PATH=/auth/confirm-email CONFIRM_TOKEN_TTL_SECONDS=86400 CONFIRM_TOKEN_RESEND_SECONDS=60 @@ -47,32 +61,27 @@ OPENAI_BASE_URL= OPENAI_ORGANIZATION= OPENAI_PROJECT= -# ATS scoring (bulk-ats engine embedded via `pip install -e ..`). -# OPENAI_API_KEY / OPENAI_MODEL / OPENAI_MAX_OUTPUT_TOKENS above are shared. +# ATS scoring (bulk-ats engine). Shared OPENAI_* names above. OPENAI_EFFORT=low OPENAI_ENABLE_PROMPT_CACHE=true +OPENAI_TIMEOUT_SECONDS=120 SCORING_CONCURRENCY=5 MAX_RESUMES_PER_REQUEST=50 MAX_PDF_SIZE_MB=10 MAX_JD_CHARS=30000 MAX_RESUME_CHARS=60000 -# Inbox intake gate (inbox_classifier/): only mail judged to be a job application -# gets an inbox_messages row; every verdict is logged to inbox_message_triage. -# Model / token / effort / cache knobs are the OPENAI_* ones above. -# false restores the pre-gate behaviour exactly — the rollback lever. +# Inbox intake gate (inbox_classifier/). INBOX_TRIAGE_ENABLED=true -# true: a provider outage or missing key ingests the mail and marks the verdict -# unclassified. false: skip it and leave it for a later /email/fetch. INBOX_TRIAGE_FAIL_OPEN=true INBOX_TRIAGE_CONCURRENCY=5 INBOX_TRIAGE_MAX_SUBJECT_CHARS=300 INBOX_TRIAGE_MAX_BODY_CHARS=4000 -# 0 disables the uncertainty branch; >0 routes low-confidence verdicts to the -# INBOX_TRIAGE_FAIL_OPEN policy. INBOX_TRIAGE_MIN_CONFIDENCE=0 -REDIS_URL=redis://localhost:6379/0 +# Compose overrides these on the network; keep docker DNS names for containers. +REDIS_URL=redis://redis:6379/0 +BACKEND_URL=http://backend-api:8000 TASKIQ_QUEUE_NAME=inbox TASKIQ_CV_QUEUE_NAME=cv_upload TASKIQ_MAX_RETRIES=3 @@ -82,3 +91,17 @@ TASKIQ_DLQ_STREAM=taskiq:dlq TASKIQ_IDLE_TIMEOUT_MS=600000 MANUAL_UPLOAD_TO_ADDRESS=manual-cv-upload@hr-ats.local APP_VERSION=dev + +# Compose host ports (docker compose --env-file ./backend/.env …). +FRONTEND_PORT=5173 +BACKEND_PORT=8000 +ATS_PORT=8100 +REDIS_PORT=6379 +POSTGRES_PORT=5433 +UVICORN_WORKERS=2 +# Empty = same-origin via nginx on :5173. For Vite on the host, use +# VITE_API_BASE=http://127.0.0.1:8000 (backend is published on BACKEND_PORT). +VITE_API_BASE= + +LOG_FORMAT=json +LOG_LEVEL=INFO diff --git a/backend/README.md b/backend/README.md index 6807e88..c023607 100644 --- a/backend/README.md +++ b/backend/README.md @@ -837,8 +837,8 @@ bind-mounted into the worker so both processes see the same files. ## Configuration Copy `.env.example` to `.env` and fill it in. `.env` is git-ignored; `.env.example` is not. -`db_setup.Settings` reads `backend/.env` or the repo-root `.env`; every other module reads its -own keys with `os.getenv`. +`db_setup.Settings` reads `backend/.env` only (no repo-root `.env`); every other module reads its +own keys with `os.getenv` from the same file. ### Database @@ -1053,7 +1053,9 @@ Local host-Postgres + reload uses the dev overlay. See repo-root **[DOCKER.md](. # Production docker compose up -d --build -# Local (host DB, published ports, --reload) +# Default (host Postgres or RDS via backend/.env PROD_ENV + DB_*) +docker compose up -d --build +# Optional live-reload / bind mounts docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build docker compose logs -f taskiq-worker docker compose logs -f taskiq-cv-worker diff --git a/backend/db_setup.py b/backend/db_setup.py index c57371e..3c09091 100644 --- a/backend/db_setup.py +++ b/backend/db_setup.py @@ -1,9 +1,9 @@ """PostgreSQL connection, async SQLAlchemy ORM and session management. -Configuration comes from the environment, with `.env` read from the repo root or -from `backend/` (`Db_USERNAME`, `Db_PASSWORD`, `Db_HOST`, `Db_PORT`, `Db_NAME`, and -the `DB_*` tuning fields below). Alembic lives in `alembic_setup.py`; `init_db()` -calls into it. +Configuration comes from the environment, with `.env` read from `backend/` +(`DB_USERNAME`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_NAME`, `PROD_ENV`, and +the `DB_*` tuning fields below). There is no repo-root `.env`. Alembic lives in +`alembic_setup.py`; `init_db()` calls into it. app = FastAPI(lifespan=lifespan) # migrate on startup async def endpoint(db: AsyncSession = Depends(get_session)): ... @@ -13,13 +13,14 @@ calls into it. from __future__ import annotations import asyncio -import os import logging +import os from contextlib import asynccontextmanager from functools import lru_cache from pathlib import Path from typing import Annotated, Any, AsyncIterator, Sequence +from dotenv import load_dotenv from pydantic import field_validator from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict from sqlalchemy import MetaData, text @@ -32,40 +33,48 @@ from sqlalchemy.ext.asyncio import ( ) from sqlalchemy.orm import DeclarativeBase from sqlmodel import SQLModel -from dotenv import load_dotenv load_dotenv() logger = logging.getLogger("db") BASE_DIR = Path(__file__).resolve().parent +_TRUE = {"1", "true", "yes", "on"} +_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1"}) + + +def _running_in_docker() -> bool: + """True inside a container (/.dockerenv) or when Compose sets IN_DOCKER=1.""" + return Path("/.dockerenv").exists() or os.environ.get("IN_DOCKER", "").strip().lower() in _TRUE + class Settings(BaseSettings): """Every field is overridden by an environment variable of the same name.""" model_config = SettingsConfigDict( - env_file=(BASE_DIR.parent / ".env", BASE_DIR / ".env"), extra="ignore" + env_file=BASE_DIR / ".env", + extra="ignore", ) - database_url: str = "" # full DSN; wins over the Db_* parts below - db_username: str = os.getenv("DB_USERNAME") - db_password: str = os.getenv("DB_PASSWORD") - db_host: str = os.getenv("DB_HOST") - db_port: int = int(os.getenv("DB_PORT")) - db_name: str = os.getenv("DB_NAME") - db_sslmode: str = "" # e.g. "require" on Azure - + database_url: str = "" # full DSN; wins over the DB_* parts below + db_username: str = "" + db_password: str = "" + db_host: str = "localhost" + db_port: int = 5432 + db_name: str = "" + db_sslmode: str = "" # blank = derive from PROD_ENV (require on RDS, off locally) + prod_env: bool = False # true → RDS (SSL); false → local psql over asyncpg db_schemas: Annotated[list[str], NoDecode] = "app" - db_default_schema: str = "app" # schema for models that declare none + db_default_schema: str = "app" db_echo: bool = False db_pool_size: int = 5 db_max_overflow: int = 10 db_pool_recycle: int = 1800 db_connect_retries: int = 10 - db_auto_migrate: bool = True # run `upgrade head` on startup - db_autogenerate: bool = True # write a revision when models drift from the schema - db_model_modules: Annotated[list[str], NoDecode] = [] # empty means auto-discover + db_auto_migrate: bool = True + db_autogenerate: bool = True + db_model_modules: Annotated[list[str], NoDecode] = [] app_name: str = "hr-ats-portal" @field_validator("db_schemas", "db_model_modules", mode="before") @@ -75,8 +84,20 @@ class Settings(BaseSettings): return [item.strip() for item in value.split(",") if item.strip()] return value + @field_validator("prod_env", mode="before") + @classmethod + def _bool(cls, value: Any) -> Any: + if isinstance(value, str): + return value.strip().lower() in _TRUE + return value + def url(self, *, async_driver: bool = True) -> URL: - """DSN with the driver forced; `sslmode` is mapped to asyncpg's `ssl` mode name.""" + """DSN with the driver forced; `sslmode` is mapped to asyncpg's `ssl` mode name. + + Local Docker: `DB_HOST=localhost` means the container itself, so rewrite to + `host.docker.internal` for the connection URL only (Settings.db_host unchanged). + Prod never rewrites — RDS hostname is used as-is. + """ url = ( make_url(self.database_url) if self.database_url @@ -89,10 +110,22 @@ class Settings(BaseSettings): self.db_name, ) ) + if ( + not self.prod_env + and _running_in_docker() + and (url.host or "").lower() in _LOOPBACK_HOSTS + ): + url = url.set(host="host.docker.internal") + query = dict(url.query) - if self.db_sslmode: - query.setdefault("sslmode", self.db_sslmode) - # asyncpg accepts ssl as an SSLMode name (require, verify-full, …), not "true". + + # PROD_ENV=true → RDS needs SSL. Local psql talks plain asyncpg (no SSL). + sslmode = self.db_sslmode.strip() if self.db_sslmode else ("require" if self.prod_env else "") + if sslmode: + query.setdefault("sslmode", sslmode) + else: + query.pop("sslmode", None) + if async_driver and (mode := query.pop("sslmode", None)) is not None: query["ssl"] = mode driver = "asyncpg" if async_driver else "psycopg2" @@ -199,11 +232,16 @@ async def close_db() -> None: async def check_connection(retries: int | None = None, delay: float = 1.0) -> None: """Wait for Postgres to answer `SELECT 1`, retrying with a capped backoff.""" attempts = get_settings().db_connect_retries if retries is None else retries + s = get_settings() for attempt in range(1, max(attempts, 1) + 1): try: async with get_engine().connect() as conn: await conn.execute(text("SELECT 1")) - logger.info("connected to %s", database_url(hide_password=True)) + logger.info( + "connected to %s [PROD_ENV=%s]", + database_url(hide_password=True), + s.prod_env, + ) return except Exception as exc: if attempt >= attempts: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 63eb9f1..19e319c 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,33 +1,16 @@ -# Local / host-Postgres overlay. Restores the previous day-to-day workflow on top -# of the production docker-compose.yml: +# Optional local overlay — NOT required for day-to-day use. # +# Default workflow (host Postgres + loopback API ports) is in docker-compose.yml: +# docker compose up -d --build +# +# Use this file only when you want --reload and bind-mounted source: # docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build # -# - DB_HOST=host.docker.internal (host Postgres; container postgres still starts -# but is unused unless you flip DB_HOST back) -# - Publishes API / ATS / Redis / frontend ports for direct host access -# - Bind-mounts CV attachments to ./backend/inbox/decoded_attachments -# - Live source mounts + --reload on API / ATS / workers -# -# Frontend stays as the built nginx image — run `npm run dev` on the host for the -# Vite loop if you need HMR. +# Do not set DB_HOST here. Keep PROD_ENV=false and DB_HOST=localhost in +# backend/.env; db_setup rewrites localhost → host.docker.internal in Docker. services: - redis: - ports: - - "${REDIS_PORT:-6379}:6379" - - postgres: - ports: - - "127.0.0.1:${POSTGRES_PORT:-5433}:5432" - backend-api: - environment: - PYTHONPATH: /app - DB_HOST: host.docker.internal - REDIS_URL: redis://redis:6379/0 - EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} - BACKEND_URL: http://backend-api:8000 command: [ "uvicorn", @@ -38,8 +21,6 @@ services: "8000", "--reload", ] - ports: - - "${BACKEND_PORT:-8000}:8000" volumes: - ./backend:/app - ./app:/app/app @@ -57,8 +38,6 @@ services: "8100", "--reload", ] - ports: - - "${ATS_PORT:-8100}:8100" volumes: - ./app:/srv/app @@ -67,66 +46,28 @@ services: - "${FRONTEND_PORT:-5173}:80" taskiq-worker: - environment: - PYTHONPATH: /app - DB_HOST: host.docker.internal - REDIS_URL: redis://redis:6379/0 - EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} - BACKEND_URL: http://backend-api:8000 - TASKIQ_QUEUE_NAME: inbox - TASKIQ_WORKER_NAME: worker-01 volumes: - ./backend:/app - ./app:/app/app - ${ATTACHMENTS_DIR:-./backend/inbox/decoded_attachments}:/app/inbox/decoded_attachments taskiq-scheduler: - environment: - PYTHONPATH: /app - DB_HOST: host.docker.internal - REDIS_URL: redis://redis:6379/0 - EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} - BACKEND_URL: http://backend-api:8000 - TASKIQ_QUEUE_NAME: inbox volumes: - ./backend:/app - ./app:/app/app taskiq-cv-worker: - environment: - PYTHONPATH: /app - DB_HOST: host.docker.internal - REDIS_URL: redis://redis:6379/0 - EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} - BACKEND_URL: http://backend-api:8000 - TASKIQ_CV_QUEUE_NAME: cv_upload - TASKIQ_WORKER_NAME: cv-worker-01 volumes: - ./backend:/app - ./app:/app/app - ${ATTACHMENTS_DIR:-./backend/inbox/decoded_attachments}:/app/inbox/decoded_attachments taskiq-cv-scheduler: - environment: - PYTHONPATH: /app - DB_HOST: host.docker.internal - REDIS_URL: redis://redis:6379/0 - EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} - BACKEND_URL: http://backend-api:8000 - TASKIQ_CV_QUEUE_NAME: cv_upload volumes: - ./backend:/app - ./app:/app/app taskiq-mailbox-sync-worker: - environment: - PYTHONPATH: /app - DB_HOST: host.docker.internal - REDIS_URL: redis://redis:6379/0 - EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} - BACKEND_URL: http://backend-api:8000 - TASKIQ_MAILBOX_SYNC_QUEUE_NAME: mailbox_sync - TASKIQ_WORKER_NAME: mailbox-sync-worker-01 volumes: - ./backend:/app - ./app:/app/app diff --git a/docker-compose.host-ports.yml b/docker-compose.host-ports.yml new file mode 100644 index 0000000..6a51f26 --- /dev/null +++ b/docker-compose.host-ports.yml @@ -0,0 +1,24 @@ +# Optional overlay: publish API / ATS / Redis on loopback for host tools +# (curl, redis-cli, Postman). Not required for the SPA - nginx proxies +# to backend-api on the Compose network. +# +# docker compose --env-file ./backend/.env -f docker-compose.yml -f docker-compose.host-ports.yml up -d +# +# If bind fails because an IDE (Cursor/VS Code) still holds the port after a +# previous run, clear Port Forwarding in the IDE or override in backend/.env: +# BACKEND_PORT=8001 +# ATS_PORT=8101 +# REDIS_PORT=6380 + +services: + redis: + ports: + - "127.0.0.1:${REDIS_PORT:-6379}:6379" + + backend-api: + ports: + - "127.0.0.1:${BACKEND_PORT:-8000}:8000" + + ats-engine: + ports: + - "127.0.0.1:${ATS_PORT:-8100}:8100" diff --git a/docker-compose.yml b/docker-compose.yml index 884bc7d..9d0a3d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,26 @@ -# HR-ATS-Portal — production Compose stack (self-contained). +# HR-ATS-Portal — single Compose file for local and production. # -# docker compose up -d --build -# docker compose ps -# docker compose logs -f backend-api +# Sole env file: backend/.env (no repo-root .env). Pass it for Compose +# variable substitution (${FRONTEND_PORT}, ${DB_*}, …): # -# Public surface: only the frontend (default host port 80). The browser talks -# same-origin to nginx, which proxies API paths to backend-api. Postgres, Redis, -# the API, ATS engine and Taskiq workers stay on the Compose network. +# docker compose --env-file ./backend/.env up -d --build +# docker compose --env-file ./backend/.env ps +# docker compose --env-file ./backend/.env logs -f backend-api # -# Local / current host-Postgres workflow (exposed ports, --reload, bind mounts): -# docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build +# Local (PROD_ENV=false, DB_HOST=localhost in backend/.env): +# Containers reach host Postgres via host.docker.internal (db_setup rewrite +# when IN_DOCKER=1). API / ATS / Redis stay on the Compose network by default +# (avoids IDE/Cursor stale port-forwards fighting Docker on Windows). Opt in: +# docker compose --env-file ./backend/.env -f docker-compose.yml -f docker-compose.host-ports.yml up -d +# +# Prod (PROD_ENV=true, DB_* = RDS in backend/.env — edit manually): +# Same command. No host rewrite; SSL require when DB_SSLMODE is blank. +# +# Optional Compose Postgres (empty volume, not host/RDS data): +# docker compose --env-file ./backend/.env --profile postgres up -d postgres +# +# Optional live-reload / bind mounts (not required day-to-day): +# docker compose --env-file ./backend/.env -f docker-compose.yml -f docker-compose.dev.yml up -d --build # # See DOCKER.md for env checklist and verification. @@ -27,26 +38,21 @@ x-backend-build: &backend-build x-backend-env: &backend-env PYTHONPATH: /app - # Production default: container Postgres on the Compose network. The dev overlay - # overrides DB_HOST to host.docker.internal. Credentials come from root .env so - # they stay in lockstep with the postgres service (backend/.env is for host runs). - DB_HOST: ${DB_HOST:-postgres} - DB_PORT: ${DB_PORT:-5432} - DB_USERNAME: ${DB_USERNAME:-postgres} - DB_PASSWORD: ${DB_PASSWORD:-postgres} - DB_NAME: ${DB_NAME:-hrms} - # On every API boot: upgrade any on-disk revisions (usually none in images), - # then apply ORM drift in-memory (no revision files written — versions stay - # gitignored and out of the image). - DB_AUTO_MIGRATE: ${DB_AUTO_MIGRATE:-true} - DB_AUTOGENERATE: ${DB_AUTOGENERATE:-true} + # Lets db_setup rewrite DB_HOST=localhost → host.docker.internal (local only). + IN_DOCKER: "1" + # Credentials and endpoints come from backend/.env via `env_file` below. + # DB_HOST is deliberately ABSENT here: an empty `environment:` override would + # blank env_file / RDS. Do not name credential keys under environment. + # + # REDIS_URL and BACKEND_URL are compose-network DNS names, correct everywhere. REDIS_URL: redis://redis:6379/0 - # Prefer root/.env EMAIL_URL. Fall back to host-gateway when mail is on this machine. - EMAIL_URL: ${EMAIL_URL:-http://host.docker.internal:5000} BACKEND_URL: http://backend-api:8000 + # Apify token, if kept at repo root rather than in backend/.env. Harmless when + # unset: the app treats empty as absent. + APIFY_API_TOKEN: ${APIFY_API_TOKEN:-${APIFY_TOKEN:-}} -# Shared CV storage. Named volume in production so API + workers see the same -# files without a host path. Dev overlay remounts ./backend/inbox/decoded_attachments. +# Shared CV storage. Named volume so API + workers see the same files. +# Optional docker-compose.dev.yml remounts ./backend/inbox/decoded_attachments. x-attachments: &attachments - attachments-data:/app/inbox/decoded_attachments @@ -55,15 +61,22 @@ x-backend-service: &backend-service image: hrms-backend:local working_dir: /app env_file: + # backend/.env is the source of truth (plain DB_* + PROD_ENV). - ./backend/.env + # Optional local overrides (required:false). Do not set DB_HOST=postgres + # here unless you intentionally start the postgres profile. + - path: ./docker.local.env + required: false environment: *backend-env extra_hosts: - "host.docker.internal:host-gateway" depends_on: redis: condition: service_healthy + # required:false — default stack never starts postgres (host Postgres or RDS). postgres: condition: service_healthy + required: false restart: unless-stopped logging: *default-logging @@ -73,7 +86,8 @@ services: image: redis:7-alpine container_name: hrms-redis command: ["redis-server", "--appendonly", "yes"] - # Not published in production. Dev overlay binds ${REDIS_PORT:-6379}:6379. + # No host publish by default (Compose DNS redis:6379). Optional loopback: + # docker-compose.host-ports.yml volumes: - redis-data:/data healthcheck: @@ -84,19 +98,29 @@ services: restart: unless-stopped logging: *default-logging - # --- Postgres (always on in production) ------------------------------------------ + # --- Postgres: DEFINED HERE, NOT STARTED BY DEFAULT ------------------------------- + # Default local path is host Postgres (DB_HOST=localhost → host.docker.internal). + # Prod uses AWS RDS. Opt in only when you want a disposable Compose DB: + # + # docker compose --profile postgres up -d postgres + # # then set DB_HOST=postgres in backend/.env and recreate backend services + # postgres: + profiles: ["postgres"] build: context: ./docker/postgres image: hrms-postgres:local container_name: hrms-postgres environment: - # Interpolated from the shell or root .env, NOT from backend/.env. + # Compose substitution needs: docker compose --env-file ./backend/.env … + # (there is no repo-root .env). Defaults apply if the flag is omitted. POSTGRES_USER: ${DB_USERNAME:-postgres} POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres} POSTGRES_DB: ${DB_NAME:-hrms} POSTGRES_INITDB_ARGS: "--encoding=UTF8" - # Not published in production. Dev overlay can bind 127.0.0.1:${POSTGRES_PORT:-5433}:5432. + ports: + # 5433: the host's own Postgres owns 5432. + - "127.0.0.1:${POSTGRES_PORT:-5433}:5432" volumes: - postgres-data:/var/lib/postgresql/data healthcheck: @@ -126,7 +150,9 @@ services: "--proxy-headers", "--forwarded-allow-ips=*", ] - # Not published in production — browsers reach the API via frontend nginx. + # Published for host tools / Vite (`VITE_API_BASE=http://127.0.0.1:8000`). + ports: + - "${BACKEND_PORT:-8000}:8000" volumes: *attachments healthcheck: test: @@ -150,9 +176,12 @@ services: container_name: hrms-ats-engine env_file: - ./backend/.env - - path: ./.env + - path: ./docker.local.env required: false - # Not published in production. + environment: + IN_DOCKER: "1" + # No host publish by default (backend uses ats-engine:8100). Optional: + # docker-compose.host-ports.yml healthcheck: test: [ @@ -168,7 +197,7 @@ services: restart: unless-stopped logging: *default-logging - # --- React portal (only published host port) ------------------------------------- + # Portal on FRONTEND_PORT (default 5173). API also on BACKEND_PORT (8000). frontend: build: context: ./frontend @@ -181,7 +210,7 @@ services: backend-api: condition: service_healthy ports: - - "${FRONTEND_PORT:-80}:80" + - "${FRONTEND_PORT:-5173}:80" healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1/"] interval: 15s diff --git a/frontend/.env.development b/frontend/.env.development index de07be0..0426bbe 100644 --- a/frontend/.env.development +++ b/frontend/.env.development @@ -1,3 +1,10 @@ -# Use 127.0.0.1, not localhost. On this machine localhost prefers ::1 and hits a -# different listener (WSL/Docker on :8000) instead of the Windows uvicorn on 127.0.0.1. +# Use 127.0.0.1, not localhost. On this machine localhost prefers ::1 and can +# hit a different listener than the Windows uvicorn / Docker publish. +# +# Docker Compose (prod-like): leave empty and run `npm run dev` with the +# host-ports overlay so Vite can proxy — or open the built SPA on :8080 +# (nginx same-origin proxy; no VITE_API_BASE needed). +# +# Hybrid (Vite on host + API in Docker): publish API first, then: +# docker compose --env-file ./backend/.env -f docker-compose.yml -f docker-compose.host-ports.yml up -d VITE_API_BASE=http://127.0.0.1:8000 diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 9ebb5f2..139069d 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -14,11 +14,15 @@ server { add_header Referrer-Policy strict-origin-when-cross-origin always; # Same-origin API proxy. The SPA is built with an empty VITE_API_BASE so - # fetch('/jobs/fetch') stays on this host; without this block nginx would - # serve index.html for those paths (200 HTML) and the Jobs board would - # parse an empty payload. OpenAPI (/docs, /redoc, /openapi.json) is - # intentionally NOT proxied — keep the schema off the public edge. - location ~ ^/(health|users|roles|permissions|permission-tags|managers|inbox|email|job|jobs|candidate|notes|interview|feedback|activity|pipeline|notifications|analytics|offers|tasks|assessments|org-settings|saved-searches|search)(/|$) { + # fetch('/jobs/fetch') stays on this host. OpenAPI (/docs, /redoc, + # /openapi.json) is intentionally NOT proxied. + # + # Paths that are BOTH React routes (/jobs, /inbox, …) and API prefixes must + # require a sub-path: otherwise a cold open / refresh of /jobs is stolen by + # the proxy and returns a FastAPI 404 instead of index.html. + + # SPA page roots that also prefix API calls — sub-path required. + location ~ ^/(jobs|inbox|pipeline|tasks|assessments|offers|managers|analytics|notifications)/ { proxy_pass http://backend-api:8000; proxy_http_version 1.1; proxy_set_header Host $host; @@ -29,6 +33,22 @@ server { proxy_connect_timeout 10s; proxy_send_timeout 120s; proxy_read_timeout 120s; + proxy_request_buffering off; + } + + # API-only prefixes (no SPA page at the bare path). + location ~ ^/(health|users|roles|permissions|permission-tags|email|job|candidate|notes|interview|feedback|activity|org-settings|saved-searches|search|documents|sheet)(/|$) { + proxy_pass http://backend-api:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Authorization $http_authorization; + proxy_connect_timeout 10s; + proxy_send_timeout 120s; + proxy_read_timeout 120s; + proxy_request_buffering off; } # One SPA at `/`. Without this, /auth/confirm-email (a router path, not a diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 99b4120..2f34a4e 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -16,6 +16,16 @@ export default defineConfig({ base: '/', build: { outDir: 'dist', emptyOutDir: true, sourcemap: true }, resolve: { alias: { '@': path.resolve(__dirname, 'src') } }, - server: { port: 5173 }, + server: { + port: 5173, + // Same-origin style for local Vite when VITE_API_BASE is empty. + // Requires API published on the host (docker-compose.host-ports.yml). + proxy: { + '^/(health|users|roles|permissions|permission-tags|email|job|jobs|candidate|notes|interview|feedback|activity|pipeline|notifications|analytics|offers|tasks|assessments|org-settings|saved-searches|search|documents|sheet|managers|inbox)(/|$)': { + target: 'http://127.0.0.1:8000', + changeOrigin: true, + }, + }, + }, preview: { port: 4173 }, }) -- 2.40.1 From f1597792ed15a441bff648206c2f22a98e23c0e9 Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Mon, 24 Aug 2026 20:40:08 +0500 Subject: [PATCH 7/7] tests rtemoved --- .gitignore | 1 + backend/inbox/mailbox_sync_tasks.py | 97 +++++++++++++++++++ .../mailbox_sync_broker_setup.py | 59 +++++++++++ 3 files changed, 157 insertions(+) create mode 100644 backend/inbox/mailbox_sync_tasks.py create mode 100644 backend/taskiq_management/mailbox_sync_broker_setup.py diff --git a/.gitignore b/.gitignore index a043f7a..2275f4b 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ Utopia-ai-hr-ats-portal 1.pem # Local-only Compose overrides (never deployed) docker.local.env +tests/** \ No newline at end of file diff --git a/backend/inbox/mailbox_sync_tasks.py b/backend/inbox/mailbox_sync_tasks.py new file mode 100644 index 0000000..e4f90d9 --- /dev/null +++ b/backend/inbox/mailbox_sync_tasks.py @@ -0,0 +1,97 @@ +"""Mailbox sync Taskiq tasks — Outlook pull + triage + ingest on own stream.""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime,timezone + +import redis.asyncio as redis +from dotenv import load_dotenv + +from db_setup import session_scope +from inbox.models import MailboxSyncRun +from inbox.views import Email +from taskiq_management.broker_setup import MAX_RETRIES,RETRY_DELAY +from taskiq_management.mailbox_sync_broker_setup import mailbox_sync_broker +from taskiq_management.middleware import PermanentTaskError + +load_dotenv() + +logger=logging.getLogger("inbox.mailbox_sync") +REDIS_URL=os.getenv("REDIS_URL","redis://localhost:6379/0") +_LOCK_KEY="inbox:mailbox_sync:lock" +_LOCK_TTL=900 + + +async def _fail(run_id:str,error:str) -> dict: + async with session_scope() as session: + await MailboxSyncRun.update_run(session,run_id,{ + "status":"failed", + "error":error, + "finished_at":datetime.now(timezone.utc), + }) + return {"status":"failed","error":error} + + +@mailbox_sync_broker.task( + task_name="inbox.sync_mailbox", + retry_on_error=True, + max_retries=MAX_RETRIES, + delay=RETRY_DELAY, +) +async def sync_mailbox(run_id:str) -> dict: + if not run_id or not str(run_id).strip(): + raise PermanentTaskError("run_id is required") + run_id=str(run_id).strip() + + client=redis.from_url(REDIS_URL,decode_responses=True) + try: + acquired=await client.set(_LOCK_KEY,run_id,nx=True,ex=_LOCK_TTL) + if not acquired: + return await _fail(run_id,"another mailbox sync is already running") + + try: + async with session_scope() as session: + row=await MailboxSyncRun.get_by_id(session,run_id) + if not row: + raise PermanentTaskError(f"sync run {run_id} not found") + await MailboxSyncRun.update_run(session,run_id,{ + "status":"running", + "started_at":datetime.now(timezone.utc), + "error":None, + }) + top=row.top or 100 + skip=row.skip or 0 + test_on=True if row.test_on is None else bool(row.test_on) + + async with session_scope() as session: + service=Email(session=session) + if not service.token: + return await _fail(run_id,"EMAIL_API_TOKEN is not configured") + try: + summary=await service.run_mailbox_sync_page( + top=top,skip=skip,test_on=test_on, + ) + except Exception as e: + logger.exception("mailbox sync failed for run %s",run_id) + return await _fail(run_id,str(e)) + + await MailboxSyncRun.update_run(session,run_id,{ + "status":"completed", + "entries":summary["entries"], + "triage":summary["triage"], + "error":None, + "finished_at":datetime.now(timezone.utc), + }) + return { + "status":"completed", + "triage":summary["triage"], + "entries":len(summary["entries"]), + } + finally: + current=await client.get(_LOCK_KEY) + if current==run_id: + await client.delete(_LOCK_KEY) + finally: + await client.aclose() diff --git a/backend/taskiq_management/mailbox_sync_broker_setup.py b/backend/taskiq_management/mailbox_sync_broker_setup.py new file mode 100644 index 0000000..2ab9a1f --- /dev/null +++ b/backend/taskiq_management/mailbox_sync_broker_setup.py @@ -0,0 +1,59 @@ +"""Taskiq mailbox-sync broker — isolated Redis stream so Outlook pull/triage +never blocks inbox match/ATS or the cv_upload queue. + +Worker: taskiq worker taskiq_management.mailbox_sync_broker_setup:mailbox_sync_broker inbox.mailbox_sync_tasks +""" + +from __future__ import annotations + +import os + +from dotenv import load_dotenv +from taskiq import TaskiqScheduler +from taskiq.middlewares import SmartRetryMiddleware +from taskiq.schedule_sources import LabelScheduleSource +from taskiq_redis import ( + ListRedisScheduleSource, + RedisAsyncResultBackend, + RedisStreamBroker, +) + +from taskiq_management.broker_setup import MAX_RETRIES,RETRY_DELAY +from taskiq_management.middleware import DeadLetterMiddleware + +load_dotenv() + +REDIS_URL=os.getenv("REDIS_URL","redis://localhost:6379/0") +MAILBOX_SYNC_QUEUE_NAME=os.getenv("TASKIQ_MAILBOX_SYNC_QUEUE_NAME","mailbox_sync") + +result_backend=RedisAsyncResultBackend(redis_url=REDIS_URL) +mailbox_sync_schedule_source=ListRedisScheduleSource( + url=REDIS_URL,prefix="taskiq:schedule:mailbox_sync", +) + +mailbox_sync_broker=( + RedisStreamBroker( + url=REDIS_URL, + queue_name=MAILBOX_SYNC_QUEUE_NAME, + consumer_group_name=os.getenv("TASKIQ_CONSUMER_GROUP","taskiq"), + idle_timeout=int(os.getenv("TASKIQ_IDLE_TIMEOUT_MS","600000")), + ) + .with_result_backend(result_backend) + .with_middlewares( + DeadLetterMiddleware(redis_url=REDIS_URL), + SmartRetryMiddleware( + default_retry_count=MAX_RETRIES, + default_retry_label=True, + default_delay=RETRY_DELAY, + use_jitter=True, + use_delay_exponent=True, + max_delay_exponent=float(os.getenv("TASKIQ_MAX_DELAY","120")), + schedule_source=mailbox_sync_schedule_source, + ), + ) +) + +mailbox_sync_scheduler=TaskiqScheduler( + broker=mailbox_sync_broker, + sources=[mailbox_sync_schedule_source,LabelScheduleSource(mailbox_sync_broker)], +) -- 2.40.1