Cv_extractor of g_sheet
parent
75943cf03f
commit
1399cf3fec
|
|
@ -5,7 +5,7 @@
|
|||
"refresh_token": "1//03C8LMkk-9mSyCgYIARAAGAMSNwF-L9IrAcPhShsp3cDeprSlqI-P6lArpmbyzu-PcKiLfQ5gC3H_MEk930IaKfewy3cxP3T0Oo8",
|
||||
"universe_domain": "googleapis.com",
|
||||
"account": "ahmed.mujtaba@utopiabrands.com",
|
||||
"token": "ya29.a0AdMD6EgILeNb9UszC7bQJbAcqX709J5ky3eM8MEuQayGwhDStmfnR5t7o192x-FPdt53Q29rYL69zqYrgofUqpwxoI_sPjBsb0wrLqYDo6zwJMTx4P5svM4jZJd9nrXzUEyp3uI81e9DQ3z6lIDKtm6aUTPWQ3fm33i2hxJM7i-svhi3OwjnLtpOUHDyw--v8rQLPHdfaCgYKATkSARASFQHGX2MiXDnutGLllH9DnNCBUKWi4Q0207",
|
||||
"expiry": "2026-09-02T08:29:45Z",
|
||||
"token": "ya29.a0AdMD6EgKB22VSy--W0qCtRkMOYECCDhvL4c14xNUSvizbooOBC-ctQeyWR_XybUqa6PZvQ0csVqrIDR6e_uQazKuTAxKPLgpgbTmJ96-sHWbj_981xNrcWe6JsxIpQuGLX9GiKSGa8y5t50ZgWbDy0ECUoOQDvzlUq-hgNdLve1ECxDG4twpL3-2ZpGgskHlhuRL4-PQaCgYKARISARASFQHGX2MiOWOjgYvNnX5ZWhBoYqO9Cg0207",
|
||||
"expiry": "2026-09-02T16:27:05Z",
|
||||
"quota_project_id": "hrms-ats-portal"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,331 @@
|
|||
"""Drive CV extract wrapper for sheet ingest.
|
||||
|
||||
Hang `@extract_drive_cvs` on `SheetImport.import_sheet` only (the worker).
|
||||
HTTP enqueue routes must not run this — FormData rows do not exist yet.
|
||||
|
||||
Worker job pattern (same as a Taskiq message): create a temp dir for the run,
|
||||
stream each Drive CV to a file, extract, write extracted_data, delete that file.
|
||||
A finally block removes the job dir so a successful run leaves no CVs on disk.
|
||||
One row at a time — a plain sequential loop, no extra locks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime,timezone
|
||||
from functools import wraps
|
||||
from inspect import signature
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from g_sheet.models import FormData
|
||||
from g_sheet.plugins import (
|
||||
SheetsApiError,
|
||||
drive_file_id,
|
||||
download_drive_file,
|
||||
ensure_fresh,
|
||||
load_credentials,
|
||||
)
|
||||
|
||||
load_dotenv(Path(__file__).resolve().parent.parent/".env")
|
||||
|
||||
logger=logging.getLogger("g_sheet.decorators")
|
||||
|
||||
_MAX_RESUME_CHARS=int(os.getenv("MAX_RESUME_CHARS","60000"))
|
||||
_MAX_PDF_SIZE_MB=int(os.getenv("MAX_PDF_SIZE_MB","10"))
|
||||
_TEMP_ROOT=Path(__file__).resolve().parent/"tmp"/"cv_extract"
|
||||
|
||||
|
||||
def build_extracted_data(
|
||||
*,
|
||||
status,
|
||||
resume_link,
|
||||
file_id=None,
|
||||
filename=None,
|
||||
mime_type=None,
|
||||
text=None,
|
||||
page_count=None,
|
||||
truncated=None,
|
||||
error_code=None,
|
||||
error_message=None,
|
||||
):
|
||||
"""Stable JSON blob stored on form_data.extracted_data."""
|
||||
return {
|
||||
"status":status,
|
||||
"resume_link":resume_link or "",
|
||||
"file_id":file_id,
|
||||
"filename":filename,
|
||||
"mime_type":mime_type,
|
||||
"text":text,
|
||||
"page_count":page_count,
|
||||
"truncated":truncated,
|
||||
"char_count":len(text) if isinstance(text,str) else None,
|
||||
"error_code":error_code,
|
||||
"error_message":error_message,
|
||||
"extracted_at":datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def _drive_error_code(status_code):
|
||||
if status_code in (401,403):
|
||||
return "DRIVE_FORBIDDEN"
|
||||
if status_code==404:
|
||||
return "DRIVE_FILE_NOT_FOUND"
|
||||
if status_code==413:
|
||||
return "PAYLOAD_TOO_LARGE"
|
||||
if status_code in (400,415):
|
||||
return "UNSUPPORTED_FILE_TYPE"
|
||||
return "DRIVE_DOWNLOAD_FAILED"
|
||||
|
||||
|
||||
def _prepare_drive_credentials(service):
|
||||
"""Load/refresh the Google session. Never raises — None means skip extract."""
|
||||
try:
|
||||
if service is None:
|
||||
return load_credentials()
|
||||
creds=getattr(service,"credentials",None)
|
||||
path=getattr(service,"credentials_path",None)
|
||||
scopes=getattr(service,"scopes",None)
|
||||
if creds is not None:
|
||||
return ensure_fresh(creds,path)
|
||||
return load_credentials(path,scopes)
|
||||
except Exception:
|
||||
logger.warning("Google Drive session unavailable; skipping CV extract")
|
||||
return None
|
||||
|
||||
|
||||
def _is_sheet_service(obj):
|
||||
return obj is not None and hasattr(obj,"session") and hasattr(obj,"spreadsheet_id")
|
||||
|
||||
|
||||
def _tab_from(result,args,kwargs):
|
||||
if isinstance(result,dict) and result.get("tab"):
|
||||
return result.get("tab")
|
||||
if kwargs.get("tab"):
|
||||
return kwargs.get("tab")
|
||||
if args:
|
||||
return args[0]
|
||||
return None
|
||||
|
||||
|
||||
def _should_ingest(result):
|
||||
if not isinstance(result,dict):
|
||||
return False
|
||||
if result.get("error"):
|
||||
return False
|
||||
if result.get("status") in ("queued","running","failed"):
|
||||
return False
|
||||
if result.get("rows_read",1)==0 and result.get("inserted",1)==0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def make_job_temp_dir(root=None):
|
||||
"""Temp dir for one extract job. Caller must remove_job_temp_dir in finally."""
|
||||
base=Path(root) if root else _TEMP_ROOT
|
||||
base.mkdir(parents=True,exist_ok=True)
|
||||
job_dir=base/uuid.uuid4().hex
|
||||
job_dir.mkdir()
|
||||
return job_dir
|
||||
|
||||
|
||||
def remove_job_temp_dir(job_dir):
|
||||
"""Delete leftover CVs and the job dir. No-op if missing."""
|
||||
if not job_dir:
|
||||
return
|
||||
path=Path(job_dir)
|
||||
if not path.exists():
|
||||
return
|
||||
shutil.rmtree(path,ignore_errors=True)
|
||||
|
||||
|
||||
def _unlink(path):
|
||||
if path is None:
|
||||
return
|
||||
try:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning("could not delete temp CV %s: %s",path,e)
|
||||
|
||||
|
||||
def _extract_pdf(data,filename,max_chars):
|
||||
from app.services.pdf import extract_resume,sanitize_filename
|
||||
from job.candidate.plugins import normalize_spaced_text
|
||||
|
||||
resume=extract_resume(data,sanitize_filename(filename),max_chars)
|
||||
return {
|
||||
"text":normalize_spaced_text(resume.text),
|
||||
"page_count":resume.page_count,
|
||||
"truncated":resume.truncated,
|
||||
}
|
||||
|
||||
|
||||
def _download_and_extract(credentials,link,max_chars,max_bytes,dest_dir):
|
||||
"""Stream one Drive file into dest_dir, extract, then delete that file."""
|
||||
file_id=drive_file_id(link)
|
||||
if not file_id:
|
||||
return build_extracted_data(
|
||||
status="skipped",
|
||||
resume_link=link,
|
||||
error_code="NOT_DRIVE_URL",
|
||||
error_message="Resume link is not a Google Drive file URL",
|
||||
)
|
||||
dest=None
|
||||
try:
|
||||
downloaded=download_drive_file(
|
||||
credentials,link,max_bytes=max_bytes,dest_dir=dest_dir,
|
||||
)
|
||||
dest=downloaded.get("path")
|
||||
if dest is None:
|
||||
return build_extracted_data(
|
||||
status="failed",
|
||||
resume_link=link,
|
||||
file_id=downloaded.get("file_id") or file_id,
|
||||
error_code="DRIVE_DOWNLOAD_FAILED",
|
||||
error_message="Drive download did not write a file",
|
||||
)
|
||||
data=Path(dest).read_bytes()
|
||||
try:
|
||||
parsed=_extract_pdf(data,downloaded.get("filename") or "resume.pdf",max_chars)
|
||||
finally:
|
||||
data=b""
|
||||
return build_extracted_data(
|
||||
status="completed",
|
||||
resume_link=link,
|
||||
file_id=downloaded.get("file_id") or file_id,
|
||||
filename=downloaded.get("filename"),
|
||||
mime_type=downloaded.get("mime_type"),
|
||||
text=parsed["text"],
|
||||
page_count=parsed["page_count"],
|
||||
truncated=parsed["truncated"],
|
||||
)
|
||||
except SheetsApiError as e:
|
||||
return build_extracted_data(
|
||||
status="failed",
|
||||
resume_link=link,
|
||||
file_id=file_id,
|
||||
error_code=_drive_error_code(e.status_code),
|
||||
error_message=(e.message or "")[:300],
|
||||
)
|
||||
except Exception as e:
|
||||
from app.core.errors import ATSError
|
||||
if isinstance(e,ATSError):
|
||||
return build_extracted_data(
|
||||
status="failed",
|
||||
resume_link=link,
|
||||
file_id=file_id,
|
||||
error_code=e.error_code,
|
||||
error_message=e.public_message,
|
||||
)
|
||||
logger.exception("drive download/extract failed for file_id=%s",file_id)
|
||||
return build_extracted_data(
|
||||
status="failed",
|
||||
resume_link=link,
|
||||
file_id=file_id,
|
||||
error_code="DRIVE_DOWNLOAD_FAILED",
|
||||
error_message="Drive download failed",
|
||||
)
|
||||
finally:
|
||||
_unlink(dest)
|
||||
|
||||
|
||||
async def extract_one_resume(credentials,resume_link,dest_dir,max_chars=None,max_bytes=None):
|
||||
"""Download one Drive URL into dest_dir and return extracted_data JSON."""
|
||||
if max_chars is None:
|
||||
max_chars=_MAX_RESUME_CHARS
|
||||
if max_bytes is None:
|
||||
max_bytes=_MAX_PDF_SIZE_MB*1024*1024
|
||||
link=(resume_link or "").strip()
|
||||
return await asyncio.to_thread(
|
||||
_download_and_extract,credentials,link,max_chars,max_bytes,dest_dir,
|
||||
)
|
||||
|
||||
|
||||
async def ingest_form_resume_links(session,sheet,credentials,temp_root=None):
|
||||
"""One Drive file per resume_link: download → extract → DB → delete file.
|
||||
|
||||
The job temp dir is created at start and removed in finally so a finished
|
||||
run leaves no CVs on disk (Taskiq worker cleanup).
|
||||
"""
|
||||
rows=await FormData.fetch_resume_links(session,sheet)
|
||||
completed=0
|
||||
failed=0
|
||||
max_chars=_MAX_RESUME_CHARS
|
||||
max_bytes=_MAX_PDF_SIZE_MB*1024*1024
|
||||
job_dir=make_job_temp_dir(temp_root)
|
||||
logger.info(
|
||||
"drive CV extract starting tab=%s resumes=%s temp=%s",
|
||||
sheet,len(rows),job_dir,
|
||||
)
|
||||
try:
|
||||
for record_id,resume_link in rows:
|
||||
payload=await extract_one_resume(
|
||||
credentials,resume_link,job_dir,max_chars,max_bytes,
|
||||
)
|
||||
try:
|
||||
await FormData.set_extracted_data(session,record_id,payload)
|
||||
except Exception:
|
||||
logger.exception("could not persist extracted_data for %s",record_id)
|
||||
failed+=1
|
||||
try:
|
||||
await session.rollback()
|
||||
except Exception:
|
||||
logger.exception("rollback after extracted_data persist failed")
|
||||
continue
|
||||
status=payload.get("status")
|
||||
if status=="completed":
|
||||
completed+=1
|
||||
elif status=="failed":
|
||||
failed+=1
|
||||
logger.info(
|
||||
"drive CV extract finished tab=%s extracted=%s failed=%s",
|
||||
sheet,completed,failed,
|
||||
)
|
||||
return {"extracted":completed,"extract_failed":failed}
|
||||
finally:
|
||||
remove_job_temp_dir(job_dir)
|
||||
|
||||
|
||||
def extract_drive_cvs(func):
|
||||
"""Hang on SheetImport.import_sheet (worker ingest), not on HTTP enqueue.
|
||||
|
||||
After rows are inserted: one Drive download + extract per resume_link,
|
||||
written to form_data.extracted_data at the end of each row.
|
||||
Credentials load only if ingest will actually run.
|
||||
"""
|
||||
|
||||
@wraps(func)
|
||||
async def wrapper(*args,**kwargs):
|
||||
result=await func(*args,**kwargs)
|
||||
service=args[0] if args and _is_sheet_service(args[0]) else None
|
||||
session=getattr(service,"session",None) if service is not None else None
|
||||
rest=args[1:] if service is not None else args
|
||||
tab=_tab_from(result,rest,kwargs)
|
||||
if session is None or not tab or not _should_ingest(result):
|
||||
logger.info(
|
||||
"drive CV extract skipped tab=%s session=%s ingest=%s",
|
||||
tab,session is not None,
|
||||
_should_ingest(result) if isinstance(result,dict) else False,
|
||||
)
|
||||
return result
|
||||
credentials=await asyncio.to_thread(_prepare_drive_credentials,service)
|
||||
if credentials is None:
|
||||
logger.warning("Google Drive session unavailable; skipping CV extract")
|
||||
return result
|
||||
try:
|
||||
stats=await ingest_form_resume_links(session,tab,credentials)
|
||||
except Exception:
|
||||
logger.exception("drive CV extract after import of %s failed",tab)
|
||||
stats={"extracted":0,"extract_failed":0}
|
||||
if isinstance(result,dict):
|
||||
result["extracted"]=stats.get("extracted",0)
|
||||
result["extract_failed"]=stats.get("extract_failed",0)
|
||||
return result
|
||||
|
||||
wrapper.__signature__=signature(func)
|
||||
return wrapper
|
||||
|
|
@ -189,6 +189,7 @@ class FormDataColumn(str, Enum):
|
|||
CANDIDATE_EMAIL = "candidate_email"
|
||||
PROFILE_LINK = "profile_link"
|
||||
RESUME_LINK = "resume_link"
|
||||
EXTRACTED_DATA = "extracted_data"
|
||||
AREA_OF_EXPERTISE = "area_of_expertise"
|
||||
REQUISITION_NUMBER = "requisition_number"
|
||||
POSITION_APPLIED_FOR = "position_applied_for"
|
||||
|
|
|
|||
|
|
@ -51,6 +51,8 @@ class FormData(SQLModel, table=True):
|
|||
candidate_email: str | None = Field(default=None, index=True)
|
||||
profile_link: str | None = Field(default=None)
|
||||
resume_link: str | None = Field(default=None)
|
||||
# Drive CV extract JSON written by @extract_drive_cvs after sheet ingest.
|
||||
extracted_data: dict | None = Field(default=None, sa_column=Column(JSONB))
|
||||
area_of_expertise: str | None = Field(default=None)
|
||||
requisition_number: str | None = Field(default=None, index=True)
|
||||
position_applied_for: str | None = Field(default=None)
|
||||
|
|
@ -190,6 +192,38 @@ class FormData(SQLModel, table=True):
|
|||
await session.refresh(row)
|
||||
return row
|
||||
|
||||
@classmethod
|
||||
async def set_extracted_data(
|
||||
cls, session: AsyncSession, record_id, extracted_data, *, commit: bool = True,
|
||||
):
|
||||
row = await cls.get_form_data_by_id(session, record_id)
|
||||
if not row:
|
||||
return None
|
||||
row.extracted_data = extracted_data
|
||||
row.updated_at = _now()
|
||||
session.add(row)
|
||||
if commit:
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return row
|
||||
|
||||
@classmethod
|
||||
async def fetch_resume_links(cls, session: AsyncSession, sheet: str):
|
||||
"""(id, resume_link) for one tab. Blank links are dropped."""
|
||||
statement = (
|
||||
select(cls.id, cls.resume_link)
|
||||
.where(cls.sheet == sheet)
|
||||
.where(cls.resume_link.is_not(None))
|
||||
.order_by(cls.row_number)
|
||||
)
|
||||
result = await session.execute(statement)
|
||||
rows = []
|
||||
for record_id, link in result.all():
|
||||
text = (link or "").strip()
|
||||
if text:
|
||||
rows.append((record_id, text))
|
||||
return rows
|
||||
|
||||
@classmethod
|
||||
async def fetch_form_data(
|
||||
cls, session: AsyncSession, *, sheet=None, search=None,
|
||||
|
|
@ -423,6 +457,14 @@ class SheetImportRun(SQLModel, table=True):
|
|||
)
|
||||
return result.scalars().first()
|
||||
|
||||
@classmethod
|
||||
async def delete_failed(cls, session: AsyncSession, *, commit: bool = True):
|
||||
"""Drop failed import rows so a new job is not blocked by them."""
|
||||
result = await session.execute(delete(cls).where(cls.status == "failed"))
|
||||
if commit:
|
||||
await session.commit()
|
||||
return result.rowcount
|
||||
|
||||
@classmethod
|
||||
async def insert_run(cls, session: AsyncSession, fields: dict, *, commit: bool = True):
|
||||
row = cls(**fields)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ the session can be copied to Linux prod. Re-auth lives in g_sheet/store_session.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -25,6 +26,7 @@ from google.auth import default as google_auth_default
|
|||
from google.auth.transport.requests import Request
|
||||
from googleapiclient.discovery import build
|
||||
from googleapiclient.errors import HttpError
|
||||
from googleapiclient.http import MediaIoBaseDownload
|
||||
|
||||
from g_sheet.enums import (
|
||||
ALIAS_TO_FIELD,
|
||||
|
|
@ -206,6 +208,130 @@ def build_sheets_client(credentials):
|
|||
raise SheetsApiError(f"Could not build the Sheets client: {e}")
|
||||
|
||||
|
||||
def build_drive_client(credentials):
|
||||
"""Drive v3 client. Same ADC session as Sheets; cache_discovery=False under threads."""
|
||||
try:
|
||||
return build("drive","v3",credentials=credentials,cache_discovery=False)
|
||||
except Exception as e:
|
||||
raise SheetsApiError(f"Could not build the Drive client: {e}")
|
||||
|
||||
|
||||
_DRIVE_FILE_ID_PATTERNS=(
|
||||
re.compile(r"/file/d/([a-zA-Z0-9_-]+)"),
|
||||
re.compile(r"/document/d/([a-zA-Z0-9_-]+)"),
|
||||
re.compile(r"[?&]id=([a-zA-Z0-9_-]+)"),
|
||||
re.compile(r"/d/([a-zA-Z0-9_-]+)"),
|
||||
)
|
||||
_GOOGLE_APPS_SHORTCUT="application/vnd.google-apps.shortcut"
|
||||
_GOOGLE_APPS_DOCUMENT="application/vnd.google-apps.document"
|
||||
_GOOGLE_APPS_PREFIX="application/vnd.google-apps."
|
||||
|
||||
|
||||
def drive_file_id(url):
|
||||
"""Extract a Drive/Docs file id from a Google URL, or None if it is not one."""
|
||||
raw=(url or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
lowered=raw.lower()
|
||||
if "drive.google.com" not in lowered and "docs.google.com" not in lowered:
|
||||
return None
|
||||
if "/folders/" in lowered:
|
||||
return None
|
||||
for pattern in _DRIVE_FILE_ID_PATTERNS:
|
||||
match=pattern.search(raw)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def _drive_file_meta(drive,file_id):
|
||||
request=drive.files().get(
|
||||
fileId=file_id,
|
||||
fields="id,name,mimeType,size,shortcutDetails",
|
||||
supportsAllDrives=True,
|
||||
)
|
||||
return execute(request,"drive file metadata")
|
||||
|
||||
|
||||
def _download_media(request,dest_path=None):
|
||||
"""Stream a Drive media request. dest_path set → write that file; else return bytes."""
|
||||
try:
|
||||
if dest_path is None:
|
||||
buf=io.BytesIO()
|
||||
downloader=MediaIoBaseDownload(buf,request)
|
||||
done=False
|
||||
while not done:
|
||||
_,done=downloader.next_chunk()
|
||||
return buf.getvalue()
|
||||
path=Path(dest_path)
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
with path.open("wb") as fh:
|
||||
downloader=MediaIoBaseDownload(fh,request)
|
||||
done=False
|
||||
while not done:
|
||||
_,done=downloader.next_chunk()
|
||||
return path
|
||||
except HttpError as e:
|
||||
status=_status_of(e)
|
||||
raise SheetsApiError(f"drive download failed: {_reason_of(e)}",status or 502)
|
||||
|
||||
|
||||
def _cv_dest_path(dest_dir,file_id,filename):
|
||||
dest_dir=Path(dest_dir).resolve()
|
||||
suffix=Path(filename or "resume.pdf").suffix.lower() or ".pdf"
|
||||
if suffix not in (".pdf",".doc",".docx"):
|
||||
suffix=".pdf"
|
||||
safe_id=re.sub(r"[^a-zA-Z0-9_-]","",file_id or "") or "file"
|
||||
dest=(dest_dir/f"{safe_id}{suffix}").resolve()
|
||||
if dest.parent!=dest_dir:
|
||||
raise SheetsApiError("invalid download path",400)
|
||||
return dest
|
||||
|
||||
|
||||
def download_drive_file(credentials,url,*,max_bytes=None,dest_dir=None):
|
||||
"""Download one Drive file via the existing Google session.
|
||||
|
||||
When dest_dir is set the file is streamed to disk and `path` is returned
|
||||
(`data` is None). Otherwise `data` holds the bytes (tests / callers without a
|
||||
work dir).
|
||||
"""
|
||||
file_id=drive_file_id(url)
|
||||
if not file_id:
|
||||
raise SheetsApiError("not a Google Drive file URL",400)
|
||||
ensure_fresh(credentials)
|
||||
drive=build_drive_client(credentials)
|
||||
meta=_drive_file_meta(drive,file_id)
|
||||
if (meta.get("mimeType") or "")==_GOOGLE_APPS_SHORTCUT:
|
||||
target=(meta.get("shortcutDetails") or {}).get("targetId")
|
||||
if not target:
|
||||
raise SheetsApiError("Drive shortcut has no target",400)
|
||||
file_id=target
|
||||
meta=_drive_file_meta(drive,file_id)
|
||||
mime=meta.get("mimeType") or ""
|
||||
name=meta.get("name") or "resume.pdf"
|
||||
size=meta.get("size")
|
||||
if max_bytes is not None and size is not None:
|
||||
try:
|
||||
if int(size)>max_bytes:
|
||||
raise SheetsApiError("The file exceeds the size limit.",413)
|
||||
except (TypeError,ValueError):
|
||||
pass
|
||||
if mime==_GOOGLE_APPS_DOCUMENT:
|
||||
request=drive.files().export_media(fileId=file_id,mimeType="application/pdf")
|
||||
if not name.lower().endswith(".pdf"):
|
||||
name=f"{name}.pdf"
|
||||
elif mime.startswith(_GOOGLE_APPS_PREFIX):
|
||||
raise SheetsApiError("unsupported Google file type",415)
|
||||
else:
|
||||
request=drive.files().get_media(fileId=file_id,supportsAllDrives=True)
|
||||
if dest_dir is None:
|
||||
data=_download_media(request)
|
||||
return {"file_id":file_id,"filename":name,"mime_type":mime,"data":data,"path":None}
|
||||
dest=_cv_dest_path(dest_dir,file_id,name)
|
||||
_download_media(request,dest)
|
||||
return {"file_id":file_id,"filename":name,"mime_type":mime,"data":None,"path":dest}
|
||||
|
||||
|
||||
def _status_of(error):
|
||||
status=getattr(getattr(error,"resp",None),"status",None)
|
||||
if status is None:
|
||||
|
|
|
|||
|
|
@ -128,6 +128,8 @@ def serialize_import(report: dict) -> dict:
|
|||
"ages_parsed": report.get("ages_parsed", 0),
|
||||
"salaries_parsed": report.get("salaries_parsed", 0),
|
||||
"unmapped_headers": report.get("unmapped_headers") or [],
|
||||
"extracted": report.get("extracted", 0),
|
||||
"extract_failed": report.get("extract_failed", 0),
|
||||
"error": report.get("error"),
|
||||
}
|
||||
|
||||
|
|
@ -142,6 +144,8 @@ def serialize_import_all(reports: list[dict]) -> dict:
|
|||
"failed": len(failed),
|
||||
"inserted": sum(r.get("inserted", 0) for r in ok),
|
||||
"deleted": sum(r.get("deleted", 0) for r in ok),
|
||||
"extracted": sum(r.get("extracted", 0) for r in ok),
|
||||
"extract_failed": sum(r.get("extract_failed", 0) for r in reports),
|
||||
"reports": [serialize_import(r) for r in reports],
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -49,13 +49,30 @@ async def import_sheets(run_id:str) -> dict:
|
|||
try:
|
||||
acquired=await client.set(_LOCK_KEY,run_id,nx=True,ex=_LOCK_TTL)
|
||||
if not acquired:
|
||||
return await _fail(run_id,"another sheet import is already running")
|
||||
holder=await client.get(_LOCK_KEY)
|
||||
# Crash/restart redelivers the same run_id while the TTL lock is
|
||||
# still set. Failing that as "another import" strands the lock
|
||||
# until expiry and every later click also bounces.
|
||||
if holder==run_id:
|
||||
await client.expire(_LOCK_KEY,_LOCK_TTL)
|
||||
logger.warning("sheet import %s reclaimed its own stale lock",run_id)
|
||||
else:
|
||||
logger.warning(
|
||||
"sheet import %s skipped: lock held by %s",run_id,holder,
|
||||
)
|
||||
return await _fail(run_id,"another sheet import is already running")
|
||||
|
||||
try:
|
||||
async with session_scope() as session:
|
||||
row=await SheetImportRun.get_by_id(session,run_id)
|
||||
if not row:
|
||||
raise PermanentTaskError(f"import run {run_id} not found")
|
||||
if row.status=="failed":
|
||||
await SheetImportRun.delete_failed(session)
|
||||
return {"status":"failed","error":row.error}
|
||||
if row.status=="completed":
|
||||
return {"status":"completed","report":row.report}
|
||||
await SheetImportRun.delete_failed(session)
|
||||
await SheetImportRun.update_run(session,run_id,{
|
||||
"status":"running",
|
||||
"started_at":datetime.now(timezone.utc),
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from datetime import datetime,timezone
|
|||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from g_sheet.decorators import extract_drive_cvs
|
||||
from g_sheet.plugins import (
|
||||
SCOPES,
|
||||
SPREADSHEET_ID,
|
||||
|
|
@ -232,6 +233,7 @@ class SheetHealth(SheetRead):
|
|||
class SheetImport(SheetRead):
|
||||
"""Google Sheet → FormData import + import-run tracking."""
|
||||
|
||||
@extract_drive_cvs
|
||||
async def import_sheet(self,tab):
|
||||
"""Read one tab from Google Sheets and replace its FormData rows."""
|
||||
session=self._require_session()
|
||||
|
|
@ -279,15 +281,18 @@ class SheetImport(SheetRead):
|
|||
return serialize_import_all(reports)
|
||||
|
||||
async def start_import(self,current_user=None,tab=None):
|
||||
"""Enqueue a sheet import on the shared Taskiq worker; return the run row.
|
||||
"""Enqueue a sheet import.
|
||||
|
||||
If a queued/running import already exists, return it instead of stacking another.
|
||||
At the start of every new job: queued/running → keep that job;
|
||||
failed → delete those rows and start this one; completed → start this one.
|
||||
"""
|
||||
session=self._require_session()
|
||||
active=await SheetImportRun.get_active(session)
|
||||
if active:
|
||||
return serialize_import_run(active)
|
||||
|
||||
await SheetImportRun.delete_failed(session)
|
||||
|
||||
created_by=None
|
||||
if isinstance(current_user,dict) and current_user.get("id"):
|
||||
created_by=SheetImportRun._as_uuid(current_user.get("id"))
|
||||
|
|
|
|||
|
|
@ -151,3 +151,32 @@ def test_rows_to_indexed_records_keeps_true_sheet_row_across_blank():
|
|||
|
||||
def test_form_data_fields_match_model():
|
||||
assert set(FORM_DATA_FIELDS) == set(FormData.model_fields)
|
||||
|
||||
|
||||
def test_drive_file_id_from_form_open_url():
|
||||
url = "https://drive.google.com/open?id=1l5HOY5R6KiL6270A_sV56FkdX6EIGfI4"
|
||||
assert plugins.drive_file_id(url) == "1l5HOY5R6KiL6270A_sV56FkdX6EIGfI4"
|
||||
|
||||
|
||||
def test_drive_file_id_from_file_view_url():
|
||||
url = "https://drive.google.com/file/d/abc123XYZ/view?usp=sharing"
|
||||
assert plugins.drive_file_id(url) == "abc123XYZ"
|
||||
|
||||
|
||||
def test_drive_file_id_from_docs_url():
|
||||
url = "https://docs.google.com/document/d/docFileId99/edit"
|
||||
assert plugins.drive_file_id(url) == "docFileId99"
|
||||
|
||||
|
||||
def test_drive_file_id_rejects_folder_and_non_drive():
|
||||
assert plugins.drive_file_id("https://drive.google.com/drive/folders/abc") is None
|
||||
assert plugins.drive_file_id("https://example.com/cv.pdf") is None
|
||||
assert plugins.drive_file_id("") is None
|
||||
assert plugins.drive_file_id(None) is None
|
||||
|
||||
|
||||
def test_cv_dest_path_stays_inside_dir(tmp_path):
|
||||
dest = plugins._cv_dest_path(tmp_path, "1l5HOY5R6KiL6270A_sV56FkdX6EIGfI4", "Resume.PDF")
|
||||
assert dest.parent == tmp_path.resolve()
|
||||
assert dest.name.endswith(".pdf")
|
||||
assert dest.name.startswith("1l5HOY5R6KiL6270A_sV56FkdX6EIGfI4")
|
||||
|
|
|
|||
|
|
@ -154,19 +154,8 @@ function sourceFrom(messageTo) {
|
|||
return { source: raw.split(',')[0].trim(), sourceMeta: null }
|
||||
}
|
||||
|
||||
/**
|
||||
* Form `source_of_application` is a free-text label (LinkedIn, Indeed, …), not a
|
||||
* To-address. Reuse the email source palette when the spelling matches; otherwise
|
||||
* tag the row as a Sheet Forms entry so the chip still paints.
|
||||
*/
|
||||
function formSourceFrom(raw) {
|
||||
const label = (raw || '').trim()
|
||||
if (!label) return { source: 'Google Forms', sourceMeta: SHEET_SOURCE_META }
|
||||
const flat = label.toLowerCase().replace(/[^a-z]/g, '')
|
||||
const hit = inboxSources.find((s) => flat.includes(s.toLowerCase().replace(/[^a-z]/g, '')))
|
||||
if (hit) return { source: hit, sourceMeta: sourceMeta[hit] }
|
||||
return { source: label, sourceMeta: SHEET_SOURCE_META }
|
||||
}
|
||||
/** Sheet Forms always chip as Google Sheet — not the form's "where did you hear" answer. */
|
||||
const FORM_LIST_SOURCE = { source: 'Google Sheet', sourceMeta: SHEET_SOURCE_META }
|
||||
|
||||
/** entry_date is midnight UTC; entry_time is a separate "HH:MM" string from the sheet. */
|
||||
function formReceivedAt(entryDate, entryTime) {
|
||||
|
|
@ -195,7 +184,7 @@ function mapFormRow(row) {
|
|||
email: row.candidate_email || '',
|
||||
phone: row.candidate_number || '',
|
||||
position: row.position_applied_for || '—',
|
||||
...formSourceFrom(row.source_of_application),
|
||||
...FORM_LIST_SOURCE,
|
||||
received: formReceivedAt(row.entry_date, row.entry_time),
|
||||
screenedBy: row.screened_by || '',
|
||||
hrComments: row.hr_comments || '',
|
||||
|
|
@ -1188,9 +1177,6 @@ export default function Inbox() {
|
|||
{i.atsScore != null && (
|
||||
<div style={{ marginTop: 6 }}><ScoreChip score={i.atsScore} /></div>
|
||||
)}
|
||||
{isForms && i.noticePeriod && (
|
||||
<div className="cell-sub" style={{ marginTop: 6 }}>{i.noticePeriod}</div>
|
||||
)}
|
||||
</div>
|
||||
<div className="ii-meta">
|
||||
<SourceChip item={i} /> <Badge>{i.processing}</Badge>
|
||||
|
|
|
|||
Loading…
Reference in New Issue