From 86531c77db745a9d9986a7beaae1ac8e19d7b853 Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Mon, 7 Sep 2026 19:59:28 +0500 Subject: [PATCH] commitred with correct city name --- backend/employment_agent/decorators.py | 9 ++-- backend/employment_agent/execute_agent.py | 54 ++++++++++++++++++- backend/employment_agent/prompt.py | 32 ++++++++++- backend/g_sheet/models.py | 17 +++++- backend/inbox/models.py | 16 +++++- backend/inbox/views.py | 6 ++- backend/tests/test_employment_agent.py | 24 +++++++++ .../test_employment_extraction_clamps.py | 10 ++++ backend/tests/test_form_data_filters.py | 7 +++ frontend/src/screens/Inbox.jsx | 2 +- 10 files changed, 162 insertions(+), 15 deletions(-) diff --git a/backend/employment_agent/decorators.py b/backend/employment_agent/decorators.py index 49128ae..39c4658 100644 --- a/backend/employment_agent/decorators.py +++ b/backend/employment_agent/decorators.py @@ -103,17 +103,14 @@ def _clean_phone(value,resume_text): def _clean_city(value,resume_text): - """Optional residence city. Sentinel / invented → None. Never rejects the CV. + """Optional residence city. Sentinel → None. Never rejects the CV. - The prompt forbids work-experience cities; this only drops a value that is - absent from the resume text or is the explicit empty sentinel. + Proper city names (Karachi, not Karachi(Malir)) come from the OpenAI parse + in run_employment_agent. This clamp does not rewrite place names. """ text=(value or "").strip() if not text or text.lower() in (NO_CITY.lower(),"none","null","n/a","-"): return None - haystack=(resume_text or "").lower() - if haystack and text.lower() not in haystack: - return None return text diff --git a/backend/employment_agent/execute_agent.py b/backend/employment_agent/execute_agent.py index b240284..7ecee57 100644 --- a/backend/employment_agent/execute_agent.py +++ b/backend/employment_agent/execute_agent.py @@ -6,15 +6,67 @@ Called from inbox.tasks.match_inbox_message; no HTTP surface. from __future__ import annotations +import json import logging from employment_agent.decorators import parse_employment_response -from employment_agent.prompt import CURRENT_TITLE,EDUCATION,NO_COMPANY,prompt,user_prompt +from employment_agent.prompt import CURRENT_TITLE,EDUCATION,NO_CITY,NO_COMPANY,city_list_prompt,prompt,user_prompt from llm_setup import llm_call logger=logging.getLogger("employment_agent") +def parse_normalized_cities(data,fallback=None): + """Keep unique proper city names from the list-normalizer JSON.""" + rows=None + if isinstance(data,dict): + rows=data.get("cities") + if not isinstance(rows,list): + return list(fallback or []) + out=[] + seen=set() + for item in rows: + if not isinstance(item,str): + continue + text=item.strip() + if not text or text.lower() in (NO_CITY.lower(),"none","null","n/a","-"): + continue + key=text.lower() + if key in seen: + continue + seen.add(key) + out.append(text) + out.sort(key=str.lower) + return out or list(fallback or []) + + +async def normalize_cities(values): + """OpenAI: messy stored places → the same proper city names the CV agent writes.""" + places=[] + seen=set() + for raw in values or []: + text=(raw or "").strip() + if not text: + continue + key=text.lower() + if key in seen: + continue + seen.add(key) + places.append(text) + if not places: + return [] + try: + data=await llm_call( + city_list_prompt(), + json.dumps({"places":places},ensure_ascii=False), + json_mode=True, + ) + except Exception: + logger.exception("city list normalize failed") + return places + return parse_normalized_cities(data,fallback=places) + + async def run_employment_agent(*,resume_text=""): text=(resume_text or "").strip() if not text: diff --git a/backend/employment_agent/prompt.py b/backend/employment_agent/prompt.py index aa7ffb1..c5d0c2a 100644 --- a/backend/employment_agent/prompt.py +++ b/backend/employment_agent/prompt.py @@ -14,6 +14,14 @@ NO_LINKEDIN="no linkedin url mentioned" NO_PHONE="no phone number mentioned" NO_CITY="no city mentioned" +CITY_POLICY=f"""- Return ONE proper city name only — the city, not an area, town, sector, housing society, cantonment, district, or parenthetical locality. +- Correct values look like "Karachi", "Lahore", "Islamabad", "Rawalpindi", "Peshawar". Not "Karachi(Malir)", not "Wah Cantt", not "Gulberg Lahore". +- If the text names a neighborhood or area of a city, return that city: "Karachi (Malir)" / "Karachi(Malir)" / "DHA Karachi" → "Karachi". "Gulberg, Lahore" → "Lahore". "F-10 Islamabad" → "Islamabad". +- Drop "Cantt" / "Cantonment": "Lahore Cantt" → "Lahore", "Rawalpindi Cantt" → "Rawalpindi", "Wah Cantt" → "Wah". +- Never concatenate two places. If the string is messy (for example "Karachi(Malir) Wah Cantt"), return the single residence city, not both strings glued together. +- Do not return province, country, street, house number, or text inside parentheses. +- Drop junk tokens such as KA, KAR, KARA, empty values, and unintelligible strings.""" + def prompt(): return f"""You are an HR-ATS recruiting assistant. @@ -57,11 +65,10 @@ linkedin_url (its own key — extract this separately from the other fields): - Never guess a slug or construct linkedin.com/in/ from the candidate's name. The stored value will be null when this sentinel is returned. city (its own key — OPTIONAL. A missing city must not fail the candidate): -- Return the city of residence only, city name alone (for example "Karachi", "Lahore", "Islamabad"), using the resume's own spelling. +{CITY_POLICY} - Extract city ONLY from the candidate's contact / location / address header (the block with name, phone, email, LinkedIn, "Address", "Location", "based in", "currently living in"). - Do NOT extract city from Work Experience. A job that lists Karachi, UAE, USA, or any other city is the employer's location, not proof the candidate lives there. - If the contact/location section does not name a city, return exactly: {NO_CITY}. Leave it blank rather than guessing from jobs, education, or nationality. -- The city string you return MUST appear verbatim (or as a clear substring) in that contact/location section of the resume text. phone (its own key — extract this separately; copy EVERY digit): - Return the candidate's own mobile / phone exactly as written, including country code when present. @@ -118,6 +125,14 @@ Example 9 — contact/location city is residence: Resume: "Ali Khan | Location: Lahore | 0321-5551234\\nExperience: Acme, Karachi, Engineer" JSON city must be "Lahore". Not "Karachi". +Example 10 — neighborhood / cantonment is not the city: +Resume: "Ali Khan | Karachi(Malir) | 0321-5551234" +JSON city must be "Karachi". Not "Karachi(Malir)" and not "Malir". + +Example 11 — do not glue two place fragments: +Resume: "Address: Karachi(Malir) Wah Cantt" +JSON city must be "Karachi" (one city). Not "Karachi(Malir) Wah Cantt" and not "Wah Cantt". + Respond with JSON only: {{ "current_employment": "Company Name", @@ -135,3 +150,16 @@ If the contact/location section has no city, city must be "{NO_CITY}" — still def user_prompt(resume_text:str) -> str: return json.dumps({"resume_text":resume_text or ""},ensure_ascii=False) + + +def city_list_prompt(): + """Map messy stored place strings to the same proper city names the CV agent writes.""" + return f"""You map messy residence strings to proper city names for an Inbox City filter. + +{CITY_POLICY} + +Input is JSON: {{"places": ["Karachi(Malir)", "Wah Cantt", "Lahore"]}} +Respond with JSON only: +{{"cities": ["Karachi", "Wah", "Lahore"]}} +Unique proper city names only. Do not copy raw neighborhood or cantonment strings into cities. +""" diff --git a/backend/g_sheet/models.py b/backend/g_sheet/models.py index d6374ad..8018093 100644 --- a/backend/g_sheet/models.py +++ b/backend/g_sheet/models.py @@ -126,6 +126,18 @@ class FormData(SQLModel, table=True): else_=False, ) + @staticmethod + def _cities_match(column, cities): + """Agent city name vs stored raw text: Karachi matches Karachi(Malir).""" + clauses = [] + for city in cities or []: + text = (city or "").strip() + if not text: + continue + safe = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + clauses.append(column.ilike(f"%{safe}%", escape="\\")) + return or_(*clauses) if clauses else None + @classmethod def _filters( cls, *, sheet=None, search=None, processing_state=None, is_duplicate=None, @@ -160,8 +172,9 @@ class FormData(SQLModel, table=True): ) cities = [c.strip() for c in (city or []) if (c or "").strip()] if cities: - city_col = func.lower(func.coalesce(cls.city, cls.residing_city)) - filters.append(city_col.in_([c.lower() for c in cities])) + clause = cls._cities_match(func.coalesce(cls.city, cls.residing_city), cities) + if clause is not None: + filters.append(clause) if source: text = source.strip() lowered = text.lower() diff --git a/backend/inbox/models.py b/backend/inbox/models.py index 3c09a32..a1e20a2 100644 --- a/backend/inbox/models.py +++ b/backend/inbox/models.py @@ -1022,6 +1022,18 @@ class Inbox_Messages(SQLModel, table=True): else_=False, ) + @staticmethod + def _cities_match(column, cities): + """Agent city name vs stored raw text: Karachi matches Karachi(Malir).""" + clauses = [] + for city in cities or []: + text = (city or "").strip() + if not text: + continue + safe = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + clauses.append(column.ilike(f"%{safe}%", escape="\\")) + return or_(*clauses) if clauses else None + @classmethod def _apply_filters( cls, statement, search: str | None=None, isread: bool=True, @@ -1061,7 +1073,9 @@ class Inbox_Messages(SQLModel, table=True): statement = statement.where(cls.processing_state == processing_state) cities = [c.strip() for c in (city or []) if (c or "").strip()] if cities: - statement = statement.where(func.lower(cls.city).in_([c.lower() for c in cities])) + clause = cls._cities_match(cls.city, cities) + if clause is not None: + statement = statement.where(clause) if source: text = source.strip() lowered = text.lower() diff --git a/backend/inbox/views.py b/backend/inbox/views.py index 1a0df7f..0c0933c 100644 --- a/backend/inbox/views.py +++ b/backend/inbox/views.py @@ -513,11 +513,13 @@ class Email: return await Inbox_Messages.count_inbox_messages(self.session,search,assigned=assigned,is_duplicate=is_duplicate,no_suggestions=no_suggestions,processing_state=processing_state,city=city,source=source) async def list_cities(self): - """Distinct cities from inbox_messages and form_data, merged in Python.""" + """Proper city names for the Inbox filter — same OpenAI mapping as CV parse.""" + from employment_agent.execute_agent import normalize_cities from g_sheet.models import FormData inbox=await Inbox_Messages.distinct_cities(self.session) forms=await FormData.distinct_cities(self.session) - return Reapplied(session=self.session).merge_cities(inbox,forms) + merged=Reapplied(session=self.session).merge_cities(inbox,forms) + return await normalize_cities(merged) async def list_sources(self): """Source / platform labels: seeded channels, Google Sheet, form sources.""" diff --git a/backend/tests/test_employment_agent.py b/backend/tests/test_employment_agent.py index 4c33cf4..0063829 100644 --- a/backend/tests/test_employment_agent.py +++ b/backend/tests/test_employment_agent.py @@ -89,3 +89,27 @@ def test_adds_scheme_and_rejects_company_page(): "", ) assert company_page["linkedin_url"] is None + + +def test_city_prompt_asks_openai_for_a_proper_city_name(): + from employment_agent.prompt import city_list_prompt, prompt + text = prompt() + assert "Karachi(Malir)" in text + assert 'JSON city must be "Karachi"' in text + assert "Return ONE proper city name only" in text + listed = city_list_prompt() + assert "Karachi(Malir)" in listed + assert '"cities"' in listed + + +def test_parse_normalized_cities_keeps_agent_names(): + from employment_agent.execute_agent import parse_normalized_cities + assert parse_normalized_cities( + {"cities": ["Karachi", "Karachi", "Lahore", "no city mentioned"]}, + fallback=["Karachi(Malir)"], + ) == ["Karachi", "Lahore"] + + +def test_parse_normalized_cities_falls_back_when_the_model_shape_is_wrong(): + from employment_agent.execute_agent import parse_normalized_cities + assert parse_normalized_cities({"oops": True}, fallback=["Karachi(Malir)"]) == ["Karachi(Malir)"] diff --git a/backend/tests/test_employment_extraction_clamps.py b/backend/tests/test_employment_extraction_clamps.py index d4bdacd..a252333 100644 --- a/backend/tests/test_employment_extraction_clamps.py +++ b/backend/tests/test_employment_extraction_clamps.py @@ -163,3 +163,13 @@ def test_existing_sentinels_still_normalize(): assert fields["current_employment"] == NO_COMPANY assert fields["education"] == EDUCATION assert fields["skills"] == ["Python"] + + +def test_city_from_the_model_is_kept_as_returned(): + resume = "Ali Khan | Karachi(Malir) | 0321-5551234" + fields = parse_employment_response({"city": "Karachi"}, resume) + assert fields["city"] == "Karachi" + + +def test_city_sentinel_is_dropped(): + assert parse({"city": "no city mentioned"})["city"] is None diff --git a/backend/tests/test_form_data_filters.py b/backend/tests/test_form_data_filters.py index 9eef6e3..8f1d9c4 100644 --- a/backend/tests/test_form_data_filters.py +++ b/backend/tests/test_form_data_filters.py @@ -96,6 +96,13 @@ class TestListAndBadgesAgree: assert name in params, f"count_processing should narrow on {name}" +class TestCity: + def test_agent_city_matches_raw_stored_text(self): + out = sql(*FormData._filters(city=["Karachi"])) + assert "ilike" in out + assert "karachi" in out + + class TestPlumbing: @pytest.mark.parametrize( "func", [FormData.fetch_form_data, FormData.count_form_data], diff --git a/frontend/src/screens/Inbox.jsx b/frontend/src/screens/Inbox.jsx index 4a2631c..515f7b9 100644 --- a/frontend/src/screens/Inbox.jsx +++ b/frontend/src/screens/Inbox.jsx @@ -977,7 +977,7 @@ function BulkReadBar({ rows, selection, onSetRead, onSetAllRead, busy, canEdit, * than a refactor of theirs.) */ /** Junk location tokens — hide from the dropdown, not from the list query. */ -const HIDDEN_LOCATION = /^(KA|KAR|KARA|WAH)$/i +const HIDDEN_LOCATION = /^(KA|KAR|KARA)$/i function InboxFilters({ filters, cities, sources, showLinkFilters, active, open, onToggle, onChange, onClear }) { const locations = (cities || []).filter((name) => !HIDDEN_LOCATION.test(String(name).trim()))