From cd4743e95325eb63a695a51bfd2fc8659eabc0e9 Mon Sep 17 00:00:00 2001 From: "ahmed.mujtaba" Date: Tue, 8 Sep 2026 17:10:08 +0500 Subject: [PATCH] global countires --- backend/employment_agent/decorators.py | 63 ++++- backend/employment_agent/execute_agent.py | 54 +--- backend/employment_agent/prompt.py | 36 ++- backend/g_sheet/models.py | 9 +- backend/global_cities.py | 247 ++++++++++++++++++ backend/inbox/views.py | 6 +- backend/tests/test_employment_agent.py | 84 +++++- .../test_employment_extraction_clamps.py | 8 + backend/tests/test_g_sheet_plugins.py | 8 + 9 files changed, 413 insertions(+), 102 deletions(-) create mode 100644 backend/global_cities.py diff --git a/backend/employment_agent/decorators.py b/backend/employment_agent/decorators.py index 39c4658..41cc9a5 100644 --- a/backend/employment_agent/decorators.py +++ b/backend/employment_agent/decorators.py @@ -18,6 +18,17 @@ import re from functools import wraps from employment_agent.prompt import EDUCATION,NO_CITY,NO_COMPANY,NO_LINKEDIN,NO_PHONE +from global_cities import CITY_BY_KEY,CITY_RE + +_CITY_SENTINELS=frozenset({ + NO_CITY.lower(),"none","null","n/a","-","na","n.a.","n.a", +}) +_CITY_DROP=frozenset({ + "dha","cantt","cantonment","cant","phase","sector","area","district", + "tehsil","division","housing","society","scheme","block","street","house", + "near","colony","neighborhood","neighbourhood","suburb", +}) +_SECTOR_RE=re.compile(r"^(?:[a-z]-?\d+[a-z]?|\d+[a-z]?)$",re.I) def require_json_object(func): @@ -102,16 +113,54 @@ def _clean_phone(value,resume_text): return text -def _clean_city(value,resume_text): - """Optional residence city. Sentinel → None. Never rejects the CV. +def canonical_city(text): + """Write-time only: messy locality → one proper city name, or None. - Proper city names (Karachi, not Karachi(Malir)) come from the OpenAI parse - in run_employment_agent. This clamp does not rewrite place names. + Looks up `global_cities.Countries` (every country, Pakistan included). + "Karachi(Malir)" / "London(Westminster)" / "DHA Karachi" / "Wah Cantt" + map to the listed city. Sentinels and blanks are None. Never rejects a CV. """ - text=(value or "").strip() - if not text or text.lower() in (NO_CITY.lower(),"none","null","n/a","-"): + raw=(text or "").strip() + if not raw or raw.lower() in _CITY_SENTINELS: return None - return text + known=CITY_BY_KEY.get(raw.lower()) + if known: + return known + normalised=re.sub(r"[()\[\]{}]"," ",raw) + normalised=re.sub(r"[,/;|]+"," ",normalised) + normalised=re.sub(r"\s+"," ",normalised).strip() + if not normalised: + return None + known=CITY_BY_KEY.get(normalised.lower()) + if known: + return known + match=CITY_RE.search(normalised.lower()) + if match: + return CITY_BY_KEY[match.group(0)] + leftover=[] + for token in normalised.split(): + lowered=token.lower() + if lowered in _CITY_DROP or _SECTOR_RE.fullmatch(token): + continue + leftover.append(token) + if not leftover: + return None + cleaned=" ".join(leftover) + known=CITY_BY_KEY.get(cleaned.lower()) + if known: + return known + if len(cleaned)>40 or len(leftover)>3: + return leftover[0][:1].upper()+leftover[0][1:] + return " ".join(t[:1].upper()+t[1:] for t in leftover) + + +def _clean_city(value,resume_text): + """Optional residence city. Sentinel / blank → None. Never rejects the CV. + + After the employment-agent JSON is parsed, clamp to a proper city name so + a model that still returns "Karachi(Malir)" is stored as "Karachi". + """ + return canonical_city(value) def _clean_skills(value,resume_text): diff --git a/backend/employment_agent/execute_agent.py b/backend/employment_agent/execute_agent.py index 7ecee57..b240284 100644 --- a/backend/employment_agent/execute_agent.py +++ b/backend/employment_agent/execute_agent.py @@ -6,67 +6,15 @@ Called from inbox.tasks.match_inbox_message; no HTTP surface. from __future__ import annotations -import json import logging from employment_agent.decorators import parse_employment_response -from employment_agent.prompt import CURRENT_TITLE,EDUCATION,NO_CITY,NO_COMPANY,city_list_prompt,prompt,user_prompt +from employment_agent.prompt import CURRENT_TITLE,EDUCATION,NO_COMPANY,prompt,user_prompt from llm_setup import llm_call logger=logging.getLogger("employment_agent") -def parse_normalized_cities(data,fallback=None): - """Keep unique proper city names from the list-normalizer JSON.""" - rows=None - if isinstance(data,dict): - rows=data.get("cities") - if not isinstance(rows,list): - return list(fallback or []) - out=[] - seen=set() - for item in rows: - if not isinstance(item,str): - continue - text=item.strip() - if not text or text.lower() in (NO_CITY.lower(),"none","null","n/a","-"): - continue - key=text.lower() - if key in seen: - continue - seen.add(key) - out.append(text) - out.sort(key=str.lower) - return out or list(fallback or []) - - -async def normalize_cities(values): - """OpenAI: messy stored places → the same proper city names the CV agent writes.""" - places=[] - seen=set() - for raw in values or []: - text=(raw or "").strip() - if not text: - continue - key=text.lower() - if key in seen: - continue - seen.add(key) - places.append(text) - if not places: - return [] - try: - data=await llm_call( - city_list_prompt(), - json.dumps({"places":places},ensure_ascii=False), - json_mode=True, - ) - except Exception: - logger.exception("city list normalize failed") - return places - return parse_normalized_cities(data,fallback=places) - - async def run_employment_agent(*,resume_text=""): text=(resume_text or "").strip() if not text: diff --git a/backend/employment_agent/prompt.py b/backend/employment_agent/prompt.py index c5d0c2a..5dfd420 100644 --- a/backend/employment_agent/prompt.py +++ b/backend/employment_agent/prompt.py @@ -7,6 +7,8 @@ from __future__ import annotations import json +from global_cities import countries_prompt_block + NO_COMPANY="no company was mentioned" EDUCATION="No Education Mentioned" CURRENT_TITLE="No JOB POSITION MENTIONED" @@ -14,13 +16,14 @@ NO_LINKEDIN="no linkedin url mentioned" NO_PHONE="no phone number mentioned" NO_CITY="no city mentioned" -CITY_POLICY=f"""- Return ONE proper city name only — the city, not an area, town, sector, housing society, cantonment, district, or parenthetical locality. -- Correct values look like "Karachi", "Lahore", "Islamabad", "Rawalpindi", "Peshawar". Not "Karachi(Malir)", not "Wah Cantt", not "Gulberg Lahore". -- If the text names a neighborhood or area of a city, return that city: "Karachi (Malir)" / "Karachi(Malir)" / "DHA Karachi" → "Karachi". "Gulberg, Lahore" → "Lahore". "F-10 Islamabad" → "Islamabad". -- Drop "Cantt" / "Cantonment": "Lahore Cantt" → "Lahore", "Rawalpindi Cantt" → "Rawalpindi", "Wah Cantt" → "Wah". +CITY_POLICY="""- Return ONE proper city name only — the city, not an area, town, sector, housing society, cantonment, district, or parenthetical locality. +- Identify the city if possible. Map it to exactly one city name from the country→cities list supplied below. Pakistan is in that list along with every other country — do not prefer one country. +- Return the city name only, never the country. If the text names a neighborhood or area of a listed city, return that city: "Karachi(Malir)" / "Karachi Malir" / "DHA Karachi" → "Karachi". "London(Westminster)" → "London". "Gulberg, Lahore" → "Lahore". "F-10 Islamabad" → "Islamabad". +- Drop "Cantt" / "Cantonment" and housing-society prefixes: "Lahore Cantt" → "Lahore", "Wah Cantt" → "Wah". - Never concatenate two places. If the string is messy (for example "Karachi(Malir) Wah Cantt"), return the single residence city, not both strings glued together. -- Do not return province, country, street, house number, or text inside parentheses. -- Drop junk tokens such as KA, KAR, KARA, empty values, and unintelligible strings.""" +- Do not return province, country, street, house number, neighborhood, cantonment, or text inside parentheses. +- Drop junk tokens, empty values, and unintelligible strings. +- If you cannot map the residence to a listed city, still return a single proper city name. If none is stated, use the no-city sentinel.""" def prompt(): @@ -70,6 +73,9 @@ city (its own key — OPTIONAL. A missing city must not fail the candidate): - Do NOT extract city from Work Experience. A job that lists Karachi, UAE, USA, or any other city is the employer's location, not proof the candidate lives there. - If the contact/location section does not name a city, return exactly: {NO_CITY}. Leave it blank rather than guessing from jobs, education, or nationality. +Country → cities (map messy locality to exactly one city from this list; return the city, never the country): +{countries_prompt_block()} + phone (its own key — extract this separately; copy EVERY digit): - Return the candidate's own mobile / phone exactly as written, including country code when present. - Pakistani mobiles are 11 digits local (03XX-XXXXXXX / 03XX XXXXXXX) or +92 3XX XXXXXXX (12 digits with country code). Copy the last group in full — never stop after 7 or 8 digits. @@ -129,7 +135,10 @@ Example 10 — neighborhood / cantonment is not the city: Resume: "Ali Khan | Karachi(Malir) | 0321-5551234" JSON city must be "Karachi". Not "Karachi(Malir)" and not "Malir". -Example 11 — do not glue two place fragments: +Example 11 — DHA / sector / cantonment still collapse to the city: +Resume: "Address: DHA Karachi" → "Karachi". "Lahore Cantt" → "Lahore". "F-10 Islamabad" → "Islamabad". "Wah Cantt" → "Wah". "London(Westminster)" → "London". + +Example 12 — do not glue two place fragments: Resume: "Address: Karachi(Malir) Wah Cantt" JSON city must be "Karachi" (one city). Not "Karachi(Malir) Wah Cantt" and not "Wah Cantt". @@ -150,16 +159,3 @@ If the contact/location section has no city, city must be "{NO_CITY}" — still def user_prompt(resume_text:str) -> str: return json.dumps({"resume_text":resume_text or ""},ensure_ascii=False) - - -def city_list_prompt(): - """Map messy stored place strings to the same proper city names the CV agent writes.""" - return f"""You map messy residence strings to proper city names for an Inbox City filter. - -{CITY_POLICY} - -Input is JSON: {{"places": ["Karachi(Malir)", "Wah Cantt", "Lahore"]}} -Respond with JSON only: -{{"cities": ["Karachi", "Wah", "Lahore"]}} -Unique proper city names only. Do not copy raw neighborhood or cantonment strings into cities. -""" diff --git a/backend/g_sheet/models.py b/backend/g_sheet/models.py index a06c006..a470a12 100644 --- a/backend/g_sheet/models.py +++ b/backend/g_sheet/models.py @@ -562,9 +562,8 @@ class FormData(SQLModel, table=True): @classmethod async def distinct_cities(cls, session: AsyncSession): """Non-blank city values on this table. Distinct only within form_data.""" - col = func.coalesce(cls.city, cls.residing_city) result = await session.execute( - select(col).where(col.is_not(None), col != "").distinct() + select(cls.city).where(cls.city.is_not(None), cls.city != "").distinct() ) return [text.strip() for text in result.scalars().all() if (text or "").strip()] @@ -660,6 +659,7 @@ class FormData(SQLModel, table=True): Year of Graduation: prefer the second column when present; else the first; else None. Duplicate headers are renamed Year of Graduation_1 by normalise_headers. """ + from employment_agent.decorators import canonical_city from g_sheet.plugins import parse_date, parse_date_time, parse_salary first_year = cls._cell(data, "Year of Graduation") @@ -676,6 +676,7 @@ class FormData(SQLModel, table=True): current_salary = cls._cell(data, "Current Salary") expected_salary = cls._cell(data, "Expected Salary") + residing_city = cls._cell(data, "Residing City") return { "sheet": sheet, @@ -694,8 +695,8 @@ class FormData(SQLModel, table=True): "position_applied_for": cls._cell(data, "Position Applied For"), "profile_link": cls._cell(data, "LinkedIn Profile Link"), "residing_country": cls._cell(data, "Residing Country"), - "residing_city": cls._cell(data, "Residing City"), - "city": cls._cell(data, "Residing City"), + "residing_city": residing_city, + "city": canonical_city(residing_city), "ho_availability": cls._cell(data, "Are you willing to relocate?"), "degree": cls._cell(data, "Educational Degree"), "university": cls._cell(data, "University"), diff --git a/backend/global_cities.py b/backend/global_cities.py new file mode 100644 index 0000000..5922764 --- /dev/null +++ b/backend/global_cities.py @@ -0,0 +1,247 @@ +"""Global country → cities dataset for residence canonicalization. + +Pakistan is one country in this map, not a special case. The employment-agent +prompt receives `countries_prompt_block()` so the model can map a messy +locality to exactly one city name. `canonical_city` uses the same index. +""" + +from __future__ import annotations + +import re + +Countries={ + "Afghanistan":["Kabul","Kandahar","Herat","Mazar-i-Sharif","Jalalabad"], + "Albania":["Tirana","Durres","Vlore","Shkoder"], + "Algeria":["Algiers","Oran","Constantine","Annaba"], + "Andorra":["Andorra la Vella"], + "Angola":["Luanda","Huambo","Lobito","Benguela"], + "Argentina":["Buenos Aires","Cordoba","Rosario","Mendoza","La Plata"], + "Armenia":["Yerevan","Gyumri","Vanadzor"], + "Australia":["Sydney","Melbourne","Brisbane","Perth","Adelaide","Canberra","Gold Coast","Hobart","Darwin"], + "Austria":["Vienna","Graz","Linz","Salzburg","Innsbruck"], + "Azerbaijan":["Baku","Ganja","Sumqayit"], + "Bahamas":["Nassau","Freeport"], + "Bahrain":["Manama","Riffa","Muharraq"], + "Bangladesh":["Dhaka","Chittagong","Khulna","Rajshahi","Sylhet","Gazipur","Narayanganj"], + "Belarus":["Minsk","Gomel","Mogilev","Vitebsk"], + "Belgium":["Brussels","Antwerp","Ghent","Charleroi","Liege","Bruges"], + "Belize":["Belmopan","Belize City"], + "Benin":["Porto-Novo","Cotonou"], + "Bhutan":["Thimphu","Phuntsholing"], + "Bolivia":["La Paz","Santa Cruz","Cochabamba","Sucre"], + "Bosnia and Herzegovina":["Sarajevo","Banja Luka","Mostar","Tuzla"], + "Botswana":["Gaborone","Francistown"], + "Brazil":["Sao Paulo","Rio de Janeiro","Brasilia","Salvador","Fortaleza","Belo Horizonte","Manaus","Curitiba","Recife","Porto Alegre"], + "Brunei":["Bandar Seri Begawan"], + "Bulgaria":["Sofia","Plovdiv","Varna","Burgas"], + "Burkina Faso":["Ouagadougou","Bobo-Dioulasso"], + "Burundi":["Gitega","Bujumbura"], + "Cambodia":["Phnom Penh","Siem Reap","Sihanoukville"], + "Cameroon":["Yaounde","Douala","Garoua"], + "Canada":["Toronto","Montreal","Vancouver","Calgary","Ottawa","Edmonton","Winnipeg","Quebec City","Hamilton","Halifax"], + "Cape Verde":["Praia","Mindelo"], + "Central African Republic":["Bangui"], + "Chad":["N'Djamena","Moundou"], + "Chile":["Santiago","Valparaiso","Concepcion","Antofagasta"], + "China":["Beijing","Shanghai","Guangzhou","Shenzhen","Chengdu","Chongqing","Tianjin","Wuhan","Hangzhou","Nanjing","Xi'an","Suzhou","Dongguan","Qingdao","Dalian"], + "Colombia":["Bogota","Medellin","Cali","Barranquilla","Cartagena"], + "Comoros":["Moroni"], + "Congo":["Brazzaville","Pointe-Noire"], + "Costa Rica":["San Jose","Alajuela","Cartago"], + "Croatia":["Zagreb","Split","Rijeka","Osijek"], + "Cuba":["Havana","Santiago de Cuba","Camaguey"], + "Cyprus":["Nicosia","Limassol","Larnaca","Paphos"], + "Czech Republic":["Prague","Brno","Ostrava","Plzen"], + "Democratic Republic of the Congo":["Kinshasa","Lubumbashi","Mbuji-Mayi"], + "Denmark":["Copenhagen","Aarhus","Odense","Aalborg"], + "Djibouti":["Djibouti"], + "Dominican Republic":["Santo Domingo","Santiago"], + "Ecuador":["Quito","Guayaquil","Cuenca"], + "Egypt":["Cairo","Alexandria","Giza","Shubra El Kheima","Port Said","Suez","Luxor"], + "El Salvador":["San Salvador","Santa Ana","San Miguel"], + "Equatorial Guinea":["Malabo","Bata"], + "Eritrea":["Asmara"], + "Estonia":["Tallinn","Tartu"], + "Eswatini":["Mbabane","Manzini"], + "Ethiopia":["Addis Ababa","Dire Dawa","Mekelle"], + "Fiji":["Suva","Nadi"], + "Finland":["Helsinki","Espoo","Tampere","Oulu","Turku"], + "France":["Paris","Marseille","Lyon","Toulouse","Nice","Nantes","Strasbourg","Bordeaux","Lille","Rennes"], + "Gabon":["Libreville"], + "Gambia":["Banjul","Serekunda"], + "Georgia":["Tbilisi","Batumi","Kutaisi"], + "Germany":["Berlin","Hamburg","Munich","Cologne","Frankfurt","Stuttgart","Dusseldorf","Dortmund","Essen","Leipzig","Dresden","Hanover","Nuremberg"], + "Ghana":["Accra","Kumasi","Tamale","Takoradi"], + "Greece":["Athens","Thessaloniki","Patras","Heraklion"], + "Guatemala":["Guatemala City","Quetzaltenango"], + "Guinea":["Conakry"], + "Guyana":["Georgetown"], + "Haiti":["Port-au-Prince","Cap-Haitien"], + "Honduras":["Tegucigalpa","San Pedro Sula"], + "Hungary":["Budapest","Debrecen","Szeged","Miskolc"], + "Iceland":["Reykjavik"], + "India":["Mumbai","Delhi","Bengaluru","Hyderabad","Ahmedabad","Chennai","Kolkata","Pune","Jaipur","Surat","Lucknow","Kanpur","Nagpur","Indore","Bhopal","Patna","Chandigarh","Noida","Gurgaon","Kochi","Coimbatore"], + "Indonesia":["Jakarta","Surabaya","Bandung","Medan","Bekasi","Depok","Tangerang","Semarang","Makassar","Palembang"], + "Iran":["Tehran","Mashhad","Isfahan","Karaj","Shiraz","Tabriz","Qom","Ahvaz"], + "Iraq":["Baghdad","Basra","Mosul","Erbil","Najaf","Karbala","Sulaymaniyah"], + "Ireland":["Dublin","Cork","Limerick","Galway","Waterford"], + "Israel":["Jerusalem","Tel Aviv","Haifa","Rishon LeZion","Petah Tikva"], + "Italy":["Rome","Milan","Naples","Turin","Palermo","Genoa","Bologna","Florence","Venice","Bari"], + "Ivory Coast":["Yamoussoukro","Abidjan"], + "Jamaica":["Kingston","Montego Bay"], + "Japan":["Tokyo","Yokohama","Osaka","Nagoya","Sapporo","Fukuoka","Kobe","Kyoto","Kawasaki","Saitama","Hiroshima","Sendai"], + "Jordan":["Amman","Zarqa","Irbid","Aqaba"], + "Kazakhstan":["Astana","Almaty","Shymkent","Aktobe"], + "Kenya":["Nairobi","Mombasa","Kisumu","Nakuru"], + "Kuwait":["Kuwait City","Hawalli","Salmiya","Jahra"], + "Kyrgyzstan":["Bishkek","Osh"], + "Laos":["Vientiane","Luang Prabang"], + "Latvia":["Riga","Daugavpils"], + "Lebanon":["Beirut","Tripoli","Sidon","Zahle"], + "Lesotho":["Maseru"], + "Liberia":["Monrovia"], + "Libya":["Tripoli","Benghazi","Misrata"], + "Liechtenstein":["Vaduz"], + "Lithuania":["Vilnius","Kaunas","Klaipeda"], + "Luxembourg":["Luxembourg"], + "Madagascar":["Antananarivo","Toamasina"], + "Malawi":["Lilongwe","Blantyre"], + "Malaysia":["Kuala Lumpur","George Town","Johor Bahru","Ipoh","Shah Alam","Petaling Jaya","Kota Kinabalu","Kuching","Malacca"], + "Maldives":["Male"], + "Mali":["Bamako"], + "Malta":["Valletta","Birkirkara"], + "Mauritania":["Nouakchott"], + "Mauritius":["Port Louis"], + "Mexico":["Mexico City","Guadalajara","Monterrey","Puebla","Tijuana","Leon","Juarez","Merida","Cancun","Queretaro"], + "Moldova":["Chisinau"], + "Monaco":["Monaco"], + "Mongolia":["Ulaanbaatar"], + "Montenegro":["Podgorica","Niksic"], + "Morocco":["Rabat","Casablanca","Fes","Marrakesh","Tangier","Agadir","Meknes"], + "Mozambique":["Maputo","Beira","Nampula"], + "Myanmar":["Naypyidaw","Yangon","Mandalay"], + "Namibia":["Windhoek","Walvis Bay"], + "Nepal":["Kathmandu","Pokhara","Lalitpur","Biratnagar"], + "Netherlands":["Amsterdam","Rotterdam","The Hague","Utrecht","Eindhoven","Groningen"], + "New Zealand":["Auckland","Wellington","Christchurch","Hamilton","Dunedin"], + "Nicaragua":["Managua"], + "Niger":["Niamey"], + "Nigeria":["Abuja","Lagos","Kano","Ibadan","Port Harcourt","Benin City","Kaduna"], + "North Korea":["Pyongyang"], + "North Macedonia":["Skopje"], + "Norway":["Oslo","Bergen","Trondheim","Stavanger"], + "Oman":["Muscat","Salalah","Sohar","Nizwa"], + "Pakistan":[ + "Karachi","Lahore","Islamabad","Rawalpindi","Peshawar","Quetta","Faisalabad", + "Multan","Hyderabad","Sialkot","Gujranwala","Sargodha","Bahawalpur", + "Sukkur","Larkana","Sheikhupura","Rahim Yar Khan","Sahiwal","Jhang","Okara", + "Gujrat","Kasur","Dera Ghazi Khan","Mardan","Abbottabad","Mingora","Nawabshah", + "Mirpur","Muzaffarabad","Gilgit","Skardu","Wah","Attock","Jhelum","Chakwal", + "Taxila","Kamra","Haripur","Mansehra","Kohat","Bannu","Dera Ismail Khan", + "Charsadda","Nowshera","Swat","Chitral","Swabi","Jacobabad","Khairpur","Thatta", + "Gwadar","Turbat","Hub","Kotri","Jamshoro","Shikarpur","Dadu","Badin","Khuzdar", + "Chaman","Kamoke","Muridke","Hafizabad","Narowal","Pakpattan","Vehari","Khanewal", + "Layyah","Burewala","Gojra","Chiniot","Bhakkar","Mianwali","Khushab","Murree", + "Kotli","Bhimber","Rawalakot","Toba Tek Singh","Mandi Bahauddin","Muzaffargarh", + "Mirpur Khas","Hasan Abdal", + ], + "Palestine":["Gaza","Ramallah","Hebron","Nablus"], + "Panama":["Panama City","Colon"], + "Papua New Guinea":["Port Moresby"], + "Paraguay":["Asuncion","Ciudad del Este"], + "Peru":["Lima","Arequipa","Trujillo","Cusco"], + "Philippines":["Manila","Quezon City","Davao","Cebu","Zamboanga","Taguig","Pasig","Cagayan de Oro"], + "Poland":["Warsaw","Krakow","Lodz","Wroclaw","Poznan","Gdansk","Szczecin"], + "Portugal":["Lisbon","Porto","Braga","Coimbra","Faro"], + "Qatar":["Doha","Al Rayyan","Al Wakrah"], + "Romania":["Bucharest","Cluj-Napoca","Timisoara","Iasi","Constanta","Brasov"], + "Russia":["Moscow","Saint Petersburg","Novosibirsk","Yekaterinburg","Kazan","Nizhny Novgorod","Chelyabinsk","Samara","Rostov-on-Don","Ufa"], + "Rwanda":["Kigali"], + "Saudi Arabia":["Riyadh","Jeddah","Mecca","Medina","Dammam","Khobar","Dhahran","Tabuk","Abha","Taif"], + "Senegal":["Dakar","Touba","Thies"], + "Serbia":["Belgrade","Novi Sad","Nis"], + "Seychelles":["Victoria"], + "Sierra Leone":["Freetown"], + "Singapore":["Singapore"], + "Slovakia":["Bratislava","Kosice"], + "Slovenia":["Ljubljana","Maribor"], + "Somalia":["Mogadishu","Hargeisa"], + "South Africa":["Johannesburg","Cape Town","Durban","Pretoria","Port Elizabeth","Bloemfontein","East London","Soweto"], + "South Korea":["Seoul","Busan","Incheon","Daegu","Daejeon","Gwangju","Suwon","Ulsan"], + "South Sudan":["Juba"], + "Spain":["Madrid","Barcelona","Valencia","Seville","Zaragoza","Malaga","Murcia","Palma","Bilbao","Alicante"], + "Sri Lanka":["Colombo","Kandy","Galle","Jaffna","Negombo"], + "Sudan":["Khartoum","Omdurman","Port Sudan"], + "Suriname":["Paramaribo"], + "Sweden":["Stockholm","Gothenburg","Malmo","Uppsala"], + "Switzerland":["Zurich","Geneva","Basel","Bern","Lausanne","Lucerne"], + "Syria":["Damascus","Aleppo","Homs","Latakia"], + "Taiwan":["Taipei","Kaohsiung","Taichung","Tainan"], + "Tajikistan":["Dushanbe"], + "Tanzania":["Dodoma","Dar es Salaam","Mwanza","Arusha","Zanzibar"], + "Thailand":["Bangkok","Chiang Mai","Pattaya","Phuket","Nonthaburi","Hat Yai"], + "Togo":["Lome"], + "Trinidad and Tobago":["Port of Spain","San Fernando"], + "Tunisia":["Tunis","Sfax","Sousse"], + "Turkey":["Istanbul","Ankara","Izmir","Bursa","Antalya","Adana","Gaziantep","Konya","Mersin"], + "Turkmenistan":["Ashgabat"], + "Uganda":["Kampala","Gulu"], + "Ukraine":["Kyiv","Kharkiv","Odesa","Dnipro","Lviv","Zaporizhzhia"], + "United Arab Emirates":["Dubai","Abu Dhabi","Sharjah","Ajman","Ras Al Khaimah","Fujairah","Al Ain","Umm Al Quwain"], + "United Kingdom":["London","Birmingham","Manchester","Glasgow","Liverpool","Leeds","Sheffield","Edinburgh","Bristol","Leicester","Newcastle","Cardiff","Belfast","Nottingham","Southampton"], + "United States":[ + "New York","Los Angeles","Chicago","Houston","Phoenix","Philadelphia","San Antonio", + "San Diego","Dallas","San Jose","Austin","Jacksonville","Fort Worth","Columbus", + "Charlotte","San Francisco","Indianapolis","Seattle","Denver","Washington", + "Boston","Nashville","Detroit","Portland","Las Vegas","Baltimore","Milwaukee", + "Albuquerque","Atlanta","Miami","Minneapolis","Tampa","Orlando","Cleveland", + "Pittsburgh","Cincinnati","Kansas City","St. Louis","Raleigh","Salt Lake City", + ], + "Uruguay":["Montevideo"], + "Uzbekistan":["Tashkent","Samarkand","Bukhara"], + "Venezuela":["Caracas","Maracaibo","Valencia"], + "Vietnam":["Hanoi","Ho Chi Minh City","Da Nang","Hai Phong","Can Tho"], + "Yemen":["Sanaa","Aden","Taiz"], + "Zambia":["Lusaka","Ndola","Kitwe"], + "Zimbabwe":["Harare","Bulawayo"], +} + + +def _city_index(): + """First spelling of each city name wins. Longest names are matched first.""" + out={} + for cities in Countries.values(): + for city in cities: + name=(city or "").strip() + if not name: + continue + out.setdefault(name.lower(),name) + return out + + +CITY_BY_KEY=_city_index() +CITY_RE=re.compile( + r"\b(?:"+"|".join( + re.escape(name) for name in sorted(CITY_BY_KEY,key=len,reverse=True) + )+r")\b", +) + + +def countries_prompt_block(): + """Compact country → cities block fed into the employment-agent prompt.""" + lines=[] + for country,cities in Countries.items(): + names=[c.strip() for c in cities if (c or "").strip()] + if not names: + continue + # De-dupe while keeping order — Pakistan lists Peshawar twice above. + seen=set() + unique=[] + for name in names: + key=name.lower() + if key in seen: + continue + seen.add(key) + unique.append(name) + lines.append(f"{country}: {', '.join(unique)}") + return "\n".join(lines) diff --git a/backend/inbox/views.py b/backend/inbox/views.py index 834bdba..efa551a 100644 --- a/backend/inbox/views.py +++ b/backend/inbox/views.py @@ -747,13 +747,11 @@ class Email: return await Inbox_Messages.count_inbox_messages(self.session,search,assigned=assigned,is_duplicate=is_duplicate,no_suggestions=no_suggestions,processing_state=processing_state,city=city,source=source) async def list_cities(self): - """Proper city names for the Inbox filter — same OpenAI mapping as CV parse.""" - from employment_agent.execute_agent import normalize_cities + """Proper city names for the Inbox filter — DISTINCT of the stored city column.""" from g_sheet.models import FormData inbox=await Inbox_Messages.distinct_cities(self.session) forms=await FormData.distinct_cities(self.session) - merged=Reapplied(session=self.session).merge_cities(inbox,forms) - return await normalize_cities(merged) + return Reapplied(session=self.session).merge_cities(inbox,forms) async def list_sources(self): """Source / platform labels: seeded channels, Google Sheet, form sources.""" diff --git a/backend/tests/test_employment_agent.py b/backend/tests/test_employment_agent.py index 0063829..8d07c1b 100644 --- a/backend/tests/test_employment_agent.py +++ b/backend/tests/test_employment_agent.py @@ -91,25 +91,81 @@ def test_adds_scheme_and_rejects_company_page(): assert company_page["linkedin_url"] is None -def test_city_prompt_asks_openai_for_a_proper_city_name(): - from employment_agent.prompt import city_list_prompt, prompt +def test_city_prompt_asks_for_a_proper_city_name(): + from employment_agent.prompt import prompt text = prompt() assert "Karachi(Malir)" in text assert 'JSON city must be "Karachi"' in text assert "Return ONE proper city name only" in text - listed = city_list_prompt() - assert "Karachi(Malir)" in listed - assert '"cities"' in listed + assert "city_list_prompt" not in text + assert "Pakistan:" in text + assert "United Kingdom:" in text + assert "Karachi" in text + assert "London" in text -def test_parse_normalized_cities_keeps_agent_names(): - from employment_agent.execute_agent import parse_normalized_cities - assert parse_normalized_cities( - {"cities": ["Karachi", "Karachi", "Lahore", "no city mentioned"]}, - fallback=["Karachi(Malir)"], - ) == ["Karachi", "Lahore"] +def test_countries_dataset_is_global_and_includes_pakistan(): + from global_cities import CITY_BY_KEY, Countries, countries_prompt_block + + assert isinstance(Countries, dict) + assert "Pakistan" in Countries + assert "Karachi" in Countries["Pakistan"] + assert "United Kingdom" in Countries + assert "London" in Countries["United Kingdom"] + assert "United States" in Countries + assert "New York" in Countries["United States"] + assert CITY_BY_KEY["karachi"] == "Karachi" + assert CITY_BY_KEY["london"] == "London" + block = countries_prompt_block() + assert block.startswith("Afghanistan:") + assert "Pakistan: " in block + assert "Karachi" in block -def test_parse_normalized_cities_falls_back_when_the_model_shape_is_wrong(): - from employment_agent.execute_agent import parse_normalized_cities - assert parse_normalized_cities({"oops": True}, fallback=["Karachi(Malir)"]) == ["Karachi(Malir)"] +def test_canonical_city_maps_messy_localities(): + from employment_agent.decorators import canonical_city + from employment_agent.prompt import NO_CITY + + assert canonical_city("Karachi(Malir)") == "Karachi" + assert canonical_city("Karachi (Malir)") == "Karachi" + assert canonical_city("Karachi Malir") == "Karachi" + assert canonical_city("DHA Karachi") == "Karachi" + assert canonical_city("Karachi DHA") == "Karachi" + assert canonical_city("Lahore Cantt") == "Lahore" + assert canonical_city("Gulberg, Lahore") == "Lahore" + assert canonical_city("F-10 Islamabad") == "Islamabad" + assert canonical_city("Wah Cantt") == "Wah" + assert canonical_city("Karachi") == "Karachi" + assert canonical_city("Karachi Malir Wah Cantt") == "Karachi" + assert canonical_city("London(Westminster)") == "London" + assert canonical_city("New York") == "New York" + assert canonical_city("Dubai Marina") == "Dubai" + assert canonical_city("") is None + assert canonical_city(" ") is None + assert canonical_city(NO_CITY) is None + assert canonical_city("none") is None + assert canonical_city("n/a") is None + + +def test_list_cities_is_distinct_without_openai(monkeypatch): + import asyncio + import inspect + + from g_sheet.models import FormData + from inbox.models import Inbox_Messages + from inbox.views import Email + + source = inspect.getsource(Email.list_cities) + assert "normalize_cities" not in source + assert "llm_call" not in source + + async def inbox_cities(session): + return ["Karachi", "Lahore", "Karachi"] + + async def form_cities(session): + return ["Lahore", "Islamabad", " ", None] + + monkeypatch.setattr(Inbox_Messages, "distinct_cities", inbox_cities) + monkeypatch.setattr(FormData, "distinct_cities", form_cities) + cities = asyncio.run(Email(session=object()).list_cities()) + assert cities == ["Islamabad", "Karachi", "Lahore"] diff --git a/backend/tests/test_employment_extraction_clamps.py b/backend/tests/test_employment_extraction_clamps.py index a252333..858e1ee 100644 --- a/backend/tests/test_employment_extraction_clamps.py +++ b/backend/tests/test_employment_extraction_clamps.py @@ -171,5 +171,13 @@ def test_city_from_the_model_is_kept_as_returned(): assert fields["city"] == "Karachi" +def test_messy_model_city_is_clamped_to_canonical_before_persist(): + resume = "Ali Khan | Karachi(Malir) | 0321-5551234" + fields = parse_employment_response({"city": "Karachi(Malir)"}, resume) + assert fields["city"] == "Karachi" + assert parse({"city": "DHA Karachi"})["city"] == "Karachi" + assert parse({"city": "Wah Cantt"})["city"] == "Wah" + + def test_city_sentinel_is_dropped(): assert parse({"city": "no city mentioned"})["city"] is None diff --git a/backend/tests/test_g_sheet_plugins.py b/backend/tests/test_g_sheet_plugins.py index 0775b2c..63bea0c 100644 --- a/backend/tests/test_g_sheet_plugins.py +++ b/backend/tests/test_g_sheet_plugins.py @@ -66,6 +66,7 @@ def test_from_sheet_row_maps_form_response_keys(): assert mapped["ho_availability"] == "Yes" assert mapped["marital_status"] == "Single" assert mapped["residing_city"] == "Karachi" + assert mapped["city"] == "Karachi" assert mapped["residing_country"] == "Pakistan" assert mapped["director_poc_category"] == "Operations" assert mapped["hr_comments"] == "Good profile" @@ -116,6 +117,13 @@ def test_year_of_graduation_single_column(): assert mapped["entry_year"] == "2012" +def test_from_sheet_row_canonicalizes_city_keeps_raw_residing(): + data = {**FORM_RESPONSE_RECORD, "Residing City": "Karachi(Malir)"} + mapped = FormData.from_sheet_row("tab", 9, data) + assert mapped["city"] == "Karachi" + assert mapped["residing_city"] == "Karachi(Malir)" + + def test_duplicate_year_header_becomes_year_of_graduation_1(): headers = plugins.normalise_headers( ["Full Name", "Year of Graduation", "Year of Graduation"],