Merge pull request 'global countires' (#82) from SQS_BROKER into main

Reviewed-on: #82

backend/employment_agent/decorators.py

@@ -18,6 +18,17 @@ import re
 from functools import wraps
 
 from employment_agent.prompt import EDUCATION,NO_CITY,NO_COMPANY,NO_LINKEDIN,NO_PHONE
+from global_cities import CITY_BY_KEY,CITY_RE
+
+_CITY_SENTINELS=frozenset({
+    NO_CITY.lower(),"none","null","n/a","-","na","n.a.","n.a",
+})
+_CITY_DROP=frozenset({
+    "dha","cantt","cantonment","cant","phase","sector","area","district",
+    "tehsil","division","housing","society","scheme","block","street","house",
+    "near","colony","neighborhood","neighbourhood","suburb",
+})
+_SECTOR_RE=re.compile(r"^(?:[a-z]-?\d+[a-z]?|\d+[a-z]?)$",re.I)
 
 
 def require_json_object(func):
@@ -102,16 +113,54 @@ def _clean_phone(value,resume_text):
     return text
 
 
-def _clean_city(value,resume_text):
+def canonical_city(text):
-    """Optional residence city. Sentinel → None. Never rejects the CV.
+    """Write-time only: messy locality → one proper city name, or None.
 
-    Proper city names (Karachi, not Karachi(Malir)) come from the OpenAI parse
+    Looks up `global_cities.Countries` (every country, Pakistan included).
-    in run_employment_agent. This clamp does not rewrite place names.
+    "Karachi(Malir)" / "London(Westminster)" / "DHA Karachi" / "Wah Cantt"
+    map to the listed city. Sentinels and blanks are None. Never rejects a CV.
     """
-    text=(value or "").strip()
+    raw=(text or "").strip()
-    if not text or text.lower() in (NO_CITY.lower(),"none","null","n/a","-"):
+    if not raw or raw.lower() in _CITY_SENTINELS:
         return None
-    return text
+    known=CITY_BY_KEY.get(raw.lower())
+    if known:
+        return known
+    normalised=re.sub(r"[()\[\]{}]"," ",raw)
+    normalised=re.sub(r"[,/;|]+"," ",normalised)
+    normalised=re.sub(r"\s+"," ",normalised).strip()
+    if not normalised:
+        return None
+    known=CITY_BY_KEY.get(normalised.lower())
+    if known:
+        return known
+    match=CITY_RE.search(normalised.lower())
+    if match:
+        return CITY_BY_KEY[match.group(0)]
+    leftover=[]
+    for token in normalised.split():
+        lowered=token.lower()
+        if lowered in _CITY_DROP or _SECTOR_RE.fullmatch(token):
+            continue
+        leftover.append(token)
+    if not leftover:
+        return None
+    cleaned=" ".join(leftover)
+    known=CITY_BY_KEY.get(cleaned.lower())
+    if known:
+        return known
+    if len(cleaned)>40 or len(leftover)>3:
+        return leftover[0][:1].upper()+leftover[0][1:]
+    return " ".join(t[:1].upper()+t[1:] for t in leftover)
+
+
+def _clean_city(value,resume_text):
+    """Optional residence city. Sentinel / blank → None. Never rejects the CV.
+
+    After the employment-agent JSON is parsed, clamp to a proper city name so
+    a model that still returns "Karachi(Malir)" is stored as "Karachi".
+    """
+    return canonical_city(value)
 
 
 def _clean_skills(value,resume_text):

backend/employment_agent/execute_agent.py

@@ -6,67 +6,15 @@ Called from inbox.tasks.match_inbox_message; no HTTP surface.
 
 from __future__ import annotations
 
-import json
 import logging
 
 from employment_agent.decorators import parse_employment_response
-from employment_agent.prompt import CURRENT_TITLE,EDUCATION,NO_CITY,NO_COMPANY,city_list_prompt,prompt,user_prompt
+from employment_agent.prompt import CURRENT_TITLE,EDUCATION,NO_COMPANY,prompt,user_prompt
 from llm_setup import llm_call
 
 logger=logging.getLogger("employment_agent")
 
 
-def parse_normalized_cities(data,fallback=None):
-    """Keep unique proper city names from the list-normalizer JSON."""
-    rows=None
-    if isinstance(data,dict):
-        rows=data.get("cities")
-    if not isinstance(rows,list):
-        return list(fallback or [])
-    out=[]
-    seen=set()
-    for item in rows:
-        if not isinstance(item,str):
-            continue
-        text=item.strip()
-        if not text or text.lower() in (NO_CITY.lower(),"none","null","n/a","-"):
-            continue
-        key=text.lower()
-        if key in seen:
-            continue
-        seen.add(key)
-        out.append(text)
-    out.sort(key=str.lower)
-    return out or list(fallback or [])
-
-
-async def normalize_cities(values):
-    """OpenAI: messy stored places → the same proper city names the CV agent writes."""
-    places=[]
-    seen=set()
-    for raw in values or []:
-        text=(raw or "").strip()
-        if not text:
-            continue
-        key=text.lower()
-        if key in seen:
-            continue
-        seen.add(key)
-        places.append(text)
-    if not places:
-        return []
-    try:
-        data=await llm_call(
-            city_list_prompt(),
-            json.dumps({"places":places},ensure_ascii=False),
-            json_mode=True,
-        )
-    except Exception:
-        logger.exception("city list normalize failed")
-        return places
-    return parse_normalized_cities(data,fallback=places)
-
-
 async def run_employment_agent(*,resume_text=""):
     text=(resume_text or "").strip()
     if not text:

backend/employment_agent/prompt.py

@@ -7,6 +7,8 @@ from __future__ import annotations
 
 import json
 
+from global_cities import countries_prompt_block
+
 NO_COMPANY="no company was mentioned"
 EDUCATION="No Education Mentioned"
 CURRENT_TITLE="No JOB POSITION MENTIONED"
@@ -14,13 +16,14 @@ NO_LINKEDIN="no linkedin url mentioned"
 NO_PHONE="no phone number mentioned"
 NO_CITY="no city mentioned"
 
-CITY_POLICY=f"""- Return ONE proper city name only — the city, not an area, town, sector, housing society, cantonment, district, or parenthetical locality.
+CITY_POLICY="""- Return ONE proper city name only — the city, not an area, town, sector, housing society, cantonment, district, or parenthetical locality.
-- Correct values look like "Karachi", "Lahore", "Islamabad", "Rawalpindi", "Peshawar". Not "Karachi(Malir)", not "Wah Cantt", not "Gulberg Lahore".
+- Identify the city if possible. Map it to exactly one city name from the country→cities list supplied below. Pakistan is in that list along with every other country — do not prefer one country.
-- If the text names a neighborhood or area of a city, return that city: "Karachi (Malir)" / "Karachi(Malir)" / "DHA Karachi" → "Karachi". "Gulberg, Lahore" → "Lahore". "F-10 Islamabad" → "Islamabad".
+- Return the city name only, never the country. If the text names a neighborhood or area of a listed city, return that city: "Karachi(Malir)" / "Karachi Malir" / "DHA Karachi" → "Karachi". "London(Westminster)" → "London". "Gulberg, Lahore" → "Lahore". "F-10 Islamabad" → "Islamabad".
-- Drop "Cantt" / "Cantonment": "Lahore Cantt" → "Lahore", "Rawalpindi Cantt" → "Rawalpindi", "Wah Cantt" → "Wah".
+- Drop "Cantt" / "Cantonment" and housing-society prefixes: "Lahore Cantt" → "Lahore", "Wah Cantt" → "Wah".
 - Never concatenate two places. If the string is messy (for example "Karachi(Malir) Wah Cantt"), return the single residence city, not both strings glued together.
-- Do not return province, country, street, house number, or text inside parentheses.
+- Do not return province, country, street, house number, neighborhood, cantonment, or text inside parentheses.
-- Drop junk tokens such as KA, KAR, KARA, empty values, and unintelligible strings."""
+- Drop junk tokens, empty values, and unintelligible strings.
+- If you cannot map the residence to a listed city, still return a single proper city name. If none is stated, use the no-city sentinel."""
 
 
 def prompt():
@@ -70,6 +73,9 @@ city (its own key — OPTIONAL. A missing city must not fail the candidate):
 - Do NOT extract city from Work Experience. A job that lists Karachi, UAE, USA, or any other city is the employer's location, not proof the candidate lives there.
 - If the contact/location section does not name a city, return exactly: {NO_CITY}. Leave it blank rather than guessing from jobs, education, or nationality.
 
+Country → cities (map messy locality to exactly one city from this list; return the city, never the country):
+{countries_prompt_block()}
+
 phone (its own key — extract this separately; copy EVERY digit):
 - Return the candidate's own mobile / phone exactly as written, including country code when present.
 - Pakistani mobiles are 11 digits local (03XX-XXXXXXX / 03XX XXXXXXX) or +92 3XX XXXXXXX (12 digits with country code). Copy the last group in full — never stop after 7 or 8 digits.
@@ -129,7 +135,10 @@ Example 10 — neighborhood / cantonment is not the city:
 Resume: "Ali Khan | Karachi(Malir) | 0321-5551234"
 JSON city must be "Karachi". Not "Karachi(Malir)" and not "Malir".
 
-Example 11 — do not glue two place fragments:
+Example 11 — DHA / sector / cantonment still collapse to the city:
+Resume: "Address: DHA Karachi" → "Karachi". "Lahore Cantt" → "Lahore". "F-10 Islamabad" → "Islamabad". "Wah Cantt" → "Wah". "London(Westminster)" → "London".
+
+Example 12 — do not glue two place fragments:
 Resume: "Address: Karachi(Malir) Wah Cantt"
 JSON city must be "Karachi" (one city). Not "Karachi(Malir) Wah Cantt" and not "Wah Cantt".
 
@@ -150,16 +159,3 @@ If the contact/location section has no city, city must be "{NO_CITY}" — still
 
 def user_prompt(resume_text:str) -> str:
     return json.dumps({"resume_text":resume_text or ""},ensure_ascii=False)
-
-
-def city_list_prompt():
-    """Map messy stored place strings to the same proper city names the CV agent writes."""
-    return f"""You map messy residence strings to proper city names for an Inbox City filter.
-
-{CITY_POLICY}
-
-Input is JSON: {{"places": ["Karachi(Malir)", "Wah Cantt", "Lahore"]}}
-Respond with JSON only:
-{{"cities": ["Karachi", "Wah", "Lahore"]}}
-Unique proper city names only. Do not copy raw neighborhood or cantonment strings into cities.
-"""

backend/g_sheet/models.py

@@ -562,9 +562,8 @@ class FormData(SQLModel, table=True):
     @classmethod
     async def distinct_cities(cls, session: AsyncSession):
         """Non-blank city values on this table. Distinct only within form_data."""
-        col = func.coalesce(cls.city, cls.residing_city)
         result = await session.execute(
-            select(col).where(col.is_not(None), col != "").distinct()
+            select(cls.city).where(cls.city.is_not(None), cls.city != "").distinct()
         )
         return [text.strip() for text in result.scalars().all() if (text or "").strip()]
 
@@ -660,6 +659,7 @@ class FormData(SQLModel, table=True):
         Year of Graduation: prefer the second column when present; else the first;
         else None. Duplicate headers are renamed Year of Graduation_1 by normalise_headers.
         """
+        from employment_agent.decorators import canonical_city
         from g_sheet.plugins import parse_date, parse_date_time, parse_salary
 
         first_year = cls._cell(data, "Year of Graduation")
@@ -676,6 +676,7 @@ class FormData(SQLModel, table=True):
 
         current_salary = cls._cell(data, "Current Salary")
         expected_salary = cls._cell(data, "Expected Salary")
+        residing_city = cls._cell(data, "Residing City")
 
         return {
             "sheet": sheet,
@@ -694,8 +695,8 @@ class FormData(SQLModel, table=True):
             "position_applied_for": cls._cell(data, "Position Applied For"),
             "profile_link": cls._cell(data, "LinkedIn Profile Link"),
             "residing_country": cls._cell(data, "Residing Country"),
-            "residing_city": cls._cell(data, "Residing City"),
+            "residing_city": residing_city,
-            "city": cls._cell(data, "Residing City"),
+            "city": canonical_city(residing_city),
             "ho_availability": cls._cell(data, "Are you willing to relocate?"),
             "degree": cls._cell(data, "Educational Degree"),
             "university": cls._cell(data, "University"),

backend/global_cities.py

@@ -0,0 +1,247 @@
+"""Global country → cities dataset for residence canonicalization.
+
+Pakistan is one country in this map, not a special case. The employment-agent
+prompt receives `countries_prompt_block()` so the model can map a messy
+locality to exactly one city name. `canonical_city` uses the same index.
+"""
+
+from __future__ import annotations
+
+import re
+
+Countries={
+    "Afghanistan":["Kabul","Kandahar","Herat","Mazar-i-Sharif","Jalalabad"],
+    "Albania":["Tirana","Durres","Vlore","Shkoder"],
+    "Algeria":["Algiers","Oran","Constantine","Annaba"],
+    "Andorra":["Andorra la Vella"],
+    "Angola":["Luanda","Huambo","Lobito","Benguela"],
+    "Argentina":["Buenos Aires","Cordoba","Rosario","Mendoza","La Plata"],
+    "Armenia":["Yerevan","Gyumri","Vanadzor"],
+    "Australia":["Sydney","Melbourne","Brisbane","Perth","Adelaide","Canberra","Gold Coast","Hobart","Darwin"],
+    "Austria":["Vienna","Graz","Linz","Salzburg","Innsbruck"],
+    "Azerbaijan":["Baku","Ganja","Sumqayit"],
+    "Bahamas":["Nassau","Freeport"],
+    "Bahrain":["Manama","Riffa","Muharraq"],
+    "Bangladesh":["Dhaka","Chittagong","Khulna","Rajshahi","Sylhet","Gazipur","Narayanganj"],
+    "Belarus":["Minsk","Gomel","Mogilev","Vitebsk"],
+    "Belgium":["Brussels","Antwerp","Ghent","Charleroi","Liege","Bruges"],
+    "Belize":["Belmopan","Belize City"],
+    "Benin":["Porto-Novo","Cotonou"],
+    "Bhutan":["Thimphu","Phuntsholing"],
+    "Bolivia":["La Paz","Santa Cruz","Cochabamba","Sucre"],
+    "Bosnia and Herzegovina":["Sarajevo","Banja Luka","Mostar","Tuzla"],
+    "Botswana":["Gaborone","Francistown"],
+    "Brazil":["Sao Paulo","Rio de Janeiro","Brasilia","Salvador","Fortaleza","Belo Horizonte","Manaus","Curitiba","Recife","Porto Alegre"],
+    "Brunei":["Bandar Seri Begawan"],
+    "Bulgaria":["Sofia","Plovdiv","Varna","Burgas"],
+    "Burkina Faso":["Ouagadougou","Bobo-Dioulasso"],
+    "Burundi":["Gitega","Bujumbura"],
+    "Cambodia":["Phnom Penh","Siem Reap","Sihanoukville"],
+    "Cameroon":["Yaounde","Douala","Garoua"],
+    "Canada":["Toronto","Montreal","Vancouver","Calgary","Ottawa","Edmonton","Winnipeg","Quebec City","Hamilton","Halifax"],
+    "Cape Verde":["Praia","Mindelo"],
+    "Central African Republic":["Bangui"],
+    "Chad":["N'Djamena","Moundou"],
+    "Chile":["Santiago","Valparaiso","Concepcion","Antofagasta"],
+    "China":["Beijing","Shanghai","Guangzhou","Shenzhen","Chengdu","Chongqing","Tianjin","Wuhan","Hangzhou","Nanjing","Xi'an","Suzhou","Dongguan","Qingdao","Dalian"],
+    "Colombia":["Bogota","Medellin","Cali","Barranquilla","Cartagena"],
+    "Comoros":["Moroni"],
+    "Congo":["Brazzaville","Pointe-Noire"],
+    "Costa Rica":["San Jose","Alajuela","Cartago"],
+    "Croatia":["Zagreb","Split","Rijeka","Osijek"],
+    "Cuba":["Havana","Santiago de Cuba","Camaguey"],
+    "Cyprus":["Nicosia","Limassol","Larnaca","Paphos"],
+    "Czech Republic":["Prague","Brno","Ostrava","Plzen"],
+    "Democratic Republic of the Congo":["Kinshasa","Lubumbashi","Mbuji-Mayi"],
+    "Denmark":["Copenhagen","Aarhus","Odense","Aalborg"],
+    "Djibouti":["Djibouti"],
+    "Dominican Republic":["Santo Domingo","Santiago"],
+    "Ecuador":["Quito","Guayaquil","Cuenca"],
+    "Egypt":["Cairo","Alexandria","Giza","Shubra El Kheima","Port Said","Suez","Luxor"],
+    "El Salvador":["San Salvador","Santa Ana","San Miguel"],
+    "Equatorial Guinea":["Malabo","Bata"],
+    "Eritrea":["Asmara"],
+    "Estonia":["Tallinn","Tartu"],
+    "Eswatini":["Mbabane","Manzini"],
+    "Ethiopia":["Addis Ababa","Dire Dawa","Mekelle"],
+    "Fiji":["Suva","Nadi"],
+    "Finland":["Helsinki","Espoo","Tampere","Oulu","Turku"],
+    "France":["Paris","Marseille","Lyon","Toulouse","Nice","Nantes","Strasbourg","Bordeaux","Lille","Rennes"],
+    "Gabon":["Libreville"],
+    "Gambia":["Banjul","Serekunda"],
+    "Georgia":["Tbilisi","Batumi","Kutaisi"],
+    "Germany":["Berlin","Hamburg","Munich","Cologne","Frankfurt","Stuttgart","Dusseldorf","Dortmund","Essen","Leipzig","Dresden","Hanover","Nuremberg"],
+    "Ghana":["Accra","Kumasi","Tamale","Takoradi"],
+    "Greece":["Athens","Thessaloniki","Patras","Heraklion"],
+    "Guatemala":["Guatemala City","Quetzaltenango"],
+    "Guinea":["Conakry"],
+    "Guyana":["Georgetown"],
+    "Haiti":["Port-au-Prince","Cap-Haitien"],
+    "Honduras":["Tegucigalpa","San Pedro Sula"],
+    "Hungary":["Budapest","Debrecen","Szeged","Miskolc"],
+    "Iceland":["Reykjavik"],
+    "India":["Mumbai","Delhi","Bengaluru","Hyderabad","Ahmedabad","Chennai","Kolkata","Pune","Jaipur","Surat","Lucknow","Kanpur","Nagpur","Indore","Bhopal","Patna","Chandigarh","Noida","Gurgaon","Kochi","Coimbatore"],
+    "Indonesia":["Jakarta","Surabaya","Bandung","Medan","Bekasi","Depok","Tangerang","Semarang","Makassar","Palembang"],
+    "Iran":["Tehran","Mashhad","Isfahan","Karaj","Shiraz","Tabriz","Qom","Ahvaz"],
+    "Iraq":["Baghdad","Basra","Mosul","Erbil","Najaf","Karbala","Sulaymaniyah"],
+    "Ireland":["Dublin","Cork","Limerick","Galway","Waterford"],
+    "Israel":["Jerusalem","Tel Aviv","Haifa","Rishon LeZion","Petah Tikva"],
+    "Italy":["Rome","Milan","Naples","Turin","Palermo","Genoa","Bologna","Florence","Venice","Bari"],
+    "Ivory Coast":["Yamoussoukro","Abidjan"],
+    "Jamaica":["Kingston","Montego Bay"],
+    "Japan":["Tokyo","Yokohama","Osaka","Nagoya","Sapporo","Fukuoka","Kobe","Kyoto","Kawasaki","Saitama","Hiroshima","Sendai"],
+    "Jordan":["Amman","Zarqa","Irbid","Aqaba"],
+    "Kazakhstan":["Astana","Almaty","Shymkent","Aktobe"],
+    "Kenya":["Nairobi","Mombasa","Kisumu","Nakuru"],
+    "Kuwait":["Kuwait City","Hawalli","Salmiya","Jahra"],
+    "Kyrgyzstan":["Bishkek","Osh"],
+    "Laos":["Vientiane","Luang Prabang"],
+    "Latvia":["Riga","Daugavpils"],
+    "Lebanon":["Beirut","Tripoli","Sidon","Zahle"],
+    "Lesotho":["Maseru"],
+    "Liberia":["Monrovia"],
+    "Libya":["Tripoli","Benghazi","Misrata"],
+    "Liechtenstein":["Vaduz"],
+    "Lithuania":["Vilnius","Kaunas","Klaipeda"],
+    "Luxembourg":["Luxembourg"],
+    "Madagascar":["Antananarivo","Toamasina"],
+    "Malawi":["Lilongwe","Blantyre"],
+    "Malaysia":["Kuala Lumpur","George Town","Johor Bahru","Ipoh","Shah Alam","Petaling Jaya","Kota Kinabalu","Kuching","Malacca"],
+    "Maldives":["Male"],
+    "Mali":["Bamako"],
+    "Malta":["Valletta","Birkirkara"],
+    "Mauritania":["Nouakchott"],
+    "Mauritius":["Port Louis"],
+    "Mexico":["Mexico City","Guadalajara","Monterrey","Puebla","Tijuana","Leon","Juarez","Merida","Cancun","Queretaro"],
+    "Moldova":["Chisinau"],
+    "Monaco":["Monaco"],
+    "Mongolia":["Ulaanbaatar"],
+    "Montenegro":["Podgorica","Niksic"],
+    "Morocco":["Rabat","Casablanca","Fes","Marrakesh","Tangier","Agadir","Meknes"],
+    "Mozambique":["Maputo","Beira","Nampula"],
+    "Myanmar":["Naypyidaw","Yangon","Mandalay"],
+    "Namibia":["Windhoek","Walvis Bay"],
+    "Nepal":["Kathmandu","Pokhara","Lalitpur","Biratnagar"],
+    "Netherlands":["Amsterdam","Rotterdam","The Hague","Utrecht","Eindhoven","Groningen"],
+    "New Zealand":["Auckland","Wellington","Christchurch","Hamilton","Dunedin"],
+    "Nicaragua":["Managua"],
+    "Niger":["Niamey"],
+    "Nigeria":["Abuja","Lagos","Kano","Ibadan","Port Harcourt","Benin City","Kaduna"],
+    "North Korea":["Pyongyang"],
+    "North Macedonia":["Skopje"],
+    "Norway":["Oslo","Bergen","Trondheim","Stavanger"],
+    "Oman":["Muscat","Salalah","Sohar","Nizwa"],
+    "Pakistan":[
+        "Karachi","Lahore","Islamabad","Rawalpindi","Peshawar","Quetta","Faisalabad",
+        "Multan","Hyderabad","Sialkot","Gujranwala","Sargodha","Bahawalpur",
+        "Sukkur","Larkana","Sheikhupura","Rahim Yar Khan","Sahiwal","Jhang","Okara",
+        "Gujrat","Kasur","Dera Ghazi Khan","Mardan","Abbottabad","Mingora","Nawabshah",
+        "Mirpur","Muzaffarabad","Gilgit","Skardu","Wah","Attock","Jhelum","Chakwal",
+        "Taxila","Kamra","Haripur","Mansehra","Kohat","Bannu","Dera Ismail Khan",
+        "Charsadda","Nowshera","Swat","Chitral","Swabi","Jacobabad","Khairpur","Thatta",
+        "Gwadar","Turbat","Hub","Kotri","Jamshoro","Shikarpur","Dadu","Badin","Khuzdar",
+        "Chaman","Kamoke","Muridke","Hafizabad","Narowal","Pakpattan","Vehari","Khanewal",
+        "Layyah","Burewala","Gojra","Chiniot","Bhakkar","Mianwali","Khushab","Murree",
+        "Kotli","Bhimber","Rawalakot","Toba Tek Singh","Mandi Bahauddin","Muzaffargarh",
+        "Mirpur Khas","Hasan Abdal",
+    ],
+    "Palestine":["Gaza","Ramallah","Hebron","Nablus"],
+    "Panama":["Panama City","Colon"],
+    "Papua New Guinea":["Port Moresby"],
+    "Paraguay":["Asuncion","Ciudad del Este"],
+    "Peru":["Lima","Arequipa","Trujillo","Cusco"],
+    "Philippines":["Manila","Quezon City","Davao","Cebu","Zamboanga","Taguig","Pasig","Cagayan de Oro"],
+    "Poland":["Warsaw","Krakow","Lodz","Wroclaw","Poznan","Gdansk","Szczecin"],
+    "Portugal":["Lisbon","Porto","Braga","Coimbra","Faro"],
+    "Qatar":["Doha","Al Rayyan","Al Wakrah"],
+    "Romania":["Bucharest","Cluj-Napoca","Timisoara","Iasi","Constanta","Brasov"],
+    "Russia":["Moscow","Saint Petersburg","Novosibirsk","Yekaterinburg","Kazan","Nizhny Novgorod","Chelyabinsk","Samara","Rostov-on-Don","Ufa"],
+    "Rwanda":["Kigali"],
+    "Saudi Arabia":["Riyadh","Jeddah","Mecca","Medina","Dammam","Khobar","Dhahran","Tabuk","Abha","Taif"],
+    "Senegal":["Dakar","Touba","Thies"],
+    "Serbia":["Belgrade","Novi Sad","Nis"],
+    "Seychelles":["Victoria"],
+    "Sierra Leone":["Freetown"],
+    "Singapore":["Singapore"],
+    "Slovakia":["Bratislava","Kosice"],
+    "Slovenia":["Ljubljana","Maribor"],
+    "Somalia":["Mogadishu","Hargeisa"],
+    "South Africa":["Johannesburg","Cape Town","Durban","Pretoria","Port Elizabeth","Bloemfontein","East London","Soweto"],
+    "South Korea":["Seoul","Busan","Incheon","Daegu","Daejeon","Gwangju","Suwon","Ulsan"],
+    "South Sudan":["Juba"],
+    "Spain":["Madrid","Barcelona","Valencia","Seville","Zaragoza","Malaga","Murcia","Palma","Bilbao","Alicante"],
+    "Sri Lanka":["Colombo","Kandy","Galle","Jaffna","Negombo"],
+    "Sudan":["Khartoum","Omdurman","Port Sudan"],
+    "Suriname":["Paramaribo"],
+    "Sweden":["Stockholm","Gothenburg","Malmo","Uppsala"],
+    "Switzerland":["Zurich","Geneva","Basel","Bern","Lausanne","Lucerne"],
+    "Syria":["Damascus","Aleppo","Homs","Latakia"],
+    "Taiwan":["Taipei","Kaohsiung","Taichung","Tainan"],
+    "Tajikistan":["Dushanbe"],
+    "Tanzania":["Dodoma","Dar es Salaam","Mwanza","Arusha","Zanzibar"],
+    "Thailand":["Bangkok","Chiang Mai","Pattaya","Phuket","Nonthaburi","Hat Yai"],
+    "Togo":["Lome"],
+    "Trinidad and Tobago":["Port of Spain","San Fernando"],
+    "Tunisia":["Tunis","Sfax","Sousse"],
+    "Turkey":["Istanbul","Ankara","Izmir","Bursa","Antalya","Adana","Gaziantep","Konya","Mersin"],
+    "Turkmenistan":["Ashgabat"],
+    "Uganda":["Kampala","Gulu"],
+    "Ukraine":["Kyiv","Kharkiv","Odesa","Dnipro","Lviv","Zaporizhzhia"],
+    "United Arab Emirates":["Dubai","Abu Dhabi","Sharjah","Ajman","Ras Al Khaimah","Fujairah","Al Ain","Umm Al Quwain"],
+    "United Kingdom":["London","Birmingham","Manchester","Glasgow","Liverpool","Leeds","Sheffield","Edinburgh","Bristol","Leicester","Newcastle","Cardiff","Belfast","Nottingham","Southampton"],
+    "United States":[
+        "New York","Los Angeles","Chicago","Houston","Phoenix","Philadelphia","San Antonio",
+        "San Diego","Dallas","San Jose","Austin","Jacksonville","Fort Worth","Columbus",
+        "Charlotte","San Francisco","Indianapolis","Seattle","Denver","Washington",
+        "Boston","Nashville","Detroit","Portland","Las Vegas","Baltimore","Milwaukee",
+        "Albuquerque","Atlanta","Miami","Minneapolis","Tampa","Orlando","Cleveland",
+        "Pittsburgh","Cincinnati","Kansas City","St. Louis","Raleigh","Salt Lake City",
+    ],
+    "Uruguay":["Montevideo"],
+    "Uzbekistan":["Tashkent","Samarkand","Bukhara"],
+    "Venezuela":["Caracas","Maracaibo","Valencia"],
+    "Vietnam":["Hanoi","Ho Chi Minh City","Da Nang","Hai Phong","Can Tho"],
+    "Yemen":["Sanaa","Aden","Taiz"],
+    "Zambia":["Lusaka","Ndola","Kitwe"],
+    "Zimbabwe":["Harare","Bulawayo"],
+}
+
+
+def _city_index():
+    """First spelling of each city name wins. Longest names are matched first."""
+    out={}
+    for cities in Countries.values():
+        for city in cities:
+            name=(city or "").strip()
+            if not name:
+                continue
+            out.setdefault(name.lower(),name)
+    return out
+
+
+CITY_BY_KEY=_city_index()
+CITY_RE=re.compile(
+    r"\b(?:"+"|".join(
+        re.escape(name) for name in sorted(CITY_BY_KEY,key=len,reverse=True)
+    )+r")\b",
+)
+
+
+def countries_prompt_block():
+    """Compact country → cities block fed into the employment-agent prompt."""
+    lines=[]
+    for country,cities in Countries.items():
+        names=[c.strip() for c in cities if (c or "").strip()]
+        if not names:
+            continue
+        # De-dupe while keeping order — Pakistan lists Peshawar twice above.
+        seen=set()
+        unique=[]
+        for name in names:
+            key=name.lower()
+            if key in seen:
+                continue
+            seen.add(key)
+            unique.append(name)
+        lines.append(f"{country}: {', '.join(unique)}")
+    return "\n".join(lines)

backend/inbox/views.py

@@ -747,13 +747,11 @@ class Email:
             return await Inbox_Messages.count_inbox_messages(self.session,search,assigned=assigned,is_duplicate=is_duplicate,no_suggestions=no_suggestions,processing_state=processing_state,city=city,source=source)
 
     async def list_cities(self):
-        """Proper city names for the Inbox filter — same OpenAI mapping as CV parse."""
+        """Proper city names for the Inbox filter — DISTINCT of the stored city column."""
-        from employment_agent.execute_agent import normalize_cities
         from g_sheet.models import FormData
         inbox=await Inbox_Messages.distinct_cities(self.session)
         forms=await FormData.distinct_cities(self.session)
-        merged=Reapplied(session=self.session).merge_cities(inbox,forms)
+        return Reapplied(session=self.session).merge_cities(inbox,forms)
-        return await normalize_cities(merged)
 
     async def list_sources(self):
         """Source / platform labels: seeded channels, Google Sheet, form sources."""

backend/tests/test_employment_agent.py

@@ -91,25 +91,81 @@ def test_adds_scheme_and_rejects_company_page():
     assert company_page["linkedin_url"] is None
 
 
-def test_city_prompt_asks_openai_for_a_proper_city_name():
+def test_city_prompt_asks_for_a_proper_city_name():
-    from employment_agent.prompt import city_list_prompt, prompt
+    from employment_agent.prompt import prompt
     text = prompt()
     assert "Karachi(Malir)" in text
     assert 'JSON city must be "Karachi"' in text
     assert "Return ONE proper city name only" in text
-    listed = city_list_prompt()
+    assert "city_list_prompt" not in text
-    assert "Karachi(Malir)" in listed
+    assert "Pakistan:" in text
-    assert '"cities"' in listed
+    assert "United Kingdom:" in text
+    assert "Karachi" in text
+    assert "London" in text
 
 
-def test_parse_normalized_cities_keeps_agent_names():
+def test_countries_dataset_is_global_and_includes_pakistan():
-    from employment_agent.execute_agent import parse_normalized_cities
+    from global_cities import CITY_BY_KEY, Countries, countries_prompt_block
-    assert parse_normalized_cities(
+
-        {"cities": ["Karachi", "Karachi", "Lahore", "no city mentioned"]},
+    assert isinstance(Countries, dict)
-        fallback=["Karachi(Malir)"],
+    assert "Pakistan" in Countries
-    ) == ["Karachi", "Lahore"]
+    assert "Karachi" in Countries["Pakistan"]
+    assert "United Kingdom" in Countries
+    assert "London" in Countries["United Kingdom"]
+    assert "United States" in Countries
+    assert "New York" in Countries["United States"]
+    assert CITY_BY_KEY["karachi"] == "Karachi"
+    assert CITY_BY_KEY["london"] == "London"
+    block = countries_prompt_block()
+    assert block.startswith("Afghanistan:")
+    assert "Pakistan: " in block
+    assert "Karachi" in block
 
 
-def test_parse_normalized_cities_falls_back_when_the_model_shape_is_wrong():
+def test_canonical_city_maps_messy_localities():
-    from employment_agent.execute_agent import parse_normalized_cities
+    from employment_agent.decorators import canonical_city
-    assert parse_normalized_cities({"oops": True}, fallback=["Karachi(Malir)"]) == ["Karachi(Malir)"]
+    from employment_agent.prompt import NO_CITY
+
+    assert canonical_city("Karachi(Malir)") == "Karachi"
+    assert canonical_city("Karachi (Malir)") == "Karachi"
+    assert canonical_city("Karachi Malir") == "Karachi"
+    assert canonical_city("DHA Karachi") == "Karachi"
+    assert canonical_city("Karachi DHA") == "Karachi"
+    assert canonical_city("Lahore Cantt") == "Lahore"
+    assert canonical_city("Gulberg, Lahore") == "Lahore"
+    assert canonical_city("F-10 Islamabad") == "Islamabad"
+    assert canonical_city("Wah Cantt") == "Wah"
+    assert canonical_city("Karachi") == "Karachi"
+    assert canonical_city("Karachi Malir Wah Cantt") == "Karachi"
+    assert canonical_city("London(Westminster)") == "London"
+    assert canonical_city("New York") == "New York"
+    assert canonical_city("Dubai Marina") == "Dubai"
+    assert canonical_city("") is None
+    assert canonical_city("   ") is None
+    assert canonical_city(NO_CITY) is None
+    assert canonical_city("none") is None
+    assert canonical_city("n/a") is None
+
+
+def test_list_cities_is_distinct_without_openai(monkeypatch):
+    import asyncio
+    import inspect
+
+    from g_sheet.models import FormData
+    from inbox.models import Inbox_Messages
+    from inbox.views import Email
+
+    source = inspect.getsource(Email.list_cities)
+    assert "normalize_cities" not in source
+    assert "llm_call" not in source
+
+    async def inbox_cities(session):
+        return ["Karachi", "Lahore", "Karachi"]
+
+    async def form_cities(session):
+        return ["Lahore", "Islamabad", "  ", None]
+
+    monkeypatch.setattr(Inbox_Messages, "distinct_cities", inbox_cities)
+    monkeypatch.setattr(FormData, "distinct_cities", form_cities)
+    cities = asyncio.run(Email(session=object()).list_cities())
+    assert cities == ["Islamabad", "Karachi", "Lahore"]

backend/tests/test_employment_extraction_clamps.py

@@ -171,5 +171,13 @@ def test_city_from_the_model_is_kept_as_returned():
     assert fields["city"] == "Karachi"
 
 
+def test_messy_model_city_is_clamped_to_canonical_before_persist():
+    resume = "Ali Khan | Karachi(Malir) | 0321-5551234"
+    fields = parse_employment_response({"city": "Karachi(Malir)"}, resume)
+    assert fields["city"] == "Karachi"
+    assert parse({"city": "DHA Karachi"})["city"] == "Karachi"
+    assert parse({"city": "Wah Cantt"})["city"] == "Wah"
+
+
 def test_city_sentinel_is_dropped():
     assert parse({"city": "no city mentioned"})["city"] is None

backend/tests/test_g_sheet_plugins.py

@@ -66,6 +66,7 @@ def test_from_sheet_row_maps_form_response_keys():
     assert mapped["ho_availability"] == "Yes"
     assert mapped["marital_status"] == "Single"
     assert mapped["residing_city"] == "Karachi"
+    assert mapped["city"] == "Karachi"
     assert mapped["residing_country"] == "Pakistan"
     assert mapped["director_poc_category"] == "Operations"
     assert mapped["hr_comments"] == "Good profile"
@@ -116,6 +117,13 @@ def test_year_of_graduation_single_column():
     assert mapped["entry_year"] == "2012"
 
 
+def test_from_sheet_row_canonicalizes_city_keeps_raw_residing():
+    data = {**FORM_RESPONSE_RECORD, "Residing City": "Karachi(Malir)"}
+    mapped = FormData.from_sheet_row("tab", 9, data)
+    assert mapped["city"] == "Karachi"
+    assert mapped["residing_city"] == "Karachi(Malir)"
+
+
 def test_duplicate_year_header_becomes_year_of_graduation_1():
     headers = plugins.normalise_headers(
         ["Full Name", "Year of Graduation", "Year of Graduation"],