global countires
parent
9f6f453220
commit
cd4743e953
|
|
@ -18,6 +18,17 @@ import re
|
|||
from functools import wraps
|
||||
|
||||
from employment_agent.prompt import EDUCATION,NO_CITY,NO_COMPANY,NO_LINKEDIN,NO_PHONE
|
||||
from global_cities import CITY_BY_KEY,CITY_RE
|
||||
|
||||
_CITY_SENTINELS=frozenset({
|
||||
NO_CITY.lower(),"none","null","n/a","-","na","n.a.","n.a",
|
||||
})
|
||||
_CITY_DROP=frozenset({
|
||||
"dha","cantt","cantonment","cant","phase","sector","area","district",
|
||||
"tehsil","division","housing","society","scheme","block","street","house",
|
||||
"near","colony","neighborhood","neighbourhood","suburb",
|
||||
})
|
||||
_SECTOR_RE=re.compile(r"^(?:[a-z]-?\d+[a-z]?|\d+[a-z]?)$",re.I)
|
||||
|
||||
|
||||
def require_json_object(func):
|
||||
|
|
@ -102,16 +113,54 @@ def _clean_phone(value,resume_text):
|
|||
return text
|
||||
|
||||
|
||||
def _clean_city(value,resume_text):
|
||||
"""Optional residence city. Sentinel → None. Never rejects the CV.
|
||||
def canonical_city(text):
|
||||
"""Write-time only: messy locality → one proper city name, or None.
|
||||
|
||||
Proper city names (Karachi, not Karachi(Malir)) come from the OpenAI parse
|
||||
in run_employment_agent. This clamp does not rewrite place names.
|
||||
Looks up `global_cities.Countries` (every country, Pakistan included).
|
||||
"Karachi(Malir)" / "London(Westminster)" / "DHA Karachi" / "Wah Cantt"
|
||||
map to the listed city. Sentinels and blanks are None. Never rejects a CV.
|
||||
"""
|
||||
text=(value or "").strip()
|
||||
if not text or text.lower() in (NO_CITY.lower(),"none","null","n/a","-"):
|
||||
raw=(text or "").strip()
|
||||
if not raw or raw.lower() in _CITY_SENTINELS:
|
||||
return None
|
||||
return text
|
||||
known=CITY_BY_KEY.get(raw.lower())
|
||||
if known:
|
||||
return known
|
||||
normalised=re.sub(r"[()\[\]{}]"," ",raw)
|
||||
normalised=re.sub(r"[,/;|]+"," ",normalised)
|
||||
normalised=re.sub(r"\s+"," ",normalised).strip()
|
||||
if not normalised:
|
||||
return None
|
||||
known=CITY_BY_KEY.get(normalised.lower())
|
||||
if known:
|
||||
return known
|
||||
match=CITY_RE.search(normalised.lower())
|
||||
if match:
|
||||
return CITY_BY_KEY[match.group(0)]
|
||||
leftover=[]
|
||||
for token in normalised.split():
|
||||
lowered=token.lower()
|
||||
if lowered in _CITY_DROP or _SECTOR_RE.fullmatch(token):
|
||||
continue
|
||||
leftover.append(token)
|
||||
if not leftover:
|
||||
return None
|
||||
cleaned=" ".join(leftover)
|
||||
known=CITY_BY_KEY.get(cleaned.lower())
|
||||
if known:
|
||||
return known
|
||||
if len(cleaned)>40 or len(leftover)>3:
|
||||
return leftover[0][:1].upper()+leftover[0][1:]
|
||||
return " ".join(t[:1].upper()+t[1:] for t in leftover)
|
||||
|
||||
|
||||
def _clean_city(value,resume_text):
|
||||
"""Optional residence city. Sentinel / blank → None. Never rejects the CV.
|
||||
|
||||
After the employment-agent JSON is parsed, clamp to a proper city name so
|
||||
a model that still returns "Karachi(Malir)" is stored as "Karachi".
|
||||
"""
|
||||
return canonical_city(value)
|
||||
|
||||
|
||||
def _clean_skills(value,resume_text):
|
||||
|
|
|
|||
|
|
@ -6,67 +6,15 @@ Called from inbox.tasks.match_inbox_message; no HTTP surface.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from employment_agent.decorators import parse_employment_response
|
||||
from employment_agent.prompt import CURRENT_TITLE,EDUCATION,NO_CITY,NO_COMPANY,city_list_prompt,prompt,user_prompt
|
||||
from employment_agent.prompt import CURRENT_TITLE,EDUCATION,NO_COMPANY,prompt,user_prompt
|
||||
from llm_setup import llm_call
|
||||
|
||||
logger=logging.getLogger("employment_agent")
|
||||
|
||||
|
||||
def parse_normalized_cities(data,fallback=None):
|
||||
"""Keep unique proper city names from the list-normalizer JSON."""
|
||||
rows=None
|
||||
if isinstance(data,dict):
|
||||
rows=data.get("cities")
|
||||
if not isinstance(rows,list):
|
||||
return list(fallback or [])
|
||||
out=[]
|
||||
seen=set()
|
||||
for item in rows:
|
||||
if not isinstance(item,str):
|
||||
continue
|
||||
text=item.strip()
|
||||
if not text or text.lower() in (NO_CITY.lower(),"none","null","n/a","-"):
|
||||
continue
|
||||
key=text.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(text)
|
||||
out.sort(key=str.lower)
|
||||
return out or list(fallback or [])
|
||||
|
||||
|
||||
async def normalize_cities(values):
|
||||
"""OpenAI: messy stored places → the same proper city names the CV agent writes."""
|
||||
places=[]
|
||||
seen=set()
|
||||
for raw in values or []:
|
||||
text=(raw or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
key=text.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
places.append(text)
|
||||
if not places:
|
||||
return []
|
||||
try:
|
||||
data=await llm_call(
|
||||
city_list_prompt(),
|
||||
json.dumps({"places":places},ensure_ascii=False),
|
||||
json_mode=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("city list normalize failed")
|
||||
return places
|
||||
return parse_normalized_cities(data,fallback=places)
|
||||
|
||||
|
||||
async def run_employment_agent(*,resume_text=""):
|
||||
text=(resume_text or "").strip()
|
||||
if not text:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
|
||||
from global_cities import countries_prompt_block
|
||||
|
||||
NO_COMPANY="no company was mentioned"
|
||||
EDUCATION="No Education Mentioned"
|
||||
CURRENT_TITLE="No JOB POSITION MENTIONED"
|
||||
|
|
@ -14,13 +16,14 @@ NO_LINKEDIN="no linkedin url mentioned"
|
|||
NO_PHONE="no phone number mentioned"
|
||||
NO_CITY="no city mentioned"
|
||||
|
||||
CITY_POLICY=f"""- Return ONE proper city name only — the city, not an area, town, sector, housing society, cantonment, district, or parenthetical locality.
|
||||
- Correct values look like "Karachi", "Lahore", "Islamabad", "Rawalpindi", "Peshawar". Not "Karachi(Malir)", not "Wah Cantt", not "Gulberg Lahore".
|
||||
- If the text names a neighborhood or area of a city, return that city: "Karachi (Malir)" / "Karachi(Malir)" / "DHA Karachi" → "Karachi". "Gulberg, Lahore" → "Lahore". "F-10 Islamabad" → "Islamabad".
|
||||
- Drop "Cantt" / "Cantonment": "Lahore Cantt" → "Lahore", "Rawalpindi Cantt" → "Rawalpindi", "Wah Cantt" → "Wah".
|
||||
CITY_POLICY="""- Return ONE proper city name only — the city, not an area, town, sector, housing society, cantonment, district, or parenthetical locality.
|
||||
- Identify the city if possible. Map it to exactly one city name from the country→cities list supplied below. Pakistan is in that list along with every other country — do not prefer one country.
|
||||
- Return the city name only, never the country. If the text names a neighborhood or area of a listed city, return that city: "Karachi(Malir)" / "Karachi Malir" / "DHA Karachi" → "Karachi". "London(Westminster)" → "London". "Gulberg, Lahore" → "Lahore". "F-10 Islamabad" → "Islamabad".
|
||||
- Drop "Cantt" / "Cantonment" and housing-society prefixes: "Lahore Cantt" → "Lahore", "Wah Cantt" → "Wah".
|
||||
- Never concatenate two places. If the string is messy (for example "Karachi(Malir) Wah Cantt"), return the single residence city, not both strings glued together.
|
||||
- Do not return province, country, street, house number, or text inside parentheses.
|
||||
- Drop junk tokens such as KA, KAR, KARA, empty values, and unintelligible strings."""
|
||||
- Do not return province, country, street, house number, neighborhood, cantonment, or text inside parentheses.
|
||||
- Drop junk tokens, empty values, and unintelligible strings.
|
||||
- If you cannot map the residence to a listed city, still return a single proper city name. If none is stated, use the no-city sentinel."""
|
||||
|
||||
|
||||
def prompt():
|
||||
|
|
@ -70,6 +73,9 @@ city (its own key — OPTIONAL. A missing city must not fail the candidate):
|
|||
- Do NOT extract city from Work Experience. A job that lists Karachi, UAE, USA, or any other city is the employer's location, not proof the candidate lives there.
|
||||
- If the contact/location section does not name a city, return exactly: {NO_CITY}. Leave it blank rather than guessing from jobs, education, or nationality.
|
||||
|
||||
Country → cities (map messy locality to exactly one city from this list; return the city, never the country):
|
||||
{countries_prompt_block()}
|
||||
|
||||
phone (its own key — extract this separately; copy EVERY digit):
|
||||
- Return the candidate's own mobile / phone exactly as written, including country code when present.
|
||||
- Pakistani mobiles are 11 digits local (03XX-XXXXXXX / 03XX XXXXXXX) or +92 3XX XXXXXXX (12 digits with country code). Copy the last group in full — never stop after 7 or 8 digits.
|
||||
|
|
@ -129,7 +135,10 @@ Example 10 — neighborhood / cantonment is not the city:
|
|||
Resume: "Ali Khan | Karachi(Malir) | 0321-5551234"
|
||||
JSON city must be "Karachi". Not "Karachi(Malir)" and not "Malir".
|
||||
|
||||
Example 11 — do not glue two place fragments:
|
||||
Example 11 — DHA / sector / cantonment still collapse to the city:
|
||||
Resume: "Address: DHA Karachi" → "Karachi". "Lahore Cantt" → "Lahore". "F-10 Islamabad" → "Islamabad". "Wah Cantt" → "Wah". "London(Westminster)" → "London".
|
||||
|
||||
Example 12 — do not glue two place fragments:
|
||||
Resume: "Address: Karachi(Malir) Wah Cantt"
|
||||
JSON city must be "Karachi" (one city). Not "Karachi(Malir) Wah Cantt" and not "Wah Cantt".
|
||||
|
||||
|
|
@ -150,16 +159,3 @@ If the contact/location section has no city, city must be "{NO_CITY}" — still
|
|||
|
||||
def user_prompt(resume_text:str) -> str:
|
||||
return json.dumps({"resume_text":resume_text or ""},ensure_ascii=False)
|
||||
|
||||
|
||||
def city_list_prompt():
|
||||
"""Map messy stored place strings to the same proper city names the CV agent writes."""
|
||||
return f"""You map messy residence strings to proper city names for an Inbox City filter.
|
||||
|
||||
{CITY_POLICY}
|
||||
|
||||
Input is JSON: {{"places": ["Karachi(Malir)", "Wah Cantt", "Lahore"]}}
|
||||
Respond with JSON only:
|
||||
{{"cities": ["Karachi", "Wah", "Lahore"]}}
|
||||
Unique proper city names only. Do not copy raw neighborhood or cantonment strings into cities.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -562,9 +562,8 @@ class FormData(SQLModel, table=True):
|
|||
@classmethod
|
||||
async def distinct_cities(cls, session: AsyncSession):
|
||||
"""Non-blank city values on this table. Distinct only within form_data."""
|
||||
col = func.coalesce(cls.city, cls.residing_city)
|
||||
result = await session.execute(
|
||||
select(col).where(col.is_not(None), col != "").distinct()
|
||||
select(cls.city).where(cls.city.is_not(None), cls.city != "").distinct()
|
||||
)
|
||||
return [text.strip() for text in result.scalars().all() if (text or "").strip()]
|
||||
|
||||
|
|
@ -660,6 +659,7 @@ class FormData(SQLModel, table=True):
|
|||
Year of Graduation: prefer the second column when present; else the first;
|
||||
else None. Duplicate headers are renamed Year of Graduation_1 by normalise_headers.
|
||||
"""
|
||||
from employment_agent.decorators import canonical_city
|
||||
from g_sheet.plugins import parse_date, parse_date_time, parse_salary
|
||||
|
||||
first_year = cls._cell(data, "Year of Graduation")
|
||||
|
|
@ -676,6 +676,7 @@ class FormData(SQLModel, table=True):
|
|||
|
||||
current_salary = cls._cell(data, "Current Salary")
|
||||
expected_salary = cls._cell(data, "Expected Salary")
|
||||
residing_city = cls._cell(data, "Residing City")
|
||||
|
||||
return {
|
||||
"sheet": sheet,
|
||||
|
|
@ -694,8 +695,8 @@ class FormData(SQLModel, table=True):
|
|||
"position_applied_for": cls._cell(data, "Position Applied For"),
|
||||
"profile_link": cls._cell(data, "LinkedIn Profile Link"),
|
||||
"residing_country": cls._cell(data, "Residing Country"),
|
||||
"residing_city": cls._cell(data, "Residing City"),
|
||||
"city": cls._cell(data, "Residing City"),
|
||||
"residing_city": residing_city,
|
||||
"city": canonical_city(residing_city),
|
||||
"ho_availability": cls._cell(data, "Are you willing to relocate?"),
|
||||
"degree": cls._cell(data, "Educational Degree"),
|
||||
"university": cls._cell(data, "University"),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,247 @@
|
|||
"""Global country → cities dataset for residence canonicalization.
|
||||
|
||||
Pakistan is one country in this map, not a special case. The employment-agent
|
||||
prompt receives `countries_prompt_block()` so the model can map a messy
|
||||
locality to exactly one city name. `canonical_city` uses the same index.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
Countries={
|
||||
"Afghanistan":["Kabul","Kandahar","Herat","Mazar-i-Sharif","Jalalabad"],
|
||||
"Albania":["Tirana","Durres","Vlore","Shkoder"],
|
||||
"Algeria":["Algiers","Oran","Constantine","Annaba"],
|
||||
"Andorra":["Andorra la Vella"],
|
||||
"Angola":["Luanda","Huambo","Lobito","Benguela"],
|
||||
"Argentina":["Buenos Aires","Cordoba","Rosario","Mendoza","La Plata"],
|
||||
"Armenia":["Yerevan","Gyumri","Vanadzor"],
|
||||
"Australia":["Sydney","Melbourne","Brisbane","Perth","Adelaide","Canberra","Gold Coast","Hobart","Darwin"],
|
||||
"Austria":["Vienna","Graz","Linz","Salzburg","Innsbruck"],
|
||||
"Azerbaijan":["Baku","Ganja","Sumqayit"],
|
||||
"Bahamas":["Nassau","Freeport"],
|
||||
"Bahrain":["Manama","Riffa","Muharraq"],
|
||||
"Bangladesh":["Dhaka","Chittagong","Khulna","Rajshahi","Sylhet","Gazipur","Narayanganj"],
|
||||
"Belarus":["Minsk","Gomel","Mogilev","Vitebsk"],
|
||||
"Belgium":["Brussels","Antwerp","Ghent","Charleroi","Liege","Bruges"],
|
||||
"Belize":["Belmopan","Belize City"],
|
||||
"Benin":["Porto-Novo","Cotonou"],
|
||||
"Bhutan":["Thimphu","Phuntsholing"],
|
||||
"Bolivia":["La Paz","Santa Cruz","Cochabamba","Sucre"],
|
||||
"Bosnia and Herzegovina":["Sarajevo","Banja Luka","Mostar","Tuzla"],
|
||||
"Botswana":["Gaborone","Francistown"],
|
||||
"Brazil":["Sao Paulo","Rio de Janeiro","Brasilia","Salvador","Fortaleza","Belo Horizonte","Manaus","Curitiba","Recife","Porto Alegre"],
|
||||
"Brunei":["Bandar Seri Begawan"],
|
||||
"Bulgaria":["Sofia","Plovdiv","Varna","Burgas"],
|
||||
"Burkina Faso":["Ouagadougou","Bobo-Dioulasso"],
|
||||
"Burundi":["Gitega","Bujumbura"],
|
||||
"Cambodia":["Phnom Penh","Siem Reap","Sihanoukville"],
|
||||
"Cameroon":["Yaounde","Douala","Garoua"],
|
||||
"Canada":["Toronto","Montreal","Vancouver","Calgary","Ottawa","Edmonton","Winnipeg","Quebec City","Hamilton","Halifax"],
|
||||
"Cape Verde":["Praia","Mindelo"],
|
||||
"Central African Republic":["Bangui"],
|
||||
"Chad":["N'Djamena","Moundou"],
|
||||
"Chile":["Santiago","Valparaiso","Concepcion","Antofagasta"],
|
||||
"China":["Beijing","Shanghai","Guangzhou","Shenzhen","Chengdu","Chongqing","Tianjin","Wuhan","Hangzhou","Nanjing","Xi'an","Suzhou","Dongguan","Qingdao","Dalian"],
|
||||
"Colombia":["Bogota","Medellin","Cali","Barranquilla","Cartagena"],
|
||||
"Comoros":["Moroni"],
|
||||
"Congo":["Brazzaville","Pointe-Noire"],
|
||||
"Costa Rica":["San Jose","Alajuela","Cartago"],
|
||||
"Croatia":["Zagreb","Split","Rijeka","Osijek"],
|
||||
"Cuba":["Havana","Santiago de Cuba","Camaguey"],
|
||||
"Cyprus":["Nicosia","Limassol","Larnaca","Paphos"],
|
||||
"Czech Republic":["Prague","Brno","Ostrava","Plzen"],
|
||||
"Democratic Republic of the Congo":["Kinshasa","Lubumbashi","Mbuji-Mayi"],
|
||||
"Denmark":["Copenhagen","Aarhus","Odense","Aalborg"],
|
||||
"Djibouti":["Djibouti"],
|
||||
"Dominican Republic":["Santo Domingo","Santiago"],
|
||||
"Ecuador":["Quito","Guayaquil","Cuenca"],
|
||||
"Egypt":["Cairo","Alexandria","Giza","Shubra El Kheima","Port Said","Suez","Luxor"],
|
||||
"El Salvador":["San Salvador","Santa Ana","San Miguel"],
|
||||
"Equatorial Guinea":["Malabo","Bata"],
|
||||
"Eritrea":["Asmara"],
|
||||
"Estonia":["Tallinn","Tartu"],
|
||||
"Eswatini":["Mbabane","Manzini"],
|
||||
"Ethiopia":["Addis Ababa","Dire Dawa","Mekelle"],
|
||||
"Fiji":["Suva","Nadi"],
|
||||
"Finland":["Helsinki","Espoo","Tampere","Oulu","Turku"],
|
||||
"France":["Paris","Marseille","Lyon","Toulouse","Nice","Nantes","Strasbourg","Bordeaux","Lille","Rennes"],
|
||||
"Gabon":["Libreville"],
|
||||
"Gambia":["Banjul","Serekunda"],
|
||||
"Georgia":["Tbilisi","Batumi","Kutaisi"],
|
||||
"Germany":["Berlin","Hamburg","Munich","Cologne","Frankfurt","Stuttgart","Dusseldorf","Dortmund","Essen","Leipzig","Dresden","Hanover","Nuremberg"],
|
||||
"Ghana":["Accra","Kumasi","Tamale","Takoradi"],
|
||||
"Greece":["Athens","Thessaloniki","Patras","Heraklion"],
|
||||
"Guatemala":["Guatemala City","Quetzaltenango"],
|
||||
"Guinea":["Conakry"],
|
||||
"Guyana":["Georgetown"],
|
||||
"Haiti":["Port-au-Prince","Cap-Haitien"],
|
||||
"Honduras":["Tegucigalpa","San Pedro Sula"],
|
||||
"Hungary":["Budapest","Debrecen","Szeged","Miskolc"],
|
||||
"Iceland":["Reykjavik"],
|
||||
"India":["Mumbai","Delhi","Bengaluru","Hyderabad","Ahmedabad","Chennai","Kolkata","Pune","Jaipur","Surat","Lucknow","Kanpur","Nagpur","Indore","Bhopal","Patna","Chandigarh","Noida","Gurgaon","Kochi","Coimbatore"],
|
||||
"Indonesia":["Jakarta","Surabaya","Bandung","Medan","Bekasi","Depok","Tangerang","Semarang","Makassar","Palembang"],
|
||||
"Iran":["Tehran","Mashhad","Isfahan","Karaj","Shiraz","Tabriz","Qom","Ahvaz"],
|
||||
"Iraq":["Baghdad","Basra","Mosul","Erbil","Najaf","Karbala","Sulaymaniyah"],
|
||||
"Ireland":["Dublin","Cork","Limerick","Galway","Waterford"],
|
||||
"Israel":["Jerusalem","Tel Aviv","Haifa","Rishon LeZion","Petah Tikva"],
|
||||
"Italy":["Rome","Milan","Naples","Turin","Palermo","Genoa","Bologna","Florence","Venice","Bari"],
|
||||
"Ivory Coast":["Yamoussoukro","Abidjan"],
|
||||
"Jamaica":["Kingston","Montego Bay"],
|
||||
"Japan":["Tokyo","Yokohama","Osaka","Nagoya","Sapporo","Fukuoka","Kobe","Kyoto","Kawasaki","Saitama","Hiroshima","Sendai"],
|
||||
"Jordan":["Amman","Zarqa","Irbid","Aqaba"],
|
||||
"Kazakhstan":["Astana","Almaty","Shymkent","Aktobe"],
|
||||
"Kenya":["Nairobi","Mombasa","Kisumu","Nakuru"],
|
||||
"Kuwait":["Kuwait City","Hawalli","Salmiya","Jahra"],
|
||||
"Kyrgyzstan":["Bishkek","Osh"],
|
||||
"Laos":["Vientiane","Luang Prabang"],
|
||||
"Latvia":["Riga","Daugavpils"],
|
||||
"Lebanon":["Beirut","Tripoli","Sidon","Zahle"],
|
||||
"Lesotho":["Maseru"],
|
||||
"Liberia":["Monrovia"],
|
||||
"Libya":["Tripoli","Benghazi","Misrata"],
|
||||
"Liechtenstein":["Vaduz"],
|
||||
"Lithuania":["Vilnius","Kaunas","Klaipeda"],
|
||||
"Luxembourg":["Luxembourg"],
|
||||
"Madagascar":["Antananarivo","Toamasina"],
|
||||
"Malawi":["Lilongwe","Blantyre"],
|
||||
"Malaysia":["Kuala Lumpur","George Town","Johor Bahru","Ipoh","Shah Alam","Petaling Jaya","Kota Kinabalu","Kuching","Malacca"],
|
||||
"Maldives":["Male"],
|
||||
"Mali":["Bamako"],
|
||||
"Malta":["Valletta","Birkirkara"],
|
||||
"Mauritania":["Nouakchott"],
|
||||
"Mauritius":["Port Louis"],
|
||||
"Mexico":["Mexico City","Guadalajara","Monterrey","Puebla","Tijuana","Leon","Juarez","Merida","Cancun","Queretaro"],
|
||||
"Moldova":["Chisinau"],
|
||||
"Monaco":["Monaco"],
|
||||
"Mongolia":["Ulaanbaatar"],
|
||||
"Montenegro":["Podgorica","Niksic"],
|
||||
"Morocco":["Rabat","Casablanca","Fes","Marrakesh","Tangier","Agadir","Meknes"],
|
||||
"Mozambique":["Maputo","Beira","Nampula"],
|
||||
"Myanmar":["Naypyidaw","Yangon","Mandalay"],
|
||||
"Namibia":["Windhoek","Walvis Bay"],
|
||||
"Nepal":["Kathmandu","Pokhara","Lalitpur","Biratnagar"],
|
||||
"Netherlands":["Amsterdam","Rotterdam","The Hague","Utrecht","Eindhoven","Groningen"],
|
||||
"New Zealand":["Auckland","Wellington","Christchurch","Hamilton","Dunedin"],
|
||||
"Nicaragua":["Managua"],
|
||||
"Niger":["Niamey"],
|
||||
"Nigeria":["Abuja","Lagos","Kano","Ibadan","Port Harcourt","Benin City","Kaduna"],
|
||||
"North Korea":["Pyongyang"],
|
||||
"North Macedonia":["Skopje"],
|
||||
"Norway":["Oslo","Bergen","Trondheim","Stavanger"],
|
||||
"Oman":["Muscat","Salalah","Sohar","Nizwa"],
|
||||
"Pakistan":[
|
||||
"Karachi","Lahore","Islamabad","Rawalpindi","Peshawar","Quetta","Faisalabad",
|
||||
"Multan","Hyderabad","Sialkot","Gujranwala","Sargodha","Bahawalpur",
|
||||
"Sukkur","Larkana","Sheikhupura","Rahim Yar Khan","Sahiwal","Jhang","Okara",
|
||||
"Gujrat","Kasur","Dera Ghazi Khan","Mardan","Abbottabad","Mingora","Nawabshah",
|
||||
"Mirpur","Muzaffarabad","Gilgit","Skardu","Wah","Attock","Jhelum","Chakwal",
|
||||
"Taxila","Kamra","Haripur","Mansehra","Kohat","Bannu","Dera Ismail Khan",
|
||||
"Charsadda","Nowshera","Swat","Chitral","Swabi","Jacobabad","Khairpur","Thatta",
|
||||
"Gwadar","Turbat","Hub","Kotri","Jamshoro","Shikarpur","Dadu","Badin","Khuzdar",
|
||||
"Chaman","Kamoke","Muridke","Hafizabad","Narowal","Pakpattan","Vehari","Khanewal",
|
||||
"Layyah","Burewala","Gojra","Chiniot","Bhakkar","Mianwali","Khushab","Murree",
|
||||
"Kotli","Bhimber","Rawalakot","Toba Tek Singh","Mandi Bahauddin","Muzaffargarh",
|
||||
"Mirpur Khas","Hasan Abdal",
|
||||
],
|
||||
"Palestine":["Gaza","Ramallah","Hebron","Nablus"],
|
||||
"Panama":["Panama City","Colon"],
|
||||
"Papua New Guinea":["Port Moresby"],
|
||||
"Paraguay":["Asuncion","Ciudad del Este"],
|
||||
"Peru":["Lima","Arequipa","Trujillo","Cusco"],
|
||||
"Philippines":["Manila","Quezon City","Davao","Cebu","Zamboanga","Taguig","Pasig","Cagayan de Oro"],
|
||||
"Poland":["Warsaw","Krakow","Lodz","Wroclaw","Poznan","Gdansk","Szczecin"],
|
||||
"Portugal":["Lisbon","Porto","Braga","Coimbra","Faro"],
|
||||
"Qatar":["Doha","Al Rayyan","Al Wakrah"],
|
||||
"Romania":["Bucharest","Cluj-Napoca","Timisoara","Iasi","Constanta","Brasov"],
|
||||
"Russia":["Moscow","Saint Petersburg","Novosibirsk","Yekaterinburg","Kazan","Nizhny Novgorod","Chelyabinsk","Samara","Rostov-on-Don","Ufa"],
|
||||
"Rwanda":["Kigali"],
|
||||
"Saudi Arabia":["Riyadh","Jeddah","Mecca","Medina","Dammam","Khobar","Dhahran","Tabuk","Abha","Taif"],
|
||||
"Senegal":["Dakar","Touba","Thies"],
|
||||
"Serbia":["Belgrade","Novi Sad","Nis"],
|
||||
"Seychelles":["Victoria"],
|
||||
"Sierra Leone":["Freetown"],
|
||||
"Singapore":["Singapore"],
|
||||
"Slovakia":["Bratislava","Kosice"],
|
||||
"Slovenia":["Ljubljana","Maribor"],
|
||||
"Somalia":["Mogadishu","Hargeisa"],
|
||||
"South Africa":["Johannesburg","Cape Town","Durban","Pretoria","Port Elizabeth","Bloemfontein","East London","Soweto"],
|
||||
"South Korea":["Seoul","Busan","Incheon","Daegu","Daejeon","Gwangju","Suwon","Ulsan"],
|
||||
"South Sudan":["Juba"],
|
||||
"Spain":["Madrid","Barcelona","Valencia","Seville","Zaragoza","Malaga","Murcia","Palma","Bilbao","Alicante"],
|
||||
"Sri Lanka":["Colombo","Kandy","Galle","Jaffna","Negombo"],
|
||||
"Sudan":["Khartoum","Omdurman","Port Sudan"],
|
||||
"Suriname":["Paramaribo"],
|
||||
"Sweden":["Stockholm","Gothenburg","Malmo","Uppsala"],
|
||||
"Switzerland":["Zurich","Geneva","Basel","Bern","Lausanne","Lucerne"],
|
||||
"Syria":["Damascus","Aleppo","Homs","Latakia"],
|
||||
"Taiwan":["Taipei","Kaohsiung","Taichung","Tainan"],
|
||||
"Tajikistan":["Dushanbe"],
|
||||
"Tanzania":["Dodoma","Dar es Salaam","Mwanza","Arusha","Zanzibar"],
|
||||
"Thailand":["Bangkok","Chiang Mai","Pattaya","Phuket","Nonthaburi","Hat Yai"],
|
||||
"Togo":["Lome"],
|
||||
"Trinidad and Tobago":["Port of Spain","San Fernando"],
|
||||
"Tunisia":["Tunis","Sfax","Sousse"],
|
||||
"Turkey":["Istanbul","Ankara","Izmir","Bursa","Antalya","Adana","Gaziantep","Konya","Mersin"],
|
||||
"Turkmenistan":["Ashgabat"],
|
||||
"Uganda":["Kampala","Gulu"],
|
||||
"Ukraine":["Kyiv","Kharkiv","Odesa","Dnipro","Lviv","Zaporizhzhia"],
|
||||
"United Arab Emirates":["Dubai","Abu Dhabi","Sharjah","Ajman","Ras Al Khaimah","Fujairah","Al Ain","Umm Al Quwain"],
|
||||
"United Kingdom":["London","Birmingham","Manchester","Glasgow","Liverpool","Leeds","Sheffield","Edinburgh","Bristol","Leicester","Newcastle","Cardiff","Belfast","Nottingham","Southampton"],
|
||||
"United States":[
|
||||
"New York","Los Angeles","Chicago","Houston","Phoenix","Philadelphia","San Antonio",
|
||||
"San Diego","Dallas","San Jose","Austin","Jacksonville","Fort Worth","Columbus",
|
||||
"Charlotte","San Francisco","Indianapolis","Seattle","Denver","Washington",
|
||||
"Boston","Nashville","Detroit","Portland","Las Vegas","Baltimore","Milwaukee",
|
||||
"Albuquerque","Atlanta","Miami","Minneapolis","Tampa","Orlando","Cleveland",
|
||||
"Pittsburgh","Cincinnati","Kansas City","St. Louis","Raleigh","Salt Lake City",
|
||||
],
|
||||
"Uruguay":["Montevideo"],
|
||||
"Uzbekistan":["Tashkent","Samarkand","Bukhara"],
|
||||
"Venezuela":["Caracas","Maracaibo","Valencia"],
|
||||
"Vietnam":["Hanoi","Ho Chi Minh City","Da Nang","Hai Phong","Can Tho"],
|
||||
"Yemen":["Sanaa","Aden","Taiz"],
|
||||
"Zambia":["Lusaka","Ndola","Kitwe"],
|
||||
"Zimbabwe":["Harare","Bulawayo"],
|
||||
}
|
||||
|
||||
|
||||
def _city_index():
|
||||
"""First spelling of each city name wins. Longest names are matched first."""
|
||||
out={}
|
||||
for cities in Countries.values():
|
||||
for city in cities:
|
||||
name=(city or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
out.setdefault(name.lower(),name)
|
||||
return out
|
||||
|
||||
|
||||
CITY_BY_KEY=_city_index()
|
||||
CITY_RE=re.compile(
|
||||
r"\b(?:"+"|".join(
|
||||
re.escape(name) for name in sorted(CITY_BY_KEY,key=len,reverse=True)
|
||||
)+r")\b",
|
||||
)
|
||||
|
||||
|
||||
def countries_prompt_block():
|
||||
"""Compact country → cities block fed into the employment-agent prompt."""
|
||||
lines=[]
|
||||
for country,cities in Countries.items():
|
||||
names=[c.strip() for c in cities if (c or "").strip()]
|
||||
if not names:
|
||||
continue
|
||||
# De-dupe while keeping order — Pakistan lists Peshawar twice above.
|
||||
seen=set()
|
||||
unique=[]
|
||||
for name in names:
|
||||
key=name.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
unique.append(name)
|
||||
lines.append(f"{country}: {', '.join(unique)}")
|
||||
return "\n".join(lines)
|
||||
|
|
@ -747,13 +747,11 @@ class Email:
|
|||
return await Inbox_Messages.count_inbox_messages(self.session,search,assigned=assigned,is_duplicate=is_duplicate,no_suggestions=no_suggestions,processing_state=processing_state,city=city,source=source)
|
||||
|
||||
async def list_cities(self):
|
||||
"""Proper city names for the Inbox filter — same OpenAI mapping as CV parse."""
|
||||
from employment_agent.execute_agent import normalize_cities
|
||||
"""Proper city names for the Inbox filter — DISTINCT of the stored city column."""
|
||||
from g_sheet.models import FormData
|
||||
inbox=await Inbox_Messages.distinct_cities(self.session)
|
||||
forms=await FormData.distinct_cities(self.session)
|
||||
merged=Reapplied(session=self.session).merge_cities(inbox,forms)
|
||||
return await normalize_cities(merged)
|
||||
return Reapplied(session=self.session).merge_cities(inbox,forms)
|
||||
|
||||
async def list_sources(self):
|
||||
"""Source / platform labels: seeded channels, Google Sheet, form sources."""
|
||||
|
|
|
|||
|
|
@ -91,25 +91,81 @@ def test_adds_scheme_and_rejects_company_page():
|
|||
assert company_page["linkedin_url"] is None
|
||||
|
||||
|
||||
def test_city_prompt_asks_openai_for_a_proper_city_name():
|
||||
from employment_agent.prompt import city_list_prompt, prompt
|
||||
def test_city_prompt_asks_for_a_proper_city_name():
|
||||
from employment_agent.prompt import prompt
|
||||
text = prompt()
|
||||
assert "Karachi(Malir)" in text
|
||||
assert 'JSON city must be "Karachi"' in text
|
||||
assert "Return ONE proper city name only" in text
|
||||
listed = city_list_prompt()
|
||||
assert "Karachi(Malir)" in listed
|
||||
assert '"cities"' in listed
|
||||
assert "city_list_prompt" not in text
|
||||
assert "Pakistan:" in text
|
||||
assert "United Kingdom:" in text
|
||||
assert "Karachi" in text
|
||||
assert "London" in text
|
||||
|
||||
|
||||
def test_parse_normalized_cities_keeps_agent_names():
|
||||
from employment_agent.execute_agent import parse_normalized_cities
|
||||
assert parse_normalized_cities(
|
||||
{"cities": ["Karachi", "Karachi", "Lahore", "no city mentioned"]},
|
||||
fallback=["Karachi(Malir)"],
|
||||
) == ["Karachi", "Lahore"]
|
||||
def test_countries_dataset_is_global_and_includes_pakistan():
|
||||
from global_cities import CITY_BY_KEY, Countries, countries_prompt_block
|
||||
|
||||
assert isinstance(Countries, dict)
|
||||
assert "Pakistan" in Countries
|
||||
assert "Karachi" in Countries["Pakistan"]
|
||||
assert "United Kingdom" in Countries
|
||||
assert "London" in Countries["United Kingdom"]
|
||||
assert "United States" in Countries
|
||||
assert "New York" in Countries["United States"]
|
||||
assert CITY_BY_KEY["karachi"] == "Karachi"
|
||||
assert CITY_BY_KEY["london"] == "London"
|
||||
block = countries_prompt_block()
|
||||
assert block.startswith("Afghanistan:")
|
||||
assert "Pakistan: " in block
|
||||
assert "Karachi" in block
|
||||
|
||||
|
||||
def test_parse_normalized_cities_falls_back_when_the_model_shape_is_wrong():
|
||||
from employment_agent.execute_agent import parse_normalized_cities
|
||||
assert parse_normalized_cities({"oops": True}, fallback=["Karachi(Malir)"]) == ["Karachi(Malir)"]
|
||||
def test_canonical_city_maps_messy_localities():
|
||||
from employment_agent.decorators import canonical_city
|
||||
from employment_agent.prompt import NO_CITY
|
||||
|
||||
assert canonical_city("Karachi(Malir)") == "Karachi"
|
||||
assert canonical_city("Karachi (Malir)") == "Karachi"
|
||||
assert canonical_city("Karachi Malir") == "Karachi"
|
||||
assert canonical_city("DHA Karachi") == "Karachi"
|
||||
assert canonical_city("Karachi DHA") == "Karachi"
|
||||
assert canonical_city("Lahore Cantt") == "Lahore"
|
||||
assert canonical_city("Gulberg, Lahore") == "Lahore"
|
||||
assert canonical_city("F-10 Islamabad") == "Islamabad"
|
||||
assert canonical_city("Wah Cantt") == "Wah"
|
||||
assert canonical_city("Karachi") == "Karachi"
|
||||
assert canonical_city("Karachi Malir Wah Cantt") == "Karachi"
|
||||
assert canonical_city("London(Westminster)") == "London"
|
||||
assert canonical_city("New York") == "New York"
|
||||
assert canonical_city("Dubai Marina") == "Dubai"
|
||||
assert canonical_city("") is None
|
||||
assert canonical_city(" ") is None
|
||||
assert canonical_city(NO_CITY) is None
|
||||
assert canonical_city("none") is None
|
||||
assert canonical_city("n/a") is None
|
||||
|
||||
|
||||
def test_list_cities_is_distinct_without_openai(monkeypatch):
|
||||
import asyncio
|
||||
import inspect
|
||||
|
||||
from g_sheet.models import FormData
|
||||
from inbox.models import Inbox_Messages
|
||||
from inbox.views import Email
|
||||
|
||||
source = inspect.getsource(Email.list_cities)
|
||||
assert "normalize_cities" not in source
|
||||
assert "llm_call" not in source
|
||||
|
||||
async def inbox_cities(session):
|
||||
return ["Karachi", "Lahore", "Karachi"]
|
||||
|
||||
async def form_cities(session):
|
||||
return ["Lahore", "Islamabad", " ", None]
|
||||
|
||||
monkeypatch.setattr(Inbox_Messages, "distinct_cities", inbox_cities)
|
||||
monkeypatch.setattr(FormData, "distinct_cities", form_cities)
|
||||
cities = asyncio.run(Email(session=object()).list_cities())
|
||||
assert cities == ["Islamabad", "Karachi", "Lahore"]
|
||||
|
|
|
|||
|
|
@ -171,5 +171,13 @@ def test_city_from_the_model_is_kept_as_returned():
|
|||
assert fields["city"] == "Karachi"
|
||||
|
||||
|
||||
def test_messy_model_city_is_clamped_to_canonical_before_persist():
|
||||
resume = "Ali Khan | Karachi(Malir) | 0321-5551234"
|
||||
fields = parse_employment_response({"city": "Karachi(Malir)"}, resume)
|
||||
assert fields["city"] == "Karachi"
|
||||
assert parse({"city": "DHA Karachi"})["city"] == "Karachi"
|
||||
assert parse({"city": "Wah Cantt"})["city"] == "Wah"
|
||||
|
||||
|
||||
def test_city_sentinel_is_dropped():
|
||||
assert parse({"city": "no city mentioned"})["city"] is None
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ def test_from_sheet_row_maps_form_response_keys():
|
|||
assert mapped["ho_availability"] == "Yes"
|
||||
assert mapped["marital_status"] == "Single"
|
||||
assert mapped["residing_city"] == "Karachi"
|
||||
assert mapped["city"] == "Karachi"
|
||||
assert mapped["residing_country"] == "Pakistan"
|
||||
assert mapped["director_poc_category"] == "Operations"
|
||||
assert mapped["hr_comments"] == "Good profile"
|
||||
|
|
@ -116,6 +117,13 @@ def test_year_of_graduation_single_column():
|
|||
assert mapped["entry_year"] == "2012"
|
||||
|
||||
|
||||
def test_from_sheet_row_canonicalizes_city_keeps_raw_residing():
|
||||
data = {**FORM_RESPONSE_RECORD, "Residing City": "Karachi(Malir)"}
|
||||
mapped = FormData.from_sheet_row("tab", 9, data)
|
||||
assert mapped["city"] == "Karachi"
|
||||
assert mapped["residing_city"] == "Karachi(Malir)"
|
||||
|
||||
|
||||
def test_duplicate_year_header_becomes_year_of_graduation_1():
|
||||
headers = plugins.normalise_headers(
|
||||
["Full Name", "Year of Graduation", "Year of Graduation"],
|
||||
|
|
|
|||
Loading…
Reference in New Issue