HR-ATS-Portal/fix_inbox_cities.py

"""One-off: rewrite inbox_messages.city to a proper city name.

Uses backend/global_cities.py as the city list (every country, not Pakistan-only).
Standalone: does not import the app. Reads backend/.env for DB settings.

  python fix_inbox_cities.py
"""

from __future__ import annotations

import os
import re
import sys
from pathlib import Path

import psycopg2

ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT / "backend"))
from global_cities import CITY_BY_KEY, CITY_RE  # noqa: E402

# Localities that do not contain the city name (Shahrah-e-Faisal, Malir, …).
ALIASES = {
    "malir": "Karachi",
    "clifton": "Karachi",
    "korangi": "Karachi",
    "landhi": "Karachi",
    "pechs": "Karachi",
    "saddar": "Karachi",
    "lyari": "Karachi",
    "orangi": "Karachi",
    "nazimabad": "Karachi",
    "north nazimabad": "Karachi",
    "gulshan": "Karachi",
    "gulshan e iqbal": "Karachi",
    "gulistan e jauhar": "Karachi",
    "jauhar": "Karachi",
    "shah faisal": "Karachi",
    "shah re faisal": "Karachi",
    "shah rae faisal": "Karachi",
    "shahrah e faisal": "Karachi",
    "shahrah faisal": "Karachi",
    "shahrae faisal": "Karachi",
    "defence": "Karachi",
    "johar town": "Lahore",
    "model town": "Lahore",
    "gulberg": "Lahore",
    "township": "Lahore",
    "blue area": "Islamabad",
}
DROP = {
    "dha", "cantt", "cantonment", "cant", "phase", "sector", "area", "district",
    "tehsil", "malir", "gulberg", "clifton", "defence",
}
SECTOR_RE = re.compile(r"^(?:[a-z]-?\d+[a-z]?|\d+[a-z]?)$", re.I)
SENTINELS = {"", "none", "null", "n/a", "-", "na", "n.a.", "n.a"}


def canonical_city(text):
    raw = (text or "").strip()
    if not raw or raw.lower() in SENTINELS:
        return None
    known = CITY_BY_KEY.get(raw.lower())
    if known:
        return known
    cleaned = re.sub(r"[()\[\]{}]", " ", raw)
    cleaned = re.sub(r"[,/;|]+", " ", cleaned)
    cleaned = re.sub(r"\s+", " ", cleaned).strip()
    if not cleaned:
        return None
    known = CITY_BY_KEY.get(cleaned.lower())
    if known:
        return known
    lowered = cleaned.lower()
    match = CITY_RE.search(lowered)
    if match:
        return CITY_BY_KEY[match.group(0)]
    hyphen_fold = re.sub(r"[-]+", " ", lowered)
    hyphen_fold = re.sub(r"\s+", " ", hyphen_fold).strip()
    for alias, city in sorted(ALIASES.items(), key=lambda item: len(item[0]), reverse=True):
        if alias in hyphen_fold:
            return city
    leftover = []
    for token in cleaned.split():
        lowered_token = token.lower()
        if lowered_token in DROP or SECTOR_RE.fullmatch(token):
            continue
        leftover.append(token)
    if leftover:
        known = CITY_BY_KEY.get(" ".join(leftover).lower())
        if known:
            return known
    return None


def load_env():
    path = ROOT / "backend" / ".env"
    out = {}
    if not path.exists():
        return out
    for line in path.read_text(encoding="utf-8-sig").splitlines():
        line = line.strip()
        if not line or line.startswith("#") or "=" not in line:
            continue
        key, _, value = line.partition("=")
        out[key.strip()] = value.strip().strip('"').strip("'")
    return out


def rewrite_table(cur, table):
    cur.execute(
        f"SELECT id, city FROM {table} WHERE city IS NOT NULL AND btrim(city) <> ''"
    )
    rows = cur.fetchall()
    updated = 0
    skipped = 0
    unchanged = 0
    samples = []
    unmatched = []
    for record_id, city in rows:
        new = canonical_city(city)
        if new is None:
            cur.execute(f"UPDATE {table} SET city = NULL WHERE id = %s", (record_id,))
            skipped += 1
            if len(unmatched) < 30:
                unmatched.append(city)
            continue
        if new == city:
            unchanged += 1
            continue
        cur.execute(f"UPDATE {table} SET city = %s WHERE id = %s", (new, record_id))
        updated += 1
        if len(samples) < 20:
            samples.append((city, new))
    print(f"{table}: read={len(rows)} updated={updated} already_ok={unchanged} cleared={skipped}")
    for old, new in samples:
        print(f"  {old!r} -> {new!r}")
    for city in unmatched:
        print(f"  cleared {city!r}")


def dropdown_cities(cur):
    cur.execute(
        """
        SELECT city FROM inbox_messages
        WHERE city IS NOT NULL AND btrim(city) <> '' AND attachment = true
        UNION
        SELECT city FROM form_data
        WHERE city IS NOT NULL AND btrim(city) <> ''
        ORDER BY 1
        """
    )
    return [row[0] for row in cur.fetchall()]


def main():
    env = {**os.environ, **load_env()}
    kwargs = dict(
        host=env.get("DB_HOST", "localhost"),
        port=int(env.get("DB_PORT") or 5432),
        dbname=env.get("DB_NAME", "hrms"),
        user=env.get("DB_USERNAME", "postgres"),
        password=env.get("DB_PASSWORD", ""),
        options="-c search_path=app,public",
    )
    sslmode = (env.get("DB_SSLMODE") or "").strip()
    if sslmode:
        kwargs["sslmode"] = sslmode
    conn = psycopg2.connect(**kwargs)
    conn.autocommit = False
    print(f"db={kwargs['user']}@{kwargs['host']}:{kwargs['port']}/{kwargs['dbname']}")
    print(f"cities={len(CITY_BY_KEY)}")
    cur = conn.cursor()
    rewrite_table(cur, "inbox_messages")
    rewrite_table(cur, "form_data")
    conn.commit()
    names = dropdown_cities(cur)
    print(f"inbox city filter ({len(names)}):")
    for name in names:
        print(f"  {name}")
    cur.close()
    conn.close()


if __name__ == "__main__":
    main()