sheheryarsoomro12 2026-08-17 15:41:46 +05:00
parent bacd13c8b5
commit 60c4489415
7 changed files with 481 additions and 64 deletions

View File

@ -8,7 +8,8 @@ from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from sqlalchemy.orm import Session as OrmSession from sqlalchemy.orm import Session as OrmSession
from ...config import ALLOWED_EXTENSIONS, MAX_UPLOAD_BYTES, UPLOAD_DIR from ...config import ALLOWED_EXTENSIONS, MAX_UPLOAD_BYTES, UPLOAD_DIR
from ...core.xlsx_reader import TransactionReader, ParseError from ...core.readers import make_reader
from ...core.xlsx_reader import ParseError
from ...db import models from ...db import models
from ..deps import db_dep, file_dict, get_session_or_404, sanitize_filename from ..deps import db_dep, file_dict, get_session_or_404, sanitize_filename
@ -53,14 +54,19 @@ async def upload_files(session_id: int, files: list[UploadFile] = File(...),
session_id=session_id, filename=safe, stored_path=str(path), session_id=session_id, filename=safe, stored_path=str(path),
size_bytes=size, sha256=h.hexdigest(), status="uploaded", size_bytes=size, sha256=h.hexdigest(), status="uploaded",
) )
# light validation: detect sheet + required columns (no full row scan) # light validation: detect sheet/header + required columns (no full row scan)
try: try:
reader = TransactionReader(str(path)) reader = make_reader(str(path))
reader.detect() reader.detect()
rec.data_sheet = reader.sheet_name rec.data_sheet = reader.sheet_name
rec.status = "invalid" if reader.column_mapping.missing_required else "parsed" rec.status = "invalid" if reader.column_mapping.missing_required else "parsed"
if reader.column_mapping.missing_required: if reader.column_mapping.missing_required:
rec.message = f"missing required columns: {reader.column_mapping.missing_required}" rec.message = f"missing required columns: {reader.column_mapping.missing_required}"
# Surface marketplace / date span when cheap (CSV already has rows in memory;
# for xlsx this stays blank until processing).
meta = reader.file_meta
if getattr(meta, "currency", None):
rec.currency = meta.currency
reader.close() reader.close()
except ParseError as e: except ParseError as e:
rec.status = "invalid" rec.status = "invalid"

View File

@ -276,8 +276,18 @@ CANONICAL_FIELDS: list[tuple[str, str, tuple[str, ...]]] = [
"inne", # PL "inne", # PL
"övrigt", # SV "övrigt", # SV
)), )),
("transaction_status", "text", ("transaction status",)), ("transaction_status", "text", (
("transaction_release_date", "date", ("transaction release date",)), "transaction status",
"statut de la transaction", # FR / BE CSV
"transaktionsstatus", # DE
"stato della transazione", # IT
"estado de la transacción", # ES
)),
("transaction_release_date", "date", (
"transaction release date",
"date de délivrance de la transaction", # FR / BE CSV
"transaktionsfreigabedatum", # DE
)),
("total", "amount", ( ("total", "amount", (
"total", "total amount", "amount", "total", "total amount", "amount",
"gesamt", # DE "gesamt", # DE

View File

@ -0,0 +1,256 @@
"""
Amazon Custom Unified Transaction reports delivered as CSV (UTF-8, often with BOM).
Same public interface as TransactionReader / CalamineReader:
detect() / iter_records() / file_meta / column_mapping / close()
Amazon CSVs typically begin with a short preamble (scope, currency, definitions) before the
real header row. Amounts use a European decimal comma in localized EU reports ("13,49").
"""
from __future__ import annotations
import csv
import os
import re
from datetime import date
from typing import Iterator
from .column_map import FIELD_KIND, ColumnMapping, build_mapping
from .dates import parse_amazon_date_fast
from .xlsx_reader import FileMeta, ParseError
_DETECT_REQUIRED = {"settlement_id", "total", "date_time"}
_CURRENCY_RE = re.compile(
r"\b(USD|EUR|GBP|CAD|AUD|PLN|SEK|TRY|JPY)\b", re.IGNORECASE
)
def parse_amount(raw) -> float:
"""Parse Amazon amount cells, including European '1.234,56' / '13,49' forms."""
if raw is None or raw == "":
return 0.0
if isinstance(raw, bool):
return 0.0
if isinstance(raw, (int, float)):
return float(raw)
s = str(raw).strip().replace("\u00a0", "").replace(" ", "")
if not s:
return 0.0
# European: decimal comma, optional thousands dots / spaces.
if "," in s and "." in s:
if s.rfind(",") > s.rfind("."):
s = s.replace(".", "").replace(",", ".")
else:
s = s.replace(",", "")
elif "," in s:
# "13,49" or "1.234" — if one comma and digits after, treat as decimal.
left, _, right = s.partition(",")
if right.isdigit() and 1 <= len(right) <= 2:
s = f"{left.replace('.', '')}.{right}"
else:
s = s.replace(",", "")
try:
return float(s)
except ValueError:
return 0.0
def _index_to_col(idx: int) -> str:
idx += 1
s = ""
while idx:
idx, r = divmod(idx - 1, 26)
s = chr(65 + r) + s
return s
def _convert(field_name: str, raw):
kind = FIELD_KIND.get(field_name, "text")
if kind == "amount":
return parse_amount(raw)
if kind == "int":
if raw in (None, ""):
return None
try:
return int(parse_amount(raw))
except (TypeError, ValueError):
return None
if raw is None:
return None
s = str(raw).strip()
if kind == "id" and s.endswith(".0"):
s = s[:-2]
return s or None
class CsvReader:
def __init__(self, path: str, saved_overrides: dict[str, str] | None = None):
self.path = path
self.filename = os.path.basename(path)
self.saved_overrides = saved_overrides
self._rows: list[list[str]] | None = None
self.sheet_name = "CSV"
self.header_row = 0 # 1-based, matching Excel readers
self.column_mapping: ColumnMapping | None = None
self._field_to_idx: dict[str, int] = {}
self.file_meta = FileMeta(filename=self.filename)
def open(self) -> None:
if self._rows is not None:
return
try:
raw = open(self.path, "rb").read()
except OSError as e:
raise ParseError(f"'{self.filename}' could not be read: {e}") from e
if not raw:
raise ParseError(f"'{self.filename}' is empty.")
# Strip UTF-8 BOM; fall back through common Amazon encodings.
if raw.startswith(b"\xef\xbb\xbf"):
text = raw.decode("utf-8-sig")
else:
text = None
for enc in ("utf-8", "utf-16", "cp1252", "latin-1"):
try:
text = raw.decode(enc)
break
except UnicodeDecodeError:
continue
if text is None:
raise ParseError(f"'{self.filename}' is not a readable text/CSV file.")
# Sniff delimiter from the densest early line (comma vs semicolon EU exports).
sample = "\n".join(text.splitlines()[:40])
try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
delimiter = dialect.delimiter
except csv.Error:
delimiter = ";" if sample.count(";") > sample.count(",") else ","
self._rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
self.file_meta.size_bytes = os.path.getsize(self.path)
self.file_meta.worksheets = [self.sheet_name]
# Currency hint from the preamble ("Tous les montants sont en EUR…").
for row in self._rows[:15]:
joined = " ".join(row)
m = _CURRENCY_RE.search(joined)
if m and ("montant" in joined.lower() or "amount" in joined.lower()
or "currency" in joined.lower() or "en " in joined.lower()):
self.file_meta.currency = m.group(1).upper()
break
def close(self) -> None:
self._rows = None
def __enter__(self):
self.open()
return self
def __exit__(self, *exc):
self.close()
def detect(self) -> ColumnMapping:
self.open()
assert self._rows is not None
best = None
best_score = -1
# Scan the first ~40 rows for the real Amazon header (skip preamble / definitions).
for r_idx, row in enumerate(self._rows[:40]):
cells = [(_index_to_col(i), str(v)) for i, v in enumerate(row) if str(v).strip()]
if len(cells) < 5:
continue
mapping = build_mapping(cells, r_idx + 1, self.saved_overrides)
if _DETECT_REQUIRED.issubset(set(mapping.field_to_col)):
score = len(mapping.field_to_col)
if score > best_score:
best, best_score = (r_idx, mapping), score
if not best:
raise ParseError(
f"'{self.filename}': could not find an Amazon transaction header row "
f"(need columns: date/time, settlement id, total)."
)
r_idx, mapping = best
self.header_row = r_idx + 1
self.column_mapping = mapping
self._field_to_idx = {
fld: _col_to_idx(col) for col, fld in mapping.col_to_field.items()
}
self.file_meta.data_sheet = self.sheet_name
self.file_meta.header_row = self.header_row
self.file_meta.unmapped_headers = mapping.unmapped
self.file_meta.missing_required = mapping.missing_required
self.file_meta.duplicate_fields = mapping.duplicate_fields
self.file_meta.sheet_last_row = len(self._rows)
return mapping
def iter_records(self, only_fields: set[str] | None = None) -> Iterator[dict]:
if self.column_mapping is None:
self.detect()
assert self._rows is not None
idx_map = self._field_to_idx
if only_fields is not None:
idx_map = {f: i for f, i in idx_map.items() if f in only_fields}
items = list(idx_map.items())
mapped_idx = set(self._field_to_idx.values())
unmapped_sums = self.file_meta.unmapped_amount_sums
hdr = self.header_row
min_d: date | None = None
max_d: date | None = None
marketplace: str | None = None
count = 0
want_date = "date_time" in self._field_to_idx
for excel_row, row in enumerate(self._rows, start=1):
if excel_row <= hdr:
continue
rec = {
"_source_file": self.filename,
"_source_sheet": self.sheet_name,
"_source_row": excel_row,
}
has_value = False
for fld, i in items:
v = row[i] if i < len(row) else None
rec[fld] = _convert(fld, v)
if v not in (None, ""):
has_value = True
if not has_value:
self.file_meta.blank_rows_skipped += 1
continue
if excel_row <= hdr + 2:
raw_sid = row[self._field_to_idx["settlement_id"]] \
if self._field_to_idx.get("settlement_id", 99999) < len(row) else None
sid = str(raw_sid or "").strip()
d_probe = parse_amazon_date_fast(rec.get("date_time")) \
if rec.get("date_time") else None
if d_probe is None and not sid.replace(".", "").isdigit():
self.file_meta.helper_rows_skipped += 1
continue
if len(row) > len(mapped_idx):
for i, v in enumerate(row):
if i not in mapped_idx and v not in (None, ""):
amt = parse_amount(v)
if amt:
col = _index_to_col(i)
unmapped_sums[col] = unmapped_sums.get(col, 0.0) + amt
if want_date and rec.get("date_time"):
d = parse_amazon_date_fast(rec["date_time"])
rec["_date"] = d
if d:
if min_d is None or d < min_d:
min_d = d
if max_d is None or d > max_d:
max_d = d
if marketplace is None and rec.get("marketplace"):
marketplace = rec["marketplace"]
count += 1
yield rec
self.file_meta.imported_rows = count
self.file_meta.min_date = min_d
self.file_meta.max_date = max_d
self.file_meta.marketplace = marketplace
def _col_to_idx(letters: str) -> int:
n = 0
for ch in letters:
n = n * 26 + (ord(ch) - 64)
return n - 1

View File

@ -1,4 +1,4 @@
"""Reader factory: fast calamine engine by default, streaming iterparse as fallback.""" """Reader factory: CSV, fast calamine (xlsx), or streaming iterparse fallback."""
from __future__ import annotations from __future__ import annotations
import os import os
@ -19,9 +19,15 @@ def _calamine_available() -> bool:
def make_reader(path: str, saved_overrides: dict[str, str] | None = None): def make_reader(path: str, saved_overrides: dict[str, str] | None = None):
""" """
Return a reader with the TransactionReader interface (detect/iter_records/file_meta/close). Return a reader with the TransactionReader interface (detect/iter_records/file_meta/close).
Uses python-calamine when available (much faster, higher peak memory); otherwise the
CSV reports (Amazon "Custom Unified Transaction" downloads) use CsvReader. Spreadsheets
use python-calamine when available (much faster, higher peak memory); otherwise the
low-memory streaming reader. Set AR_USE_CALAMINE=0 to force the streaming reader. low-memory streaming reader. Set AR_USE_CALAMINE=0 to force the streaming reader.
""" """
ext = os.path.splitext(path)[1].lower()
if ext == ".csv":
from .csv_reader import CsvReader
return CsvReader(path, saved_overrides=saved_overrides)
if _calamine_available(): if _calamine_available():
from .calamine_reader import CalamineReader from .calamine_reader import CalamineReader
return CalamineReader(path, saved_overrides=saved_overrides) return CalamineReader(path, saved_overrides=saved_overrides)

View File

@ -350,10 +350,20 @@ class TransactionReader:
def quick_expected_rows(path: str) -> int: def quick_expected_rows(path: str) -> int:
""" """
Fast (KB-sized) estimate of data-row count without loading sharedStrings, used to drive Fast estimate of data-row count for the progress bar.
the progress bar. Reads the _xlnm._FilterDatabase defined name (or the largest sheet's
<dimension>) to find the last row. Spreadsheets: reads the _xlnm._FilterDatabase defined name (or the largest sheet's
<dimension>) without loading sharedStrings. CSV: line count minus a small preamble
allowance (exact count comes later from the reader).
""" """
if path.lower().endswith(".csv"):
try:
with open(path, "rb") as fh:
# Cheap line count; header/preamble typically ≤ 15 rows.
n = sum(1 for _ in fh)
return max(0, n - 12)
except OSError:
return 0
try: try:
z = zipfile.ZipFile(path) z = zipfile.ZipFile(path)
except Exception: except Exception:

View File

@ -0,0 +1,66 @@
"""CSV Amazon transaction reader — Belgium/FR June sample shape."""
from __future__ import annotations
from pathlib import Path
from app.core.csv_reader import CsvReader, parse_amount
from app.core.readers import make_reader
def test_parse_amount_european():
assert parse_amount("13,49") == 13.49
assert parse_amount("1.234,56") == 1234.56
assert parse_amount("1,234.56") == 1234.56
assert parse_amount("-6,24") == -6.24
assert parse_amount("0") == 0.0
assert parse_amount("") == 0.0
def test_make_reader_routes_csv(tmp_path: Path):
p = tmp_path / "sample.csv"
p.write_text(
"preamble\n"
"date/time,settlement id,type,total\n"
"1 Jun 2026 00:00:00 UTC,123,Order,10.00\n",
encoding="utf-8",
)
r = make_reader(str(p))
assert type(r).__name__ == "CsvReader"
def test_csv_reader_belgium_headers(tmp_path: Path):
# Minimal FR/BE Custom Unified Transaction CSV (preamble + header + 1 row).
body = (
'"Comprend les transactions Amazon Marketplace"\n'
'"Tous les montants sont en EUR, sauf indication contraire"\n'
'"date/heure","Identifiant du paiement","type","Numéro de la commande","SKU",'
'"description","quantité","site de vente","expédition","ville de la commande",'
'"état de la commande","commande postale","ventes de produits",'
'"crédits dexpédition","crédits demballage-cadeau","Total des réductions",'
'"taxe de ventes prélevée","Taxe Marketplace Facilitator","frais de vente",'
'"Frais pour le service Expédié par Amazon","autres frais de transaction",'
'"autres","total","Statut de la transaction","Date de délivrance de la transaction"\n'
'"31 mai 2026 22:00:44 UTC","27177484042","Commande","405-9354558-3629905",'
'"SKU1","desc","2","amazon.com.be","Amazon","Enines","","1350","29,74","0","0","0",'
'"6,24","-6,24","-4,68","-11,57","0","0","13,49","Effectuée","8 juin 2026 15:26:40 UTC"\n'
)
p = tmp_path / "2026JunMonthlyTransaction.csv"
p.write_text(body, encoding="utf-8-sig")
reader = CsvReader(str(p))
mapping = reader.detect()
assert not mapping.missing_required
assert reader.sheet_name == "CSV"
assert reader.file_meta.currency == "EUR"
rows = list(reader.iter_records())
assert len(rows) == 1
rec = rows[0]
assert rec["settlement_id"] == "27177484042"
assert rec["txn_type"] == "Commande"
assert rec["marketplace"] == "amazon.com.be"
assert rec["total"] == 13.49
assert rec["product_sales"] == 29.74
assert rec["_date"].isoformat() == "2026-05-31"
assert reader.file_meta.imported_rows == 1
reader.close()

View File

@ -18,6 +18,7 @@ cd -- "$HERE" || exit 1
APP_DIR="$HERE/ar-aging-app" APP_DIR="$HERE/ar-aging-app"
BACKEND_DIR="$APP_DIR/backend" BACKEND_DIR="$APP_DIR/backend"
FRONTEND_DIR="$APP_DIR/frontend" FRONTEND_DIR="$APP_DIR/frontend"
VENV_DIR="$APP_DIR/.venv"
LOG_DIR="$APP_DIR/backend/data/logs" LOG_DIR="$APP_DIR/backend/data/logs"
BACKEND_LOG="$LOG_DIR/backend.log" BACKEND_LOG="$LOG_DIR/backend.log"
FRONTEND_LOG="$LOG_DIR/frontend.log" FRONTEND_LOG="$LOG_DIR/frontend.log"
@ -31,6 +32,23 @@ DASHBOARD_URL="http://localhost:$FRONTEND_PORT"
# ~/.local/bin or Homebrew would otherwise be "command not found". # ~/.local/bin or Homebrew would otherwise be "command not found".
export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:/Library/Frameworks/Python.framework/Versions/3.11/bin:$PATH" export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:/Library/Frameworks/Python.framework/Versions/3.11/bin:$PATH"
# Auto-answer prompts when stdin is not a Terminal (CI / Cursor / piped runs).
INTERACTIVE=0
[ -t 0 ] && INTERACTIVE=1
ask_yes() {
local prompt="$1"
if [ "$INTERACTIVE" -eq 0 ]; then
printf ' %s Y (non-interactive)\n' "$prompt"
return 0
fi
printf ' %s [Y/n] ' "$prompt"
read -r reply
case "${reply:-Y}" in
[Nn]*) return 1 ;;
*) return 0 ;;
esac
}
# --- pretty output ------------------------------------------------------------------ # --- pretty output ------------------------------------------------------------------
if [ -t 1 ]; then if [ -t 1 ]; then
B=$'\033[1m'; DIM=$'\033[2m'; R=$'\033[0m' B=$'\033[1m'; DIM=$'\033[2m'; R=$'\033[0m'
@ -48,8 +66,10 @@ die() {
printf '\n%s%sCould not start the dashboard.%s\n\n' "$ERR" "$B" "$R" printf '\n%s%sCould not start the dashboard.%s\n\n' "$ERR" "$B" "$R"
printf ' %s\n\n' "$1" printf ' %s\n\n' "$1"
[ $# -gt 1 ] && printf ' Try: %s%s%s\n\n' "$B" "$2" "$R" [ $# -gt 1 ] && printf ' Try: %s%s%s\n\n' "$B" "$2" "$R"
printf '%sPress Return to close this window.%s\n' "$DIM" "$R" if [ "$INTERACTIVE" -eq 1 ]; then
read -r _ printf '%sPress Return to close this window.%s\n' "$DIM" "$R"
read -r _
fi
exit 1 exit 1
} }
@ -68,43 +88,91 @@ command -v npm >/dev/null 2>&1 || die "npm was not found." \
"install Node.js 20+ from nodejs.org" "install Node.js 20+ from nodejs.org"
good "python $(python3 -V 2>&1 | awk '{print $2}') · node $(node -v 2>/dev/null) · npm $(npm -v 2>/dev/null)" good "python $(python3 -V 2>&1 | awk '{print $2}') · node $(node -v 2>/dev/null) · npm $(npm -v 2>/dev/null)"
# Each check below diagnoses ONE thing. A single "does the app import?" test used to stand in # Isolated venv — required. Global site-packages (e.g. streamlit's Starlette 1.x) break
# for all of them, so a database that was merely switched off was reported as broken Python # FastAPI 0.115's Router(on_startup=...) and make "import fastapi" look fine while the app dies.
# packages — and the launcher then ran pip install, which of course changed nothing. ensure_venv() {
if [ ! -x "$VENV_DIR/bin/python" ]; then
step "creating project virtualenv at $VENV_DIR…"
python3 -m venv "$VENV_DIR" \
|| die "Could not create the virtualenv." "python3 -m venv '$VENV_DIR'"
good "virtualenv created"
else
good "virtualenv present"
fi
PYTHON="$VENV_DIR/bin/python"
PIP="$VENV_DIR/bin/pip"
}
# 1a. Python packages — libraries only, no app code, so this cannot fail for config reasons. install_python_deps() {
if ! python3 -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1; then step "installing Python packages into the project venv…"
warn "Python packages are missing or incomplete." "$PIP" install -q --upgrade pip \
printf ' Install them now? [Y/n] ' || die "pip upgrade failed." "'$PIP' install --upgrade pip"
read -r reply "$PIP" install -q -r "$BACKEND_DIR/requirements.txt" \
case "${reply:-Y}" in
[Nn]*) die "Python dependencies are not installed." \
"python3 -m pip install -r '$BACKEND_DIR/requirements.txt'" ;;
esac
step "installing Python packages (this can take a minute)…"
python3 -m pip install -q -r "$BACKEND_DIR/requirements.txt" \
|| die "pip install failed — see the messages above." \ || die "pip install failed — see the messages above." \
"python3 -m pip install -r '$BACKEND_DIR/requirements.txt'" "'$PIP' install -r '$BACKEND_DIR/requirements.txt'"
python3 -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1 \ }
|| die "Python packages are still incomplete after installing." "python3 -m pip check"
backend_imports_ok() {
"$PYTHON" -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1
}
# Prove the pin that Streamlit commonly breaks: Starlette must stay <0.42 for this FastAPI.
backend_versions_ok() {
"$PYTHON" -c "
import fastapi, starlette
from packaging.version import Version
assert Version(fastapi.__version__) >= Version('0.115.0')
assert Version(starlette.__version__) < Version('0.42.0'), starlette.__version__
" >/dev/null 2>&1
}
backend_app_loads() {
( cd -- "$BACKEND_DIR" && "$PYTHON" -c "from app.api.main import app" ) >/dev/null 2>&1
}
ensure_venv
need_install=0
if ! backend_imports_ok; then
warn "Python packages are missing or incomplete in the project venv."
need_install=1
elif ! backend_versions_ok; then
warn "Wrong Starlette/FastAPI versions in the venv (often after a global pip upgrade)."
need_install=1
elif ! backend_app_loads; then
warn "Backend failed to import — reinstalling pinned dependencies."
need_install=1
fi
if [ "$need_install" -eq 1 ]; then
ask_yes "Install / repair Python packages now?" \
|| die "Python dependencies are not installed." \
"'$PIP' install -r '$BACKEND_DIR/requirements.txt'"
install_python_deps
# packaging is used only for the version check; requirements may not list it.
"$PIP" install -q packaging >/dev/null 2>&1 || true
backend_imports_ok \
|| die "Python packages are still incomplete after installing." "'$PIP' check"
backend_versions_ok \
|| die "Starlette is still too new for this FastAPI pin." \
"'$PIP' install 'starlette==0.41.3'"
good "Python packages installed" good "Python packages installed"
else else
good "Python packages present" good "Python packages present (pinned versions)"
fi fi
# 1b. Which database? SQLite needs nothing; MySQL needs a server and credentials. The app # 1b. Which database? SQLite needs nothing; MySQL needs a server and credentials. The app
# picks MySQL only when ar-aging-app/.env names a real host, so a machine with no # picks MySQL only when ar-aging-app/.env names a real host, so a machine with no
# database installed still runs off the local file instead of refusing to start. # database installed still runs off the local file instead of refusing to start.
ENV_FILE="$APP_DIR/.env" ENV_FILE="$APP_DIR/.env"
db_backend="$(cd -- "$BACKEND_DIR" && python3 -c " db_backend="$(cd -- "$BACKEND_DIR" && "$PYTHON" -c "
from app.config import DB_BACKEND; print(DB_BACKEND)" 2>/dev/null)" from app.config import DB_BACKEND; print(DB_BACKEND)" 2>/dev/null)"
db_label="$(cd -- "$BACKEND_DIR" && python3 -c " db_label="$(cd -- "$BACKEND_DIR" && "$PYTHON" -c "
from app.config import database_label; print(database_label())" 2>/dev/null)" from app.config import database_label; print(database_label())" 2>/dev/null)"
if [ "$db_backend" = "mysql" ]; then if [ "$db_backend" = "mysql" ]; then
good "database: $db_label" good "database: $db_label"
# Reachable? A refused connection is a server/credentials problem, never a Python one. db_err="$(cd -- "$BACKEND_DIR" && "$PYTHON" -c "
db_err="$(cd -- "$BACKEND_DIR" && python3 -c "
import sys, pymysql import sys, pymysql
from app.config import MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD from app.config import MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD
try: try:
@ -120,22 +188,23 @@ except Exception as e:
say " Start the server and run this again, or fall back to the local file by setting" say " Start the server and run this again, or fall back to the local file by setting"
say " ${B}AR_DB_BACKEND=sqlite${R} in $ENV_FILE." say " ${B}AR_DB_BACKEND=sqlite${R} in $ENV_FILE."
say "" say ""
printf '%sPress Return to close this window.%s\n' "$DIM" "$R" if [ "$INTERACTIVE" -eq 1 ]; then
read -r _ printf '%sPress Return to close this window.%s\n' "$DIM" "$R"
read -r _
fi
exit 1 exit 1
} }
good "MySQL reachable" good "MySQL reachable"
else else
good "database: $db_label" good "database: ${db_label:-SQLite (local file)}"
[ -f "$ENV_FILE" ] || warn "no .env — using the local file (fine for a demo or one user)" [ -f "$ENV_FILE" ] || warn "no .env — using the local file (fine for a demo or one user)"
fi fi
# 1d. Everything above is fine — now prove the app itself loads. Anything failing here is a # 1d. Everything above is fine — now prove the app itself loads.
# genuine code/dependency problem, so the message points at that rather than at config. if ! import_error="$(cd -- "$BACKEND_DIR" && "$PYTHON" -c "from app.api.main import app" 2>&1)"; then
if ! import_error="$(cd -- "$BACKEND_DIR" && python3 -c "from app.api.main import app" 2>&1)"; then
fail "The backend failed to load even though its packages and database are fine:" fail "The backend failed to load even though its packages and database are fine:"
printf '%s\n' "$import_error" | tail -n 6 | sed 's/^/ /' printf '%s\n' "$import_error" | tail -n 8 | sed 's/^/ /'
die "This looks like a code or dependency-version problem." "python3 -m pip check" die "This looks like a code or dependency-version problem." "'$PIP' check"
fi fi
good "backend loads cleanly" good "backend loads cleanly"
@ -155,7 +224,7 @@ fi
# The Vite proxy points at a fixed localhost:8000, so we can't just pick another port. # The Vite proxy points at a fixed localhost:8000, so we can't just pick another port.
free_port() { free_port() {
local port="$1" label="$2" pids local port="$1" label="$2" pids
pids="$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null)" pids="$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true)"
[ -z "$pids" ] && { good "port $port free ($label)"; return 0; } [ -z "$pids" ] && { good "port $port free ($label)"; return 0; }
warn "port $port is already in use ($label):" warn "port $port is already in use ($label):"
@ -163,24 +232,20 @@ free_port() {
for pid in $pids; do for pid in $pids; do
printf ' pid %-7s %s\n' "$pid" "$(ps -p "$pid" -o command= 2>/dev/null | cut -c1-88)" printf ' pid %-7s %s\n' "$pid" "$(ps -p "$pid" -o command= 2>/dev/null | cut -c1-88)"
done done
printf ' Stop it and continue? [Y/n] ' ask_yes "Stop it and continue?" \
read -r reply || die "Port $port is in use, so the dashboard cannot start." \
case "${reply:-Y}" in "quit the other program, or close the old dashboard window"
[Nn]*) die "Port $port is in use, so the dashboard cannot start." \ for pid in $pids; do kill "$pid" 2>/dev/null || true; done
"quit the other program, or close the old dashboard window" ;;
esac
for pid in $pids; do kill "$pid" 2>/dev/null; done
for _ in 1 2 3 4 5 6 7 8 9 10; do for _ in 1 2 3 4 5 6 7 8 9 10; do
sleep 0.3 sleep 0.3
[ -z "$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null)" ] && break [ -z "$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true)" ] && break
done done
# Still holding on? Escalate once. pids="$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true)"
pids="$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null)"
if [ -n "$pids" ]; then if [ -n "$pids" ]; then
for pid in $pids; do kill -9 "$pid" 2>/dev/null; done for pid in $pids; do kill -9 "$pid" 2>/dev/null || true; done
sleep 1 sleep 1
fi fi
[ -n "$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null)" ] \ [ -n "$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true)" ] \
&& die "Port $port is still in use after trying to stop it." \ && die "Port $port is still in use after trying to stop it." \
"restart your Mac, or find the process with: lsof -i :$port" "restart your Mac, or find the process with: lsof -i :$port"
good "port $port freed" good "port $port freed"
@ -199,11 +264,11 @@ FRONTEND_PID=""
shutdown() { shutdown() {
printf '\n%sStopping…%s\n' "$DIM" "$R" printf '\n%sStopping…%s\n' "$DIM" "$R"
# Kill the whole process group of each server: uvicorn --reload and vite both fork. # Kill the whole process group of each server: uvicorn --reload and vite both fork.
[ -n "$FRONTEND_PID" ] && kill -- "-$FRONTEND_PID" 2>/dev/null [ -n "$FRONTEND_PID" ] && kill -- "-$FRONTEND_PID" 2>/dev/null || true
[ -n "$BACKEND_PID" ] && kill -- "-$BACKEND_PID" 2>/dev/null [ -n "$BACKEND_PID" ] && kill -- "-$BACKEND_PID" 2>/dev/null || true
sleep 0.5 sleep 0.5
[ -n "$FRONTEND_PID" ] && kill -9 -- "-$FRONTEND_PID" 2>/dev/null [ -n "$FRONTEND_PID" ] && kill -9 -- "-$FRONTEND_PID" 2>/dev/null || true
[ -n "$BACKEND_PID" ] && kill -9 -- "-$BACKEND_PID" 2>/dev/null [ -n "$BACKEND_PID" ] && kill -9 -- "-$BACKEND_PID" 2>/dev/null || true
printf '%sBoth servers stopped.%s\n\n' "$OK" "$R" printf '%sBoth servers stopped.%s\n\n' "$OK" "$R"
exit 0 exit 0
} }
@ -213,16 +278,15 @@ say ""
say "${B}3. Starting servers${R}" say "${B}3. Starting servers${R}"
step "backend (FastAPI on :$BACKEND_PORT)" step "backend (FastAPI on :$BACKEND_PORT)"
# setsid-style: run in its own process group so shutdown() can take down the reloader too. # Own process group so shutdown() can take down the reloader children too.
set -m set -m
python3 -m uvicorn app.api.main:app \ "$PYTHON" -m uvicorn app.api.main:app \
--app-dir "$BACKEND_DIR" \ --app-dir "$BACKEND_DIR" \
--host 127.0.0.1 --port "$BACKEND_PORT" \ --host 127.0.0.1 --port "$BACKEND_PORT" \
>"$BACKEND_LOG" 2>&1 & >"$BACKEND_LOG" 2>&1 &
BACKEND_PID=$! BACKEND_PID=$!
set +m set +m
# Wait for it to actually answer — "process started" is not the same as "server ready".
backend_ready="" backend_ready=""
for _ in $(seq 1 60); do for _ in $(seq 1 60); do
if ! kill -0 "$BACKEND_PID" 2>/dev/null; then if ! kill -0 "$BACKEND_PID" 2>/dev/null; then
@ -261,7 +325,6 @@ done
die "The frontend did not respond within 30 seconds." "full log: $FRONTEND_LOG"; } die "The frontend did not respond within 30 seconds." "full log: $FRONTEND_LOG"; }
good "frontend ready" good "frontend ready"
# End-to-end check: the browser reaches the API *through* the Vite proxy, not directly.
if curl -fsS -o /dev/null "$DASHBOARD_URL/api/sessions" 2>/dev/null; then if curl -fsS -o /dev/null "$DASHBOARD_URL/api/sessions" 2>/dev/null; then
good "dashboard is talking to the API" good "dashboard is talking to the API"
else else
@ -280,12 +343,12 @@ say ""
say " Dashboard ${B}$DASHBOARD_URL${R}" say " Dashboard ${B}$DASHBOARD_URL${R}"
say " API docs ${DIM}http://localhost:$BACKEND_PORT/docs${R}" say " API docs ${DIM}http://localhost:$BACKEND_PORT/docs${R}"
say " Logs ${DIM}$LOG_DIR${R}" say " Logs ${DIM}$LOG_DIR${R}"
say " Python ${DIM}$PYTHON${R}"
say "" say ""
say "${DIM} Keep this window open while you work.${R}" say "${DIM} Keep this window open while you work.${R}"
say "${DIM} Press Ctrl-C to stop both servers.${R}" say "${DIM} Press Ctrl-C to stop both servers.${R}"
say "" say ""
# Stay alive until a server dies or the user interrupts.
while kill -0 "$BACKEND_PID" 2>/dev/null && kill -0 "$FRONTEND_PID" 2>/dev/null; do while kill -0 "$BACKEND_PID" 2>/dev/null && kill -0 "$FRONTEND_PID" 2>/dev/null; do
sleep 1 sleep 1
done done