New
parent
bacd13c8b5
commit
60c4489415
|
|
@ -8,7 +8,8 @@ from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||||
from sqlalchemy.orm import Session as OrmSession
|
from sqlalchemy.orm import Session as OrmSession
|
||||||
|
|
||||||
from ...config import ALLOWED_EXTENSIONS, MAX_UPLOAD_BYTES, UPLOAD_DIR
|
from ...config import ALLOWED_EXTENSIONS, MAX_UPLOAD_BYTES, UPLOAD_DIR
|
||||||
from ...core.xlsx_reader import TransactionReader, ParseError
|
from ...core.readers import make_reader
|
||||||
|
from ...core.xlsx_reader import ParseError
|
||||||
from ...db import models
|
from ...db import models
|
||||||
from ..deps import db_dep, file_dict, get_session_or_404, sanitize_filename
|
from ..deps import db_dep, file_dict, get_session_or_404, sanitize_filename
|
||||||
|
|
||||||
|
|
@ -53,14 +54,19 @@ async def upload_files(session_id: int, files: list[UploadFile] = File(...),
|
||||||
session_id=session_id, filename=safe, stored_path=str(path),
|
session_id=session_id, filename=safe, stored_path=str(path),
|
||||||
size_bytes=size, sha256=h.hexdigest(), status="uploaded",
|
size_bytes=size, sha256=h.hexdigest(), status="uploaded",
|
||||||
)
|
)
|
||||||
# light validation: detect sheet + required columns (no full row scan)
|
# light validation: detect sheet/header + required columns (no full row scan)
|
||||||
try:
|
try:
|
||||||
reader = TransactionReader(str(path))
|
reader = make_reader(str(path))
|
||||||
reader.detect()
|
reader.detect()
|
||||||
rec.data_sheet = reader.sheet_name
|
rec.data_sheet = reader.sheet_name
|
||||||
rec.status = "invalid" if reader.column_mapping.missing_required else "parsed"
|
rec.status = "invalid" if reader.column_mapping.missing_required else "parsed"
|
||||||
if reader.column_mapping.missing_required:
|
if reader.column_mapping.missing_required:
|
||||||
rec.message = f"missing required columns: {reader.column_mapping.missing_required}"
|
rec.message = f"missing required columns: {reader.column_mapping.missing_required}"
|
||||||
|
# Surface marketplace / date span when cheap (CSV already has rows in memory;
|
||||||
|
# for xlsx this stays blank until processing).
|
||||||
|
meta = reader.file_meta
|
||||||
|
if getattr(meta, "currency", None):
|
||||||
|
rec.currency = meta.currency
|
||||||
reader.close()
|
reader.close()
|
||||||
except ParseError as e:
|
except ParseError as e:
|
||||||
rec.status = "invalid"
|
rec.status = "invalid"
|
||||||
|
|
|
||||||
|
|
@ -276,8 +276,18 @@ CANONICAL_FIELDS: list[tuple[str, str, tuple[str, ...]]] = [
|
||||||
"inne", # PL
|
"inne", # PL
|
||||||
"övrigt", # SV
|
"övrigt", # SV
|
||||||
)),
|
)),
|
||||||
("transaction_status", "text", ("transaction status",)),
|
("transaction_status", "text", (
|
||||||
("transaction_release_date", "date", ("transaction release date",)),
|
"transaction status",
|
||||||
|
"statut de la transaction", # FR / BE CSV
|
||||||
|
"transaktionsstatus", # DE
|
||||||
|
"stato della transazione", # IT
|
||||||
|
"estado de la transacción", # ES
|
||||||
|
)),
|
||||||
|
("transaction_release_date", "date", (
|
||||||
|
"transaction release date",
|
||||||
|
"date de délivrance de la transaction", # FR / BE CSV
|
||||||
|
"transaktionsfreigabedatum", # DE
|
||||||
|
)),
|
||||||
("total", "amount", (
|
("total", "amount", (
|
||||||
"total", "total amount", "amount",
|
"total", "total amount", "amount",
|
||||||
"gesamt", # DE
|
"gesamt", # DE
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,256 @@
|
||||||
|
"""
|
||||||
|
Amazon Custom Unified Transaction reports delivered as CSV (UTF-8, often with BOM).
|
||||||
|
|
||||||
|
Same public interface as TransactionReader / CalamineReader:
|
||||||
|
detect() / iter_records() / file_meta / column_mapping / close()
|
||||||
|
|
||||||
|
Amazon CSVs typically begin with a short preamble (scope, currency, definitions) before the
|
||||||
|
real header row. Amounts use a European decimal comma in localized EU reports ("13,49").
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from datetime import date
|
||||||
|
from typing import Iterator
|
||||||
|
|
||||||
|
from .column_map import FIELD_KIND, ColumnMapping, build_mapping
|
||||||
|
from .dates import parse_amazon_date_fast
|
||||||
|
from .xlsx_reader import FileMeta, ParseError
|
||||||
|
|
||||||
|
_DETECT_REQUIRED = {"settlement_id", "total", "date_time"}
|
||||||
|
_CURRENCY_RE = re.compile(
|
||||||
|
r"\b(USD|EUR|GBP|CAD|AUD|PLN|SEK|TRY|JPY)\b", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_amount(raw) -> float:
|
||||||
|
"""Parse Amazon amount cells, including European '1.234,56' / '13,49' forms."""
|
||||||
|
if raw is None or raw == "":
|
||||||
|
return 0.0
|
||||||
|
if isinstance(raw, bool):
|
||||||
|
return 0.0
|
||||||
|
if isinstance(raw, (int, float)):
|
||||||
|
return float(raw)
|
||||||
|
s = str(raw).strip().replace("\u00a0", "").replace(" ", "")
|
||||||
|
if not s:
|
||||||
|
return 0.0
|
||||||
|
# European: decimal comma, optional thousands dots / spaces.
|
||||||
|
if "," in s and "." in s:
|
||||||
|
if s.rfind(",") > s.rfind("."):
|
||||||
|
s = s.replace(".", "").replace(",", ".")
|
||||||
|
else:
|
||||||
|
s = s.replace(",", "")
|
||||||
|
elif "," in s:
|
||||||
|
# "13,49" or "1.234" — if one comma and digits after, treat as decimal.
|
||||||
|
left, _, right = s.partition(",")
|
||||||
|
if right.isdigit() and 1 <= len(right) <= 2:
|
||||||
|
s = f"{left.replace('.', '')}.{right}"
|
||||||
|
else:
|
||||||
|
s = s.replace(",", "")
|
||||||
|
try:
|
||||||
|
return float(s)
|
||||||
|
except ValueError:
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _index_to_col(idx: int) -> str:
|
||||||
|
idx += 1
|
||||||
|
s = ""
|
||||||
|
while idx:
|
||||||
|
idx, r = divmod(idx - 1, 26)
|
||||||
|
s = chr(65 + r) + s
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def _convert(field_name: str, raw):
|
||||||
|
kind = FIELD_KIND.get(field_name, "text")
|
||||||
|
if kind == "amount":
|
||||||
|
return parse_amount(raw)
|
||||||
|
if kind == "int":
|
||||||
|
if raw in (None, ""):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(parse_amount(raw))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if raw is None:
|
||||||
|
return None
|
||||||
|
s = str(raw).strip()
|
||||||
|
if kind == "id" and s.endswith(".0"):
|
||||||
|
s = s[:-2]
|
||||||
|
return s or None
|
||||||
|
|
||||||
|
|
||||||
|
class CsvReader:
|
||||||
|
def __init__(self, path: str, saved_overrides: dict[str, str] | None = None):
|
||||||
|
self.path = path
|
||||||
|
self.filename = os.path.basename(path)
|
||||||
|
self.saved_overrides = saved_overrides
|
||||||
|
self._rows: list[list[str]] | None = None
|
||||||
|
self.sheet_name = "CSV"
|
||||||
|
self.header_row = 0 # 1-based, matching Excel readers
|
||||||
|
self.column_mapping: ColumnMapping | None = None
|
||||||
|
self._field_to_idx: dict[str, int] = {}
|
||||||
|
self.file_meta = FileMeta(filename=self.filename)
|
||||||
|
|
||||||
|
def open(self) -> None:
|
||||||
|
if self._rows is not None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
raw = open(self.path, "rb").read()
|
||||||
|
except OSError as e:
|
||||||
|
raise ParseError(f"'{self.filename}' could not be read: {e}") from e
|
||||||
|
if not raw:
|
||||||
|
raise ParseError(f"'{self.filename}' is empty.")
|
||||||
|
# Strip UTF-8 BOM; fall back through common Amazon encodings.
|
||||||
|
if raw.startswith(b"\xef\xbb\xbf"):
|
||||||
|
text = raw.decode("utf-8-sig")
|
||||||
|
else:
|
||||||
|
text = None
|
||||||
|
for enc in ("utf-8", "utf-16", "cp1252", "latin-1"):
|
||||||
|
try:
|
||||||
|
text = raw.decode(enc)
|
||||||
|
break
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
continue
|
||||||
|
if text is None:
|
||||||
|
raise ParseError(f"'{self.filename}' is not a readable text/CSV file.")
|
||||||
|
# Sniff delimiter from the densest early line (comma vs semicolon EU exports).
|
||||||
|
sample = "\n".join(text.splitlines()[:40])
|
||||||
|
try:
|
||||||
|
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t")
|
||||||
|
delimiter = dialect.delimiter
|
||||||
|
except csv.Error:
|
||||||
|
delimiter = ";" if sample.count(";") > sample.count(",") else ","
|
||||||
|
self._rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
|
||||||
|
self.file_meta.size_bytes = os.path.getsize(self.path)
|
||||||
|
self.file_meta.worksheets = [self.sheet_name]
|
||||||
|
# Currency hint from the preamble ("Tous les montants sont en EUR…").
|
||||||
|
for row in self._rows[:15]:
|
||||||
|
joined = " ".join(row)
|
||||||
|
m = _CURRENCY_RE.search(joined)
|
||||||
|
if m and ("montant" in joined.lower() or "amount" in joined.lower()
|
||||||
|
or "currency" in joined.lower() or "en " in joined.lower()):
|
||||||
|
self.file_meta.currency = m.group(1).upper()
|
||||||
|
break
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self._rows = None
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
self.open()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
def detect(self) -> ColumnMapping:
|
||||||
|
self.open()
|
||||||
|
assert self._rows is not None
|
||||||
|
best = None
|
||||||
|
best_score = -1
|
||||||
|
# Scan the first ~40 rows for the real Amazon header (skip preamble / definitions).
|
||||||
|
for r_idx, row in enumerate(self._rows[:40]):
|
||||||
|
cells = [(_index_to_col(i), str(v)) for i, v in enumerate(row) if str(v).strip()]
|
||||||
|
if len(cells) < 5:
|
||||||
|
continue
|
||||||
|
mapping = build_mapping(cells, r_idx + 1, self.saved_overrides)
|
||||||
|
if _DETECT_REQUIRED.issubset(set(mapping.field_to_col)):
|
||||||
|
score = len(mapping.field_to_col)
|
||||||
|
if score > best_score:
|
||||||
|
best, best_score = (r_idx, mapping), score
|
||||||
|
if not best:
|
||||||
|
raise ParseError(
|
||||||
|
f"'{self.filename}': could not find an Amazon transaction header row "
|
||||||
|
f"(need columns: date/time, settlement id, total)."
|
||||||
|
)
|
||||||
|
r_idx, mapping = best
|
||||||
|
self.header_row = r_idx + 1
|
||||||
|
self.column_mapping = mapping
|
||||||
|
self._field_to_idx = {
|
||||||
|
fld: _col_to_idx(col) for col, fld in mapping.col_to_field.items()
|
||||||
|
}
|
||||||
|
self.file_meta.data_sheet = self.sheet_name
|
||||||
|
self.file_meta.header_row = self.header_row
|
||||||
|
self.file_meta.unmapped_headers = mapping.unmapped
|
||||||
|
self.file_meta.missing_required = mapping.missing_required
|
||||||
|
self.file_meta.duplicate_fields = mapping.duplicate_fields
|
||||||
|
self.file_meta.sheet_last_row = len(self._rows)
|
||||||
|
return mapping
|
||||||
|
|
||||||
|
def iter_records(self, only_fields: set[str] | None = None) -> Iterator[dict]:
|
||||||
|
if self.column_mapping is None:
|
||||||
|
self.detect()
|
||||||
|
assert self._rows is not None
|
||||||
|
idx_map = self._field_to_idx
|
||||||
|
if only_fields is not None:
|
||||||
|
idx_map = {f: i for f, i in idx_map.items() if f in only_fields}
|
||||||
|
items = list(idx_map.items())
|
||||||
|
mapped_idx = set(self._field_to_idx.values())
|
||||||
|
unmapped_sums = self.file_meta.unmapped_amount_sums
|
||||||
|
hdr = self.header_row
|
||||||
|
min_d: date | None = None
|
||||||
|
max_d: date | None = None
|
||||||
|
marketplace: str | None = None
|
||||||
|
count = 0
|
||||||
|
want_date = "date_time" in self._field_to_idx
|
||||||
|
|
||||||
|
for excel_row, row in enumerate(self._rows, start=1):
|
||||||
|
if excel_row <= hdr:
|
||||||
|
continue
|
||||||
|
rec = {
|
||||||
|
"_source_file": self.filename,
|
||||||
|
"_source_sheet": self.sheet_name,
|
||||||
|
"_source_row": excel_row,
|
||||||
|
}
|
||||||
|
has_value = False
|
||||||
|
for fld, i in items:
|
||||||
|
v = row[i] if i < len(row) else None
|
||||||
|
rec[fld] = _convert(fld, v)
|
||||||
|
if v not in (None, ""):
|
||||||
|
has_value = True
|
||||||
|
if not has_value:
|
||||||
|
self.file_meta.blank_rows_skipped += 1
|
||||||
|
continue
|
||||||
|
if excel_row <= hdr + 2:
|
||||||
|
raw_sid = row[self._field_to_idx["settlement_id"]] \
|
||||||
|
if self._field_to_idx.get("settlement_id", 99999) < len(row) else None
|
||||||
|
sid = str(raw_sid or "").strip()
|
||||||
|
d_probe = parse_amazon_date_fast(rec.get("date_time")) \
|
||||||
|
if rec.get("date_time") else None
|
||||||
|
if d_probe is None and not sid.replace(".", "").isdigit():
|
||||||
|
self.file_meta.helper_rows_skipped += 1
|
||||||
|
continue
|
||||||
|
if len(row) > len(mapped_idx):
|
||||||
|
for i, v in enumerate(row):
|
||||||
|
if i not in mapped_idx and v not in (None, ""):
|
||||||
|
amt = parse_amount(v)
|
||||||
|
if amt:
|
||||||
|
col = _index_to_col(i)
|
||||||
|
unmapped_sums[col] = unmapped_sums.get(col, 0.0) + amt
|
||||||
|
if want_date and rec.get("date_time"):
|
||||||
|
d = parse_amazon_date_fast(rec["date_time"])
|
||||||
|
rec["_date"] = d
|
||||||
|
if d:
|
||||||
|
if min_d is None or d < min_d:
|
||||||
|
min_d = d
|
||||||
|
if max_d is None or d > max_d:
|
||||||
|
max_d = d
|
||||||
|
if marketplace is None and rec.get("marketplace"):
|
||||||
|
marketplace = rec["marketplace"]
|
||||||
|
count += 1
|
||||||
|
yield rec
|
||||||
|
|
||||||
|
self.file_meta.imported_rows = count
|
||||||
|
self.file_meta.min_date = min_d
|
||||||
|
self.file_meta.max_date = max_d
|
||||||
|
self.file_meta.marketplace = marketplace
|
||||||
|
|
||||||
|
|
||||||
|
def _col_to_idx(letters: str) -> int:
|
||||||
|
n = 0
|
||||||
|
for ch in letters:
|
||||||
|
n = n * 26 + (ord(ch) - 64)
|
||||||
|
return n - 1
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
"""Reader factory: fast calamine engine by default, streaming iterparse as fallback."""
|
"""Reader factory: CSV, fast calamine (xlsx), or streaming iterparse fallback."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
@ -19,9 +19,15 @@ def _calamine_available() -> bool:
|
||||||
def make_reader(path: str, saved_overrides: dict[str, str] | None = None):
|
def make_reader(path: str, saved_overrides: dict[str, str] | None = None):
|
||||||
"""
|
"""
|
||||||
Return a reader with the TransactionReader interface (detect/iter_records/file_meta/close).
|
Return a reader with the TransactionReader interface (detect/iter_records/file_meta/close).
|
||||||
Uses python-calamine when available (much faster, higher peak memory); otherwise the
|
|
||||||
|
CSV reports (Amazon "Custom Unified Transaction" downloads) use CsvReader. Spreadsheets
|
||||||
|
use python-calamine when available (much faster, higher peak memory); otherwise the
|
||||||
low-memory streaming reader. Set AR_USE_CALAMINE=0 to force the streaming reader.
|
low-memory streaming reader. Set AR_USE_CALAMINE=0 to force the streaming reader.
|
||||||
"""
|
"""
|
||||||
|
ext = os.path.splitext(path)[1].lower()
|
||||||
|
if ext == ".csv":
|
||||||
|
from .csv_reader import CsvReader
|
||||||
|
return CsvReader(path, saved_overrides=saved_overrides)
|
||||||
if _calamine_available():
|
if _calamine_available():
|
||||||
from .calamine_reader import CalamineReader
|
from .calamine_reader import CalamineReader
|
||||||
return CalamineReader(path, saved_overrides=saved_overrides)
|
return CalamineReader(path, saved_overrides=saved_overrides)
|
||||||
|
|
|
||||||
|
|
@ -350,10 +350,20 @@ class TransactionReader:
|
||||||
|
|
||||||
def quick_expected_rows(path: str) -> int:
|
def quick_expected_rows(path: str) -> int:
|
||||||
"""
|
"""
|
||||||
Fast (KB-sized) estimate of data-row count without loading sharedStrings, used to drive
|
Fast estimate of data-row count for the progress bar.
|
||||||
the progress bar. Reads the _xlnm._FilterDatabase defined name (or the largest sheet's
|
|
||||||
<dimension>) to find the last row.
|
Spreadsheets: reads the _xlnm._FilterDatabase defined name (or the largest sheet's
|
||||||
|
<dimension>) without loading sharedStrings. CSV: line count minus a small preamble
|
||||||
|
allowance (exact count comes later from the reader).
|
||||||
"""
|
"""
|
||||||
|
if path.lower().endswith(".csv"):
|
||||||
|
try:
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
# Cheap line count; header/preamble typically ≤ 15 rows.
|
||||||
|
n = sum(1 for _ in fh)
|
||||||
|
return max(0, n - 12)
|
||||||
|
except OSError:
|
||||||
|
return 0
|
||||||
try:
|
try:
|
||||||
z = zipfile.ZipFile(path)
|
z = zipfile.ZipFile(path)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
"""CSV Amazon transaction reader — Belgium/FR June sample shape."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.core.csv_reader import CsvReader, parse_amount
|
||||||
|
from app.core.readers import make_reader
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_amount_european():
|
||||||
|
assert parse_amount("13,49") == 13.49
|
||||||
|
assert parse_amount("1.234,56") == 1234.56
|
||||||
|
assert parse_amount("1,234.56") == 1234.56
|
||||||
|
assert parse_amount("-6,24") == -6.24
|
||||||
|
assert parse_amount("0") == 0.0
|
||||||
|
assert parse_amount("") == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_reader_routes_csv(tmp_path: Path):
|
||||||
|
p = tmp_path / "sample.csv"
|
||||||
|
p.write_text(
|
||||||
|
"preamble\n"
|
||||||
|
"date/time,settlement id,type,total\n"
|
||||||
|
"1 Jun 2026 00:00:00 UTC,123,Order,10.00\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
r = make_reader(str(p))
|
||||||
|
assert type(r).__name__ == "CsvReader"
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_reader_belgium_headers(tmp_path: Path):
|
||||||
|
# Minimal FR/BE Custom Unified Transaction CSV (preamble + header + 1 row).
|
||||||
|
body = (
|
||||||
|
'"Comprend les transactions Amazon Marketplace"\n'
|
||||||
|
'"Tous les montants sont en EUR, sauf indication contraire"\n'
|
||||||
|
'"date/heure","Identifiant du paiement","type","Numéro de la commande","SKU",'
|
||||||
|
'"description","quantité","site de vente","expédition","ville de la commande",'
|
||||||
|
'"état de la commande","commande postale","ventes de produits",'
|
||||||
|
'"crédits d’expédition","crédits d’emballage-cadeau","Total des réductions",'
|
||||||
|
'"taxe de ventes prélevée","Taxe Marketplace Facilitator","frais de vente",'
|
||||||
|
'"Frais pour le service Expédié par Amazon","autres frais de transaction",'
|
||||||
|
'"autres","total","Statut de la transaction","Date de délivrance de la transaction"\n'
|
||||||
|
'"31 mai 2026 22:00:44 UTC","27177484042","Commande","405-9354558-3629905",'
|
||||||
|
'"SKU1","desc","2","amazon.com.be","Amazon","Enines","","1350","29,74","0","0","0",'
|
||||||
|
'"6,24","-6,24","-4,68","-11,57","0","0","13,49","Effectuée","8 juin 2026 15:26:40 UTC"\n'
|
||||||
|
)
|
||||||
|
p = tmp_path / "2026JunMonthlyTransaction.csv"
|
||||||
|
p.write_text(body, encoding="utf-8-sig")
|
||||||
|
|
||||||
|
reader = CsvReader(str(p))
|
||||||
|
mapping = reader.detect()
|
||||||
|
assert not mapping.missing_required
|
||||||
|
assert reader.sheet_name == "CSV"
|
||||||
|
assert reader.file_meta.currency == "EUR"
|
||||||
|
|
||||||
|
rows = list(reader.iter_records())
|
||||||
|
assert len(rows) == 1
|
||||||
|
rec = rows[0]
|
||||||
|
assert rec["settlement_id"] == "27177484042"
|
||||||
|
assert rec["txn_type"] == "Commande"
|
||||||
|
assert rec["marketplace"] == "amazon.com.be"
|
||||||
|
assert rec["total"] == 13.49
|
||||||
|
assert rec["product_sales"] == 29.74
|
||||||
|
assert rec["_date"].isoformat() == "2026-05-31"
|
||||||
|
assert reader.file_meta.imported_rows == 1
|
||||||
|
reader.close()
|
||||||
163
start.command
163
start.command
|
|
@ -18,6 +18,7 @@ cd -- "$HERE" || exit 1
|
||||||
APP_DIR="$HERE/ar-aging-app"
|
APP_DIR="$HERE/ar-aging-app"
|
||||||
BACKEND_DIR="$APP_DIR/backend"
|
BACKEND_DIR="$APP_DIR/backend"
|
||||||
FRONTEND_DIR="$APP_DIR/frontend"
|
FRONTEND_DIR="$APP_DIR/frontend"
|
||||||
|
VENV_DIR="$APP_DIR/.venv"
|
||||||
LOG_DIR="$APP_DIR/backend/data/logs"
|
LOG_DIR="$APP_DIR/backend/data/logs"
|
||||||
BACKEND_LOG="$LOG_DIR/backend.log"
|
BACKEND_LOG="$LOG_DIR/backend.log"
|
||||||
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
FRONTEND_LOG="$LOG_DIR/frontend.log"
|
||||||
|
|
@ -31,6 +32,23 @@ DASHBOARD_URL="http://localhost:$FRONTEND_PORT"
|
||||||
# ~/.local/bin or Homebrew would otherwise be "command not found".
|
# ~/.local/bin or Homebrew would otherwise be "command not found".
|
||||||
export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:/Library/Frameworks/Python.framework/Versions/3.11/bin:$PATH"
|
export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:/Library/Frameworks/Python.framework/Versions/3.11/bin:$PATH"
|
||||||
|
|
||||||
|
# Auto-answer prompts when stdin is not a Terminal (CI / Cursor / piped runs).
|
||||||
|
INTERACTIVE=0
|
||||||
|
[ -t 0 ] && INTERACTIVE=1
|
||||||
|
ask_yes() {
|
||||||
|
local prompt="$1"
|
||||||
|
if [ "$INTERACTIVE" -eq 0 ]; then
|
||||||
|
printf ' %s Y (non-interactive)\n' "$prompt"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
printf ' %s [Y/n] ' "$prompt"
|
||||||
|
read -r reply
|
||||||
|
case "${reply:-Y}" in
|
||||||
|
[Nn]*) return 1 ;;
|
||||||
|
*) return 0 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
# --- pretty output ------------------------------------------------------------------
|
# --- pretty output ------------------------------------------------------------------
|
||||||
if [ -t 1 ]; then
|
if [ -t 1 ]; then
|
||||||
B=$'\033[1m'; DIM=$'\033[2m'; R=$'\033[0m'
|
B=$'\033[1m'; DIM=$'\033[2m'; R=$'\033[0m'
|
||||||
|
|
@ -48,8 +66,10 @@ die() {
|
||||||
printf '\n%s%sCould not start the dashboard.%s\n\n' "$ERR" "$B" "$R"
|
printf '\n%s%sCould not start the dashboard.%s\n\n' "$ERR" "$B" "$R"
|
||||||
printf ' %s\n\n' "$1"
|
printf ' %s\n\n' "$1"
|
||||||
[ $# -gt 1 ] && printf ' Try: %s%s%s\n\n' "$B" "$2" "$R"
|
[ $# -gt 1 ] && printf ' Try: %s%s%s\n\n' "$B" "$2" "$R"
|
||||||
|
if [ "$INTERACTIVE" -eq 1 ]; then
|
||||||
printf '%sPress Return to close this window.%s\n' "$DIM" "$R"
|
printf '%sPress Return to close this window.%s\n' "$DIM" "$R"
|
||||||
read -r _
|
read -r _
|
||||||
|
fi
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -68,43 +88,91 @@ command -v npm >/dev/null 2>&1 || die "npm was not found." \
|
||||||
"install Node.js 20+ from nodejs.org"
|
"install Node.js 20+ from nodejs.org"
|
||||||
good "python $(python3 -V 2>&1 | awk '{print $2}') · node $(node -v 2>/dev/null) · npm $(npm -v 2>/dev/null)"
|
good "python $(python3 -V 2>&1 | awk '{print $2}') · node $(node -v 2>/dev/null) · npm $(npm -v 2>/dev/null)"
|
||||||
|
|
||||||
# Each check below diagnoses ONE thing. A single "does the app import?" test used to stand in
|
# Isolated venv — required. Global site-packages (e.g. streamlit's Starlette 1.x) break
|
||||||
# for all of them, so a database that was merely switched off was reported as broken Python
|
# FastAPI 0.115's Router(on_startup=...) and make "import fastapi" look fine while the app dies.
|
||||||
# packages — and the launcher then ran pip install, which of course changed nothing.
|
ensure_venv() {
|
||||||
|
if [ ! -x "$VENV_DIR/bin/python" ]; then
|
||||||
|
step "creating project virtualenv at $VENV_DIR…"
|
||||||
|
python3 -m venv "$VENV_DIR" \
|
||||||
|
|| die "Could not create the virtualenv." "python3 -m venv '$VENV_DIR'"
|
||||||
|
good "virtualenv created"
|
||||||
|
else
|
||||||
|
good "virtualenv present"
|
||||||
|
fi
|
||||||
|
PYTHON="$VENV_DIR/bin/python"
|
||||||
|
PIP="$VENV_DIR/bin/pip"
|
||||||
|
}
|
||||||
|
|
||||||
# 1a. Python packages — libraries only, no app code, so this cannot fail for config reasons.
|
install_python_deps() {
|
||||||
if ! python3 -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1; then
|
step "installing Python packages into the project venv…"
|
||||||
warn "Python packages are missing or incomplete."
|
"$PIP" install -q --upgrade pip \
|
||||||
printf ' Install them now? [Y/n] '
|
|| die "pip upgrade failed." "'$PIP' install --upgrade pip"
|
||||||
read -r reply
|
"$PIP" install -q -r "$BACKEND_DIR/requirements.txt" \
|
||||||
case "${reply:-Y}" in
|
|
||||||
[Nn]*) die "Python dependencies are not installed." \
|
|
||||||
"python3 -m pip install -r '$BACKEND_DIR/requirements.txt'" ;;
|
|
||||||
esac
|
|
||||||
step "installing Python packages (this can take a minute)…"
|
|
||||||
python3 -m pip install -q -r "$BACKEND_DIR/requirements.txt" \
|
|
||||||
|| die "pip install failed — see the messages above." \
|
|| die "pip install failed — see the messages above." \
|
||||||
"python3 -m pip install -r '$BACKEND_DIR/requirements.txt'"
|
"'$PIP' install -r '$BACKEND_DIR/requirements.txt'"
|
||||||
python3 -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1 \
|
}
|
||||||
|| die "Python packages are still incomplete after installing." "python3 -m pip check"
|
|
||||||
|
backend_imports_ok() {
|
||||||
|
"$PYTHON" -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Prove the pin that Streamlit commonly breaks: Starlette must stay <0.42 for this FastAPI.
|
||||||
|
backend_versions_ok() {
|
||||||
|
"$PYTHON" -c "
|
||||||
|
import fastapi, starlette
|
||||||
|
from packaging.version import Version
|
||||||
|
assert Version(fastapi.__version__) >= Version('0.115.0')
|
||||||
|
assert Version(starlette.__version__) < Version('0.42.0'), starlette.__version__
|
||||||
|
" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
backend_app_loads() {
|
||||||
|
( cd -- "$BACKEND_DIR" && "$PYTHON" -c "from app.api.main import app" ) >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_venv
|
||||||
|
|
||||||
|
need_install=0
|
||||||
|
if ! backend_imports_ok; then
|
||||||
|
warn "Python packages are missing or incomplete in the project venv."
|
||||||
|
need_install=1
|
||||||
|
elif ! backend_versions_ok; then
|
||||||
|
warn "Wrong Starlette/FastAPI versions in the venv (often after a global pip upgrade)."
|
||||||
|
need_install=1
|
||||||
|
elif ! backend_app_loads; then
|
||||||
|
warn "Backend failed to import — reinstalling pinned dependencies."
|
||||||
|
need_install=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$need_install" -eq 1 ]; then
|
||||||
|
ask_yes "Install / repair Python packages now?" \
|
||||||
|
|| die "Python dependencies are not installed." \
|
||||||
|
"'$PIP' install -r '$BACKEND_DIR/requirements.txt'"
|
||||||
|
install_python_deps
|
||||||
|
# packaging is used only for the version check; requirements may not list it.
|
||||||
|
"$PIP" install -q packaging >/dev/null 2>&1 || true
|
||||||
|
backend_imports_ok \
|
||||||
|
|| die "Python packages are still incomplete after installing." "'$PIP' check"
|
||||||
|
backend_versions_ok \
|
||||||
|
|| die "Starlette is still too new for this FastAPI pin." \
|
||||||
|
"'$PIP' install 'starlette==0.41.3'"
|
||||||
good "Python packages installed"
|
good "Python packages installed"
|
||||||
else
|
else
|
||||||
good "Python packages present"
|
good "Python packages present (pinned versions)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 1b. Which database? SQLite needs nothing; MySQL needs a server and credentials. The app
|
# 1b. Which database? SQLite needs nothing; MySQL needs a server and credentials. The app
|
||||||
# picks MySQL only when ar-aging-app/.env names a real host, so a machine with no
|
# picks MySQL only when ar-aging-app/.env names a real host, so a machine with no
|
||||||
# database installed still runs off the local file instead of refusing to start.
|
# database installed still runs off the local file instead of refusing to start.
|
||||||
ENV_FILE="$APP_DIR/.env"
|
ENV_FILE="$APP_DIR/.env"
|
||||||
db_backend="$(cd -- "$BACKEND_DIR" && python3 -c "
|
db_backend="$(cd -- "$BACKEND_DIR" && "$PYTHON" -c "
|
||||||
from app.config import DB_BACKEND; print(DB_BACKEND)" 2>/dev/null)"
|
from app.config import DB_BACKEND; print(DB_BACKEND)" 2>/dev/null)"
|
||||||
db_label="$(cd -- "$BACKEND_DIR" && python3 -c "
|
db_label="$(cd -- "$BACKEND_DIR" && "$PYTHON" -c "
|
||||||
from app.config import database_label; print(database_label())" 2>/dev/null)"
|
from app.config import database_label; print(database_label())" 2>/dev/null)"
|
||||||
|
|
||||||
if [ "$db_backend" = "mysql" ]; then
|
if [ "$db_backend" = "mysql" ]; then
|
||||||
good "database: $db_label"
|
good "database: $db_label"
|
||||||
# Reachable? A refused connection is a server/credentials problem, never a Python one.
|
db_err="$(cd -- "$BACKEND_DIR" && "$PYTHON" -c "
|
||||||
db_err="$(cd -- "$BACKEND_DIR" && python3 -c "
|
|
||||||
import sys, pymysql
|
import sys, pymysql
|
||||||
from app.config import MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD
|
from app.config import MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD
|
||||||
try:
|
try:
|
||||||
|
|
@ -120,22 +188,23 @@ except Exception as e:
|
||||||
say " Start the server and run this again, or fall back to the local file by setting"
|
say " Start the server and run this again, or fall back to the local file by setting"
|
||||||
say " ${B}AR_DB_BACKEND=sqlite${R} in $ENV_FILE."
|
say " ${B}AR_DB_BACKEND=sqlite${R} in $ENV_FILE."
|
||||||
say ""
|
say ""
|
||||||
|
if [ "$INTERACTIVE" -eq 1 ]; then
|
||||||
printf '%sPress Return to close this window.%s\n' "$DIM" "$R"
|
printf '%sPress Return to close this window.%s\n' "$DIM" "$R"
|
||||||
read -r _
|
read -r _
|
||||||
|
fi
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
good "MySQL reachable"
|
good "MySQL reachable"
|
||||||
else
|
else
|
||||||
good "database: $db_label"
|
good "database: ${db_label:-SQLite (local file)}"
|
||||||
[ -f "$ENV_FILE" ] || warn "no .env — using the local file (fine for a demo or one user)"
|
[ -f "$ENV_FILE" ] || warn "no .env — using the local file (fine for a demo or one user)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 1d. Everything above is fine — now prove the app itself loads. Anything failing here is a
|
# 1d. Everything above is fine — now prove the app itself loads.
|
||||||
# genuine code/dependency problem, so the message points at that rather than at config.
|
if ! import_error="$(cd -- "$BACKEND_DIR" && "$PYTHON" -c "from app.api.main import app" 2>&1)"; then
|
||||||
if ! import_error="$(cd -- "$BACKEND_DIR" && python3 -c "from app.api.main import app" 2>&1)"; then
|
|
||||||
fail "The backend failed to load even though its packages and database are fine:"
|
fail "The backend failed to load even though its packages and database are fine:"
|
||||||
printf '%s\n' "$import_error" | tail -n 6 | sed 's/^/ /'
|
printf '%s\n' "$import_error" | tail -n 8 | sed 's/^/ /'
|
||||||
die "This looks like a code or dependency-version problem." "python3 -m pip check"
|
die "This looks like a code or dependency-version problem." "'$PIP' check"
|
||||||
fi
|
fi
|
||||||
good "backend loads cleanly"
|
good "backend loads cleanly"
|
||||||
|
|
||||||
|
|
@ -155,7 +224,7 @@ fi
|
||||||
# The Vite proxy points at a fixed localhost:8000, so we can't just pick another port.
|
# The Vite proxy points at a fixed localhost:8000, so we can't just pick another port.
|
||||||
free_port() {
|
free_port() {
|
||||||
local port="$1" label="$2" pids
|
local port="$1" label="$2" pids
|
||||||
pids="$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null)"
|
pids="$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true)"
|
||||||
[ -z "$pids" ] && { good "port $port free ($label)"; return 0; }
|
[ -z "$pids" ] && { good "port $port free ($label)"; return 0; }
|
||||||
|
|
||||||
warn "port $port is already in use ($label):"
|
warn "port $port is already in use ($label):"
|
||||||
|
|
@ -163,24 +232,20 @@ free_port() {
|
||||||
for pid in $pids; do
|
for pid in $pids; do
|
||||||
printf ' pid %-7s %s\n' "$pid" "$(ps -p "$pid" -o command= 2>/dev/null | cut -c1-88)"
|
printf ' pid %-7s %s\n' "$pid" "$(ps -p "$pid" -o command= 2>/dev/null | cut -c1-88)"
|
||||||
done
|
done
|
||||||
printf ' Stop it and continue? [Y/n] '
|
ask_yes "Stop it and continue?" \
|
||||||
read -r reply
|
|| die "Port $port is in use, so the dashboard cannot start." \
|
||||||
case "${reply:-Y}" in
|
"quit the other program, or close the old dashboard window"
|
||||||
[Nn]*) die "Port $port is in use, so the dashboard cannot start." \
|
for pid in $pids; do kill "$pid" 2>/dev/null || true; done
|
||||||
"quit the other program, or close the old dashboard window" ;;
|
|
||||||
esac
|
|
||||||
for pid in $pids; do kill "$pid" 2>/dev/null; done
|
|
||||||
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
||||||
sleep 0.3
|
sleep 0.3
|
||||||
[ -z "$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null)" ] && break
|
[ -z "$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true)" ] && break
|
||||||
done
|
done
|
||||||
# Still holding on? Escalate once.
|
pids="$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true)"
|
||||||
pids="$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null)"
|
|
||||||
if [ -n "$pids" ]; then
|
if [ -n "$pids" ]; then
|
||||||
for pid in $pids; do kill -9 "$pid" 2>/dev/null; done
|
for pid in $pids; do kill -9 "$pid" 2>/dev/null || true; done
|
||||||
sleep 1
|
sleep 1
|
||||||
fi
|
fi
|
||||||
[ -n "$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null)" ] \
|
[ -n "$(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true)" ] \
|
||||||
&& die "Port $port is still in use after trying to stop it." \
|
&& die "Port $port is still in use after trying to stop it." \
|
||||||
"restart your Mac, or find the process with: lsof -i :$port"
|
"restart your Mac, or find the process with: lsof -i :$port"
|
||||||
good "port $port freed"
|
good "port $port freed"
|
||||||
|
|
@ -199,11 +264,11 @@ FRONTEND_PID=""
|
||||||
shutdown() {
|
shutdown() {
|
||||||
printf '\n%sStopping…%s\n' "$DIM" "$R"
|
printf '\n%sStopping…%s\n' "$DIM" "$R"
|
||||||
# Kill the whole process group of each server: uvicorn --reload and vite both fork.
|
# Kill the whole process group of each server: uvicorn --reload and vite both fork.
|
||||||
[ -n "$FRONTEND_PID" ] && kill -- "-$FRONTEND_PID" 2>/dev/null
|
[ -n "$FRONTEND_PID" ] && kill -- "-$FRONTEND_PID" 2>/dev/null || true
|
||||||
[ -n "$BACKEND_PID" ] && kill -- "-$BACKEND_PID" 2>/dev/null
|
[ -n "$BACKEND_PID" ] && kill -- "-$BACKEND_PID" 2>/dev/null || true
|
||||||
sleep 0.5
|
sleep 0.5
|
||||||
[ -n "$FRONTEND_PID" ] && kill -9 -- "-$FRONTEND_PID" 2>/dev/null
|
[ -n "$FRONTEND_PID" ] && kill -9 -- "-$FRONTEND_PID" 2>/dev/null || true
|
||||||
[ -n "$BACKEND_PID" ] && kill -9 -- "-$BACKEND_PID" 2>/dev/null
|
[ -n "$BACKEND_PID" ] && kill -9 -- "-$BACKEND_PID" 2>/dev/null || true
|
||||||
printf '%sBoth servers stopped.%s\n\n' "$OK" "$R"
|
printf '%sBoth servers stopped.%s\n\n' "$OK" "$R"
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
|
|
@ -213,16 +278,15 @@ say ""
|
||||||
say "${B}3. Starting servers${R}"
|
say "${B}3. Starting servers${R}"
|
||||||
|
|
||||||
step "backend (FastAPI on :$BACKEND_PORT)"
|
step "backend (FastAPI on :$BACKEND_PORT)"
|
||||||
# setsid-style: run in its own process group so shutdown() can take down the reloader too.
|
# Own process group so shutdown() can take down the reloader children too.
|
||||||
set -m
|
set -m
|
||||||
python3 -m uvicorn app.api.main:app \
|
"$PYTHON" -m uvicorn app.api.main:app \
|
||||||
--app-dir "$BACKEND_DIR" \
|
--app-dir "$BACKEND_DIR" \
|
||||||
--host 127.0.0.1 --port "$BACKEND_PORT" \
|
--host 127.0.0.1 --port "$BACKEND_PORT" \
|
||||||
>"$BACKEND_LOG" 2>&1 &
|
>"$BACKEND_LOG" 2>&1 &
|
||||||
BACKEND_PID=$!
|
BACKEND_PID=$!
|
||||||
set +m
|
set +m
|
||||||
|
|
||||||
# Wait for it to actually answer — "process started" is not the same as "server ready".
|
|
||||||
backend_ready=""
|
backend_ready=""
|
||||||
for _ in $(seq 1 60); do
|
for _ in $(seq 1 60); do
|
||||||
if ! kill -0 "$BACKEND_PID" 2>/dev/null; then
|
if ! kill -0 "$BACKEND_PID" 2>/dev/null; then
|
||||||
|
|
@ -261,7 +325,6 @@ done
|
||||||
die "The frontend did not respond within 30 seconds." "full log: $FRONTEND_LOG"; }
|
die "The frontend did not respond within 30 seconds." "full log: $FRONTEND_LOG"; }
|
||||||
good "frontend ready"
|
good "frontend ready"
|
||||||
|
|
||||||
# End-to-end check: the browser reaches the API *through* the Vite proxy, not directly.
|
|
||||||
if curl -fsS -o /dev/null "$DASHBOARD_URL/api/sessions" 2>/dev/null; then
|
if curl -fsS -o /dev/null "$DASHBOARD_URL/api/sessions" 2>/dev/null; then
|
||||||
good "dashboard is talking to the API"
|
good "dashboard is talking to the API"
|
||||||
else
|
else
|
||||||
|
|
@ -280,12 +343,12 @@ say ""
|
||||||
say " Dashboard ${B}$DASHBOARD_URL${R}"
|
say " Dashboard ${B}$DASHBOARD_URL${R}"
|
||||||
say " API docs ${DIM}http://localhost:$BACKEND_PORT/docs${R}"
|
say " API docs ${DIM}http://localhost:$BACKEND_PORT/docs${R}"
|
||||||
say " Logs ${DIM}$LOG_DIR${R}"
|
say " Logs ${DIM}$LOG_DIR${R}"
|
||||||
|
say " Python ${DIM}$PYTHON${R}"
|
||||||
say ""
|
say ""
|
||||||
say "${DIM} Keep this window open while you work.${R}"
|
say "${DIM} Keep this window open while you work.${R}"
|
||||||
say "${DIM} Press Ctrl-C to stop both servers.${R}"
|
say "${DIM} Press Ctrl-C to stop both servers.${R}"
|
||||||
say ""
|
say ""
|
||||||
|
|
||||||
# Stay alive until a server dies or the user interrupts.
|
|
||||||
while kill -0 "$BACKEND_PID" 2>/dev/null && kill -0 "$FRONTEND_PID" 2>/dev/null; do
|
while kill -0 "$BACKEND_PID" 2>/dev/null && kill -0 "$FRONTEND_PID" 2>/dev/null; do
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue