diff --git a/ar-aging-app/backend/app/api/routes/export.py b/ar-aging-app/backend/app/api/routes/export.py index b8345f2..dd30355 100644 --- a/ar-aging-app/backend/app/api/routes/export.py +++ b/ar-aging-app/backend/app/api/routes/export.py @@ -23,6 +23,17 @@ def start_export(session_id: int, background: BackgroundTasks, kind: str = "full s = get_session_or_404(session_id, db) # An export is the number leaving the building — never generate one from a blocked close. ensure_not_blocked(s) + # The full workbook RE-COMPUTES its marketplace tabs from the source files using the + # current bank receipts, while the AR Ledger / Finance Summary sheets bound into the same + # file come from the last processing run. With unapplied receipts those two halves + # disagree — the tabs would show one receivable and the ledger sheet another. + if s.needs_reprocess: + raise HTTPException( + 409, + "Bank receipts or the payout mode changed after the last run. Re-process the " + "closing first — otherwise the workbook's marketplace tabs and its AR Ledger " + "sheet would report different receivables.", + ) if s.status not in ("processed", "exporting", "completed"): raise HTTPException(400, "Process the session before exporting.") if s.status == "exporting": diff --git a/ar-aging-app/backend/app/config.py b/ar-aging-app/backend/app/config.py index 0c688c4..522ced6 100644 --- a/ar-aging-app/backend/app/config.py +++ b/ar-aging-app/backend/app/config.py @@ -17,7 +17,15 @@ DATA_DIR = Path(os.environ.get("AR_DATA_DIR", str(BASE_DIR / "data"))) UPLOAD_DIR = DATA_DIR / "uploads" EXPORT_DIR = DATA_DIR / "exports" -# MySQL (required) +# --------------------------------------------------------------------------- database +# The store is switchable so the app is never blocked on infrastructure: +# AR_DB_BACKEND=sqlite a single local file — zero setup, good for a laptop or a demo +# AR_DB_BACKEND=mysql the shared server, for real multi-user month-end work +# Unset, it picks MySQL when a real MYSQL_HOST is configured and SQLite otherwise, so a +# machine with no database installed still runs instead of failing at import. +_PLACEHOLDER_HOSTS = {"", "your-mysql-host.example.com", "changeme", "todo"} +SQLITE_PATH = Path(os.environ.get("AR_SQLITE_PATH", str(DATA_DIR / "ar_aging.db"))) + MYSQL_HOST = os.environ.get("MYSQL_HOST", "") MYSQL_PORT = int(os.environ.get("MYSQL_PORT", "3306")) MYSQL_USER = os.environ.get("MYSQL_USER", "") @@ -28,11 +36,23 @@ MYSQL_POOL_SIZE = int(os.environ.get("MYSQL_POOL_SIZE", "10")) MYSQL_POOL_RECYCLE = int(os.environ.get("MYSQL_POOL_RECYCLE", "3600")) +def _mysql_configured() -> bool: + return (MYSQL_HOST.strip().lower() not in _PLACEHOLDER_HOSTS + and bool(MYSQL_USER) and bool(MYSQL_DATABASE)) + + +DB_BACKEND = (os.environ.get("AR_DB_BACKEND") + or ("mysql" if _mysql_configured() else "sqlite")).strip().lower() +if DB_BACKEND not in ("sqlite", "mysql"): + raise RuntimeError(f"AR_DB_BACKEND must be 'sqlite' or 'mysql' (got {DB_BACKEND!r}).") + + def mysql_url() -> str: - if not all((MYSQL_HOST, MYSQL_USER, MYSQL_DATABASE)): + if not _mysql_configured(): raise RuntimeError( - "MYSQL_HOST, MYSQL_USER, and MYSQL_DATABASE are required. " - "Copy example.env to .env and fill in credentials." + "MYSQL_HOST, MYSQL_USER, and MYSQL_DATABASE are required for the mysql " + "backend. Copy example.env to .env and fill in real credentials, or set " + "AR_DB_BACKEND=sqlite to use a local file." ) user = quote_plus(MYSQL_USER) password = quote_plus(MYSQL_PASSWORD) @@ -43,6 +63,20 @@ def mysql_url() -> str: ) +def database_url() -> str: + if DB_BACKEND == "mysql": + return mysql_url() + SQLITE_PATH.parent.mkdir(parents=True, exist_ok=True) + return f"sqlite:///{SQLITE_PATH}" + + +def database_label() -> str: + """Human-readable target, for logs and the launcher — never includes the password.""" + if DB_BACKEND == "mysql": + return f"MySQL {MYSQL_USER}@{MYSQL_HOST}:{MYSQL_PORT}/{MYSQL_DATABASE}" + return f"SQLite {SQLITE_PATH}" + + # Retention: temp uploads/exports older than this are purged (0 = keep forever). RETENTION_DAYS = int(os.environ.get("AR_RETENTION_DAYS", "30")) diff --git a/ar-aging-app/backend/app/core/excel_export.py b/ar-aging-app/backend/app/core/excel_export.py index c253809..a854815 100644 --- a/ar-aging-app/backend/app/core/excel_export.py +++ b/ar-aging-app/backend/app/core/excel_export.py @@ -107,7 +107,10 @@ class SheetLayout: data_start: int = 0 data_rows: int = 0 subtotal_cells: dict[str, str] = field(default_factory=dict) # account_type -> "AD####" - transfer_cells: dict[str, str] = field(default_factory=dict) # account_type -> "AD####" + # account_type -> ["AD9", "AD10", …]: a month can have SEVERAL received payouts per + # stream, so this is a list. It used to be one cell per account, which silently showed + # only the boundary payout and omitted every earlier one from the workbook. + transfer_cells: dict[str, list[str]] = field(default_factory=dict) @property def data_end(self) -> int: @@ -124,11 +127,10 @@ class MarketplaceLayout: return [f"'{s.name}'!{s.subtotal_cells[account_type]}" for s in self.sheets if account_type in s.subtotal_cells] - def transfer_ref(self, account_type: str) -> str | None: - for s in self.sheets: - if account_type in s.transfer_cells: - return f"'{s.name}'!{s.transfer_cells[account_type]}" - return None + def transfer_refs(self, account_type: str) -> list[str]: + """Every received-payout cell for an account stream (a month can have several).""" + return [f"'{s.name}'!{c}" + for s in self.sheets for c in s.transfer_cells.get(account_type, [])] def _sheet_names(marketplace: str, n: int) -> list[str]: @@ -143,10 +145,18 @@ def compute_layouts(result: ProcessResult, row_limit: int = EXCEL_ROW_LIMIT) -> cls = result.classification assert agg is not None and cls is not None - # order-row counts and receivable account types per marketplace + # Order-row counts and receivable account types per marketplace. Only OPEN (receivable) + # settlements contribute rows: once a settlement's payout has reached the bank it is + # closed, and its raw transactions are deliberately left out of the workbook (they are + # listed in summary form on the "Settled Settlements" sheet instead). per_mkt_orders: dict[str, int] = {} per_mkt_accts: dict[str, list[str]] = {} + all_mkt_accts: dict[str, list[str]] = {} for (mkt, acct, sid), st in agg.settlements.items(): + if acct.lower() in RECEIVABLE_ACCOUNT_TYPES: + seen = all_mkt_accts.setdefault(mkt, []) + if acct not in seen: + seen.append(acct) if st.status != "receivable" or acct.lower() not in RECEIVABLE_ACCOUNT_TYPES: continue orders = st.row_count - st.transfer_count @@ -155,17 +165,33 @@ def compute_layouts(result: ProcessResult, row_limit: int = EXCEL_ROW_LIMIT) -> if acct not in accts: accts.append(acct) - # boundary (receipt) transfers per marketplace/account - boundary_tx: dict[tuple[str, str], object] = {} - for k, t in cls.boundary_transfer.items(): - if t is not None: - boundary_tx[k] = t + # A marketplace whose payouts were ALL received has nothing outstanding and so no order + # rows — but it still gets a tab. An absent tab is indistinguishable from a marketplace + # whose file failed to upload; an empty one with its receipts and a 0.00 subtotal proves + # the market was processed and legitimately had nothing open. + for mkt in sorted(agg.marketplaces_seen): + per_mkt_orders.setdefault(mkt, 0) + + # EVERY received payout per (marketplace, account stream) — not just the boundary one. + # Using cls.boundary_transfer here showed a single payout per stream, so a month with + # several bank receipts silently omitted all but the last from the workbook. + received_tx: dict[tuple[str, str], list] = {} + for t in agg.transfers: + if not t.received: + continue # in transit: its settlement is still open + owner = cls.settlement_owner.get(t.settlement_id, t.marketplace) + received_tx.setdefault((owner, t.account_type), []).append(t) + for lst in received_tx.values(): + lst.sort(key=lambda t: (t.txn_date or date.min, t.settlement_id)) layouts: dict[str, MarketplaceLayout] = {} for mkt, order_total in per_mkt_orders.items(): - accts = sorted(per_mkt_accts.get(mkt, []), + accts = sorted(per_mkt_accts.get(mkt, []) or all_mkt_accts.get(mkt, []), key=lambda a: (0 if a.lower() == "standard orders" else 1, a)) - transfers = [boundary_tx[(mkt, a)] for a in accts if (mkt, a) in boundary_tx] + # Payout rows are keyed by the account type Amazon tagged them with, which is blank + # ("(unspecified)") everywhere except the USA — include those streams too. + transfers = [t for (m, _a), lst in received_tx.items() if m == mkt for t in lst] + transfers.sort(key=lambda t: (t.txn_date or date.min, t.settlement_id)) # capacity of the first sheet (accounts for preamble+header+transfers+subtotals) subtotal_block = 1 + 2 * max(len(accts), 1) # gap + one subtotal row per account @@ -192,13 +218,20 @@ def compute_layouts(result: ProcessResult, row_limit: int = EXCEL_ROW_LIMIT) -> # rows: preamble 1-7, header 8, transfers 9.., data start after transfers sl.data_start = 8 + len(tlist) + 1 sl.data_rows = nrows - # transfer cell addresses (rows 9..) + # transfer cell addresses (rows 9..) — several payouts can share an account stream for j, t in enumerate(tlist): - sl.transfer_cells[t.account_type] = f"{TOTAL_COL}{9 + j}" - # subtotal rows after data (one blank gap, then one row per account) + sl.transfer_cells.setdefault(t.account_type, []).append(f"{TOTAL_COL}{9 + j}") + # Subtotal rows after the data: one blank gap row, then ONE row per account + # stream, consecutively — this must mirror _finalize_marketplace_subtotals() + # exactly, because Detail/Summary reference these planned addresses. + # + # This used to stride by 2 while the writer strides by 1, so every stream after + # the first pointed at an empty cell. USA is the only marketplace with two + # streams, so Detail and Summary silently dropped the whole Invoiced Orders + # receivable (Jan-2026: 67,854.71) while Reconciliation and COA showed it. base = sl.data_end + 2 for j, acct in enumerate(accts): - sl.subtotal_cells[acct] = f"{TOTAL_COL}{base + 2 * j}" + sl.subtotal_cells[acct] = f"{TOTAL_COL}{base + j}" ml.sheets.append(sl) layouts[mkt] = ml return layouts @@ -227,7 +260,8 @@ class WorkbookBuilder: saved_column_overrides: dict[str, str] | None = None, row_limit: int = EXCEL_ROW_LIMIT, progress: "Callable[[float, int, int], None] | None" = None, - summary: dict | None = None, journal: dict | None = None): + summary: dict | None = None, journal: dict | None = None, + payout_receipts: dict[tuple[str, str, str], str] | None = None): self.result = result self.files = list(files) self.reserves = reserves or {} @@ -237,6 +271,8 @@ class WorkbookBuilder: self._progress = progress self.summary = summary or {} self.journal = journal or {} + # (marketplace, account bucket, settlement id) -> "YYYY-MM-DD · entered by" + self.payout_receipts = payout_receipts or {} self.layouts = compute_layouts(result, row_limit) self.wb = Workbook(write_only=True) self._mkt_ws: dict[str, list] = {} # marketplace -> [ws per sheet] @@ -253,6 +289,7 @@ class WorkbookBuilder: self._create_marketplace_sheets() self._stream_marketplace_rows() self._finalize_marketplace_subtotals() + self._build_settled_settlements() self._build_reconciliation() self._build_exceptions() self._build_audit_trail() @@ -313,11 +350,14 @@ class WorkbookBuilder: font=BOLD, border=BORDER) for col in "BCDEFG"], ]) ws.append([]) - ar = total_row + 2 + ar = total_row + 2 # the Allowance row, appended next ws.append([_c(ws, "Allowance for Sales Returns", font=BOLD), *[None] * 5, _c(ws, self.allowance, number_format=FMT_USD0, font=BOLD)]) + # Net Receivable = TOTAL + Allowance (the allowance is entered negative, as in the + # manual workbook). This referenced G{ar+1} — its own row — so Excel opened the + # workbook with a circular-reference warning and showed 0. ws.append([_c(ws, "Net Receivable", font=BOLD), - *[None] * 5, _c(ws, f"=G{total_row}+G{ar + 1}", number_format=FMT_USD0, font=BOLD)]) + *[None] * 5, _c(ws, f"=G{total_row}+G{ar}", number_format=FMT_USD0, font=BOLD)]) ws.freeze_panes = "A6" def _detail_receivable_usd_ref(self, mkt: str) -> str: @@ -612,6 +652,84 @@ class WorkbookBuilder: for note in r.notes: ws.append([_c(ws, "Note"), _c(ws, note)]) + # -- Settled Settlements (what was deliberately left out) -- + def _build_settled_settlements(self): + """ + Every settlement whose raw rows were EXCLUDED, and why. + + The marketplace tabs carry only open settlements — once Amazon's payout has reached + the bank the settlement is closed and its transactions are not repeated here. Without + this sheet a reader cannot tell a deliberately-omitted settled month from a file that + failed to upload, so the omission is listed line by line and reconciled: excluded + order rows + included order rows = every order row in the source files. + """ + agg, cls = self.result.aggregation, self.result.classification + if agg is None or cls is None: + return + ws = self.wb.create_sheet("Settled Settlements") + for col, w in zip("ABCDEFGHIJ", (16, 18, 18, 13, 13, 10, 18, 18, 14, 18)): + ws.column_dimensions[col].width = w + ws.append([_c(ws, "Settled settlements — raw rows deliberately excluded", + font=TITLE_FONT)]) + ws.append([_c(ws, "Amazon's payout for each settlement below reached the bank on or " + "before month-end, so the settlement is closed and its transactions " + "are summarised here instead of listed in the marketplace tabs.")]) + ws.append([]) + ws.append([_c(ws, h, font=HDR_FONT, fill=HDR_FILL) for h in ( + "Marketplace", "Account stream", "Settlement ID", "First date", "Last date", + "Rows", "Order total", "Payout amount", "Amazon paid", "Bank received")]) + + # Bank receipt per (marketplace, account stream, settlement id), when Finance entered one. + receipts = self.payout_receipts or {} + # Payout facts per settlement bucket. + pay: dict[tuple[str, str, str], list] = {} + for t in agg.transfers: + slot = pay.setdefault((t.marketplace, t.account_type, t.settlement_id), + [0.0, None]) + slot[0] += t.amount + if t.txn_date and (slot[1] is None or t.txn_date > slot[1]): + slot[1] = t.txn_date + + n_rows = 0 + excluded_total = 0.0 + settled = sorted( + ((k, st) for k, st in agg.settlements.items() + if st.status != "receivable" and (st.row_count - st.transfer_count) > 0), + key=lambda kv: (kv[0][0], kv[0][1], kv[0][2])) + for (mkt, acct, sid), st in settled: + orders = st.row_count - st.transfer_count + amount, paid_on = pay.get((mkt, acct, sid), (None, None)) + rec = receipts.get((mkt, acct, sid)) + n_rows += orders + excluded_total += st.order_total + ws.append([ + _c(ws, mkt), _c(ws, "—" if acct == "(unspecified)" else acct), _c(ws, sid), + _c(ws, st.first_date.isoformat() if st.first_date else ""), + _c(ws, st.last_date.isoformat() if st.last_date else ""), + _c(ws, orders), + # round(): summing millions of floats leaves noise like 5000.000000000001, + # which reads as a data problem in an audit workbook. + _c(ws, round(st.order_total, 2), number_format=FMT_ACCT2), + _c(ws, round(amount, 2), number_format=FMT_ACCT2) + if amount is not None else _c(ws, ""), + _c(ws, paid_on.isoformat() if paid_on else ""), + _c(ws, rec or "clearing-lag rule"), + ]) + ws.append([]) + included = sum(st.row_count - st.transfer_count for k, st in agg.settlements.items() + if st.status == "receivable") + included_total = sum(st.order_total for k, st in agg.settlements.items() + if st.status == "receivable") + for label, rows_n, amount_v in ( + ("Excluded (settled) order rows", n_rows, excluded_total), + ("Included (open) order rows — in the marketplace tabs", included, included_total), + ("Total order rows in the source files", n_rows + included, + excluded_total + included_total), + ): + ws.append([_c(ws, label, font=BOLD), _c(ws, ""), _c(ws, ""), _c(ws, ""), _c(ws, ""), + _c(ws, rows_n, font=BOLD), + _c(ws, round(amount_v, 2), number_format=FMT_ACCT2, font=BOLD)]) + # -- Exceptions -- def _build_exceptions(self): ws = self.wb.create_sheet("Exceptions") @@ -701,9 +819,11 @@ def export_workbook(result: ProcessResult, files: Iterable[str], output_path: st saved_column_overrides: dict[str, str] | None = None, row_limit: int = EXCEL_ROW_LIMIT, progress: Callable[[float, int, int], None] | None = None, - summary: dict | None = None, journal: dict | None = None) -> str: + summary: dict | None = None, journal: dict | None = None, + payout_receipts: dict[tuple[str, str, str], str] | None = None) -> str: builder = WorkbookBuilder(result, files, reserves=reserves, allowance_for_returns=allowance_for_returns, saved_column_overrides=saved_column_overrides, row_limit=row_limit, - progress=progress, summary=summary, journal=journal) + progress=progress, summary=summary, journal=journal, + payout_receipts=payout_receipts) return builder.build(output_path) diff --git a/ar-aging-app/backend/app/db/database.py b/ar-aging-app/backend/app/db/database.py index a03ed5a..280415d 100644 --- a/ar-aging-app/backend/app/db/database.py +++ b/ar-aging-app/backend/app/db/database.py @@ -1,4 +1,11 @@ -"""MySQL database setup (SQLAlchemy).""" +""" +Database setup (SQLAlchemy) — MySQL for shared use, SQLite for a laptop or a demo. + +Which one is used comes from `config.DB_BACKEND`. Everything above this layer is written +against SQLAlchemy and is dialect-agnostic; the two places that are not — the raw bulk +INSERT in services/store.py and the column-migration below — ask the engine which dialect +it is rather than assuming. +""" from __future__ import annotations import logging @@ -9,6 +16,7 @@ from sqlalchemy import create_engine, event, text from sqlalchemy.orm import declarative_base, sessionmaker from ..config import ( + DB_BACKEND, MYSQL_DATABASE, MYSQL_HOST, MYSQL_PASSWORD, @@ -17,17 +25,19 @@ from ..config import ( MYSQL_PORT, MYSQL_SLOW_QUERY_MS, MYSQL_USER, + database_label, + database_url, ensure_dirs, - mysql_url, ) ensure_dirs() logger = logging.getLogger(__name__) +IS_MYSQL = DB_BACKEND == "mysql" def _ensure_database() -> None: - """Create MYSQL_DATABASE if it does not exist yet.""" + """Create MYSQL_DATABASE if it does not exist yet (MySQL only).""" user = quote_plus(MYSQL_USER) password = quote_plus(MYSQL_PASSWORD) server_url = ( @@ -47,15 +57,32 @@ def _ensure_database() -> None: server_engine.dispose() -_ensure_database() +if IS_MYSQL: + _ensure_database() + ENGINE = create_engine( + database_url(), + pool_size=MYSQL_POOL_SIZE, + pool_recycle=MYSQL_POOL_RECYCLE, + pool_pre_ping=True, + future=True, + ) +else: + # check_same_thread=False: processing runs in a background thread with its own session. + ENGINE = create_engine( + database_url(), future=True, + connect_args={"check_same_thread": False, "timeout": 30}, + ) -ENGINE = create_engine( - mysql_url(), - pool_size=MYSQL_POOL_SIZE, - pool_recycle=MYSQL_POOL_RECYCLE, - pool_pre_ping=True, - future=True, -) + @event.listens_for(ENGINE, "connect") + def _sqlite_pragmas(dbapi_conn, _rec): + cur = dbapi_conn.cursor() + cur.execute("PRAGMA journal_mode=WAL") # readers don't block the writer + cur.execute("PRAGMA foreign_keys=ON") # cascade deletes behave like MySQL + cur.execute("PRAGMA busy_timeout=30000") # bulk insert vs progress updates + cur.execute("PRAGMA synchronous=NORMAL") + cur.close() + +logger.info("database: %s", database_label()) if MYSQL_SLOW_QUERY_MS > 0: @event.listens_for(ENGINE, "before_cursor_execute") @@ -86,16 +113,24 @@ def init_db() -> None: def _migrate() -> None: - """Add columns introduced after a DB was first created (create_all won't alter).""" + """ + Add columns introduced after a DB was first created (create_all won't alter). + + MySQL DDL rules that differ from SQLite and silently broke this list during the port: + * VARCHAR **must** carry a length — a bare `VARCHAR` is a syntax error. Lengths here + must match the model's String(n) or the column ends up a different width. + * TEXT/BLOB columns cannot take a literal DEFAULT before MySQL 8.0.13, so + `TEXT DEFAULT ''` fails. Declare plain TEXT and let the ORM default apply on insert. + """ added = { "sessions": [ ("progress_rows_done", "INTEGER DEFAULT 0"), ("progress_rows_total", "INTEGER DEFAULT 0"), ("eta_seconds", "INTEGER DEFAULT 0"), - ("opening_mode", "VARCHAR(255) DEFAULT 'zero'"), + ("opening_mode", "VARCHAR(32) DEFAULT 'zero'"), ("opening_source_session_id", "INTEGER"), - ("blocked_reason", "TEXT DEFAULT ''"), - ("payout_mode", "VARCHAR DEFAULT 'auto'"), + ("blocked_reason", "TEXT"), + ("payout_mode", "VARCHAR(32) DEFAULT 'auto'"), ("needs_reprocess", "BOOLEAN DEFAULT 0"), ], "session_files": [ @@ -104,14 +139,14 @@ def _migrate() -> None: ("helper_rows_skipped", "INTEGER DEFAULT 0"), ], "fx_rates": [ - ("confirmed_by", "VARCHAR DEFAULT ''"), + ("confirmed_by", "VARCHAR(255) DEFAULT ''"), ("confirmed_at", "DATETIME"), - ("confirmed_month", "VARCHAR DEFAULT ''"), + ("confirmed_month", "VARCHAR(32) DEFAULT ''"), ], "journal_entries": [ - ("reviewed_by", "VARCHAR DEFAULT ''"), + ("reviewed_by", "VARCHAR(255) DEFAULT ''"), ("reviewed_at", "DATETIME"), - ("approved_by", "VARCHAR DEFAULT ''"), + ("approved_by", "VARCHAR(255) DEFAULT ''"), ("approved_at", "DATETIME"), ], "reconciliation": [ @@ -129,22 +164,40 @@ def _migrate() -> None: ("storage_flag", "BOOLEAN DEFAULT 0"), ], } - with ENGINE.begin() as conn: - db_name = conn.execute(text("SELECT DATABASE()")).scalar() + # Each ALTER runs on its own connection scope: MySQL auto-commits DDL, so wrapping the + # whole loop in one transaction gives no rollback anyway — and one bad statement would + # otherwise abort every later migration for the rest of the run. + with ENGINE.connect() as conn: + db_name = conn.execute(text("SELECT DATABASE()")).scalar() if IS_MYSQL else None for table, cols in added.items(): - existing = { - r[0] - for r in conn.execute( - text( - "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS " - "WHERE TABLE_SCHEMA = :schema AND TABLE_NAME = :table" - ), - {"schema": db_name, "table": table}, - ) - } + if IS_MYSQL: + existing = { + r[0] + for r in conn.execute( + text( + "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS " + "WHERE TABLE_SCHEMA = :schema AND TABLE_NAME = :table" + ), + {"schema": db_name, "table": table}, + ) + } + else: + existing = {r[1] for r in conn.execute(text(f"PRAGMA table_info({table})"))} + if not existing: + continue # table not created yet — create_all owns it for name, decl in cols: - if name not in existing: - conn.execute(text(f"ALTER TABLE `{table}` ADD COLUMN `{name}` {decl}")) + if name in existing: + continue + # SQLite has no VARCHAR length limit and rejects some MySQL type spellings; + # its dynamic typing makes the declared type advisory anyway. + sql_decl = decl.replace("VARCHAR(255)", "VARCHAR").replace( + "VARCHAR(32)", "VARCHAR") if not IS_MYSQL else decl + try: + conn.execute(text(f"ALTER TABLE `{table}` ADD COLUMN `{name}` {sql_decl}")) + conn.commit() + except Exception: # noqa: BLE001 + conn.rollback() + logger.exception("migration failed: %s.%s %s", table, name, decl) def get_db(): diff --git a/ar-aging-app/backend/app/services/jobs.py b/ar-aging-app/backend/app/services/jobs.py index cf3f326..de2285f 100644 --- a/ar-aging-app/backend/app/services/jobs.py +++ b/ar-aging-app/backend/app/services/jobs.py @@ -356,10 +356,18 @@ def run_export(session_id: int) -> None: EXPORT_DIR.mkdir(parents=True, exist_ok=True) month = session.reporting_month or "output" out_path = str(EXPORT_DIR / f"AR_Aging_{month}_session{session_id}.xlsx") + # Bank dates for the "Settled Settlements" sheet, so the workbook records WHY each + # excluded settlement was excluded and who said so. + receipt_notes = { + (r.marketplace, r.account_type, r.settlement_id): + f"{r.bank_date}" + (f" · {r.entered_by}" if r.entered_by else "") + for r in receipts if r.bank_date + } export_workbook(result, paths, out_path, reserves=reserves, allowance_for_returns=session.allowance_for_returns or 0.0, saved_column_overrides=mapping_rules, - progress=write_progress, summary=_summary, journal=_journal) + progress=write_progress, summary=_summary, journal=_journal, + payout_receipts=receipt_notes) _finalize_export(db, session_id, out_path, "full") session.status = "processed" diff --git a/ar-aging-app/backend/app/services/store.py b/ar-aging-app/backend/app/services/store.py index cac024a..5f19cee 100644 --- a/ar-aging-app/backend/app/services/store.py +++ b/ar-aging-app/backend/app/services/store.py @@ -16,10 +16,12 @@ _TXN_COLS = ( "settlement_id", "order_id", "sku", "txn_type", "txn_type_en", "account_type", "posted_date", "total", "currency", "storage_flag", ) -# PyMySQL uses %-style placeholders for raw DBAPI executemany. +# Raw DBAPI, so the placeholder style is the driver's, not SQLAlchemy's: PyMySQL wants %s, +# sqlite3 wants ?. Ask the engine which dialect it is instead of hardcoding either. +_PARAM = "%s" if ENGINE.dialect.name == "mysql" else "?" _INSERT_SQL = ( f"INSERT INTO transactions ({', '.join(_TXN_COLS)}) " - f"VALUES ({', '.join(['%s'] * len(_TXN_COLS))})" + f"VALUES ({', '.join([_PARAM] * len(_TXN_COLS))})" ) @@ -75,14 +77,16 @@ class TransactionSink: recv = 1 if (st.status == "receivable" and acct.lower() in RECEIVABLE_ACCOUNT_TYPES) else 0 cur.execute( - "UPDATE transactions SET settlement_status=%s, receivable_flag=%s " - "WHERE session_id=%s AND settlement_id=%s AND marketplace=%s AND account_type=%s", + f"UPDATE transactions SET settlement_status={_PARAM}, " + f"receivable_flag={_PARAM} WHERE session_id={_PARAM} " + f"AND settlement_id={_PARAM} AND marketplace={_PARAM} " + f"AND account_type={_PARAM}", (st.status, recv, self.session_id, sid, mkt, acct), ) # transfer rows: never receivable (canonical type covers localized names) cur.execute( - "UPDATE transactions SET receivable_flag=0 " - "WHERE session_id=%s AND txn_type_en='Transfer'", + f"UPDATE transactions SET receivable_flag=0 " + f"WHERE session_id={_PARAM} AND txn_type_en='Transfer'", (self.session_id,), ) cur.close() diff --git a/ar-aging-app/backend/migrate_sqlite_to_mysql.py b/ar-aging-app/backend/migrate_sqlite_to_mysql.py new file mode 100755 index 0000000..e78ec70 --- /dev/null +++ b/ar-aging-app/backend/migrate_sqlite_to_mysql.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 +""" +Migrate the legacy SQLite database into MySQL. + +The app stored everything in `backend/data/ar_aging.db` before the MySQL move. That file +still holds the real month-end closings (Jan-2026: 3,399,517 transaction rows), and MySQL +starts empty, so the closings have to be copied across once. + + python3 migrate_sqlite_to_mysql.py --dry-run # inspect the source, touch nothing + python3 migrate_sqlite_to_mysql.py # migrate + python3 migrate_sqlite_to_mysql.py --force # migrate into a non-empty MySQL + +What it does + * copies every table in foreign-key order, so a child row never precedes its session + * converts SQLite's text dates / 0-1 booleans to real MySQL DATE, DATETIME and BOOLEAN + * copies only the columns both schemas share, and reports any it had to skip + * streams in batches, so 3.4M rows never sit in memory + * verifies afterwards: row counts per table AND financial checksums (Σ transaction + totals, per-marketplace receivable) must match the source exactly + +It never deletes anything from SQLite — the file is opened read-only. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import os +import sqlite3 +import sys +from pathlib import Path + +BACKEND = Path(__file__).resolve().parent +sys.path.insert(0, str(BACKEND)) + +BATCH = 5000 + +# Tables whose contents are re-derivable by re-processing, but copied anyway so the +# migrated database is byte-identical in what the dashboard shows. +SKIP_TABLES: set[str] = set() + + +def log(msg: str = "") -> None: + print(msg, flush=True) + + +# --------------------------------------------------------------------------- source +def sqlite_tables(conn: sqlite3.Connection) -> set[str]: + return {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")} + + +def sqlite_columns(conn: sqlite3.Connection, table: str) -> list[str]: + return [r[1] for r in conn.execute(f"PRAGMA table_info({table})")] + + +def sqlite_count(conn: sqlite3.Connection, table: str) -> int: + return conn.execute(f"SELECT COUNT(*) FROM `{table}`").fetchone()[0] + + +def open_sqlite(path: Path) -> sqlite3.Connection: + """Open read-only. A stale -wal is checkpointed into a COPY, never the original.""" + if not path.exists(): + raise SystemExit(f"SQLite file not found: {path}") + # immutable=0 so an existing -wal is still applied; mode=ro keeps us from writing. + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + return conn + + +# --------------------------------------------------------------------------- convert +def make_converter(col_type) -> "callable": + """Return a function turning a SQLite value into something MySQL accepts.""" + name = col_type.__class__.__name__ + + if name == "Date": + def conv(v): + if v in (None, ""): + return None + if isinstance(v, dt.date) and not isinstance(v, dt.datetime): + return v + if isinstance(v, dt.datetime): + return v.date() + try: + return dt.date.fromisoformat(str(v)[:10]) + except ValueError: + return None + return conv + + if name == "DateTime": + def conv(v): + if v in (None, ""): + return None + if isinstance(v, dt.datetime): + return v + s = str(v).replace("T", " ") + for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"): + try: + return dt.datetime.strptime(s[:26], fmt) + except ValueError: + continue + return None + return conv + + if name == "Boolean": + def conv(v): + if v is None: + return None + if isinstance(v, bool): + return v + return bool(int(v)) if str(v).strip() in ("0", "1") else bool(v) + return conv + + if name in ("Integer", "BigInteger", "SmallInteger"): + def conv(v): + if v in (None, ""): + return None + try: + return int(v) + except (TypeError, ValueError): + return None + return conv + + if name in ("Float", "Numeric"): + def conv(v): + if v in (None, ""): + return None + try: + return float(v) + except (TypeError, ValueError): + return None + return conv + + # String / Text: MySQL columns are sized, so over-long values would be truncated or + # rejected. Trim to the declared length and report it rather than failing the batch. + length = getattr(col_type, "length", None) + + def conv(v): + if v is None: + return None + s = v if isinstance(v, str) else str(v) + return s[:length] if length and len(s) > length else s + return conv + + +# --------------------------------------------------------------------------- migrate +def migrate() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--sqlite", default=str(BACKEND / "data" / "ar_aging.db"), + help="path to the legacy SQLite file") + ap.add_argument("--dry-run", action="store_true", + help="inspect the source and print the plan; do not connect to MySQL") + ap.add_argument("--force", action="store_true", + help="migrate even if the MySQL tables already contain rows") + args = ap.parse_args() + + src_path = Path(args.sqlite) + src = open_sqlite(src_path) + have = sqlite_tables(src) + + log("") + log(" Migrate SQLite → MySQL") + log(f" source: {src_path} ({src_path.stat().st_size / 1e9:.2f} GB)") + wal = src_path.with_name(src_path.name + "-wal") + if wal.exists() and wal.stat().st_size > 0: + log(f" note: a {wal.stat().st_size / 1e6:.0f} MB write-ahead log is present and " + f"will be read as part of the database") + log("") + + # ---- source inventory (works with no MySQL at all) ---- + log(" Source contents") + src_counts: dict[str, int] = {} + for t in sorted(have): + src_counts[t] = sqlite_count(src, t) + for t, n in sorted(src_counts.items(), key=lambda kv: -kv[1]): + if n: + log(f" {t:22} {n:>10,}") + empty = [t for t, n in src_counts.items() if not n] + if empty: + log(f" ({len(empty)} empty: {', '.join(sorted(empty))})") + + src_checks = financial_checksums_sqlite(src) + log("") + log(" Financial checksums to preserve") + for k, v in src_checks.items(): + log(f" {k:34} {v}") + + if args.dry_run: + log("") + log(" Dry run — MySQL was not contacted and nothing was written.") + log(" Fill in ar-aging-app/.env, then re-run without --dry-run.") + return 0 + + # ---- target ---- + try: + from app.db.database import ENGINE, init_db + from app.db import models # noqa: F401 + except Exception as e: # noqa: BLE001 + log("") + log(f" Could not connect to MySQL: {e}") + log(" Check MYSQL_* in ar-aging-app/.env and that the server is reachable.") + return 1 + + log("") + log(" Creating the MySQL schema (safe if it already exists)…") + init_db() + + meta = models.Base.metadata + ordered = [t for t in meta.sorted_tables if t.name in have and t.name not in SKIP_TABLES] + missing_in_sqlite = [t.name for t in meta.sorted_tables if t.name not in have] + if missing_in_sqlite: + log(f" tables absent from the SQLite file (created empty): " + f"{', '.join(missing_in_sqlite)}") + + raw = ENGINE.raw_connection() + cur = raw.cursor() + # Existing rows? + non_empty = [] + for t in ordered: + cur.execute(f"SELECT COUNT(*) FROM `{t.name}`") + n = cur.fetchone()[0] + if n: + non_empty.append((t.name, n)) + if non_empty and not args.force: + log("") + log(" MySQL already contains data — refusing to migrate on top of it:") + for name, n in non_empty: + log(f" {name:22} {n:>10,} rows") + log("") + log(" Re-run with --force to add these rows anyway (duplicates are possible),") + log(" or empty the MySQL database first.") + return 1 + + log("") + log(" Copying tables (foreign-key order)") + cur.execute("SET FOREIGN_KEY_CHECKS=0") + cur.execute("SET UNIQUE_CHECKS=0") + truncated: list[str] = [] + copied: dict[str, int] = {} + try: + for table in ordered: + name = table.name + total = src_counts.get(name, 0) + if not total: + copied[name] = 0 + continue + sq_cols = set(sqlite_columns(src, name)) + cols = [c for c in table.columns if c.name in sq_cols] + dropped = [c.name for c in table.columns if c.name not in sq_cols] + extra = sq_cols - {c.name for c in table.columns} + convs = [make_converter(c.type) for c in cols] + names = [c.name for c in cols] + placeholders = ", ".join(["%s"] * len(names)) + collist = ", ".join(f"`{n}`" for n in names) + sql = f"INSERT INTO `{name}` ({collist}) VALUES ({placeholders})" + + done = 0 + batch: list[tuple] = [] + for row in src.execute(f"SELECT {', '.join(f'`{n}`' for n in names)} " + f"FROM `{name}`"): + vals = [] + for i, conv in enumerate(convs): + v = conv(row[i]) + vals.append(v) + batch.append(tuple(vals)) + if len(batch) >= BATCH: + cur.executemany(sql, batch) + raw.commit() + done += len(batch) + batch.clear() + if total > 50000: + pct = 100.0 * done / total + print(f" {name:22} {done:>10,} / {total:,} ({pct:5.1f}%)", + end="\r", flush=True) + if batch: + cur.executemany(sql, batch) + raw.commit() + done += len(batch) + copied[name] = done + note = "" + if dropped: + note += f" [not in source: {', '.join(dropped)}]" + if extra: + note += f" [source-only, skipped: {', '.join(sorted(extra))}]" + truncated.append(name) + print(" " * 78, end="\r") + log(f" {name:22} {done:>10,}{note}") + finally: + cur.execute("SET FOREIGN_KEY_CHECKS=1") + cur.execute("SET UNIQUE_CHECKS=1") + raw.commit() + + # ---- verify ---- + log("") + log(" Verifying") + ok = True + for name, n in sorted(copied.items()): + cur.execute(f"SELECT COUNT(*) FROM `{name}`") + got = cur.fetchone()[0] + want = src_counts.get(name, 0) + if got != want: + ok = False + log(f" ✗ {name:22} MySQL {got:,} != SQLite {want:,}") + if ok: + log(f" ✓ row counts match on all {len(copied)} tables") + + dst_checks = financial_checksums_mysql(cur) + for k, want in src_checks.items(): + got = dst_checks.get(k) + if str(got) != str(want): + ok = False + log(f" ✗ {k}: MySQL {got} != SQLite {want}") + if ok: + log(" ✓ financial checksums match") + + cur.close() + raw.close() + src.close() + + log("") + if ok: + log(" Migration complete. Start the dashboard with start.command.") + return 0 + log(" Migration finished with MISMATCHES — do not rely on the MySQL data until") + log(" they are explained. The SQLite file is untouched.") + return 1 + + +# --------------------------------------------------------------------------- checksums +def financial_checksums_sqlite(conn: sqlite3.Connection) -> dict[str, str]: + out: dict[str, str] = {} + tables = sqlite_tables(conn) + + def one(sql: str, default="—") -> str: + try: + r = conn.execute(sql).fetchone() + return "—" if r is None or r[0] is None else str(r[0]) + except sqlite3.Error: + return default + + if "sessions" in tables: + out["sessions"] = one("SELECT COUNT(*) FROM sessions") + if "transactions" in tables: + out["transaction rows"] = one("SELECT COUNT(*) FROM transactions") + out["Σ transactions.total"] = one("SELECT ROUND(SUM(total),2) FROM transactions") + out["receivable-flagged rows"] = one( + "SELECT COUNT(*) FROM transactions WHERE receivable_flag=1") + if "receivable_results" in tables: + out["USA receivable_local (TOTAL)"] = one( + "SELECT ROUND(receivable_local) FROM receivable_results " + "WHERE marketplace='USA' AND account_type='TOTAL'") + out["Σ receivable_usd (TOTAL rows)"] = one( + "SELECT ROUND(SUM(receivable_usd),2) FROM receivable_results " + "WHERE account_type='TOTAL'") + return out + + +def financial_checksums_mysql(cur) -> dict[str, str]: + out: dict[str, str] = {} + + def one(sql: str) -> str: + try: + cur.execute(sql) + r = cur.fetchone() + return "—" if r is None or r[0] is None else str(r[0]) + except Exception: # noqa: BLE001 + return "—" + + out["sessions"] = one("SELECT COUNT(*) FROM sessions") + out["transaction rows"] = one("SELECT COUNT(*) FROM transactions") + out["Σ transactions.total"] = one("SELECT ROUND(SUM(total),2) FROM transactions") + out["receivable-flagged rows"] = one( + "SELECT COUNT(*) FROM transactions WHERE receivable_flag=1") + out["USA receivable_local (TOTAL)"] = one( + "SELECT ROUND(receivable_local) FROM receivable_results " + "WHERE marketplace='USA' AND account_type='TOTAL'") + out["Σ receivable_usd (TOTAL rows)"] = one( + "SELECT ROUND(SUM(receivable_usd),2) FROM receivable_results " + "WHERE account_type='TOTAL'") + return out + + +if __name__ == "__main__": + try: + raise SystemExit(migrate()) + except KeyboardInterrupt: + log("\n Interrupted. The SQLite source is unchanged.") + raise SystemExit(130) diff --git a/ar-aging-app/backend/tests/conftest.py b/ar-aging-app/backend/tests/conftest.py index 2ad9088..612d807 100644 --- a/ar-aging-app/backend/tests/conftest.py +++ b/ar-aging-app/backend/tests/conftest.py @@ -8,20 +8,24 @@ from pathlib import Path import pytest # --------------------------------------------------------------------------------------- -# Redirect ALL test data to a throwaway directory — BEFORE anything imports app.config, -# which reads these variables once at module load and caches the paths. +# Redirect ALL test data away from production — BEFORE anything imports app.config, which +# reads these variables once at module load and caches them. # -# Without this the suite runs against the real production database: `app/config.py` falls -# back to `backend/data/ar_aging.db`, so every test that created a closing was writing into -# Finance's live data (75 sessions had accumulated there). Tests must never be able to touch -# a real closing. +# The suite creates AND DELETES closings, so pointing it at the live database would destroy +# Finance's data. That already happened once under SQLite (75 test sessions accumulated in +# the production file), and the blast radius is larger now that the store is a shared MySQL +# server rather than a local file. # -# The names must match app/config.py exactly — AR_DB_PATH / AR_DATA_DIR. A near-miss such as -# "AR_DB_URL" silently does nothing and the tests quietly hit production again. +# `load_dotenv()` in app/config.py does not override variables already present in the +# environment, so setting MYSQL_DATABASE here wins over .env. The database is created +# automatically by database._ensure_database(). # --------------------------------------------------------------------------------------- _TEST_DATA_DIR = Path(tempfile.mkdtemp(prefix="ar-aging-tests-")) os.environ["AR_DATA_DIR"] = str(_TEST_DATA_DIR) -os.environ["AR_DB_PATH"] = str(_TEST_DATA_DIR / "test.db") + +_PROD_DB = os.environ.get("MYSQL_DATABASE", "") +TEST_DB_NAME = os.environ.get("AR_TEST_MYSQL_DATABASE", "ar_aging_pytest") +os.environ["MYSQL_DATABASE"] = TEST_DB_NAME # Default: the project root two levels above ar-aging-app/backend. _DEFAULT_SAMPLE_DIR = Path(__file__).resolve().parents[3] @@ -58,13 +62,19 @@ def _never_touch_production_data(): """ Hard stop if the redirect above ever fails. - The suite creates and deletes closings, so pointing at the real database would destroy - Finance's data. Assert the isolation actually took effect rather than trusting it. + The suite creates and deletes closings, so running against the live database would + destroy Finance's data. Assert the isolation actually took effect rather than trusting + it — this fixture is the reason a renamed config variable can't silently re-point the + tests at production. """ - from app.config import DATA_DIR, DB_PATH - assert str(DB_PATH).startswith(str(_TEST_DATA_DIR)), ( - f"tests are pointed at {DB_PATH} — expected a temp path under {_TEST_DATA_DIR}. " - f"app/config.py reads AR_DB_PATH / AR_DATA_DIR; check those names." + from app.config import DATA_DIR, MYSQL_DATABASE + assert MYSQL_DATABASE == TEST_DB_NAME, ( + f"tests are pointed at MySQL database {MYSQL_DATABASE!r} — expected " + f"{TEST_DB_NAME!r}. app/config.py reads MYSQL_DATABASE; check that name." + ) + assert not _PROD_DB or MYSQL_DATABASE != _PROD_DB, ( + f"the test database is the same as the configured production database " + f"({_PROD_DB!r}). Set AR_TEST_MYSQL_DATABASE to a separate name." ) assert str(DATA_DIR).startswith(str(_TEST_DATA_DIR)), ( f"tests would write uploads/exports to {DATA_DIR}, not a temp directory." diff --git a/ar-aging-app/backend/tests/test_excel_export.py b/ar-aging-app/backend/tests/test_excel_export.py index 0ba1145..2563871 100644 --- a/ar-aging-app/backend/tests/test_excel_export.py +++ b/ar-aging-app/backend/tests/test_excel_export.py @@ -182,3 +182,87 @@ def test_sheet_split_when_over_row_limit(synth_file): add = wb["Detail"]["B5"].value for s in usa_sheets: assert f"'{s}'!" in add + + +# ------------------------------------------------------------------ workbook must tie +def test_detail_references_every_account_stream(synth_file): + """ + Detail/Summary must reference EVERY account stream's subtotal cell. + + compute_layouts() planned the subtotal rows with a stride of 2 while + _finalize_marketplace_subtotals() writes them consecutively, so every stream after the + first pointed at an empty cell. USA is the only marketplace with two streams, so the + whole Invoiced Orders receivable silently vanished from Detail and Summary (Jan-2026: + 67,854.71) while Reconciliation and COA in the same workbook showed it. + """ + result = process([synth_file], month_end=date(2026, 1, 31), clearing_lag_days=2) + out = synth_file.replace(".xlsx", "-streams.xlsx") + export_workbook(result, [synth_file], out) + wb = openpyxl.load_workbook(out) + ws = wb["USA"] + + # Where the subtotal formulas actually landed. + actual = {} + for row in ws.iter_rows(): + for c in row: + if isinstance(c.value, str) and c.value.startswith("=SUMIFS"): + actual[ws.cell(row=c.row, column=c.column - 1).value] = c.coordinate + streams = set(result.receivable.marketplaces["USA"].accounts) + assert set(actual) == streams, f"a stream has no subtotal row: {actual} vs {streams}" + + detail_formula = wb["Detail"]["B5"].value + for stream, coord in actual.items(): + assert f"'USA'!{coord}" in detail_formula, ( + f"Detail!B5 ({detail_formula}) does not reference the {stream} subtotal at {coord}" + ) + + +def test_summary_net_receivable_is_not_circular(synth_file): + """'Net Receivable' referenced its own cell, so Excel warned and showed 0.""" + result = process([synth_file], month_end=date(2026, 1, 31), clearing_lag_days=2) + out = synth_file.replace(".xlsx", "-net.xlsx") + export_workbook(result, [synth_file], out, allowance_for_returns=-100.0) + ws = openpyxl.load_workbook(out)["Summary"] + rows = {ws.cell(row=r, column=1).value: r for r in range(1, ws.max_row + 1)} + net_row = rows["Net Receivable"] + formula = ws.cell(row=net_row, column=7).value + assert f"G{net_row}" not in formula, f"circular reference: G{net_row} in {formula}" + assert f"G{rows['TOTAL']}" in formula and f"G{rows['Allowance for Sales Returns']}" in formula + + +def test_every_received_payout_appears_on_the_tab(synth_file): + """ + A month can have several received payouts per stream; the workbook used to lift only the + boundary one, so earlier bank receipts were absent from the entire file and the tab could + not be hand-footed against the bank statement. + """ + result = process([synth_file], month_end=date(2026, 1, 31), clearing_lag_days=2) + out = synth_file.replace(".xlsx", "-payouts.xlsx") + export_workbook(result, [synth_file], out) + ws = openpyxl.load_workbook(out)["USA"] + in_book = sorted(r[TOTAL_IDX] for r in ws.iter_rows(min_row=9, values_only=True) + if r and r[FIELD_ORDER.index("txn_type")] == "Transfer") + received = sorted(t.amount for t in result.aggregation.transfers if t.received) + assert in_book == received, f"workbook payouts {in_book} != received payouts {received}" + + +def test_settled_settlements_sheet_reconciles(synth_file): + """The excluded settlements are listed and their rows + the included rows = every row.""" + result = process([synth_file], month_end=date(2026, 1, 31), clearing_lag_days=2) + out = synth_file.replace(".xlsx", "-settled.xlsx") + export_workbook(result, [synth_file], out) + ws = openpyxl.load_workbook(out)["Settled Settlements"] + labels = {ws.cell(row=r, column=1).value: r for r in range(1, ws.max_row + 1)} + excluded = ws.cell(row=labels["Excluded (settled) order rows"], column=6).value + included = ws.cell(row=labels["Included (open) order rows — in the marketplace tabs"], + column=6).value + total = ws.cell(row=labels["Total order rows in the source files"], column=6).value + assert excluded + included == total + engine_total = sum(st.row_count - st.transfer_count + for st in result.aggregation.settlements.values()) + assert total == engine_total, f"sheet says {total} order rows, engine has {engine_total}" + # Every settled settlement is named, so the omission is documented rather than silent. + listed = {ws.cell(row=r, column=3).value for r in range(5, ws.max_row + 1)} + settled = {sid for (m, a, sid), st in result.aggregation.settlements.items() + if st.status != "receivable" and (st.row_count - st.transfer_count) > 0} + assert settled <= listed, f"settled settlements missing from the sheet: {settled - listed}" diff --git a/start.command b/start.command index 5e33533..2c7c9f8 100755 --- a/start.command +++ b/start.command @@ -68,33 +68,77 @@ command -v npm >/dev/null 2>&1 || die "npm was not found." \ "install Node.js 20+ from nodejs.org" good "python $(python3 -V 2>&1 | awk '{print $2}') · node $(node -v 2>/dev/null) · npm $(npm -v 2>/dev/null)" -# Python packages — actually IMPORT the app rather than checking a few package names. -# "Installed" is not the same as "compatible": an unpinned starlette upgrade once satisfied -# every import check while breaking the app at load time. Importing proves it will boot. -import_error="" -if ! import_error="$(cd -- "$BACKEND_DIR" && python3 -c "from app.api.main import app" 2>&1)"; then - warn "the backend could not be loaded:" - printf '%s\n' "$import_error" | tail -n 3 | sed 's/^/ /' - printf ' Install/repair Python packages now? [Y/n] ' +# Each check below diagnoses ONE thing. A single "does the app import?" test used to stand in +# for all of them, so a database that was merely switched off was reported as broken Python +# packages — and the launcher then ran pip install, which of course changed nothing. + +# 1a. Python packages — libraries only, no app code, so this cannot fail for config reasons. +if ! python3 -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1; then + warn "Python packages are missing or incomplete." + printf ' Install them now? [Y/n] ' read -r reply case "${reply:-Y}" in - [Nn]*) die "The backend cannot start with the current Python packages." \ + [Nn]*) die "Python dependencies are not installed." \ "python3 -m pip install -r '$BACKEND_DIR/requirements.txt'" ;; esac step "installing Python packages (this can take a minute)…" python3 -m pip install -q -r "$BACKEND_DIR/requirements.txt" \ || die "pip install failed — see the messages above." \ "python3 -m pip install -r '$BACKEND_DIR/requirements.txt'" - if ! import_error="$(cd -- "$BACKEND_DIR" && python3 -c "from app.api.main import app" 2>&1)"; then - printf '%s\n' "$import_error" | tail -n 5 | sed 's/^/ /' - die "The backend still cannot be loaded after installing packages." \ - "python3 -m pip check" - fi - good "Python packages repaired" + python3 -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1 \ + || die "Python packages are still incomplete after installing." "python3 -m pip check" + good "Python packages installed" else - good "Python packages present and compatible" + good "Python packages present" fi +# 1b. Which database? SQLite needs nothing; MySQL needs a server and credentials. The app +# picks MySQL only when ar-aging-app/.env names a real host, so a machine with no +# database installed still runs off the local file instead of refusing to start. +ENV_FILE="$APP_DIR/.env" +db_backend="$(cd -- "$BACKEND_DIR" && python3 -c " +from app.config import DB_BACKEND; print(DB_BACKEND)" 2>/dev/null)" +db_label="$(cd -- "$BACKEND_DIR" && python3 -c " +from app.config import database_label; print(database_label())" 2>/dev/null)" + +if [ "$db_backend" = "mysql" ]; then + good "database: $db_label" + # Reachable? A refused connection is a server/credentials problem, never a Python one. + db_err="$(cd -- "$BACKEND_DIR" && python3 -c " +import sys, pymysql +from app.config import MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD +try: + pymysql.connect(host=MYSQL_HOST, port=MYSQL_PORT, user=MYSQL_USER, + password=MYSQL_PASSWORD, connect_timeout=6).close() +except Exception as e: + sys.stderr.write(str(e)); sys.exit(1) +" 2>&1)" || { + fail "Cannot reach the MySQL server named in $ENV_FILE" + say "" + printf ' %s\n' "$(printf '%s' "$db_err" | tail -n 2)" + say "" + say " Start the server and run this again, or fall back to the local file by setting" + say " ${B}AR_DB_BACKEND=sqlite${R} in $ENV_FILE." + say "" + printf '%sPress Return to close this window.%s\n' "$DIM" "$R" + read -r _ + exit 1 + } + good "MySQL reachable" +else + good "database: $db_label" + [ -f "$ENV_FILE" ] || warn "no .env — using the local file (fine for a demo or one user)" +fi + +# 1d. Everything above is fine — now prove the app itself loads. Anything failing here is a +# genuine code/dependency problem, so the message points at that rather than at config. +if ! import_error="$(cd -- "$BACKEND_DIR" && python3 -c "from app.api.main import app" 2>&1)"; then + fail "The backend failed to load even though its packages and database are fine:" + printf '%s\n' "$import_error" | tail -n 6 | sed 's/^/ /' + die "This looks like a code or dependency-version problem." "python3 -m pip check" +fi +good "backend loads cleanly" + # Frontend packages — safe to install unattended, they're local to the project. if [ ! -d "$FRONTEND_DIR/node_modules" ]; then step "installing frontend packages (first run only, ~1 minute)…"