New
sheheryarsoomro12
parent
4e69fce7d2
commit
bacd13c8b5
|
|
@ -23,6 +23,17 @@ def start_export(session_id: int, background: BackgroundTasks, kind: str = "full
|
|||
s = get_session_or_404(session_id, db)
|
||||
# An export is the number leaving the building — never generate one from a blocked close.
|
||||
ensure_not_blocked(s)
|
||||
# The full workbook RE-COMPUTES its marketplace tabs from the source files using the
|
||||
# current bank receipts, while the AR Ledger / Finance Summary sheets bound into the same
|
||||
# file come from the last processing run. With unapplied receipts those two halves
|
||||
# disagree — the tabs would show one receivable and the ledger sheet another.
|
||||
if s.needs_reprocess:
|
||||
raise HTTPException(
|
||||
409,
|
||||
"Bank receipts or the payout mode changed after the last run. Re-process the "
|
||||
"closing first — otherwise the workbook's marketplace tabs and its AR Ledger "
|
||||
"sheet would report different receivables.",
|
||||
)
|
||||
if s.status not in ("processed", "exporting", "completed"):
|
||||
raise HTTPException(400, "Process the session before exporting.")
|
||||
if s.status == "exporting":
|
||||
|
|
|
|||
|
|
@ -17,7 +17,15 @@ DATA_DIR = Path(os.environ.get("AR_DATA_DIR", str(BASE_DIR / "data")))
|
|||
UPLOAD_DIR = DATA_DIR / "uploads"
|
||||
EXPORT_DIR = DATA_DIR / "exports"
|
||||
|
||||
# MySQL (required)
|
||||
# --------------------------------------------------------------------------- database
|
||||
# The store is switchable so the app is never blocked on infrastructure:
|
||||
# AR_DB_BACKEND=sqlite a single local file — zero setup, good for a laptop or a demo
|
||||
# AR_DB_BACKEND=mysql the shared server, for real multi-user month-end work
|
||||
# Unset, it picks MySQL when a real MYSQL_HOST is configured and SQLite otherwise, so a
|
||||
# machine with no database installed still runs instead of failing at import.
|
||||
_PLACEHOLDER_HOSTS = {"", "your-mysql-host.example.com", "changeme", "todo"}
|
||||
SQLITE_PATH = Path(os.environ.get("AR_SQLITE_PATH", str(DATA_DIR / "ar_aging.db")))
|
||||
|
||||
MYSQL_HOST = os.environ.get("MYSQL_HOST", "")
|
||||
MYSQL_PORT = int(os.environ.get("MYSQL_PORT", "3306"))
|
||||
MYSQL_USER = os.environ.get("MYSQL_USER", "")
|
||||
|
|
@ -28,11 +36,23 @@ MYSQL_POOL_SIZE = int(os.environ.get("MYSQL_POOL_SIZE", "10"))
|
|||
MYSQL_POOL_RECYCLE = int(os.environ.get("MYSQL_POOL_RECYCLE", "3600"))
|
||||
|
||||
|
||||
def _mysql_configured() -> bool:
|
||||
return (MYSQL_HOST.strip().lower() not in _PLACEHOLDER_HOSTS
|
||||
and bool(MYSQL_USER) and bool(MYSQL_DATABASE))
|
||||
|
||||
|
||||
DB_BACKEND = (os.environ.get("AR_DB_BACKEND")
|
||||
or ("mysql" if _mysql_configured() else "sqlite")).strip().lower()
|
||||
if DB_BACKEND not in ("sqlite", "mysql"):
|
||||
raise RuntimeError(f"AR_DB_BACKEND must be 'sqlite' or 'mysql' (got {DB_BACKEND!r}).")
|
||||
|
||||
|
||||
def mysql_url() -> str:
|
||||
if not all((MYSQL_HOST, MYSQL_USER, MYSQL_DATABASE)):
|
||||
if not _mysql_configured():
|
||||
raise RuntimeError(
|
||||
"MYSQL_HOST, MYSQL_USER, and MYSQL_DATABASE are required. "
|
||||
"Copy example.env to .env and fill in credentials."
|
||||
"MYSQL_HOST, MYSQL_USER, and MYSQL_DATABASE are required for the mysql "
|
||||
"backend. Copy example.env to .env and fill in real credentials, or set "
|
||||
"AR_DB_BACKEND=sqlite to use a local file."
|
||||
)
|
||||
user = quote_plus(MYSQL_USER)
|
||||
password = quote_plus(MYSQL_PASSWORD)
|
||||
|
|
@ -43,6 +63,20 @@ def mysql_url() -> str:
|
|||
)
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
if DB_BACKEND == "mysql":
|
||||
return mysql_url()
|
||||
SQLITE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
return f"sqlite:///{SQLITE_PATH}"
|
||||
|
||||
|
||||
def database_label() -> str:
|
||||
"""Human-readable target, for logs and the launcher — never includes the password."""
|
||||
if DB_BACKEND == "mysql":
|
||||
return f"MySQL {MYSQL_USER}@{MYSQL_HOST}:{MYSQL_PORT}/{MYSQL_DATABASE}"
|
||||
return f"SQLite {SQLITE_PATH}"
|
||||
|
||||
|
||||
# Retention: temp uploads/exports older than this are purged (0 = keep forever).
|
||||
RETENTION_DAYS = int(os.environ.get("AR_RETENTION_DAYS", "30"))
|
||||
|
||||
|
|
|
|||
|
|
@ -107,7 +107,10 @@ class SheetLayout:
|
|||
data_start: int = 0
|
||||
data_rows: int = 0
|
||||
subtotal_cells: dict[str, str] = field(default_factory=dict) # account_type -> "AD####"
|
||||
transfer_cells: dict[str, str] = field(default_factory=dict) # account_type -> "AD####"
|
||||
# account_type -> ["AD9", "AD10", …]: a month can have SEVERAL received payouts per
|
||||
# stream, so this is a list. It used to be one cell per account, which silently showed
|
||||
# only the boundary payout and omitted every earlier one from the workbook.
|
||||
transfer_cells: dict[str, list[str]] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def data_end(self) -> int:
|
||||
|
|
@ -124,11 +127,10 @@ class MarketplaceLayout:
|
|||
return [f"'{s.name}'!{s.subtotal_cells[account_type]}"
|
||||
for s in self.sheets if account_type in s.subtotal_cells]
|
||||
|
||||
def transfer_ref(self, account_type: str) -> str | None:
|
||||
for s in self.sheets:
|
||||
if account_type in s.transfer_cells:
|
||||
return f"'{s.name}'!{s.transfer_cells[account_type]}"
|
||||
return None
|
||||
def transfer_refs(self, account_type: str) -> list[str]:
|
||||
"""Every received-payout cell for an account stream (a month can have several)."""
|
||||
return [f"'{s.name}'!{c}"
|
||||
for s in self.sheets for c in s.transfer_cells.get(account_type, [])]
|
||||
|
||||
|
||||
def _sheet_names(marketplace: str, n: int) -> list[str]:
|
||||
|
|
@ -143,10 +145,18 @@ def compute_layouts(result: ProcessResult, row_limit: int = EXCEL_ROW_LIMIT) ->
|
|||
cls = result.classification
|
||||
assert agg is not None and cls is not None
|
||||
|
||||
# order-row counts and receivable account types per marketplace
|
||||
# Order-row counts and receivable account types per marketplace. Only OPEN (receivable)
|
||||
# settlements contribute rows: once a settlement's payout has reached the bank it is
|
||||
# closed, and its raw transactions are deliberately left out of the workbook (they are
|
||||
# listed in summary form on the "Settled Settlements" sheet instead).
|
||||
per_mkt_orders: dict[str, int] = {}
|
||||
per_mkt_accts: dict[str, list[str]] = {}
|
||||
all_mkt_accts: dict[str, list[str]] = {}
|
||||
for (mkt, acct, sid), st in agg.settlements.items():
|
||||
if acct.lower() in RECEIVABLE_ACCOUNT_TYPES:
|
||||
seen = all_mkt_accts.setdefault(mkt, [])
|
||||
if acct not in seen:
|
||||
seen.append(acct)
|
||||
if st.status != "receivable" or acct.lower() not in RECEIVABLE_ACCOUNT_TYPES:
|
||||
continue
|
||||
orders = st.row_count - st.transfer_count
|
||||
|
|
@ -155,17 +165,33 @@ def compute_layouts(result: ProcessResult, row_limit: int = EXCEL_ROW_LIMIT) ->
|
|||
if acct not in accts:
|
||||
accts.append(acct)
|
||||
|
||||
# boundary (receipt) transfers per marketplace/account
|
||||
boundary_tx: dict[tuple[str, str], object] = {}
|
||||
for k, t in cls.boundary_transfer.items():
|
||||
if t is not None:
|
||||
boundary_tx[k] = t
|
||||
# A marketplace whose payouts were ALL received has nothing outstanding and so no order
|
||||
# rows — but it still gets a tab. An absent tab is indistinguishable from a marketplace
|
||||
# whose file failed to upload; an empty one with its receipts and a 0.00 subtotal proves
|
||||
# the market was processed and legitimately had nothing open.
|
||||
for mkt in sorted(agg.marketplaces_seen):
|
||||
per_mkt_orders.setdefault(mkt, 0)
|
||||
|
||||
# EVERY received payout per (marketplace, account stream) — not just the boundary one.
|
||||
# Using cls.boundary_transfer here showed a single payout per stream, so a month with
|
||||
# several bank receipts silently omitted all but the last from the workbook.
|
||||
received_tx: dict[tuple[str, str], list] = {}
|
||||
for t in agg.transfers:
|
||||
if not t.received:
|
||||
continue # in transit: its settlement is still open
|
||||
owner = cls.settlement_owner.get(t.settlement_id, t.marketplace)
|
||||
received_tx.setdefault((owner, t.account_type), []).append(t)
|
||||
for lst in received_tx.values():
|
||||
lst.sort(key=lambda t: (t.txn_date or date.min, t.settlement_id))
|
||||
|
||||
layouts: dict[str, MarketplaceLayout] = {}
|
||||
for mkt, order_total in per_mkt_orders.items():
|
||||
accts = sorted(per_mkt_accts.get(mkt, []),
|
||||
accts = sorted(per_mkt_accts.get(mkt, []) or all_mkt_accts.get(mkt, []),
|
||||
key=lambda a: (0 if a.lower() == "standard orders" else 1, a))
|
||||
transfers = [boundary_tx[(mkt, a)] for a in accts if (mkt, a) in boundary_tx]
|
||||
# Payout rows are keyed by the account type Amazon tagged them with, which is blank
|
||||
# ("(unspecified)") everywhere except the USA — include those streams too.
|
||||
transfers = [t for (m, _a), lst in received_tx.items() if m == mkt for t in lst]
|
||||
transfers.sort(key=lambda t: (t.txn_date or date.min, t.settlement_id))
|
||||
|
||||
# capacity of the first sheet (accounts for preamble+header+transfers+subtotals)
|
||||
subtotal_block = 1 + 2 * max(len(accts), 1) # gap + one subtotal row per account
|
||||
|
|
@ -192,13 +218,20 @@ def compute_layouts(result: ProcessResult, row_limit: int = EXCEL_ROW_LIMIT) ->
|
|||
# rows: preamble 1-7, header 8, transfers 9.., data start after transfers
|
||||
sl.data_start = 8 + len(tlist) + 1
|
||||
sl.data_rows = nrows
|
||||
# transfer cell addresses (rows 9..)
|
||||
# transfer cell addresses (rows 9..) — several payouts can share an account stream
|
||||
for j, t in enumerate(tlist):
|
||||
sl.transfer_cells[t.account_type] = f"{TOTAL_COL}{9 + j}"
|
||||
# subtotal rows after data (one blank gap, then one row per account)
|
||||
sl.transfer_cells.setdefault(t.account_type, []).append(f"{TOTAL_COL}{9 + j}")
|
||||
# Subtotal rows after the data: one blank gap row, then ONE row per account
|
||||
# stream, consecutively — this must mirror _finalize_marketplace_subtotals()
|
||||
# exactly, because Detail/Summary reference these planned addresses.
|
||||
#
|
||||
# This used to stride by 2 while the writer strides by 1, so every stream after
|
||||
# the first pointed at an empty cell. USA is the only marketplace with two
|
||||
# streams, so Detail and Summary silently dropped the whole Invoiced Orders
|
||||
# receivable (Jan-2026: 67,854.71) while Reconciliation and COA showed it.
|
||||
base = sl.data_end + 2
|
||||
for j, acct in enumerate(accts):
|
||||
sl.subtotal_cells[acct] = f"{TOTAL_COL}{base + 2 * j}"
|
||||
sl.subtotal_cells[acct] = f"{TOTAL_COL}{base + j}"
|
||||
ml.sheets.append(sl)
|
||||
layouts[mkt] = ml
|
||||
return layouts
|
||||
|
|
@ -227,7 +260,8 @@ class WorkbookBuilder:
|
|||
saved_column_overrides: dict[str, str] | None = None,
|
||||
row_limit: int = EXCEL_ROW_LIMIT,
|
||||
progress: "Callable[[float, int, int], None] | None" = None,
|
||||
summary: dict | None = None, journal: dict | None = None):
|
||||
summary: dict | None = None, journal: dict | None = None,
|
||||
payout_receipts: dict[tuple[str, str, str], str] | None = None):
|
||||
self.result = result
|
||||
self.files = list(files)
|
||||
self.reserves = reserves or {}
|
||||
|
|
@ -237,6 +271,8 @@ class WorkbookBuilder:
|
|||
self._progress = progress
|
||||
self.summary = summary or {}
|
||||
self.journal = journal or {}
|
||||
# (marketplace, account bucket, settlement id) -> "YYYY-MM-DD · entered by"
|
||||
self.payout_receipts = payout_receipts or {}
|
||||
self.layouts = compute_layouts(result, row_limit)
|
||||
self.wb = Workbook(write_only=True)
|
||||
self._mkt_ws: dict[str, list] = {} # marketplace -> [ws per sheet]
|
||||
|
|
@ -253,6 +289,7 @@ class WorkbookBuilder:
|
|||
self._create_marketplace_sheets()
|
||||
self._stream_marketplace_rows()
|
||||
self._finalize_marketplace_subtotals()
|
||||
self._build_settled_settlements()
|
||||
self._build_reconciliation()
|
||||
self._build_exceptions()
|
||||
self._build_audit_trail()
|
||||
|
|
@ -313,11 +350,14 @@ class WorkbookBuilder:
|
|||
font=BOLD, border=BORDER) for col in "BCDEFG"],
|
||||
])
|
||||
ws.append([])
|
||||
ar = total_row + 2
|
||||
ar = total_row + 2 # the Allowance row, appended next
|
||||
ws.append([_c(ws, "Allowance for Sales Returns", font=BOLD),
|
||||
*[None] * 5, _c(ws, self.allowance, number_format=FMT_USD0, font=BOLD)])
|
||||
# Net Receivable = TOTAL + Allowance (the allowance is entered negative, as in the
|
||||
# manual workbook). This referenced G{ar+1} — its own row — so Excel opened the
|
||||
# workbook with a circular-reference warning and showed 0.
|
||||
ws.append([_c(ws, "Net Receivable", font=BOLD),
|
||||
*[None] * 5, _c(ws, f"=G{total_row}+G{ar + 1}", number_format=FMT_USD0, font=BOLD)])
|
||||
*[None] * 5, _c(ws, f"=G{total_row}+G{ar}", number_format=FMT_USD0, font=BOLD)])
|
||||
ws.freeze_panes = "A6"
|
||||
|
||||
def _detail_receivable_usd_ref(self, mkt: str) -> str:
|
||||
|
|
@ -612,6 +652,84 @@ class WorkbookBuilder:
|
|||
for note in r.notes:
|
||||
ws.append([_c(ws, "Note"), _c(ws, note)])
|
||||
|
||||
# -- Settled Settlements (what was deliberately left out) --
|
||||
def _build_settled_settlements(self):
|
||||
"""
|
||||
Every settlement whose raw rows were EXCLUDED, and why.
|
||||
|
||||
The marketplace tabs carry only open settlements — once Amazon's payout has reached
|
||||
the bank the settlement is closed and its transactions are not repeated here. Without
|
||||
this sheet a reader cannot tell a deliberately-omitted settled month from a file that
|
||||
failed to upload, so the omission is listed line by line and reconciled: excluded
|
||||
order rows + included order rows = every order row in the source files.
|
||||
"""
|
||||
agg, cls = self.result.aggregation, self.result.classification
|
||||
if agg is None or cls is None:
|
||||
return
|
||||
ws = self.wb.create_sheet("Settled Settlements")
|
||||
for col, w in zip("ABCDEFGHIJ", (16, 18, 18, 13, 13, 10, 18, 18, 14, 18)):
|
||||
ws.column_dimensions[col].width = w
|
||||
ws.append([_c(ws, "Settled settlements — raw rows deliberately excluded",
|
||||
font=TITLE_FONT)])
|
||||
ws.append([_c(ws, "Amazon's payout for each settlement below reached the bank on or "
|
||||
"before month-end, so the settlement is closed and its transactions "
|
||||
"are summarised here instead of listed in the marketplace tabs.")])
|
||||
ws.append([])
|
||||
ws.append([_c(ws, h, font=HDR_FONT, fill=HDR_FILL) for h in (
|
||||
"Marketplace", "Account stream", "Settlement ID", "First date", "Last date",
|
||||
"Rows", "Order total", "Payout amount", "Amazon paid", "Bank received")])
|
||||
|
||||
# Bank receipt per (marketplace, account stream, settlement id), when Finance entered one.
|
||||
receipts = self.payout_receipts or {}
|
||||
# Payout facts per settlement bucket.
|
||||
pay: dict[tuple[str, str, str], list] = {}
|
||||
for t in agg.transfers:
|
||||
slot = pay.setdefault((t.marketplace, t.account_type, t.settlement_id),
|
||||
[0.0, None])
|
||||
slot[0] += t.amount
|
||||
if t.txn_date and (slot[1] is None or t.txn_date > slot[1]):
|
||||
slot[1] = t.txn_date
|
||||
|
||||
n_rows = 0
|
||||
excluded_total = 0.0
|
||||
settled = sorted(
|
||||
((k, st) for k, st in agg.settlements.items()
|
||||
if st.status != "receivable" and (st.row_count - st.transfer_count) > 0),
|
||||
key=lambda kv: (kv[0][0], kv[0][1], kv[0][2]))
|
||||
for (mkt, acct, sid), st in settled:
|
||||
orders = st.row_count - st.transfer_count
|
||||
amount, paid_on = pay.get((mkt, acct, sid), (None, None))
|
||||
rec = receipts.get((mkt, acct, sid))
|
||||
n_rows += orders
|
||||
excluded_total += st.order_total
|
||||
ws.append([
|
||||
_c(ws, mkt), _c(ws, "—" if acct == "(unspecified)" else acct), _c(ws, sid),
|
||||
_c(ws, st.first_date.isoformat() if st.first_date else ""),
|
||||
_c(ws, st.last_date.isoformat() if st.last_date else ""),
|
||||
_c(ws, orders),
|
||||
# round(): summing millions of floats leaves noise like 5000.000000000001,
|
||||
# which reads as a data problem in an audit workbook.
|
||||
_c(ws, round(st.order_total, 2), number_format=FMT_ACCT2),
|
||||
_c(ws, round(amount, 2), number_format=FMT_ACCT2)
|
||||
if amount is not None else _c(ws, ""),
|
||||
_c(ws, paid_on.isoformat() if paid_on else ""),
|
||||
_c(ws, rec or "clearing-lag rule"),
|
||||
])
|
||||
ws.append([])
|
||||
included = sum(st.row_count - st.transfer_count for k, st in agg.settlements.items()
|
||||
if st.status == "receivable")
|
||||
included_total = sum(st.order_total for k, st in agg.settlements.items()
|
||||
if st.status == "receivable")
|
||||
for label, rows_n, amount_v in (
|
||||
("Excluded (settled) order rows", n_rows, excluded_total),
|
||||
("Included (open) order rows — in the marketplace tabs", included, included_total),
|
||||
("Total order rows in the source files", n_rows + included,
|
||||
excluded_total + included_total),
|
||||
):
|
||||
ws.append([_c(ws, label, font=BOLD), _c(ws, ""), _c(ws, ""), _c(ws, ""), _c(ws, ""),
|
||||
_c(ws, rows_n, font=BOLD),
|
||||
_c(ws, round(amount_v, 2), number_format=FMT_ACCT2, font=BOLD)])
|
||||
|
||||
# -- Exceptions --
|
||||
def _build_exceptions(self):
|
||||
ws = self.wb.create_sheet("Exceptions")
|
||||
|
|
@ -701,9 +819,11 @@ def export_workbook(result: ProcessResult, files: Iterable[str], output_path: st
|
|||
saved_column_overrides: dict[str, str] | None = None,
|
||||
row_limit: int = EXCEL_ROW_LIMIT,
|
||||
progress: Callable[[float, int, int], None] | None = None,
|
||||
summary: dict | None = None, journal: dict | None = None) -> str:
|
||||
summary: dict | None = None, journal: dict | None = None,
|
||||
payout_receipts: dict[tuple[str, str, str], str] | None = None) -> str:
|
||||
builder = WorkbookBuilder(result, files, reserves=reserves,
|
||||
allowance_for_returns=allowance_for_returns,
|
||||
saved_column_overrides=saved_column_overrides, row_limit=row_limit,
|
||||
progress=progress, summary=summary, journal=journal)
|
||||
progress=progress, summary=summary, journal=journal,
|
||||
payout_receipts=payout_receipts)
|
||||
return builder.build(output_path)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,11 @@
|
|||
"""MySQL database setup (SQLAlchemy)."""
|
||||
"""
|
||||
Database setup (SQLAlchemy) — MySQL for shared use, SQLite for a laptop or a demo.
|
||||
|
||||
Which one is used comes from `config.DB_BACKEND`. Everything above this layer is written
|
||||
against SQLAlchemy and is dialect-agnostic; the two places that are not — the raw bulk
|
||||
INSERT in services/store.py and the column-migration below — ask the engine which dialect
|
||||
it is rather than assuming.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
|
@ -9,6 +16,7 @@ from sqlalchemy import create_engine, event, text
|
|||
from sqlalchemy.orm import declarative_base, sessionmaker
|
||||
|
||||
from ..config import (
|
||||
DB_BACKEND,
|
||||
MYSQL_DATABASE,
|
||||
MYSQL_HOST,
|
||||
MYSQL_PASSWORD,
|
||||
|
|
@ -17,17 +25,19 @@ from ..config import (
|
|||
MYSQL_PORT,
|
||||
MYSQL_SLOW_QUERY_MS,
|
||||
MYSQL_USER,
|
||||
database_label,
|
||||
database_url,
|
||||
ensure_dirs,
|
||||
mysql_url,
|
||||
)
|
||||
|
||||
ensure_dirs()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
IS_MYSQL = DB_BACKEND == "mysql"
|
||||
|
||||
|
||||
def _ensure_database() -> None:
|
||||
"""Create MYSQL_DATABASE if it does not exist yet."""
|
||||
"""Create MYSQL_DATABASE if it does not exist yet (MySQL only)."""
|
||||
user = quote_plus(MYSQL_USER)
|
||||
password = quote_plus(MYSQL_PASSWORD)
|
||||
server_url = (
|
||||
|
|
@ -47,15 +57,32 @@ def _ensure_database() -> None:
|
|||
server_engine.dispose()
|
||||
|
||||
|
||||
if IS_MYSQL:
|
||||
_ensure_database()
|
||||
|
||||
ENGINE = create_engine(
|
||||
mysql_url(),
|
||||
database_url(),
|
||||
pool_size=MYSQL_POOL_SIZE,
|
||||
pool_recycle=MYSQL_POOL_RECYCLE,
|
||||
pool_pre_ping=True,
|
||||
future=True,
|
||||
)
|
||||
else:
|
||||
# check_same_thread=False: processing runs in a background thread with its own session.
|
||||
ENGINE = create_engine(
|
||||
database_url(), future=True,
|
||||
connect_args={"check_same_thread": False, "timeout": 30},
|
||||
)
|
||||
|
||||
@event.listens_for(ENGINE, "connect")
|
||||
def _sqlite_pragmas(dbapi_conn, _rec):
|
||||
cur = dbapi_conn.cursor()
|
||||
cur.execute("PRAGMA journal_mode=WAL") # readers don't block the writer
|
||||
cur.execute("PRAGMA foreign_keys=ON") # cascade deletes behave like MySQL
|
||||
cur.execute("PRAGMA busy_timeout=30000") # bulk insert vs progress updates
|
||||
cur.execute("PRAGMA synchronous=NORMAL")
|
||||
cur.close()
|
||||
|
||||
logger.info("database: %s", database_label())
|
||||
|
||||
if MYSQL_SLOW_QUERY_MS > 0:
|
||||
@event.listens_for(ENGINE, "before_cursor_execute")
|
||||
|
|
@ -86,16 +113,24 @@ def init_db() -> None:
|
|||
|
||||
|
||||
def _migrate() -> None:
|
||||
"""Add columns introduced after a DB was first created (create_all won't alter)."""
|
||||
"""
|
||||
Add columns introduced after a DB was first created (create_all won't alter).
|
||||
|
||||
MySQL DDL rules that differ from SQLite and silently broke this list during the port:
|
||||
* VARCHAR **must** carry a length — a bare `VARCHAR` is a syntax error. Lengths here
|
||||
must match the model's String(n) or the column ends up a different width.
|
||||
* TEXT/BLOB columns cannot take a literal DEFAULT before MySQL 8.0.13, so
|
||||
`TEXT DEFAULT ''` fails. Declare plain TEXT and let the ORM default apply on insert.
|
||||
"""
|
||||
added = {
|
||||
"sessions": [
|
||||
("progress_rows_done", "INTEGER DEFAULT 0"),
|
||||
("progress_rows_total", "INTEGER DEFAULT 0"),
|
||||
("eta_seconds", "INTEGER DEFAULT 0"),
|
||||
("opening_mode", "VARCHAR(255) DEFAULT 'zero'"),
|
||||
("opening_mode", "VARCHAR(32) DEFAULT 'zero'"),
|
||||
("opening_source_session_id", "INTEGER"),
|
||||
("blocked_reason", "TEXT DEFAULT ''"),
|
||||
("payout_mode", "VARCHAR DEFAULT 'auto'"),
|
||||
("blocked_reason", "TEXT"),
|
||||
("payout_mode", "VARCHAR(32) DEFAULT 'auto'"),
|
||||
("needs_reprocess", "BOOLEAN DEFAULT 0"),
|
||||
],
|
||||
"session_files": [
|
||||
|
|
@ -104,14 +139,14 @@ def _migrate() -> None:
|
|||
("helper_rows_skipped", "INTEGER DEFAULT 0"),
|
||||
],
|
||||
"fx_rates": [
|
||||
("confirmed_by", "VARCHAR DEFAULT ''"),
|
||||
("confirmed_by", "VARCHAR(255) DEFAULT ''"),
|
||||
("confirmed_at", "DATETIME"),
|
||||
("confirmed_month", "VARCHAR DEFAULT ''"),
|
||||
("confirmed_month", "VARCHAR(32) DEFAULT ''"),
|
||||
],
|
||||
"journal_entries": [
|
||||
("reviewed_by", "VARCHAR DEFAULT ''"),
|
||||
("reviewed_by", "VARCHAR(255) DEFAULT ''"),
|
||||
("reviewed_at", "DATETIME"),
|
||||
("approved_by", "VARCHAR DEFAULT ''"),
|
||||
("approved_by", "VARCHAR(255) DEFAULT ''"),
|
||||
("approved_at", "DATETIME"),
|
||||
],
|
||||
"reconciliation": [
|
||||
|
|
@ -129,9 +164,13 @@ def _migrate() -> None:
|
|||
("storage_flag", "BOOLEAN DEFAULT 0"),
|
||||
],
|
||||
}
|
||||
with ENGINE.begin() as conn:
|
||||
db_name = conn.execute(text("SELECT DATABASE()")).scalar()
|
||||
# Each ALTER runs on its own connection scope: MySQL auto-commits DDL, so wrapping the
|
||||
# whole loop in one transaction gives no rollback anyway — and one bad statement would
|
||||
# otherwise abort every later migration for the rest of the run.
|
||||
with ENGINE.connect() as conn:
|
||||
db_name = conn.execute(text("SELECT DATABASE()")).scalar() if IS_MYSQL else None
|
||||
for table, cols in added.items():
|
||||
if IS_MYSQL:
|
||||
existing = {
|
||||
r[0]
|
||||
for r in conn.execute(
|
||||
|
|
@ -142,9 +181,23 @@ def _migrate() -> None:
|
|||
{"schema": db_name, "table": table},
|
||||
)
|
||||
}
|
||||
else:
|
||||
existing = {r[1] for r in conn.execute(text(f"PRAGMA table_info({table})"))}
|
||||
if not existing:
|
||||
continue # table not created yet — create_all owns it
|
||||
for name, decl in cols:
|
||||
if name not in existing:
|
||||
conn.execute(text(f"ALTER TABLE `{table}` ADD COLUMN `{name}` {decl}"))
|
||||
if name in existing:
|
||||
continue
|
||||
# SQLite has no VARCHAR length limit and rejects some MySQL type spellings;
|
||||
# its dynamic typing makes the declared type advisory anyway.
|
||||
sql_decl = decl.replace("VARCHAR(255)", "VARCHAR").replace(
|
||||
"VARCHAR(32)", "VARCHAR") if not IS_MYSQL else decl
|
||||
try:
|
||||
conn.execute(text(f"ALTER TABLE `{table}` ADD COLUMN `{name}` {sql_decl}"))
|
||||
conn.commit()
|
||||
except Exception: # noqa: BLE001
|
||||
conn.rollback()
|
||||
logger.exception("migration failed: %s.%s %s", table, name, decl)
|
||||
|
||||
|
||||
def get_db():
|
||||
|
|
|
|||
|
|
@ -356,10 +356,18 @@ def run_export(session_id: int) -> None:
|
|||
EXPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
month = session.reporting_month or "output"
|
||||
out_path = str(EXPORT_DIR / f"AR_Aging_{month}_session{session_id}.xlsx")
|
||||
# Bank dates for the "Settled Settlements" sheet, so the workbook records WHY each
|
||||
# excluded settlement was excluded and who said so.
|
||||
receipt_notes = {
|
||||
(r.marketplace, r.account_type, r.settlement_id):
|
||||
f"{r.bank_date}" + (f" · {r.entered_by}" if r.entered_by else "")
|
||||
for r in receipts if r.bank_date
|
||||
}
|
||||
export_workbook(result, paths, out_path, reserves=reserves,
|
||||
allowance_for_returns=session.allowance_for_returns or 0.0,
|
||||
saved_column_overrides=mapping_rules,
|
||||
progress=write_progress, summary=_summary, journal=_journal)
|
||||
progress=write_progress, summary=_summary, journal=_journal,
|
||||
payout_receipts=receipt_notes)
|
||||
|
||||
_finalize_export(db, session_id, out_path, "full")
|
||||
session.status = "processed"
|
||||
|
|
|
|||
|
|
@ -16,10 +16,12 @@ _TXN_COLS = (
|
|||
"settlement_id", "order_id", "sku", "txn_type", "txn_type_en", "account_type",
|
||||
"posted_date", "total", "currency", "storage_flag",
|
||||
)
|
||||
# PyMySQL uses %-style placeholders for raw DBAPI executemany.
|
||||
# Raw DBAPI, so the placeholder style is the driver's, not SQLAlchemy's: PyMySQL wants %s,
|
||||
# sqlite3 wants ?. Ask the engine which dialect it is instead of hardcoding either.
|
||||
_PARAM = "%s" if ENGINE.dialect.name == "mysql" else "?"
|
||||
_INSERT_SQL = (
|
||||
f"INSERT INTO transactions ({', '.join(_TXN_COLS)}) "
|
||||
f"VALUES ({', '.join(['%s'] * len(_TXN_COLS))})"
|
||||
f"VALUES ({', '.join([_PARAM] * len(_TXN_COLS))})"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -75,14 +77,16 @@ class TransactionSink:
|
|||
recv = 1 if (st.status == "receivable"
|
||||
and acct.lower() in RECEIVABLE_ACCOUNT_TYPES) else 0
|
||||
cur.execute(
|
||||
"UPDATE transactions SET settlement_status=%s, receivable_flag=%s "
|
||||
"WHERE session_id=%s AND settlement_id=%s AND marketplace=%s AND account_type=%s",
|
||||
f"UPDATE transactions SET settlement_status={_PARAM}, "
|
||||
f"receivable_flag={_PARAM} WHERE session_id={_PARAM} "
|
||||
f"AND settlement_id={_PARAM} AND marketplace={_PARAM} "
|
||||
f"AND account_type={_PARAM}",
|
||||
(st.status, recv, self.session_id, sid, mkt, acct),
|
||||
)
|
||||
# transfer rows: never receivable (canonical type covers localized names)
|
||||
cur.execute(
|
||||
"UPDATE transactions SET receivable_flag=0 "
|
||||
"WHERE session_id=%s AND txn_type_en='Transfer'",
|
||||
f"UPDATE transactions SET receivable_flag=0 "
|
||||
f"WHERE session_id={_PARAM} AND txn_type_en='Transfer'",
|
||||
(self.session_id,),
|
||||
)
|
||||
cur.close()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,388 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migrate the legacy SQLite database into MySQL.
|
||||
|
||||
The app stored everything in `backend/data/ar_aging.db` before the MySQL move. That file
|
||||
still holds the real month-end closings (Jan-2026: 3,399,517 transaction rows), and MySQL
|
||||
starts empty, so the closings have to be copied across once.
|
||||
|
||||
python3 migrate_sqlite_to_mysql.py --dry-run # inspect the source, touch nothing
|
||||
python3 migrate_sqlite_to_mysql.py # migrate
|
||||
python3 migrate_sqlite_to_mysql.py --force # migrate into a non-empty MySQL
|
||||
|
||||
What it does
|
||||
* copies every table in foreign-key order, so a child row never precedes its session
|
||||
* converts SQLite's text dates / 0-1 booleans to real MySQL DATE, DATETIME and BOOLEAN
|
||||
* copies only the columns both schemas share, and reports any it had to skip
|
||||
* streams in batches, so 3.4M rows never sit in memory
|
||||
* verifies afterwards: row counts per table AND financial checksums (Σ transaction
|
||||
totals, per-marketplace receivable) must match the source exactly
|
||||
|
||||
It never deletes anything from SQLite — the file is opened read-only.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
BACKEND = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(BACKEND))
|
||||
|
||||
BATCH = 5000
|
||||
|
||||
# Tables whose contents are re-derivable by re-processing, but copied anyway so the
|
||||
# migrated database is byte-identical in what the dashboard shows.
|
||||
SKIP_TABLES: set[str] = set()
|
||||
|
||||
|
||||
def log(msg: str = "") -> None:
|
||||
print(msg, flush=True)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- source
|
||||
def sqlite_tables(conn: sqlite3.Connection) -> set[str]:
|
||||
return {r[0] for r in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")}
|
||||
|
||||
|
||||
def sqlite_columns(conn: sqlite3.Connection, table: str) -> list[str]:
|
||||
return [r[1] for r in conn.execute(f"PRAGMA table_info({table})")]
|
||||
|
||||
|
||||
def sqlite_count(conn: sqlite3.Connection, table: str) -> int:
|
||||
return conn.execute(f"SELECT COUNT(*) FROM `{table}`").fetchone()[0]
|
||||
|
||||
|
||||
def open_sqlite(path: Path) -> sqlite3.Connection:
|
||||
"""Open read-only. A stale -wal is checkpointed into a COPY, never the original."""
|
||||
if not path.exists():
|
||||
raise SystemExit(f"SQLite file not found: {path}")
|
||||
# immutable=0 so an existing -wal is still applied; mode=ro keeps us from writing.
|
||||
conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- convert
|
||||
def make_converter(col_type) -> "callable":
|
||||
"""Return a function turning a SQLite value into something MySQL accepts."""
|
||||
name = col_type.__class__.__name__
|
||||
|
||||
if name == "Date":
|
||||
def conv(v):
|
||||
if v in (None, ""):
|
||||
return None
|
||||
if isinstance(v, dt.date) and not isinstance(v, dt.datetime):
|
||||
return v
|
||||
if isinstance(v, dt.datetime):
|
||||
return v.date()
|
||||
try:
|
||||
return dt.date.fromisoformat(str(v)[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
return conv
|
||||
|
||||
if name == "DateTime":
|
||||
def conv(v):
|
||||
if v in (None, ""):
|
||||
return None
|
||||
if isinstance(v, dt.datetime):
|
||||
return v
|
||||
s = str(v).replace("T", " ")
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
|
||||
try:
|
||||
return dt.datetime.strptime(s[:26], fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
return conv
|
||||
|
||||
if name == "Boolean":
|
||||
def conv(v):
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, bool):
|
||||
return v
|
||||
return bool(int(v)) if str(v).strip() in ("0", "1") else bool(v)
|
||||
return conv
|
||||
|
||||
if name in ("Integer", "BigInteger", "SmallInteger"):
|
||||
def conv(v):
|
||||
if v in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return int(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return conv
|
||||
|
||||
if name in ("Float", "Numeric"):
|
||||
def conv(v):
|
||||
if v in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return float(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return conv
|
||||
|
||||
# String / Text: MySQL columns are sized, so over-long values would be truncated or
|
||||
# rejected. Trim to the declared length and report it rather than failing the batch.
|
||||
length = getattr(col_type, "length", None)
|
||||
|
||||
def conv(v):
|
||||
if v is None:
|
||||
return None
|
||||
s = v if isinstance(v, str) else str(v)
|
||||
return s[:length] if length and len(s) > length else s
|
||||
return conv
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- migrate
|
||||
def migrate() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--sqlite", default=str(BACKEND / "data" / "ar_aging.db"),
|
||||
help="path to the legacy SQLite file")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="inspect the source and print the plan; do not connect to MySQL")
|
||||
ap.add_argument("--force", action="store_true",
|
||||
help="migrate even if the MySQL tables already contain rows")
|
||||
args = ap.parse_args()
|
||||
|
||||
src_path = Path(args.sqlite)
|
||||
src = open_sqlite(src_path)
|
||||
have = sqlite_tables(src)
|
||||
|
||||
log("")
|
||||
log(" Migrate SQLite → MySQL")
|
||||
log(f" source: {src_path} ({src_path.stat().st_size / 1e9:.2f} GB)")
|
||||
wal = src_path.with_name(src_path.name + "-wal")
|
||||
if wal.exists() and wal.stat().st_size > 0:
|
||||
log(f" note: a {wal.stat().st_size / 1e6:.0f} MB write-ahead log is present and "
|
||||
f"will be read as part of the database")
|
||||
log("")
|
||||
|
||||
# ---- source inventory (works with no MySQL at all) ----
|
||||
log(" Source contents")
|
||||
src_counts: dict[str, int] = {}
|
||||
for t in sorted(have):
|
||||
src_counts[t] = sqlite_count(src, t)
|
||||
for t, n in sorted(src_counts.items(), key=lambda kv: -kv[1]):
|
||||
if n:
|
||||
log(f" {t:22} {n:>10,}")
|
||||
empty = [t for t, n in src_counts.items() if not n]
|
||||
if empty:
|
||||
log(f" ({len(empty)} empty: {', '.join(sorted(empty))})")
|
||||
|
||||
src_checks = financial_checksums_sqlite(src)
|
||||
log("")
|
||||
log(" Financial checksums to preserve")
|
||||
for k, v in src_checks.items():
|
||||
log(f" {k:34} {v}")
|
||||
|
||||
if args.dry_run:
|
||||
log("")
|
||||
log(" Dry run — MySQL was not contacted and nothing was written.")
|
||||
log(" Fill in ar-aging-app/.env, then re-run without --dry-run.")
|
||||
return 0
|
||||
|
||||
# ---- target ----
|
||||
try:
|
||||
from app.db.database import ENGINE, init_db
|
||||
from app.db import models # noqa: F401
|
||||
except Exception as e: # noqa: BLE001
|
||||
log("")
|
||||
log(f" Could not connect to MySQL: {e}")
|
||||
log(" Check MYSQL_* in ar-aging-app/.env and that the server is reachable.")
|
||||
return 1
|
||||
|
||||
log("")
|
||||
log(" Creating the MySQL schema (safe if it already exists)…")
|
||||
init_db()
|
||||
|
||||
meta = models.Base.metadata
|
||||
ordered = [t for t in meta.sorted_tables if t.name in have and t.name not in SKIP_TABLES]
|
||||
missing_in_sqlite = [t.name for t in meta.sorted_tables if t.name not in have]
|
||||
if missing_in_sqlite:
|
||||
log(f" tables absent from the SQLite file (created empty): "
|
||||
f"{', '.join(missing_in_sqlite)}")
|
||||
|
||||
raw = ENGINE.raw_connection()
|
||||
cur = raw.cursor()
|
||||
# Existing rows?
|
||||
non_empty = []
|
||||
for t in ordered:
|
||||
cur.execute(f"SELECT COUNT(*) FROM `{t.name}`")
|
||||
n = cur.fetchone()[0]
|
||||
if n:
|
||||
non_empty.append((t.name, n))
|
||||
if non_empty and not args.force:
|
||||
log("")
|
||||
log(" MySQL already contains data — refusing to migrate on top of it:")
|
||||
for name, n in non_empty:
|
||||
log(f" {name:22} {n:>10,} rows")
|
||||
log("")
|
||||
log(" Re-run with --force to add these rows anyway (duplicates are possible),")
|
||||
log(" or empty the MySQL database first.")
|
||||
return 1
|
||||
|
||||
log("")
|
||||
log(" Copying tables (foreign-key order)")
|
||||
cur.execute("SET FOREIGN_KEY_CHECKS=0")
|
||||
cur.execute("SET UNIQUE_CHECKS=0")
|
||||
truncated: list[str] = []
|
||||
copied: dict[str, int] = {}
|
||||
try:
|
||||
for table in ordered:
|
||||
name = table.name
|
||||
total = src_counts.get(name, 0)
|
||||
if not total:
|
||||
copied[name] = 0
|
||||
continue
|
||||
sq_cols = set(sqlite_columns(src, name))
|
||||
cols = [c for c in table.columns if c.name in sq_cols]
|
||||
dropped = [c.name for c in table.columns if c.name not in sq_cols]
|
||||
extra = sq_cols - {c.name for c in table.columns}
|
||||
convs = [make_converter(c.type) for c in cols]
|
||||
names = [c.name for c in cols]
|
||||
placeholders = ", ".join(["%s"] * len(names))
|
||||
collist = ", ".join(f"`{n}`" for n in names)
|
||||
sql = f"INSERT INTO `{name}` ({collist}) VALUES ({placeholders})"
|
||||
|
||||
done = 0
|
||||
batch: list[tuple] = []
|
||||
for row in src.execute(f"SELECT {', '.join(f'`{n}`' for n in names)} "
|
||||
f"FROM `{name}`"):
|
||||
vals = []
|
||||
for i, conv in enumerate(convs):
|
||||
v = conv(row[i])
|
||||
vals.append(v)
|
||||
batch.append(tuple(vals))
|
||||
if len(batch) >= BATCH:
|
||||
cur.executemany(sql, batch)
|
||||
raw.commit()
|
||||
done += len(batch)
|
||||
batch.clear()
|
||||
if total > 50000:
|
||||
pct = 100.0 * done / total
|
||||
print(f" {name:22} {done:>10,} / {total:,} ({pct:5.1f}%)",
|
||||
end="\r", flush=True)
|
||||
if batch:
|
||||
cur.executemany(sql, batch)
|
||||
raw.commit()
|
||||
done += len(batch)
|
||||
copied[name] = done
|
||||
note = ""
|
||||
if dropped:
|
||||
note += f" [not in source: {', '.join(dropped)}]"
|
||||
if extra:
|
||||
note += f" [source-only, skipped: {', '.join(sorted(extra))}]"
|
||||
truncated.append(name)
|
||||
print(" " * 78, end="\r")
|
||||
log(f" {name:22} {done:>10,}{note}")
|
||||
finally:
|
||||
cur.execute("SET FOREIGN_KEY_CHECKS=1")
|
||||
cur.execute("SET UNIQUE_CHECKS=1")
|
||||
raw.commit()
|
||||
|
||||
# ---- verify ----
|
||||
log("")
|
||||
log(" Verifying")
|
||||
ok = True
|
||||
for name, n in sorted(copied.items()):
|
||||
cur.execute(f"SELECT COUNT(*) FROM `{name}`")
|
||||
got = cur.fetchone()[0]
|
||||
want = src_counts.get(name, 0)
|
||||
if got != want:
|
||||
ok = False
|
||||
log(f" ✗ {name:22} MySQL {got:,} != SQLite {want:,}")
|
||||
if ok:
|
||||
log(f" ✓ row counts match on all {len(copied)} tables")
|
||||
|
||||
dst_checks = financial_checksums_mysql(cur)
|
||||
for k, want in src_checks.items():
|
||||
got = dst_checks.get(k)
|
||||
if str(got) != str(want):
|
||||
ok = False
|
||||
log(f" ✗ {k}: MySQL {got} != SQLite {want}")
|
||||
if ok:
|
||||
log(" ✓ financial checksums match")
|
||||
|
||||
cur.close()
|
||||
raw.close()
|
||||
src.close()
|
||||
|
||||
log("")
|
||||
if ok:
|
||||
log(" Migration complete. Start the dashboard with start.command.")
|
||||
return 0
|
||||
log(" Migration finished with MISMATCHES — do not rely on the MySQL data until")
|
||||
log(" they are explained. The SQLite file is untouched.")
|
||||
return 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- checksums
|
||||
def financial_checksums_sqlite(conn: sqlite3.Connection) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
tables = sqlite_tables(conn)
|
||||
|
||||
def one(sql: str, default="—") -> str:
|
||||
try:
|
||||
r = conn.execute(sql).fetchone()
|
||||
return "—" if r is None or r[0] is None else str(r[0])
|
||||
except sqlite3.Error:
|
||||
return default
|
||||
|
||||
if "sessions" in tables:
|
||||
out["sessions"] = one("SELECT COUNT(*) FROM sessions")
|
||||
if "transactions" in tables:
|
||||
out["transaction rows"] = one("SELECT COUNT(*) FROM transactions")
|
||||
out["Σ transactions.total"] = one("SELECT ROUND(SUM(total),2) FROM transactions")
|
||||
out["receivable-flagged rows"] = one(
|
||||
"SELECT COUNT(*) FROM transactions WHERE receivable_flag=1")
|
||||
if "receivable_results" in tables:
|
||||
out["USA receivable_local (TOTAL)"] = one(
|
||||
"SELECT ROUND(receivable_local) FROM receivable_results "
|
||||
"WHERE marketplace='USA' AND account_type='TOTAL'")
|
||||
out["Σ receivable_usd (TOTAL rows)"] = one(
|
||||
"SELECT ROUND(SUM(receivable_usd),2) FROM receivable_results "
|
||||
"WHERE account_type='TOTAL'")
|
||||
return out
|
||||
|
||||
|
||||
def financial_checksums_mysql(cur) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
|
||||
def one(sql: str) -> str:
|
||||
try:
|
||||
cur.execute(sql)
|
||||
r = cur.fetchone()
|
||||
return "—" if r is None or r[0] is None else str(r[0])
|
||||
except Exception: # noqa: BLE001
|
||||
return "—"
|
||||
|
||||
out["sessions"] = one("SELECT COUNT(*) FROM sessions")
|
||||
out["transaction rows"] = one("SELECT COUNT(*) FROM transactions")
|
||||
out["Σ transactions.total"] = one("SELECT ROUND(SUM(total),2) FROM transactions")
|
||||
out["receivable-flagged rows"] = one(
|
||||
"SELECT COUNT(*) FROM transactions WHERE receivable_flag=1")
|
||||
out["USA receivable_local (TOTAL)"] = one(
|
||||
"SELECT ROUND(receivable_local) FROM receivable_results "
|
||||
"WHERE marketplace='USA' AND account_type='TOTAL'")
|
||||
out["Σ receivable_usd (TOTAL rows)"] = one(
|
||||
"SELECT ROUND(SUM(receivable_usd),2) FROM receivable_results "
|
||||
"WHERE account_type='TOTAL'")
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(migrate())
|
||||
except KeyboardInterrupt:
|
||||
log("\n Interrupted. The SQLite source is unchanged.")
|
||||
raise SystemExit(130)
|
||||
|
|
@ -8,20 +8,24 @@ from pathlib import Path
|
|||
import pytest
|
||||
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# Redirect ALL test data to a throwaway directory — BEFORE anything imports app.config,
|
||||
# which reads these variables once at module load and caches the paths.
|
||||
# Redirect ALL test data away from production — BEFORE anything imports app.config, which
|
||||
# reads these variables once at module load and caches them.
|
||||
#
|
||||
# Without this the suite runs against the real production database: `app/config.py` falls
|
||||
# back to `backend/data/ar_aging.db`, so every test that created a closing was writing into
|
||||
# Finance's live data (75 sessions had accumulated there). Tests must never be able to touch
|
||||
# a real closing.
|
||||
# The suite creates AND DELETES closings, so pointing it at the live database would destroy
|
||||
# Finance's data. That already happened once under SQLite (75 test sessions accumulated in
|
||||
# the production file), and the blast radius is larger now that the store is a shared MySQL
|
||||
# server rather than a local file.
|
||||
#
|
||||
# The names must match app/config.py exactly — AR_DB_PATH / AR_DATA_DIR. A near-miss such as
|
||||
# "AR_DB_URL" silently does nothing and the tests quietly hit production again.
|
||||
# `load_dotenv()` in app/config.py does not override variables already present in the
|
||||
# environment, so setting MYSQL_DATABASE here wins over .env. The database is created
|
||||
# automatically by database._ensure_database().
|
||||
# ---------------------------------------------------------------------------------------
|
||||
_TEST_DATA_DIR = Path(tempfile.mkdtemp(prefix="ar-aging-tests-"))
|
||||
os.environ["AR_DATA_DIR"] = str(_TEST_DATA_DIR)
|
||||
os.environ["AR_DB_PATH"] = str(_TEST_DATA_DIR / "test.db")
|
||||
|
||||
_PROD_DB = os.environ.get("MYSQL_DATABASE", "")
|
||||
TEST_DB_NAME = os.environ.get("AR_TEST_MYSQL_DATABASE", "ar_aging_pytest")
|
||||
os.environ["MYSQL_DATABASE"] = TEST_DB_NAME
|
||||
|
||||
# Default: the project root two levels above ar-aging-app/backend.
|
||||
_DEFAULT_SAMPLE_DIR = Path(__file__).resolve().parents[3]
|
||||
|
|
@ -58,13 +62,19 @@ def _never_touch_production_data():
|
|||
"""
|
||||
Hard stop if the redirect above ever fails.
|
||||
|
||||
The suite creates and deletes closings, so pointing at the real database would destroy
|
||||
Finance's data. Assert the isolation actually took effect rather than trusting it.
|
||||
The suite creates and deletes closings, so running against the live database would
|
||||
destroy Finance's data. Assert the isolation actually took effect rather than trusting
|
||||
it — this fixture is the reason a renamed config variable can't silently re-point the
|
||||
tests at production.
|
||||
"""
|
||||
from app.config import DATA_DIR, DB_PATH
|
||||
assert str(DB_PATH).startswith(str(_TEST_DATA_DIR)), (
|
||||
f"tests are pointed at {DB_PATH} — expected a temp path under {_TEST_DATA_DIR}. "
|
||||
f"app/config.py reads AR_DB_PATH / AR_DATA_DIR; check those names."
|
||||
from app.config import DATA_DIR, MYSQL_DATABASE
|
||||
assert MYSQL_DATABASE == TEST_DB_NAME, (
|
||||
f"tests are pointed at MySQL database {MYSQL_DATABASE!r} — expected "
|
||||
f"{TEST_DB_NAME!r}. app/config.py reads MYSQL_DATABASE; check that name."
|
||||
)
|
||||
assert not _PROD_DB or MYSQL_DATABASE != _PROD_DB, (
|
||||
f"the test database is the same as the configured production database "
|
||||
f"({_PROD_DB!r}). Set AR_TEST_MYSQL_DATABASE to a separate name."
|
||||
)
|
||||
assert str(DATA_DIR).startswith(str(_TEST_DATA_DIR)), (
|
||||
f"tests would write uploads/exports to {DATA_DIR}, not a temp directory."
|
||||
|
|
|
|||
|
|
@ -182,3 +182,87 @@ def test_sheet_split_when_over_row_limit(synth_file):
|
|||
add = wb["Detail"]["B5"].value
|
||||
for s in usa_sheets:
|
||||
assert f"'{s}'!" in add
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ workbook must tie
|
||||
def test_detail_references_every_account_stream(synth_file):
|
||||
"""
|
||||
Detail/Summary must reference EVERY account stream's subtotal cell.
|
||||
|
||||
compute_layouts() planned the subtotal rows with a stride of 2 while
|
||||
_finalize_marketplace_subtotals() writes them consecutively, so every stream after the
|
||||
first pointed at an empty cell. USA is the only marketplace with two streams, so the
|
||||
whole Invoiced Orders receivable silently vanished from Detail and Summary (Jan-2026:
|
||||
67,854.71) while Reconciliation and COA in the same workbook showed it.
|
||||
"""
|
||||
result = process([synth_file], month_end=date(2026, 1, 31), clearing_lag_days=2)
|
||||
out = synth_file.replace(".xlsx", "-streams.xlsx")
|
||||
export_workbook(result, [synth_file], out)
|
||||
wb = openpyxl.load_workbook(out)
|
||||
ws = wb["USA"]
|
||||
|
||||
# Where the subtotal formulas actually landed.
|
||||
actual = {}
|
||||
for row in ws.iter_rows():
|
||||
for c in row:
|
||||
if isinstance(c.value, str) and c.value.startswith("=SUMIFS"):
|
||||
actual[ws.cell(row=c.row, column=c.column - 1).value] = c.coordinate
|
||||
streams = set(result.receivable.marketplaces["USA"].accounts)
|
||||
assert set(actual) == streams, f"a stream has no subtotal row: {actual} vs {streams}"
|
||||
|
||||
detail_formula = wb["Detail"]["B5"].value
|
||||
for stream, coord in actual.items():
|
||||
assert f"'USA'!{coord}" in detail_formula, (
|
||||
f"Detail!B5 ({detail_formula}) does not reference the {stream} subtotal at {coord}"
|
||||
)
|
||||
|
||||
|
||||
def test_summary_net_receivable_is_not_circular(synth_file):
|
||||
"""'Net Receivable' referenced its own cell, so Excel warned and showed 0."""
|
||||
result = process([synth_file], month_end=date(2026, 1, 31), clearing_lag_days=2)
|
||||
out = synth_file.replace(".xlsx", "-net.xlsx")
|
||||
export_workbook(result, [synth_file], out, allowance_for_returns=-100.0)
|
||||
ws = openpyxl.load_workbook(out)["Summary"]
|
||||
rows = {ws.cell(row=r, column=1).value: r for r in range(1, ws.max_row + 1)}
|
||||
net_row = rows["Net Receivable"]
|
||||
formula = ws.cell(row=net_row, column=7).value
|
||||
assert f"G{net_row}" not in formula, f"circular reference: G{net_row} in {formula}"
|
||||
assert f"G{rows['TOTAL']}" in formula and f"G{rows['Allowance for Sales Returns']}" in formula
|
||||
|
||||
|
||||
def test_every_received_payout_appears_on_the_tab(synth_file):
|
||||
"""
|
||||
A month can have several received payouts per stream; the workbook used to lift only the
|
||||
boundary one, so earlier bank receipts were absent from the entire file and the tab could
|
||||
not be hand-footed against the bank statement.
|
||||
"""
|
||||
result = process([synth_file], month_end=date(2026, 1, 31), clearing_lag_days=2)
|
||||
out = synth_file.replace(".xlsx", "-payouts.xlsx")
|
||||
export_workbook(result, [synth_file], out)
|
||||
ws = openpyxl.load_workbook(out)["USA"]
|
||||
in_book = sorted(r[TOTAL_IDX] for r in ws.iter_rows(min_row=9, values_only=True)
|
||||
if r and r[FIELD_ORDER.index("txn_type")] == "Transfer")
|
||||
received = sorted(t.amount for t in result.aggregation.transfers if t.received)
|
||||
assert in_book == received, f"workbook payouts {in_book} != received payouts {received}"
|
||||
|
||||
|
||||
def test_settled_settlements_sheet_reconciles(synth_file):
|
||||
"""The excluded settlements are listed and their rows + the included rows = every row."""
|
||||
result = process([synth_file], month_end=date(2026, 1, 31), clearing_lag_days=2)
|
||||
out = synth_file.replace(".xlsx", "-settled.xlsx")
|
||||
export_workbook(result, [synth_file], out)
|
||||
ws = openpyxl.load_workbook(out)["Settled Settlements"]
|
||||
labels = {ws.cell(row=r, column=1).value: r for r in range(1, ws.max_row + 1)}
|
||||
excluded = ws.cell(row=labels["Excluded (settled) order rows"], column=6).value
|
||||
included = ws.cell(row=labels["Included (open) order rows — in the marketplace tabs"],
|
||||
column=6).value
|
||||
total = ws.cell(row=labels["Total order rows in the source files"], column=6).value
|
||||
assert excluded + included == total
|
||||
engine_total = sum(st.row_count - st.transfer_count
|
||||
for st in result.aggregation.settlements.values())
|
||||
assert total == engine_total, f"sheet says {total} order rows, engine has {engine_total}"
|
||||
# Every settled settlement is named, so the omission is documented rather than silent.
|
||||
listed = {ws.cell(row=r, column=3).value for r in range(5, ws.max_row + 1)}
|
||||
settled = {sid for (m, a, sid), st in result.aggregation.settlements.items()
|
||||
if st.status != "receivable" and (st.row_count - st.transfer_count) > 0}
|
||||
assert settled <= listed, f"settled settlements missing from the sheet: {settled - listed}"
|
||||
|
|
|
|||
|
|
@ -68,33 +68,77 @@ command -v npm >/dev/null 2>&1 || die "npm was not found." \
|
|||
"install Node.js 20+ from nodejs.org"
|
||||
good "python $(python3 -V 2>&1 | awk '{print $2}') · node $(node -v 2>/dev/null) · npm $(npm -v 2>/dev/null)"
|
||||
|
||||
# Python packages — actually IMPORT the app rather than checking a few package names.
|
||||
# "Installed" is not the same as "compatible": an unpinned starlette upgrade once satisfied
|
||||
# every import check while breaking the app at load time. Importing proves it will boot.
|
||||
import_error=""
|
||||
if ! import_error="$(cd -- "$BACKEND_DIR" && python3 -c "from app.api.main import app" 2>&1)"; then
|
||||
warn "the backend could not be loaded:"
|
||||
printf '%s\n' "$import_error" | tail -n 3 | sed 's/^/ /'
|
||||
printf ' Install/repair Python packages now? [Y/n] '
|
||||
# Each check below diagnoses ONE thing. A single "does the app import?" test used to stand in
|
||||
# for all of them, so a database that was merely switched off was reported as broken Python
|
||||
# packages — and the launcher then ran pip install, which of course changed nothing.
|
||||
|
||||
# 1a. Python packages — libraries only, no app code, so this cannot fail for config reasons.
|
||||
if ! python3 -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1; then
|
||||
warn "Python packages are missing or incomplete."
|
||||
printf ' Install them now? [Y/n] '
|
||||
read -r reply
|
||||
case "${reply:-Y}" in
|
||||
[Nn]*) die "The backend cannot start with the current Python packages." \
|
||||
[Nn]*) die "Python dependencies are not installed." \
|
||||
"python3 -m pip install -r '$BACKEND_DIR/requirements.txt'" ;;
|
||||
esac
|
||||
step "installing Python packages (this can take a minute)…"
|
||||
python3 -m pip install -q -r "$BACKEND_DIR/requirements.txt" \
|
||||
|| die "pip install failed — see the messages above." \
|
||||
"python3 -m pip install -r '$BACKEND_DIR/requirements.txt'"
|
||||
if ! import_error="$(cd -- "$BACKEND_DIR" && python3 -c "from app.api.main import app" 2>&1)"; then
|
||||
printf '%s\n' "$import_error" | tail -n 5 | sed 's/^/ /'
|
||||
die "The backend still cannot be loaded after installing packages." \
|
||||
"python3 -m pip check"
|
||||
fi
|
||||
good "Python packages repaired"
|
||||
python3 -c "import fastapi, uvicorn, sqlalchemy, openpyxl, pymysql, dotenv" >/dev/null 2>&1 \
|
||||
|| die "Python packages are still incomplete after installing." "python3 -m pip check"
|
||||
good "Python packages installed"
|
||||
else
|
||||
good "Python packages present and compatible"
|
||||
good "Python packages present"
|
||||
fi
|
||||
|
||||
# 1b. Which database? SQLite needs nothing; MySQL needs a server and credentials. The app
|
||||
# picks MySQL only when ar-aging-app/.env names a real host, so a machine with no
|
||||
# database installed still runs off the local file instead of refusing to start.
|
||||
ENV_FILE="$APP_DIR/.env"
|
||||
db_backend="$(cd -- "$BACKEND_DIR" && python3 -c "
|
||||
from app.config import DB_BACKEND; print(DB_BACKEND)" 2>/dev/null)"
|
||||
db_label="$(cd -- "$BACKEND_DIR" && python3 -c "
|
||||
from app.config import database_label; print(database_label())" 2>/dev/null)"
|
||||
|
||||
if [ "$db_backend" = "mysql" ]; then
|
||||
good "database: $db_label"
|
||||
# Reachable? A refused connection is a server/credentials problem, never a Python one.
|
||||
db_err="$(cd -- "$BACKEND_DIR" && python3 -c "
|
||||
import sys, pymysql
|
||||
from app.config import MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD
|
||||
try:
|
||||
pymysql.connect(host=MYSQL_HOST, port=MYSQL_PORT, user=MYSQL_USER,
|
||||
password=MYSQL_PASSWORD, connect_timeout=6).close()
|
||||
except Exception as e:
|
||||
sys.stderr.write(str(e)); sys.exit(1)
|
||||
" 2>&1)" || {
|
||||
fail "Cannot reach the MySQL server named in $ENV_FILE"
|
||||
say ""
|
||||
printf ' %s\n' "$(printf '%s' "$db_err" | tail -n 2)"
|
||||
say ""
|
||||
say " Start the server and run this again, or fall back to the local file by setting"
|
||||
say " ${B}AR_DB_BACKEND=sqlite${R} in $ENV_FILE."
|
||||
say ""
|
||||
printf '%sPress Return to close this window.%s\n' "$DIM" "$R"
|
||||
read -r _
|
||||
exit 1
|
||||
}
|
||||
good "MySQL reachable"
|
||||
else
|
||||
good "database: $db_label"
|
||||
[ -f "$ENV_FILE" ] || warn "no .env — using the local file (fine for a demo or one user)"
|
||||
fi
|
||||
|
||||
# 1d. Everything above is fine — now prove the app itself loads. Anything failing here is a
|
||||
# genuine code/dependency problem, so the message points at that rather than at config.
|
||||
if ! import_error="$(cd -- "$BACKEND_DIR" && python3 -c "from app.api.main import app" 2>&1)"; then
|
||||
fail "The backend failed to load even though its packages and database are fine:"
|
||||
printf '%s\n' "$import_error" | tail -n 6 | sed 's/^/ /'
|
||||
die "This looks like a code or dependency-version problem." "python3 -m pip check"
|
||||
fi
|
||||
good "backend loads cleanly"
|
||||
|
||||
# Frontend packages — safe to install unattended, they're local to the project.
|
||||
if [ ! -d "$FRONTEND_DIR/node_modules" ]; then
|
||||
step "installing frontend packages (first run only, ~1 minute)…"
|
||||
|
|
|
|||
Loading…
Reference in New Issue