Implement MySQL support in AR aging app. Update README with configuration instructions, modify requirements for PyMySQL and dotenv, and refactor database setup to use MySQL. Adjust models and queries for compatibility with MySQL, including column size specifications. Enhance Vite config for API proxying.

main
bahawal.baloch 2026-07-29 18:52:37 +05:00
parent 73a004d2e1
commit 2135dc873c
12 changed files with 280 additions and 90 deletions

View File

@ -0,0 +1,16 @@
.git
.env
.env.local
**/.venv
**/venv
**/__pycache__
**/*.pyc
**/node_modules
frontend/dist
backend/data
*.xlsx
*.xls
*.csv
*.db
*.db-shm
*.db-wal

View File

@ -22,6 +22,14 @@ docs/
## Run the app
```bash
# 1. Configure MySQL (hosted RDS) + data paths
cp example.env .env # then fill in MYSQL_* credentials
# Option A — Docker (backend + Vite hot reload)
docker compose up --build
# → http://localhost:5173 (API on :8000; uploads/exports on the ar_data volume)
# Option B — local processes
make install # backend deps + npm install
make backend # terminal 1 → FastAPI on :8000
make frontend # terminal 2 → dashboard on http://localhost:5173
@ -29,6 +37,9 @@ make frontend # terminal 2 → dashboard on http://localhost:5173
Then open http://localhost:5173 → **New Closing** → pick the month → drag in the three Amazon
files → **Run processing** → review → **Download Full A/R Aging Excel**.
Uploads and exports are stored under `AR_DATA_DIR` (default `backend/data` locally, `/data` in Docker).
The app connects to MySQL using `MYSQL_*` variables from `.env`.
## Run the engine headless (CLI)
```bash
cd backend && pip install -r requirements.txt

View File

@ -0,0 +1,16 @@
FROM python:3.12-slim
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8000
CMD ["uvicorn", "app.api.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]

View File

@ -3,12 +3,45 @@ from __future__ import annotations
import os
from pathlib import Path
from urllib.parse import quote_plus
from dotenv import load_dotenv
# Load ar-aging-app/.env (or cwd) before reading settings.
_APP_ROOT = Path(__file__).resolve().parent.parent.parent # .../ar-aging-app
load_dotenv(_APP_ROOT / ".env")
load_dotenv() # also allow cwd overrides
BASE_DIR = Path(__file__).resolve().parent.parent # .../backend
DATA_DIR = Path(os.environ.get("AR_DATA_DIR", str(BASE_DIR / "data")))
UPLOAD_DIR = DATA_DIR / "uploads"
EXPORT_DIR = DATA_DIR / "exports"
DB_PATH = Path(os.environ.get("AR_DB_PATH", str(DATA_DIR / "ar_aging.db")))
# MySQL (required)
MYSQL_HOST = os.environ.get("MYSQL_HOST", "")
MYSQL_PORT = int(os.environ.get("MYSQL_PORT", "3306"))
MYSQL_USER = os.environ.get("MYSQL_USER", "")
MYSQL_PASSWORD = os.environ.get("MYSQL_PASSWORD", "")
MYSQL_DATABASE = os.environ.get("MYSQL_DATABASE", "")
MYSQL_SLOW_QUERY_MS = int(os.environ.get("MYSQL_SLOW_QUERY_MS", "500"))
MYSQL_POOL_SIZE = int(os.environ.get("MYSQL_POOL_SIZE", "10"))
MYSQL_POOL_RECYCLE = int(os.environ.get("MYSQL_POOL_RECYCLE", "3600"))
def mysql_url() -> str:
if not all((MYSQL_HOST, MYSQL_USER, MYSQL_DATABASE)):
raise RuntimeError(
"MYSQL_HOST, MYSQL_USER, and MYSQL_DATABASE are required. "
"Copy example.env to .env and fill in credentials."
)
user = quote_plus(MYSQL_USER)
password = quote_plus(MYSQL_PASSWORD)
return (
f"mysql+pymysql://{user}:{password}"
f"@{MYSQL_HOST}:{MYSQL_PORT}/{MYSQL_DATABASE}"
f"?charset=utf8mb4"
)
# Retention: temp uploads/exports older than this are purged (0 = keep forever).
RETENTION_DAYS = int(os.environ.get("AR_RETENTION_DAYS", "30"))

View File

@ -1,30 +1,79 @@
"""SQLite database setup (SQLAlchemy). Tuned for bulk transaction inserts."""
"""MySQL database setup (SQLAlchemy)."""
from __future__ import annotations
from sqlalchemy import create_engine, event
import logging
import time
from urllib.parse import quote_plus
from sqlalchemy import create_engine, event, text
from sqlalchemy.orm import declarative_base, sessionmaker
from ..config import DB_PATH, ensure_dirs
from ..config import (
MYSQL_DATABASE,
MYSQL_HOST,
MYSQL_PASSWORD,
MYSQL_POOL_RECYCLE,
MYSQL_POOL_SIZE,
MYSQL_PORT,
MYSQL_SLOW_QUERY_MS,
MYSQL_USER,
ensure_dirs,
mysql_url,
)
ensure_dirs()
logger = logging.getLogger(__name__)
def _ensure_database() -> None:
"""Create MYSQL_DATABASE if it does not exist yet."""
user = quote_plus(MYSQL_USER)
password = quote_plus(MYSQL_PASSWORD)
server_url = (
f"mysql+pymysql://{user}:{password}"
f"@{MYSQL_HOST}:{MYSQL_PORT}/?charset=utf8mb4"
)
server_engine = create_engine(server_url, isolation_level="AUTOCOMMIT")
try:
with server_engine.connect() as conn:
conn.execute(
text(
f"CREATE DATABASE IF NOT EXISTS `{MYSQL_DATABASE}` "
"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
)
finally:
server_engine.dispose()
_ensure_database()
ENGINE = create_engine(
f"sqlite:///{DB_PATH}",
connect_args={"check_same_thread": False},
mysql_url(),
pool_size=MYSQL_POOL_SIZE,
pool_recycle=MYSQL_POOL_RECYCLE,
pool_pre_ping=True,
future=True,
)
if MYSQL_SLOW_QUERY_MS > 0:
@event.listens_for(ENGINE, "before_cursor_execute")
def _before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
conn.info["query_start_time"] = time.perf_counter()
@event.listens_for(ENGINE, "connect")
def _sqlite_pragmas(dbapi_conn, _rec):
cur = dbapi_conn.cursor()
cur.execute("PRAGMA journal_mode=WAL")
cur.execute("PRAGMA synchronous=NORMAL")
cur.execute("PRAGMA foreign_keys=ON")
cur.execute("PRAGMA cache_size=-64000") # ~64 MB page cache
cur.execute("PRAGMA busy_timeout=30000") # wait (don't error) when another writer holds the lock
cur.close()
@event.listens_for(ENGINE, "after_cursor_execute")
def _after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
start = conn.info.pop("query_start_time", None)
if start is None:
return
elapsed_ms = (time.perf_counter() - start) * 1000
if elapsed_ms >= MYSQL_SLOW_QUERY_MS:
logger.warning(
"Slow query (%.0f ms): %s",
elapsed_ms,
statement[:500],
)
SessionLocal = sessionmaker(bind=ENGINE, autoflush=False, expire_on_commit=False, future=True)
Base = declarative_base()
@ -37,13 +86,13 @@ def init_db() -> None:
def _migrate() -> None:
"""Add columns introduced after a DB was first created (SQLite create_all won't)."""
"""Add columns introduced after a DB was first created (create_all won't alter)."""
added = {
"sessions": [
("progress_rows_done", "INTEGER DEFAULT 0"),
("progress_rows_total", "INTEGER DEFAULT 0"),
("eta_seconds", "INTEGER DEFAULT 0"),
("opening_mode", "VARCHAR DEFAULT 'zero'"),
("opening_mode", "VARCHAR(255) DEFAULT 'zero'"),
("opening_source_session_id", "INTEGER"),
],
"reconciliation": [
@ -54,19 +103,29 @@ def _migrate() -> None:
("tolerance", "FLOAT DEFAULT 1.0"),
],
"exports": [
("kind", "VARCHAR DEFAULT 'full'"),
("kind", "VARCHAR(255) DEFAULT 'full'"),
],
"transactions": [
("txn_type_en", "VARCHAR"),
("txn_type_en", "VARCHAR(255)"),
("storage_flag", "BOOLEAN DEFAULT 0"),
],
}
with ENGINE.begin() as conn:
db_name = conn.execute(text("SELECT DATABASE()")).scalar()
for table, cols in added.items():
existing = {r[1] for r in conn.exec_driver_sql(f"PRAGMA table_info({table})")}
existing = {
r[0]
for r in conn.execute(
text(
"SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS "
"WHERE TABLE_SCHEMA = :schema AND TABLE_NAME = :table"
),
{"schema": db_name, "table": table},
)
}
for name, decl in cols:
if name not in existing:
conn.exec_driver_sql(f"ALTER TABLE {table} ADD COLUMN {name} {decl}")
conn.execute(text(f"ALTER TABLE `{table}` ADD COLUMN `{name}` {decl}"))
def get_db():

View File

@ -18,20 +18,20 @@ def _now() -> dt.datetime:
class Session(Base):
__tablename__ = "sessions"
id = Column(Integer, primary_key=True)
name = Column(String, nullable=False)
reporting_month = Column(String) # "2026-01"
name = Column(String(255), nullable=False)
reporting_month = Column(String(32)) # "2026-01"
month_end_date = Column(Date)
reporting_currency = Column(String, default="USD")
reporting_currency = Column(String(16), default="USD")
clearing_lag_days = Column(Integer, default=2)
rounding_tolerance = Column(Float, default=0.01)
allowance_for_returns = Column(Float, default=0.0)
manual_adjustment = Column(Float, default=0.0)
manual_adjustment_note = Column(String, default="")
manual_adjustment_note = Column(String(512), default="")
# How the opening AR balance is established: zero (default) | carry_forward | manual
opening_mode = Column(String, default="zero")
opening_mode = Column(String(32), default="zero")
opening_source_session_id = Column(Integer)
status = Column(String, default="draft") # draft|processing|processed|error
progress_stage = Column(String, default="")
status = Column(String(32), default="draft") # draft|processing|processed|error
progress_stage = Column(String(255), default="")
progress_pct = Column(Float, default=0.0)
progress_rows_done = Column(Integer, default=0)
progress_rows_total = Column(Integer, default=0)
@ -51,18 +51,18 @@ class SessionFile(Base):
__tablename__ = "session_files"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
filename = Column(String)
stored_path = Column(String)
filename = Column(String(512))
stored_path = Column(String(1024))
size_bytes = Column(Integer)
sha256 = Column(String)
sha256 = Column(String(64))
worksheets = Column(Text) # JSON list
data_sheet = Column(String)
data_sheet = Column(String(255))
imported_rows = Column(Integer, default=0)
min_date = Column(Date)
max_date = Column(Date)
currency = Column(String)
marketplace = Column(String)
status = Column(String, default="uploaded") # uploaded|parsed|invalid
currency = Column(String(16))
marketplace = Column(String(64))
status = Column(String(32), default="uploaded") # uploaded|parsed|invalid
message = Column(Text, default="")
session = relationship("Session", back_populates="files")
@ -71,9 +71,9 @@ class Settlement(Base):
__tablename__ = "settlements"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
marketplace = Column(String)
account_type = Column(String)
settlement_id = Column(String)
marketplace = Column(String(64))
account_type = Column(String(64))
settlement_id = Column(String(255))
order_total = Column(Float, default=0.0)
transfer_total = Column(Float, default=0.0)
transfer_amount = Column(Float) # boundary/received transfer amount if any
@ -82,7 +82,7 @@ class Settlement(Base):
row_count = Column(Integer, default=0)
first_date = Column(Date)
last_date = Column(Date)
status = Column(String) # paid|receivable
status = Column(String(32)) # paid|receivable
session = relationship("Session", back_populates="settlements")
@ -93,20 +93,20 @@ class Transaction(Base):
__tablename__ = "transactions"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
source_file = Column(String)
source_sheet = Column(String)
source_file = Column(String(512))
source_sheet = Column(String(255))
source_row = Column(Integer)
marketplace = Column(String)
settlement_id = Column(String)
order_id = Column(String)
sku = Column(String)
txn_type = Column(String) # original (possibly localized) type
txn_type_en = Column(String) # canonical English type
account_type = Column(String)
marketplace = Column(String(64))
settlement_id = Column(String(255))
order_id = Column(String(255))
sku = Column(String(255))
txn_type = Column(String(255)) # original (possibly localized) type
txn_type_en = Column(String(255)) # canonical English type
account_type = Column(String(64))
posted_date = Column(Date)
total = Column(Float, default=0.0)
currency = Column(String, default="USD")
settlement_status = Column(String) # paid|receivable
currency = Column(String(16), default="USD")
settlement_status = Column(String(32)) # paid|receivable
receivable_flag = Column(Boolean, default=False)
storage_flag = Column(Boolean, default=False) # storage-fee detection drill-down
@ -119,8 +119,8 @@ class MappingRule(Base):
Lets Finance map new/renamed Amazon headers without code changes."""
__tablename__ = "mapping_rules"
id = Column(Integer, primary_key=True)
normalized_header = Column(String, unique=True, nullable=False)
field = Column(String, nullable=False)
normalized_header = Column(String(255), unique=True, nullable=False)
field = Column(String(128), nullable=False)
created_at = Column(DateTime, default=_now)
Index("ix_txn_settlement", Transaction.session_id, Transaction.settlement_id)
Index("ix_txn_type", Transaction.session_id, Transaction.txn_type)
@ -131,10 +131,10 @@ class Exception_(Base):
__tablename__ = "exceptions"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
category = Column(String)
severity = Column(String) # error|warning|info
category = Column(String(128))
severity = Column(String(32)) # error|warning|info
detail = Column(Text)
source = Column(String, default="")
source = Column(String(512), default="")
session = relationship("Session", back_populates="exceptions")
@ -142,10 +142,10 @@ class FxRate(Base):
__tablename__ = "fx_rates"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
marketplace = Column(String)
currency = Column(String, default="USD")
marketplace = Column(String(64))
currency = Column(String(16), default="USD")
rate = Column(Float, default=1.0)
source = Column(String, default="manual")
source = Column(String(64), default="manual")
rate_date = Column(Date)
session = relationship("Session", back_populates="fx_rates")
@ -154,8 +154,8 @@ class Reserve(Base):
__tablename__ = "reserves"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
marketplace = Column(String)
account_type = Column(String)
marketplace = Column(String(64))
account_type = Column(String(64))
amount = Column(Float, default=0.0)
session = relationship("Session", back_populates="reserves")
@ -164,14 +164,14 @@ class ReceivableResultRow(Base):
__tablename__ = "receivable_results"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
marketplace = Column(String)
account_type = Column(String) # or "TOTAL"
marketplace = Column(String(64))
account_type = Column(String(64)) # or "TOTAL"
additional_sales = Column(Float, default=0.0)
reserve = Column(Float, default=0.0)
receivable_local = Column(Float, default=0.0)
fx_rate = Column(Float, default=1.0)
receivable_usd = Column(Float, default=0.0)
currency = Column(String, default="USD")
currency = Column(String(16), default="USD")
class ReconciliationRow(Base):
@ -186,7 +186,7 @@ class ReconciliationRow(Base):
manual_adjustments = Column(Float)
final_receivable_usd = Column(Float)
identity_difference = Column(Float)
status = Column(String)
status = Column(String(64))
notes = Column(Text, default="")
# AR roll-forward components
received_payouts = Column(Float, default=0.0) # Σ transfers received (negative)
@ -198,10 +198,10 @@ class FxRateDaily(Base):
__tablename__ = "fx_rates_daily"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
marketplace = Column(String)
marketplace = Column(String(64))
rate_date = Column(Date)
rate = Column(Float, default=1.0)
source = Column(String, default="manual")
source = Column(String(64), default="manual")
Index("ix_fx_daily", FxRateDaily.session_id, FxRateDaily.marketplace, FxRateDaily.rate_date)
@ -217,7 +217,7 @@ class MarketPayout(Base):
__tablename__ = "market_payouts"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
marketplace = Column(String)
marketplace = Column(String(64))
received_payouts = Column(Float, default=0.0) # negative
all_payouts = Column(Float, default=0.0) # negative
@ -229,10 +229,10 @@ class OpeningBalance(Base):
__tablename__ = "opening_balances"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
marketplace = Column(String)
marketplace = Column(String(64))
amount = Column(Float, default=0.0)
reason = Column(String, default="")
source = Column(String, default="manual") # manual | carried_forward
reason = Column(String(512), default="")
source = Column(String(64), default="manual") # manual | carried_forward
updated_at = Column(DateTime, default=_now, onupdate=_now)
@ -250,7 +250,7 @@ class FinanceControl(Base):
disbursements = Column(Float)
closing_receivable = Column(Float)
tolerance = Column(Float, default=1.0)
verified_by = Column(String, default="")
verified_by = Column(String(255), default="")
verified_at = Column(DateTime)
comment = Column(Text, default="")
@ -259,7 +259,7 @@ class JournalEntry(Base):
__tablename__ = "journal_entries"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False, unique=True)
entry_no = Column(String, default="")
entry_no = Column(String(128), default="")
data = Column(Text, default="") # JSON: periods + lines + receivable
@ -267,8 +267,8 @@ class ExportRecord(Base):
__tablename__ = "exports"
id = Column(Integer, primary_key=True)
session_id = Column(Integer, ForeignKey("sessions.id"), nullable=False)
kind = Column(String, default="full") # full | summary
path = Column(String)
sha256 = Column(String)
kind = Column(String(32), default="full") # full | summary
path = Column(String(1024))
sha256 = Column(String(64))
size_bytes = Column(Integer)
generated_at = Column(DateTime, default=_now)

View File

@ -16,9 +16,10 @@ _TXN_COLS = (
"settlement_id", "order_id", "sku", "txn_type", "txn_type_en", "account_type",
"posted_date", "total", "currency", "storage_flag",
)
# PyMySQL uses %-style placeholders for raw DBAPI executemany.
_INSERT_SQL = (
f"INSERT INTO transactions ({', '.join(_TXN_COLS)}) "
f"VALUES ({', '.join('?' * len(_TXN_COLS))})"
f"VALUES ({', '.join(['%s'] * len(_TXN_COLS))})"
)
@ -30,12 +31,6 @@ class TransactionSink:
self.batch = batch
self._buf: list[tuple] = []
self._conn = ENGINE.raw_connection()
# Bulk-import tuning: durability is not critical here (we can re-run a failed job).
cur = self._conn.cursor()
cur.execute("PRAGMA synchronous=OFF")
cur.execute("PRAGMA temp_store=MEMORY")
cur.execute("PRAGMA busy_timeout=30000")
cur.close()
self.count = 0
def add(self, rec: dict[str, Any]) -> None:
@ -60,7 +55,7 @@ class TransactionSink:
cur.executemany(_INSERT_SQL, self._buf)
cur.close()
# Commit each batch so we don't hold a long write lock that starves the progress
# updater (a separate connection). synchronous=OFF keeps these commits cheap.
# updater (a separate connection).
self._conn.commit()
self._buf.clear()
@ -76,14 +71,14 @@ class TransactionSink:
recv = 1 if (st.status == "receivable"
and acct.lower() in RECEIVABLE_ACCOUNT_TYPES) else 0
cur.execute(
"UPDATE transactions SET settlement_status=?, receivable_flag=? "
"WHERE session_id=? AND settlement_id=? AND marketplace=? AND account_type=?",
"UPDATE transactions SET settlement_status=%s, receivable_flag=%s "
"WHERE session_id=%s AND settlement_id=%s AND marketplace=%s AND account_type=%s",
(st.status, recv, self.session_id, sid, mkt, acct),
)
# transfer rows: never receivable (canonical type covers localized names)
cur.execute(
"UPDATE transactions SET receivable_flag=0 "
"WHERE session_id=? AND txn_type_en='Transfer'",
"WHERE session_id=%s AND txn_type_en='Transfer'",
(self.session_id,),
)
cur.close()
@ -103,7 +98,7 @@ def clear_session_results(db: OrmSession, session_id: int) -> None:
db.commit()
# Every table that references sessions.id. Foreign keys are enforced (PRAGMA foreign_keys=ON),
# Every table that references sessions.id. Foreign keys are enforced,
# so all children must go before the parent row.
_CHILD_MODELS = (
models.Transaction, models.Settlement, models.ReceivableResultRow,

View File

@ -9,7 +9,9 @@ uvicorn[standard]==0.34.0
python-multipart==0.0.20
pydantic==2.10.4
SQLAlchemy==2.0.36
aiosqlite==0.20.0
PyMySQL==1.1.1
cryptography>=42.0.0
python-dotenv==1.0.1
# Data helpers (optional / analysis)
pandas>=2.2

View File

@ -0,0 +1,29 @@
services:
backend:
build: ./backend
env_file: .env
environment:
AR_DATA_DIR: /data
ports:
- "8000:8000"
volumes:
- ./backend:/app
- ar_data:/data
command: uvicorn app.api.main:app --host 0.0.0.0 --port 8000 --reload
frontend:
build: ./frontend
ports:
- "5173:5173"
volumes:
- ./frontend:/app
- frontend_node_modules:/app/node_modules
environment:
CHOKIDAR_USEPOLLING: "true"
VITE_API_PROXY: http://backend:8000
depends_on:
- backend
volumes:
ar_data:
frontend_node_modules:

13
ar-aging-app/example.env Normal file
View File

@ -0,0 +1,13 @@
MYSQL_HOST=your-mysql-host.example.com
MYSQL_PORT=3306
MYSQL_USER=your_mysql_user
MYSQL_PASSWORD=your_mysql_password
MYSQL_DATABASE=account_finance
MYSQL_SLOW_QUERY_MS=500
MYSQL_POOL_SIZE=10
MYSQL_POOL_RECYCLE=3600
# Uploads/exports. Docker Compose sets AR_DATA_DIR=/data (named volume).
# Leave unset locally to use backend/data.
# AR_DATA_DIR=/data
AR_CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173

View File

@ -0,0 +1,12 @@
FROM node:20-alpine
WORKDIR /app
COPY package.json package-lock.json* ./
RUN npm install
COPY . .
EXPOSE 5173
CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0", "--port", "5173"]

View File

@ -4,9 +4,13 @@ import react from "@vitejs/plugin-react";
export default defineConfig({
plugins: [react()],
server: {
host: true,
port: 5173,
proxy: {
"/api": { target: "http://localhost:8000", changeOrigin: true },
"/api": {
target: process.env.VITE_API_PROXY || "http://localhost:8000",
changeOrigin: true,
},
},
},
});