183 lines
7.4 KiB
Python
183 lines
7.4 KiB
Python
"""
|
|
End-to-end processing pipeline: parse many files -> aggregate -> classify -> receivable
|
|
-> reconcile. Streams records so memory stays bounded regardless of file size.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import date
|
|
from typing import Callable, Iterable, Iterator
|
|
|
|
from .xlsx_reader import FileMeta, ParseError, quick_expected_rows
|
|
from .readers import make_reader
|
|
from .regions import region_for
|
|
from .i18n import currency_for_region, is_storage_like, normalize_type
|
|
|
|
# Only these fields are needed for settlement/receivable + transaction persistence, so we skip
|
|
# converting the other ~20 amount columns during processing (the export re-reads all columns).
|
|
# `description` is included for the storage-fee detection rule.
|
|
PROCESS_FIELDS = {
|
|
"settlement_id", "txn_type", "account_type", "total", "order_id", "sku",
|
|
"date_time", "marketplace", "description",
|
|
}
|
|
from .settlements import aggregate, classify, AggregationResult, Classification
|
|
from .receivable import compute_receivable, ReceivableResult
|
|
from .reconciliation import reconcile, Reconciliation, DEFAULT_TOLERANCE
|
|
|
|
PROGRESS_STAGES = (
|
|
"Uploading files", "Reading workbook", "Validating columns", "Combining transactions",
|
|
"Detecting duplicates", "Calculating settlements", "Creating receivable aging",
|
|
"Reconciling totals", "Generating Excel workbook",
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class ProcessResult:
|
|
file_metas: list[FileMeta] = field(default_factory=list)
|
|
aggregation: AggregationResult | None = None
|
|
classification: Classification | None = None
|
|
receivable: ReceivableResult | None = None
|
|
reconciliation: Reconciliation | None = None
|
|
month_end: date | None = None
|
|
clearing_lag_days: int = 2
|
|
errors: list[str] = field(default_factory=list)
|
|
|
|
|
|
def iter_enriched_records(
|
|
path: str,
|
|
marketplace_override: str | None = None,
|
|
saved_overrides: dict[str, str] | None = None,
|
|
only_fields: set[str] | None = None,
|
|
metas: list[FileMeta] | None = None,
|
|
) -> Iterator[dict]:
|
|
"""
|
|
Stream a file's records enriched with `_marketplace` (region), `_type_en` (canonical
|
|
English type), `_storage`, `currency`, and a defaulted `account_type`.
|
|
|
|
Rows with a BLANK marketplace cell (transfers, some service fees) inherit the file's
|
|
dominant marketplace — not a filename guess, which misroutes combined files like
|
|
"Belgium Germany …". They are buffered and emitted after the dominant is known.
|
|
"""
|
|
reader = make_reader(path, saved_overrides=saved_overrides)
|
|
mapping = reader.detect()
|
|
filename = reader.filename
|
|
# Only the USA report has an `account type` column; other marketplaces run one stream.
|
|
has_account_col = "account_type" in mapping.field_to_col
|
|
|
|
def enrich(rec: dict, region: str) -> dict:
|
|
rec["_marketplace"] = region
|
|
type_en = normalize_type(rec.get("txn_type"))
|
|
rec["_type_en"] = type_en
|
|
if not has_account_col:
|
|
rec["account_type"] = "" if type_en == "Transfer" else "All Orders"
|
|
rec["_storage"] = is_storage_like(type_en, rec.get("description"))
|
|
rec["currency"] = currency_for_region(region)
|
|
return rec
|
|
|
|
pending: list[dict] = []
|
|
region_counts: dict[str, int] = {}
|
|
for rec in reader.iter_records(only_fields=only_fields):
|
|
if marketplace_override:
|
|
yield enrich(rec, marketplace_override)
|
|
continue
|
|
raw_mkt = rec.get("marketplace")
|
|
if raw_mkt:
|
|
region = region_for(raw_mkt, filename=filename)
|
|
region_counts[region] = region_counts.get(region, 0) + 1
|
|
yield enrich(rec, region)
|
|
else:
|
|
pending.append(rec)
|
|
if pending:
|
|
dominant = (max(region_counts.items(), key=lambda kv: kv[1])[0]
|
|
if region_counts else region_for(None, filename=filename))
|
|
for rec in pending:
|
|
yield enrich(rec, dominant)
|
|
|
|
meta = reader.file_meta
|
|
if meta.marketplace and not marketplace_override:
|
|
meta.currency = currency_for_region(region_for(meta.marketplace, filename=filename))
|
|
if metas is not None:
|
|
metas.append(meta)
|
|
reader.close()
|
|
|
|
|
|
def process(
|
|
files: Iterable[str | tuple[str, str | None]],
|
|
month_end: date,
|
|
clearing_lag_days: int = 2,
|
|
reserves: dict[tuple[str, str], float] | None = None,
|
|
fx_rates: dict[str, float] | None = None,
|
|
currencies: dict[str, str] | None = None,
|
|
received_overrides: dict[tuple[str, str, str], bool] | None = None,
|
|
manual_adjustments: float = 0.0,
|
|
expected_receivable: float | None = None,
|
|
tolerance: float = DEFAULT_TOLERANCE,
|
|
saved_column_overrides: dict[str, str] | None = None,
|
|
record_sink: Callable[[dict], None] | None = None,
|
|
progress: Callable[..., None] | None = None,
|
|
) -> ProcessResult:
|
|
"""
|
|
files: paths, or (path, marketplace_label) tuples to force a region.
|
|
progress(stage, pct, rows_done, rows_total) is called at stage boundaries and, during the
|
|
combine stage, every ~25k rows so the UI can show a true percentage and ETA.
|
|
Returns a ProcessResult with every stage's output.
|
|
"""
|
|
def emit(stage: str, pct: float, rows_done: int = 0, rows_total: int = 0) -> None:
|
|
if progress:
|
|
progress(stage, pct, rows_done, rows_total)
|
|
|
|
result = ProcessResult(month_end=month_end, clearing_lag_days=clearing_lag_days)
|
|
metas: list[FileMeta] = []
|
|
|
|
# Normalize inputs to (path, override) and combine record streams.
|
|
norm: list[tuple[str, str | None]] = []
|
|
for f in files:
|
|
if isinstance(f, tuple):
|
|
norm.append((f[0], f[1]))
|
|
else:
|
|
norm.append((f, None))
|
|
|
|
emit("Reading workbook", 0.03)
|
|
total_expected = sum(quick_expected_rows(p) for p, _ in norm) or 0
|
|
|
|
# The combine stage (parse + optional DB insert) dominates wall-clock; map it to 0.05..0.85.
|
|
seen = 0
|
|
|
|
nfiles = len(norm)
|
|
|
|
def combined() -> Iterator[dict]:
|
|
nonlocal seen
|
|
for fi, (path, override) in enumerate(norm, 1):
|
|
frac0 = min(1.0, seen / total_expected) if total_expected else 0.0
|
|
emit(f"Reading file {fi} of {nfiles}", 0.05 + 0.80 * frac0, seen, total_expected)
|
|
for rec in iter_enriched_records(path, override, saved_column_overrides,
|
|
PROCESS_FIELDS, metas):
|
|
if record_sink is not None:
|
|
record_sink(rec)
|
|
seen += 1
|
|
if seen % 25000 == 0:
|
|
frac = min(1.0, seen / total_expected) if total_expected else 0.0
|
|
emit("Combining transactions", 0.05 + 0.80 * frac, seen, total_expected)
|
|
yield rec
|
|
|
|
emit("Combining transactions", 0.05, 0, total_expected)
|
|
agg = aggregate(combined())
|
|
result.aggregation = agg
|
|
result.file_metas = metas
|
|
|
|
emit("Calculating settlements", 0.88, seen, total_expected or seen)
|
|
cls = classify(agg, month_end, clearing_lag_days, received_overrides)
|
|
result.classification = cls
|
|
|
|
emit("Creating receivable aging", 0.92, seen, total_expected or seen)
|
|
rec = compute_receivable(agg, cls, reserves=reserves, fx_rates=fx_rates, currencies=currencies)
|
|
result.receivable = rec
|
|
|
|
emit("Reconciling totals", 0.95, seen, total_expected or seen)
|
|
result.reconciliation = reconcile(
|
|
agg, cls, rec, manual_adjustments=manual_adjustments,
|
|
expected_receivable=expected_receivable, tolerance=tolerance,
|
|
)
|
|
emit("Reconciling totals", 0.98, seen, total_expected or seen)
|
|
return result
|