""" Exchange-rate fetching (the ONE deliberate network egress in the app — currency codes and dates only, never financial data). Providers frankfurter (default) free, keyless, central-bank (ECB) reference rates, historical dates supported. Weekend/holiday dates snap to the previous banking day — exactly the month-end convention Finance uses. exchangerate-api paid fallback (AR_FX_PROVIDER=exchangerate-api + AR_FX_API_KEY). ORIENTATION — the #1 way to corrupt every non-USD receivable: The app stores USD per 1 unit of LOCAL currency (usd = local * FxRate.rate; see core/money.to_usd and store.py). Providers return the opposite (local per 1 USD when base=USD), so every provider here INVERTS before returning. test_fx_service.py pins this with a known EUR fixture. Fetched rates are SUGGESTIONS: seeding writes them unconfirmed, so Control C5 still blocks the close until a person reviews and confirms them for the reporting month — identical to the manual-entry workflow, just pre-filled with a real rate instead of the Jan-26 snapshot. Daily rates: processing auto-fetches the provider's daily fixings across the closing's transaction span (auto_seed_daily_fx), so every dated movement converts at the rate effective on ITS OWN transaction date — see api/routes/analytics.py for the resolution order (exact fixing → previous banking day's fixing → month rate). Failure policy: a provider error raises FxProviderError (the route answers 502 "enter rates manually"). DEFAULT_FX_USD is never written silently — the existing merge in jobs.py is already the fallback and C5 already flags unconfirmed defaults. """ from __future__ import annotations import datetime as dt import json import logging import ssl import urllib.error import urllib.parse import urllib.request from sqlalchemy import func as sa_func from sqlalchemy.orm import Session as OrmSession from ..config import FX_API_KEY, FX_PROVIDER, FX_TIMEOUT_S from ..core.i18n import currency_for_region from ..db import models logger = logging.getLogger(__name__) class FxProviderError(RuntimeError): """The provider could not supply rates (network, quota, unknown currency...).""" def _ssl_context() -> ssl.SSLContext | None: """Prefer certifi's CA bundle: on some Windows machines loading the OS certificate store fails outright (ssl [ASN1: NOT_ENOUGH_DATA]), which would break every fetch. Fall back to the default context when certifi isn't installed (Linux containers).""" try: import certifi return ssl.create_default_context(cafile=certifi.where()) except ImportError: return None def _http_get_json(url: str) -> dict: req = urllib.request.Request(url, headers={"User-Agent": "ar-aging-app/1.0"}) try: with urllib.request.urlopen(req, timeout=FX_TIMEOUT_S, context=_ssl_context()) as resp: return json.loads(resp.read().decode("utf-8")) except urllib.error.HTTPError as e: raise FxProviderError(f"FX provider answered HTTP {e.code} for {url.split('?')[0]}") from e except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError, ssl.SSLError) as e: raise FxProviderError(f"Could not reach the FX provider: {e}") from e class FrankfurterProvider: """https://frankfurter.dev — GET /v1/{date}?base=USD&symbols=EUR,GBP,...""" name = "frankfurter" _BASE = "https://api.frankfurter.dev/v1" def rates_on(self, on: dt.date, currencies: set[str]) -> tuple[dict[str, float], dt.date]: """{currency: USD-per-local}, plus the banking day the provider actually used.""" symbols = sorted(c for c in currencies if c and c != "USD") if not symbols: return {}, on url = (f"{self._BASE}/{on.isoformat()}" f"?base=USD&symbols={urllib.parse.quote(','.join(symbols))}") data = _http_get_json(url) raw = data.get("rates") or {} # base=USD → provider returns LOCAL per USD; the app stores USD per LOCAL. Invert. out = {ccy: 1.0 / v for ccy, v in raw.items() if v} actual = dt.date.fromisoformat(data["date"]) if data.get("date") else on return out, actual def rates_series(self, date_from: dt.date, date_to: dt.date, currencies: set[str]) -> dict[dt.date, dict[str, float]]: """{date: {currency: USD-per-local}} for every banking day in the range.""" symbols = sorted(c for c in currencies if c and c != "USD") if not symbols: return {} url = (f"{self._BASE}/{date_from.isoformat()}..{date_to.isoformat()}" f"?base=USD&symbols={urllib.parse.quote(','.join(symbols))}") data = _http_get_json(url) out: dict[dt.date, dict[str, float]] = {} for day, raw in (data.get("rates") or {}).items(): out[dt.date.fromisoformat(day)] = {c: 1.0 / v for c, v in raw.items() if v} return out class ExchangeRateApiProvider: """https://www.exchangerate-api.com — paid fallback. Needs AR_FX_API_KEY.""" name = "exchangerate-api" _BASE = "https://v6.exchangerate-api.com/v6" def __init__(self) -> None: if not FX_API_KEY: raise FxProviderError( "AR_FX_PROVIDER=exchangerate-api requires AR_FX_API_KEY.") def rates_on(self, on: dt.date, currencies: set[str]) -> tuple[dict[str, float], dt.date]: symbols = {c for c in currencies if c and c != "USD"} if not symbols: return {}, on # History endpoint (paid plans); falls back to latest when the date is today. if on >= dt.date.today(): url = f"{self._BASE}/{FX_API_KEY}/latest/USD" else: url = f"{self._BASE}/{FX_API_KEY}/history/USD/{on.year}/{on.month}/{on.day}" data = _http_get_json(url) if data.get("result") != "success": raise FxProviderError(f"exchangerate-api: {data.get('error-type', 'error')}") raw = data.get("conversion_rates") or {} return {c: 1.0 / raw[c] for c in symbols if raw.get(c)}, on def rates_series(self, date_from: dt.date, date_to: dt.date, currencies: set[str]) -> dict[dt.date, dict[str, float]]: out: dict[dt.date, dict[str, float]] = {} day = date_from while day <= date_to: try: rates, actual = self.rates_on(day, currencies) out[actual] = rates except FxProviderError: pass # weekends/holidays have no fixing day += dt.timedelta(days=1) return out def get_provider(): if FX_PROVIDER == "exchangerate-api": return ExchangeRateApiProvider() if FX_PROVIDER == "frankfurter": return FrankfurterProvider() raise FxProviderError(f"Unknown AR_FX_PROVIDER {FX_PROVIDER!r} " f"(use 'frankfurter' or 'exchangerate-api').") # --------------------------------------------------------------------------- caching def _cached_rates(db: OrmSession, provider_name: str, on: dt.date, currencies: set[str]) -> dict[str, float] | None: """All requested currencies from the cache, or None on any miss.""" want = {c for c in currencies if c != "USD"} if not want: return {} rows = db.query(models.FxProviderRate).filter( models.FxProviderRate.provider == provider_name, models.FxProviderRate.rate_date == on, models.FxProviderRate.currency.in_(want)).all() got = {r.currency: r.rate for r in rows} return got if set(got) >= want else None def _cache_rates(db: OrmSession, provider_name: str, on: dt.date, rates: dict[str, float]) -> None: existing = {r.currency for r in db.query(models.FxProviderRate).filter( models.FxProviderRate.provider == provider_name, models.FxProviderRate.rate_date == on)} for ccy, rate in rates.items(): if ccy not in existing: db.add(models.FxProviderRate(provider=provider_name, rate_date=on, currency=ccy, rate=rate)) db.commit() def rates_for_date(db: OrmSession, on: dt.date, currencies: set[str]) -> tuple[dict[str, float], str]: """{currency: USD-per-local} for a date — cache first, provider on miss. Returns (rates, source_label). The label names the provider and the banking day the rates are actually for, so an FxRate row's `source` explains itself.""" provider = get_provider() cached = _cached_rates(db, provider.name, on, currencies) if cached is not None: return cached, f"{provider.name} {on.isoformat()} (cached)" rates, actual = provider.rates_on(on, currencies) # Cache under both the requested date and the provider's actual banking day, so a # weekend month-end (snapped to Friday) is served from cache next time as well. _cache_rates(db, provider.name, actual, rates) if actual != on: _cache_rates(db, provider.name, on, rates) return rates, f"{provider.name} {actual.isoformat()}" # --------------------------------------------------------------------------- seeding def _session_fx_targets(db: OrmSession, session: models.Session) -> list[models.FxRate]: """The session's existing FX rows — the marketplaces this close actually involves. Rows are created during processing for every marketplace that appears in the files (jobs.py), so 'process first' is the natural precondition; seeding rates for marketplaces the close doesn't contain would only widen what C5 asks Finance to confirm.""" return db.query(models.FxRate).filter( models.FxRate.session_id == session.id).all() def seed_session_fx(db: OrmSession, session: models.Session) -> dict: """Fetch month-end rates and pre-fill the session's FX table (UNCONFIRMED). Existing confirmations are cleared — same withdrawal semantics as editing a rate by hand (settings.put_fx): a confirmation attests to a specific number.""" if session.month_end_date is None: raise FxProviderError("Set the month-end date first.") rows = _session_fx_targets(db, session) if not rows: raise FxProviderError( "No FX rows exist yet for this closing — process it first so its " "marketplaces are known.") currencies = {(r.currency or currency_for_region(r.marketplace)) for r in rows} fetched, source = rates_for_date(db, session.month_end_date, currencies) updated, missing = [], [] for r in rows: ccy = r.currency or currency_for_region(r.marketplace) if ccy == "USD": new_rate = 1.0 elif ccy in fetched: new_rate = round(fetched[ccy], 6) else: missing.append(f"{r.marketplace} ({ccy})") continue r.rate = new_rate r.currency = ccy r.rate_date = session.month_end_date r.source = source # A fetched rate is a suggestion — it must be confirmed for THIS month (C5). r.confirmed_by = "" r.confirmed_at = None r.confirmed_month = "" updated.append({"marketplace": r.marketplace, "currency": ccy, "rate": new_rate}) db.commit() if session.status in ("processed", "blocked", "completed"): from .controls_run import run_and_persist run_and_persist(db, session.id) logger.info("fx seed: session %s, %d rate(s) from %s, %d missing", session.id, len(updated), source, len(missing)) return {"updated": updated, "missing": missing, "source": source, "rate_date": session.month_end_date.isoformat()} def _transaction_span(db: OrmSession, session_id: int) -> tuple[dt.date | None, dt.date | None]: """Earliest/latest dated transaction of the closing ((None, None) when nothing is dated).""" lo, hi = db.query(sa_func.min(models.Transaction.posted_date), sa_func.max(models.Transaction.posted_date)).filter( models.Transaction.session_id == session_id, models.Transaction.posted_date.isnot(None)).one() def _d(v): return v if (v is None or isinstance(v, dt.date)) else dt.date.fromisoformat(str(v)) return _d(lo), _d(hi) def seed_daily_fx(db: OrmSession, session: models.Session, marketplace: str | None = None, date_from: dt.date | None = None, date_to: dt.date | None = None, overwrite_manual: bool = True) -> dict: """Fill fx_rates_daily from the provider. Default range: the span of dates the files actually contain (earliest dated transaction through month-end, extended to any later transaction), widened to the start of the reporting month — so every transaction converts at its own date's rate. Clamped to a year before / a month after month-end, so one mis-parsed date can't request a decade of history. Daily rows are what the ledger converts dated movements with (analytics), marked source=provider so hand-entered rows are distinguishable. With overwrite_manual=False (the automatic post-processing seed), rows a person typed stay untouched; the explicit Fetch button replaces them.""" if session.month_end_date is None: raise FxProviderError("Set the month-end date first.") month_end = session.month_end_date if date_from is None or date_to is None: lo, hi = _transaction_span(db, session.id) if date_from is None: date_from = min(lo or month_end.replace(day=1), month_end.replace(day=1)) date_from = max(date_from, month_end - dt.timedelta(days=366)) if date_to is None: date_to = max(hi or month_end, month_end) date_to = min(date_to, month_end + dt.timedelta(days=31)) if date_from > date_to: raise FxProviderError("date_from is after date_to.") rows = _session_fx_targets(db, session) targets = [(r.marketplace, r.currency or currency_for_region(r.marketplace)) for r in rows if (marketplace is None or r.marketplace == marketplace)] targets = [(m, c) for m, c in targets if c != "USD"] if not targets: raise FxProviderError( "No non-USD marketplace to fetch daily rates for — process the closing " "first (or this closing is USD-only).") provider = get_provider() series = provider.rates_series(date_from, date_to, {c for _, c in targets}) existing = {(r.marketplace, r.rate_date): r for r in db.query(models.FxRateDaily).filter( models.FxRateDaily.session_id == session.id)} saved = 0 for day, per_ccy in sorted(series.items()): for mkt, ccy in targets: rate = per_ccy.get(ccy) if not rate: continue row = existing.get((mkt, day)) if row is None: row = models.FxRateDaily(session_id=session.id, marketplace=mkt, rate_date=day) db.add(row) existing[(mkt, day)] = row elif not overwrite_manual and (row.source or "") == "manual": continue # a person typed this rate — keep it row.rate = round(rate, 6) row.source = provider.name saved += 1 db.commit() logger.info("fx daily seed: session %s, %d row(s) %s..%s", session.id, saved, date_from, date_to) return {"saved": saved, "date_from": date_from.isoformat(), "date_to": date_to.isoformat(), "provider": provider.name, "marketplaces": sorted({m for m, _ in targets})} def auto_seed_daily_fx(db: OrmSession, session: models.Session) -> dict: """Post-processing daily-rate fetch, so every dated movement converts at the rate effective on its own transaction date without anyone clicking anything. Advisory by design — it NEVER raises: a provider outage must not fail the close (conversion falls back to the last available fixing, then the month rate, and jobs.py surfaces the shortfall as an exception). Hand-entered daily rates are preserved; only provider rows are refreshed.""" try: rows = _session_fx_targets(db, session) if not any((r.currency or currency_for_region(r.marketplace)) != "USD" for r in rows): return {"skipped": "USD-only closing", "saved": 0} return seed_daily_fx(db, session, overwrite_manual=False) except FxProviderError as e: logger.warning("daily FX auto-seed failed for session %s: %s", session.id, e) return {"error": str(e), "saved": 0} except Exception as e: # noqa: BLE001 — advisory; never fail the close over FX logger.exception("daily FX auto-seed crashed for session %s", session.id) db.rollback() return {"error": f"{type(e).__name__}: {e}", "saved": 0}