309 lines
14 KiB
Python
309 lines
14 KiB
Python
"""
|
|
Exchange-rate fetching (the ONE deliberate network egress in the app — currency codes and
|
|
dates only, never financial data).
|
|
|
|
Providers
|
|
frankfurter (default) free, keyless, central-bank (ECB) reference rates, historical
|
|
dates supported. Weekend/holiday dates snap to the previous
|
|
banking day — exactly the month-end convention Finance uses.
|
|
exchangerate-api paid fallback (AR_FX_PROVIDER=exchangerate-api + AR_FX_API_KEY).
|
|
|
|
ORIENTATION — the #1 way to corrupt every non-USD receivable:
|
|
The app stores USD per 1 unit of LOCAL currency (usd = local * FxRate.rate; see
|
|
core/money.to_usd and store.py). Providers return the opposite (local per 1 USD when
|
|
base=USD), so every provider here INVERTS before returning. test_fx_service.py pins
|
|
this with a known EUR fixture.
|
|
|
|
Fetched rates are SUGGESTIONS: seeding writes them unconfirmed, so Control C5 still blocks
|
|
the close until a person reviews and confirms them for the reporting month — identical to
|
|
the manual-entry workflow, just pre-filled with a real rate instead of the Jan-26 snapshot.
|
|
|
|
Failure policy: a provider error raises FxProviderError (the route answers 502 "enter rates
|
|
manually"). DEFAULT_FX_USD is never written silently — the existing merge in jobs.py is
|
|
already the fallback and C5 already flags unconfirmed defaults.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import json
|
|
import logging
|
|
import ssl
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from sqlalchemy.orm import Session as OrmSession
|
|
|
|
from ..config import FX_API_KEY, FX_PROVIDER, FX_TIMEOUT_S
|
|
from ..core.i18n import currency_for_region
|
|
from ..db import models
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class FxProviderError(RuntimeError):
|
|
"""The provider could not supply rates (network, quota, unknown currency...)."""
|
|
|
|
|
|
def _ssl_context() -> ssl.SSLContext | None:
|
|
"""Prefer certifi's CA bundle: on some Windows machines loading the OS certificate
|
|
store fails outright (ssl [ASN1: NOT_ENOUGH_DATA]), which would break every fetch.
|
|
Fall back to the default context when certifi isn't installed (Linux containers)."""
|
|
try:
|
|
import certifi
|
|
return ssl.create_default_context(cafile=certifi.where())
|
|
except ImportError:
|
|
return None
|
|
|
|
|
|
def _http_get_json(url: str) -> dict:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "ar-aging-app/1.0"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=FX_TIMEOUT_S,
|
|
context=_ssl_context()) as resp:
|
|
return json.loads(resp.read().decode("utf-8"))
|
|
except urllib.error.HTTPError as e:
|
|
raise FxProviderError(f"FX provider answered HTTP {e.code} for {url.split('?')[0]}") from e
|
|
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError, ssl.SSLError) as e:
|
|
raise FxProviderError(f"Could not reach the FX provider: {e}") from e
|
|
|
|
|
|
class FrankfurterProvider:
|
|
"""https://frankfurter.dev — GET /v1/{date}?base=USD&symbols=EUR,GBP,..."""
|
|
|
|
name = "frankfurter"
|
|
_BASE = "https://api.frankfurter.dev/v1"
|
|
|
|
def rates_on(self, on: dt.date, currencies: set[str]) -> tuple[dict[str, float], dt.date]:
|
|
"""{currency: USD-per-local}, plus the banking day the provider actually used."""
|
|
symbols = sorted(c for c in currencies if c and c != "USD")
|
|
if not symbols:
|
|
return {}, on
|
|
url = (f"{self._BASE}/{on.isoformat()}"
|
|
f"?base=USD&symbols={urllib.parse.quote(','.join(symbols))}")
|
|
data = _http_get_json(url)
|
|
raw = data.get("rates") or {}
|
|
# base=USD → provider returns LOCAL per USD; the app stores USD per LOCAL. Invert.
|
|
out = {ccy: 1.0 / v for ccy, v in raw.items() if v}
|
|
actual = dt.date.fromisoformat(data["date"]) if data.get("date") else on
|
|
return out, actual
|
|
|
|
def rates_series(self, date_from: dt.date, date_to: dt.date,
|
|
currencies: set[str]) -> dict[dt.date, dict[str, float]]:
|
|
"""{date: {currency: USD-per-local}} for every banking day in the range."""
|
|
symbols = sorted(c for c in currencies if c and c != "USD")
|
|
if not symbols:
|
|
return {}
|
|
url = (f"{self._BASE}/{date_from.isoformat()}..{date_to.isoformat()}"
|
|
f"?base=USD&symbols={urllib.parse.quote(','.join(symbols))}")
|
|
data = _http_get_json(url)
|
|
out: dict[dt.date, dict[str, float]] = {}
|
|
for day, raw in (data.get("rates") or {}).items():
|
|
out[dt.date.fromisoformat(day)] = {c: 1.0 / v for c, v in raw.items() if v}
|
|
return out
|
|
|
|
|
|
class ExchangeRateApiProvider:
|
|
"""https://www.exchangerate-api.com — paid fallback. Needs AR_FX_API_KEY."""
|
|
|
|
name = "exchangerate-api"
|
|
_BASE = "https://v6.exchangerate-api.com/v6"
|
|
|
|
def __init__(self) -> None:
|
|
if not FX_API_KEY:
|
|
raise FxProviderError(
|
|
"AR_FX_PROVIDER=exchangerate-api requires AR_FX_API_KEY.")
|
|
|
|
def rates_on(self, on: dt.date, currencies: set[str]) -> tuple[dict[str, float], dt.date]:
|
|
symbols = {c for c in currencies if c and c != "USD"}
|
|
if not symbols:
|
|
return {}, on
|
|
# History endpoint (paid plans); falls back to latest when the date is today.
|
|
if on >= dt.date.today():
|
|
url = f"{self._BASE}/{FX_API_KEY}/latest/USD"
|
|
else:
|
|
url = f"{self._BASE}/{FX_API_KEY}/history/USD/{on.year}/{on.month}/{on.day}"
|
|
data = _http_get_json(url)
|
|
if data.get("result") != "success":
|
|
raise FxProviderError(f"exchangerate-api: {data.get('error-type', 'error')}")
|
|
raw = data.get("conversion_rates") or {}
|
|
return {c: 1.0 / raw[c] for c in symbols if raw.get(c)}, on
|
|
|
|
def rates_series(self, date_from: dt.date, date_to: dt.date,
|
|
currencies: set[str]) -> dict[dt.date, dict[str, float]]:
|
|
out: dict[dt.date, dict[str, float]] = {}
|
|
day = date_from
|
|
while day <= date_to:
|
|
try:
|
|
rates, actual = self.rates_on(day, currencies)
|
|
out[actual] = rates
|
|
except FxProviderError:
|
|
pass # weekends/holidays have no fixing
|
|
day += dt.timedelta(days=1)
|
|
return out
|
|
|
|
|
|
def get_provider():
|
|
if FX_PROVIDER == "exchangerate-api":
|
|
return ExchangeRateApiProvider()
|
|
if FX_PROVIDER == "frankfurter":
|
|
return FrankfurterProvider()
|
|
raise FxProviderError(f"Unknown AR_FX_PROVIDER {FX_PROVIDER!r} "
|
|
f"(use 'frankfurter' or 'exchangerate-api').")
|
|
|
|
|
|
# --------------------------------------------------------------------------- caching
|
|
def _cached_rates(db: OrmSession, provider_name: str, on: dt.date,
|
|
currencies: set[str]) -> dict[str, float] | None:
|
|
"""All requested currencies from the cache, or None on any miss."""
|
|
want = {c for c in currencies if c != "USD"}
|
|
if not want:
|
|
return {}
|
|
rows = db.query(models.FxProviderRate).filter(
|
|
models.FxProviderRate.provider == provider_name,
|
|
models.FxProviderRate.rate_date == on,
|
|
models.FxProviderRate.currency.in_(want)).all()
|
|
got = {r.currency: r.rate for r in rows}
|
|
return got if set(got) >= want else None
|
|
|
|
|
|
def _cache_rates(db: OrmSession, provider_name: str, on: dt.date,
|
|
rates: dict[str, float]) -> None:
|
|
existing = {r.currency for r in db.query(models.FxProviderRate).filter(
|
|
models.FxProviderRate.provider == provider_name,
|
|
models.FxProviderRate.rate_date == on)}
|
|
for ccy, rate in rates.items():
|
|
if ccy not in existing:
|
|
db.add(models.FxProviderRate(provider=provider_name, rate_date=on,
|
|
currency=ccy, rate=rate))
|
|
db.commit()
|
|
|
|
|
|
def rates_for_date(db: OrmSession, on: dt.date,
|
|
currencies: set[str]) -> tuple[dict[str, float], str]:
|
|
"""{currency: USD-per-local} for a date — cache first, provider on miss.
|
|
|
|
Returns (rates, source_label). The label names the provider and the banking day the
|
|
rates are actually for, so an FxRate row's `source` explains itself."""
|
|
provider = get_provider()
|
|
cached = _cached_rates(db, provider.name, on, currencies)
|
|
if cached is not None:
|
|
return cached, f"{provider.name} {on.isoformat()} (cached)"
|
|
rates, actual = provider.rates_on(on, currencies)
|
|
# Cache under both the requested date and the provider's actual banking day, so a
|
|
# weekend month-end (snapped to Friday) is served from cache next time as well.
|
|
_cache_rates(db, provider.name, actual, rates)
|
|
if actual != on:
|
|
_cache_rates(db, provider.name, on, rates)
|
|
return rates, f"{provider.name} {actual.isoformat()}"
|
|
|
|
|
|
# --------------------------------------------------------------------------- seeding
|
|
def _session_fx_targets(db: OrmSession, session: models.Session) -> list[models.FxRate]:
|
|
"""The session's existing FX rows — the marketplaces this close actually involves.
|
|
|
|
Rows are created during processing for every marketplace that appears in the files
|
|
(jobs.py), so 'process first' is the natural precondition; seeding rates for
|
|
marketplaces the close doesn't contain would only widen what C5 asks Finance to
|
|
confirm."""
|
|
return db.query(models.FxRate).filter(
|
|
models.FxRate.session_id == session.id).all()
|
|
|
|
|
|
def seed_session_fx(db: OrmSession, session: models.Session) -> dict:
|
|
"""Fetch month-end rates and pre-fill the session's FX table (UNCONFIRMED).
|
|
|
|
Existing confirmations are cleared — same withdrawal semantics as editing a rate by
|
|
hand (settings.put_fx): a confirmation attests to a specific number."""
|
|
if session.month_end_date is None:
|
|
raise FxProviderError("Set the month-end date first.")
|
|
rows = _session_fx_targets(db, session)
|
|
if not rows:
|
|
raise FxProviderError(
|
|
"No FX rows exist yet for this closing — process it first so its "
|
|
"marketplaces are known.")
|
|
|
|
currencies = {(r.currency or currency_for_region(r.marketplace)) for r in rows}
|
|
fetched, source = rates_for_date(db, session.month_end_date, currencies)
|
|
|
|
updated, missing = [], []
|
|
for r in rows:
|
|
ccy = r.currency or currency_for_region(r.marketplace)
|
|
if ccy == "USD":
|
|
new_rate = 1.0
|
|
elif ccy in fetched:
|
|
new_rate = round(fetched[ccy], 6)
|
|
else:
|
|
missing.append(f"{r.marketplace} ({ccy})")
|
|
continue
|
|
r.rate = new_rate
|
|
r.currency = ccy
|
|
r.rate_date = session.month_end_date
|
|
r.source = source
|
|
# A fetched rate is a suggestion — it must be confirmed for THIS month (C5).
|
|
r.confirmed_by = ""
|
|
r.confirmed_at = None
|
|
r.confirmed_month = ""
|
|
updated.append({"marketplace": r.marketplace, "currency": ccy, "rate": new_rate})
|
|
db.commit()
|
|
|
|
if session.status in ("processed", "blocked", "completed"):
|
|
from .controls_run import run_and_persist
|
|
run_and_persist(db, session.id)
|
|
|
|
logger.info("fx seed: session %s, %d rate(s) from %s, %d missing",
|
|
session.id, len(updated), source, len(missing))
|
|
return {"updated": updated, "missing": missing, "source": source,
|
|
"rate_date": session.month_end_date.isoformat()}
|
|
|
|
|
|
def seed_daily_fx(db: OrmSession, session: models.Session, marketplace: str | None = None,
|
|
date_from: dt.date | None = None, date_to: dt.date | None = None) -> dict:
|
|
"""Fill fx_rates_daily from the provider for a date range (defaults: the whole month).
|
|
|
|
Daily rows are optional per-date OVERRIDES of the month rate (analytics fx-daily),
|
|
marked source=provider so hand-entered rows are distinguishable."""
|
|
if session.month_end_date is None:
|
|
raise FxProviderError("Set the month-end date first.")
|
|
date_to = date_to or session.month_end_date
|
|
date_from = date_from or session.month_end_date.replace(day=1)
|
|
if date_from > date_to:
|
|
raise FxProviderError("date_from is after date_to.")
|
|
|
|
rows = _session_fx_targets(db, session)
|
|
targets = [(r.marketplace, r.currency or currency_for_region(r.marketplace))
|
|
for r in rows
|
|
if (marketplace is None or r.marketplace == marketplace)]
|
|
targets = [(m, c) for m, c in targets if c != "USD"]
|
|
if not targets:
|
|
raise FxProviderError(
|
|
"No non-USD marketplace to fetch daily rates for — process the closing "
|
|
"first (or this closing is USD-only).")
|
|
|
|
provider = get_provider()
|
|
series = provider.rates_series(date_from, date_to, {c for _, c in targets})
|
|
|
|
existing = {(r.marketplace, r.rate_date): r for r in db.query(models.FxRateDaily).filter(
|
|
models.FxRateDaily.session_id == session.id)}
|
|
saved = 0
|
|
for day, per_ccy in sorted(series.items()):
|
|
for mkt, ccy in targets:
|
|
rate = per_ccy.get(ccy)
|
|
if not rate:
|
|
continue
|
|
row = existing.get((mkt, day))
|
|
if row is None:
|
|
row = models.FxRateDaily(session_id=session.id, marketplace=mkt,
|
|
rate_date=day)
|
|
db.add(row)
|
|
existing[(mkt, day)] = row
|
|
row.rate = round(rate, 6)
|
|
row.source = provider.name
|
|
saved += 1
|
|
db.commit()
|
|
logger.info("fx daily seed: session %s, %d row(s) %s..%s",
|
|
session.id, saved, date_from, date_to)
|
|
return {"saved": saved, "date_from": date_from.isoformat(),
|
|
"date_to": date_to.isoformat(), "provider": provider.name,
|
|
"marketplaces": sorted({m for m, _ in targets})}
|