446 lines
22 KiB
Python
446 lines
22 KiB
Python
"""Verdict backtest: what do the competitor rules actually change, on real SKUs?
|
|
|
|
Same shape as the storage-fix backtest (61.9% -> 45.2%), but the quantity under test is not
|
|
an error percentage -- it is the VERDICT. So this reports, per SKU, the action and the reason
|
|
code with the competitor rules blind vs. live, and names every SKU that moved and why.
|
|
|
|
Method
|
|
------
|
|
The real pipeline runs ONCE per SKU. `dashboard.live_data._decide` is wrapped so the exact
|
|
arguments it was handed in production -- the real AnalysisResult, the real 180-day history,
|
|
the real fee stack, the real scenario grid, the real inventory outlook -- are captured. Every
|
|
arm below then re-runs that same captured input through the same cascade, varying ONLY the
|
|
competitor state. Nothing is reconstructed by hand, so no arm can drift from what the live
|
|
dashboard would actually compute.
|
|
|
|
Four arms:
|
|
BLIND comp=None -- the behaviour before this change
|
|
REAL the actual competitor state -- what production would do right now
|
|
CF_UNDERCUT counterfactual: a rival holds the Buy Box `--undercut` below us
|
|
CF_SUPPRESSED counterfactual: our Buy Box is suppressed
|
|
|
|
The REAL arm is built from the Apify data ALREADY ON DISK (`data/apify_cache.json`) rather
|
|
than by scraping. The actor is pay-per-run, and a backtest is not a reason to spend: the
|
|
cached payloads are real responses for real ASINs, and reading them exercises the same
|
|
`from_apify` adapter production uses. Pass `--scrape` to fetch fresh data instead, which does
|
|
cost money -- it is opt-in for that reason, never the default.
|
|
|
|
The counterfactuals exist because the real arm can only exercise the states our data happens
|
|
to be in today. They are labelled as counterfactual everywhere and are never presented as
|
|
observed fact.
|
|
|
|
python scripts/backtest_competitor_rules.py --skus SKU1,SKU2 --undercut 0.08
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
for p in (ROOT / "src", ROOT):
|
|
if str(p) not in sys.path:
|
|
sys.path.insert(0, str(p))
|
|
|
|
import dashboard.live_data as L # noqa: E402
|
|
from pricing_agent.competitive_state import CompetitiveState # noqa: E402
|
|
from pricing_agent.schemas import BuyBoxStatus # noqa: E402
|
|
|
|
NOW = datetime.now(timezone.utc)
|
|
|
|
# Defaults: the two ASINs with real cached competitive data, plus a slice of the duvet line
|
|
# the workbook tool was last run against.
|
|
DEFAULT_SKUS = [
|
|
"UBMICROFIBERGUSSETPILLOWWHITEQUEEN",
|
|
"UBCFKFITTEDSHEETWHITECALKING",
|
|
"UBMICROFIBERDUVETQUEENWHITE",
|
|
]
|
|
|
|
|
|
def capture_decide():
|
|
"""Wrap _decide so we keep the real arguments it was called with.
|
|
|
|
Signature-agnostic on purpose. This spy was pinned to `_decide`'s exact parameter list
|
|
and silently rotted when the cascade gained `cover_days` — the wrapper then raised
|
|
TypeError on the first SKU and the whole backtest was unrunnable. Capturing *args/**kw
|
|
and replaying them verbatim means the harness cannot drift from the function under test.
|
|
"""
|
|
calls: list[dict] = []
|
|
original = L._decide
|
|
|
|
def spy(*a, **kw):
|
|
calls.append({"args": a, "kwargs": dict(kw)})
|
|
return original(*a, **kw)
|
|
|
|
L._decide = spy
|
|
return calls, original
|
|
|
|
|
|
# Positional slots in `_decide(r, cur_price, hist, fp, scen, outlook, comp, cover_days)`.
|
|
_SLOTS = ("r", "cur_price", "hist", "fp", "scen", "outlook", "comp", "cover_days")
|
|
|
|
|
|
def bound(call: dict) -> dict:
|
|
"""A captured call as a name->value dict, whichever way the args were passed."""
|
|
out = dict(zip(_SLOTS, call["args"]))
|
|
out.update(call["kwargs"])
|
|
return out
|
|
|
|
|
|
def with_comp_match(call: dict, rival: float | None):
|
|
"""The captured scenario grid, plus the `comp_match` candidate build_live_sku would add.
|
|
|
|
Without this the counterfactual arms are untestable: the undercut branch requires a priced
|
|
`comp_match` candidate to move toward, and build_live_sku only adds one when competitor
|
|
state was usable AT PIPELINE TIME. Reusing the captured grid therefore holds the rule
|
|
permanently off and the arm would report "no change" for the wrong reason.
|
|
|
|
The row is computed the same way `_scenarios` computes every other row -- same elasticity,
|
|
same fee stack -- so the candidate is priced consistently with its neighbours rather than
|
|
pasted in.
|
|
"""
|
|
scen = call["scen"]
|
|
if not rival or rival <= 0:
|
|
return scen
|
|
cur = call["cur_price"]
|
|
row_cur = scen.set_index("scenario").loc["current"]
|
|
el = ((call["r"].elasticity or {}).get("elasticity")) or L.FALLBACK_ELASTICITY
|
|
units_day = float(row_cur["units_day"])
|
|
inventory = float(row_cur["cover_days"]) * max(units_day, 0.1)
|
|
units = units_day * (rival / cur) ** el if cur > 0 else units_day
|
|
th = L._take_home(rival, call["fp"])
|
|
extra = {
|
|
"scenario": "comp_match", "price": round(rival, 2), "units_day": round(units, 1),
|
|
"revenue_30d": round(units * 30 * rival), "profit_30d": round(units * 30 * th),
|
|
"margin_pct": round(th / rival * 100, 1) if rival else 0.0,
|
|
"cover_days": round(inventory / max(units, 0.1)),
|
|
}
|
|
import pandas as pd
|
|
|
|
# REPLACE, never append. build_live_sku already adds a `comp_match` row whenever competitor
|
|
# state was usable at pipeline time; concatenating a second one gave the frame a duplicate
|
|
# index, and `.loc["comp_match", "price"]` then returned a two-row Series instead of a
|
|
# price — which crashed the whole backtest on exactly the SKUs that had real competitor
|
|
# data, i.e. the ones it most needed to report on.
|
|
scen = scen[scen["scenario"] != "comp_match"]
|
|
return pd.concat([scen, pd.DataFrame([extra])], ignore_index=True)
|
|
|
|
|
|
def rerun(call: dict, comp, *, rival: float | None = None
|
|
) -> tuple[str, str, str, float, float]:
|
|
"""Re-run the captured input through the real cascade with a different comp state.
|
|
|
|
Returns the SHIPPABLE price too, i.e. after the same guardrails build_live_sku applies:
|
|
lifted to the safe floor, then capped at one MAX_STEP_PCT move. Reporting the raw target
|
|
would let a "match the rival at $8" recommendation look like it shipped when the floor
|
|
would in fact have stopped it.
|
|
"""
|
|
scen = with_comp_match(call, rival) if rival else call["scen"]
|
|
action, rec, _cons, _aggr, reasons, _root, objective = L._decide(
|
|
call["r"], call["cur_price"], call["hist"], call["fp"], scen,
|
|
call["outlook"], comp, call.get("cover_days"),
|
|
)
|
|
cur = call["cur_price"]
|
|
# `.loc[key]` on a duplicated index yields a Series, not a scalar. Take the first match
|
|
# explicitly so a malformed grid is a wrong number rather than a TypeError deep in a run.
|
|
hit = scen.loc[scen["scenario"] == rec, "price"]
|
|
if hit.empty:
|
|
raise KeyError(f"scenario {rec!r} missing from the grid: "
|
|
f"{sorted(scen['scenario'])}")
|
|
target = float(hit.iloc[0])
|
|
floor = L.price_floor(call["r"], cur).get("floor") or 0.0
|
|
if action == "Investigate":
|
|
# Investigate HOLDS — it proposes no price, so it has no target to test against the
|
|
# floor. `None` says that explicitly. (The live engine behaves the same way and
|
|
# reports the breach in words: "selling $X BELOW the floor — but price is not the
|
|
# lever here".) Scoring the held price as a "target below break-even" turned every
|
|
# already-underpriced SKU into a fake invariant violation.
|
|
return action, ",".join(reasons), objective, round(cur, 2), None
|
|
else:
|
|
floored = max(target, floor) if floor else target
|
|
shipped = floored
|
|
step = cur * L.MAX_STEP_PCT
|
|
if abs(shipped - cur) > step + 0.005:
|
|
shipped = cur + (step if shipped > cur else -step)
|
|
# `floored` is the TARGET after the floor is applied but BEFORE the step cap. That is the
|
|
# number the break-even invariant is about. `shipped` can legitimately sit under the floor
|
|
# when a SKU is already priced below it: the cap only permits a 5% move, so the engine
|
|
# walks up over several cycles and says so ("moving $X now; on the way to $Y"). Judging
|
|
# the invariant on `shipped` reports that intended climb as a violation.
|
|
return action, ",".join(reasons), objective, round(shipped, 2), round(floored, 2)
|
|
|
|
|
|
def state_from_disk(asin: str | None, our_price: float):
|
|
"""The real competitor state for this ASIN from the Apify cache — no network, no spend.
|
|
|
|
Uses the same `item_to_competitive` -> `from_apify` path production uses, so the state is
|
|
gated (age, unknown ownership) exactly as it would be live. Returns None when the cache
|
|
holds nothing for this ASIN, which is itself a real and common condition.
|
|
"""
|
|
import json
|
|
|
|
from config.settings import get_rules, get_settings
|
|
from pricing_agent.competitive_state import from_apify
|
|
from pricing_agent.tools.amazon.apify import item_to_competitive
|
|
|
|
s, rules = get_settings(), get_rules()
|
|
try:
|
|
raw = json.loads(Path(s.apify_cache_path).read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
entry = raw.get((asin or "").upper())
|
|
if not isinstance(entry, dict) or "item" not in entry:
|
|
return None
|
|
snap = item_to_competitive(entry["item"], asin=asin, our_price=our_price,
|
|
our_seller_id=s.apify_seller_id)
|
|
return from_apify(
|
|
snap, our_price=our_price,
|
|
as_of=datetime.fromtimestamp(float(entry["ts"]), tz=timezone.utc),
|
|
max_age_hours=rules.competitor_state_max_age_hours, now=NOW)
|
|
|
|
|
|
def cf(status: BuyBoxStatus, our_price: float, rival: float | None,
|
|
basis: str | None = None) -> CompetitiveState:
|
|
"""A COUNTERFACTUAL state — fresh by construction, so the age gate cannot mask the rule."""
|
|
from pricing_agent.competitive_state import BASIS_SAME_ASIN
|
|
|
|
return CompetitiveState(
|
|
status=status, our_price=our_price, buy_box_price=rival, competitor_min=rival,
|
|
competitor_median=rival, rivals=1 if rival else 0, source="counterfactual",
|
|
basis=basis or BASIS_SAME_ASIN,
|
|
as_of=NOW, reason="counterfactual injected by the verdict backtest",
|
|
)
|
|
|
|
|
|
@contextmanager
|
|
def old_policy():
|
|
"""Restore the cascade's PRE-CHANGE behaviour for the two policy fixes under test.
|
|
|
|
Both are re-expressed through the switches the change itself introduced, rather than by
|
|
keeping a second copy of the old code around — so the "before" arm cannot drift from what
|
|
the "after" arm actually turned off.
|
|
|
|
* sheet undercuts: `competitor_sheet_requires_corroboration` off -> any material
|
|
like-for-like undercut cuts, even on a healthy Buy-Box-winning SKU.
|
|
* ad spiral: the yield set narrowed back to BUYBOX_SUPPRESSED only -> AD_SPIRAL again
|
|
overrides NO_COST_DATA and LOW_STOCK.
|
|
"""
|
|
from config.settings import get_rules
|
|
|
|
rules = get_rules()
|
|
prev_corr = rules.competitor_sheet_requires_corroboration
|
|
prev_yield = L.AD_SPIRAL_YIELDS_TO
|
|
rules.competitor_sheet_requires_corroboration = False
|
|
L.AD_SPIRAL_YIELDS_TO = frozenset({"BUYBOX_SUPPRESSED"})
|
|
try:
|
|
yield
|
|
finally:
|
|
rules.competitor_sheet_requires_corroboration = prev_corr
|
|
L.AD_SPIRAL_YIELDS_TO = prev_yield
|
|
|
|
|
|
def spiral(call: dict, cover: int | None = None) -> dict:
|
|
"""The captured call with an unrecoverable ad curve (and optionally a cover override).
|
|
|
|
`ad_curve_unrecoverable` is a property of the SKU's own fitted ad model, so most real SKUs
|
|
do not have it and the AD_SPIRAL ordering could not otherwise be exercised on live data.
|
|
Everything else about the call stays real.
|
|
"""
|
|
r = call["r"].model_copy(update={"ad_curve_unrecoverable": True,
|
|
"contribution_per_dollar": 0.05,
|
|
"break_even_ad_curve": None})
|
|
out = dict(call, r=r)
|
|
if cover is not None:
|
|
out["cover_days"] = cover
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--skus", default=",".join(DEFAULT_SKUS))
|
|
ap.add_argument("--undercut", type=float, default=0.08,
|
|
help="counterfactual rival price, as a fraction below ours")
|
|
ap.add_argument("--scrape", action="store_true",
|
|
help="fetch FRESH competitor data (COSTS MONEY — the Apify actor is "
|
|
"pay-per-run). Off by default; the cached payloads on disk are real.")
|
|
args = ap.parse_args()
|
|
skus = tuple(s.strip() for s in args.skus.split(",") if s.strip())
|
|
|
|
calls, original = capture_decide()
|
|
try:
|
|
print(f"running the real pipeline for {len(skus)} SKU(s) — this fetches live COSMOS "
|
|
f"data and may take a minute each")
|
|
print("competitor data: " + ("FRESH SCRAPE (paid)" if args.scrape
|
|
else "read from data/apify_cache.json (free)") + "\n")
|
|
data = L.get_live_data(skus, with_competitive=args.scrape)
|
|
finally:
|
|
L._decide = original
|
|
|
|
details, errors = data.get("details") or {}, data.get("errors") or {}
|
|
for sku, err in errors.items():
|
|
print(f"!! {sku}: {err}")
|
|
|
|
# Pair each captured call with its SKU by current price — build_live_sku calls _decide
|
|
# exactly once per SKU.
|
|
by_price = {round(c["cur_price"], 4): c for c in map(bound, calls)}
|
|
|
|
rows = []
|
|
for sku in skus:
|
|
d = details.get(sku)
|
|
if not d:
|
|
continue
|
|
cur = round(float(d.get("current_price") or 0), 4)
|
|
call = by_price.get(cur)
|
|
if call is None:
|
|
print(f"!! {sku}: could not pair a captured cascade call (price {cur})")
|
|
continue
|
|
# With --scrape the pipeline already built a real state; otherwise read the real
|
|
# cached payload off disk. Either way the REAL arm is real Apify data.
|
|
comp = call["comp"]
|
|
if not args.scrape or comp is None or not comp.usable:
|
|
from_disk = state_from_disk(d.get("asin"), cur)
|
|
if from_disk is not None:
|
|
comp = from_disk
|
|
rival = round(cur * (1 - args.undercut), 2)
|
|
|
|
arms = {
|
|
"BLIND": rerun(call, None),
|
|
"REAL": rerun(call, comp),
|
|
"CF_UNDERCUT": rerun(call, cf(BuyBoxStatus.LOST_PRICE, cur, rival), rival=rival),
|
|
"CF_SUPPRESSED": rerun(call, cf(BuyBoxStatus.SUPPRESSED, cur, None)),
|
|
}
|
|
|
|
# ── the two POLICY changes under test, each old-vs-new on this SKU's real data ──
|
|
from pricing_agent.competitive_state import BASIS_SHEET
|
|
|
|
sheet_won = cf(BuyBoxStatus.WON, cur, rival, basis=BASIS_SHEET)
|
|
low_stock = spiral(call, cover=20)
|
|
cases = {
|
|
# A rival BRAND is `--undercut` cheaper while we hold our own Buy Box.
|
|
"SHEET_UNDERCUT_WE_WIN": (call, sheet_won, rival),
|
|
# An unrecoverable ad curve on a SKU that is about to run out of stock.
|
|
"AD_SPIRAL_LOW_STOCK": (low_stock, None, None),
|
|
# ...and with the COGS fields empty, which is what makes the curve untrustworthy.
|
|
"AD_SPIRAL_NO_COST": (dict(spiral(call), fp={**call["fp"], "cost": 0.0,
|
|
"fba": 0.0}), None, None),
|
|
}
|
|
policy = {}
|
|
for name, (c, state_, riv) in cases.items():
|
|
new = rerun(c, state_, rival=riv)
|
|
with old_policy():
|
|
old = rerun(c, state_, rival=riv)
|
|
policy[name] = (old, new)
|
|
# Carry THIS SKU's floor and counterfactual rival on the row. They used to be read
|
|
# inside the reporting loop from `call` / `rival`, which by then were whatever the
|
|
# last iteration of THIS loop left behind — so every SKU printed the final SKU's
|
|
# break-even. On the run that found this, one SKU's floor showed as $14.12 while its
|
|
# real floor was $22.80.
|
|
floor = L.price_floor(call["r"], cur).get("floor") or 0.0
|
|
rows.append((sku, cur, comp, arms, policy, floor, rival))
|
|
|
|
print("\n" + "=" * 100)
|
|
print("VERDICT BACKTEST — competitor rules blind vs live, on real SKUs")
|
|
print("=" * 100)
|
|
|
|
for sku, cur, comp, arms, _policy, floor, rival in rows:
|
|
print(f"\n{sku} current ${cur:.2f}")
|
|
st = (f"{comp.status.value} via {comp.source}" if comp is not None else "none")
|
|
usable = "usable" if (comp is not None and comp.usable) else (
|
|
f"NOT usable — {comp.unusable_because}" if comp is not None else "n/a")
|
|
print(f" real competitor state : {st}")
|
|
print(f" : {usable}")
|
|
if comp is not None and comp.competitor_min:
|
|
print(f" : cheapest rival ${comp.competitor_min:.2f} "
|
|
f"({comp.rivals} rival offer(s))")
|
|
print(f" break-even floor : ${floor:.2f}"
|
|
+ (" — ALREADY PRICED BELOW IT; the 5% cap means several cycles to climb out"
|
|
if floor and cur < floor - 0.005 else "")
|
|
+ f" (counterfactual rival ${rival:.2f})")
|
|
for arm, (action, reasons, obj, shipped, target) in arms.items():
|
|
moved = "" if arms[arm] == arms["BLIND"] else " <<< CHANGED"
|
|
tag = " (counterfactual)" if arm.startswith("CF_") else ""
|
|
flag = ""
|
|
if floor and target is not None and target < floor - 0.005:
|
|
flag = " *** TARGETS BELOW BREAK-EVEN — INVARIANT VIOLATED ***"
|
|
elif floor and shipped < floor - 0.005:
|
|
flag = (" (holding below the floor — price is not the lever)"
|
|
if target is None else
|
|
" (step-capped, still climbing to the floor)")
|
|
print(f" {arm:14} {action:12} ${shipped:>7.2f} {reasons:42} "
|
|
f"{obj}{moved}{tag}{flag}")
|
|
|
|
# ── POLICY DELTA: what the two cascade changes do, old vs new, on these SKUs ──
|
|
print("\n" + "=" * 100)
|
|
print("POLICY DELTA — cascade BEFORE vs AFTER the two changes")
|
|
print("=" * 100)
|
|
CASE_NOTE = {
|
|
"SHEET_UNDERCUT_WE_WIN":
|
|
f"a rival BRAND is {args.undercut:.0%} cheaper, we HOLD our own Buy Box, and this "
|
|
f"SKU's real demand decides whether that is corroborated",
|
|
"AD_SPIRAL_LOW_STOCK":
|
|
"unrecoverable ad curve on a SKU with 20 days of cover (counterfactual curve)",
|
|
"AD_SPIRAL_NO_COST":
|
|
"unrecoverable ad curve with costPerUnit/fbaFee empty (counterfactual curve)",
|
|
}
|
|
deltas = {k: [] for k in CASE_NOTE}
|
|
for sku, cur, _comp, _arms, policy, _floor, _rival in rows:
|
|
print(f"\n{sku} current ${cur:.2f}")
|
|
for case, (old, new) in policy.items():
|
|
changed = old != new
|
|
if changed:
|
|
deltas[case].append(sku)
|
|
print(f" {case}")
|
|
print(f" {CASE_NOTE[case]}")
|
|
print(f" before {old[0]:12} ${old[3]:>7.2f} {old[1]}")
|
|
print(f" after {new[0]:12} ${new[3]:>7.2f} {new[1]}"
|
|
+ (" <<< CHANGED" if changed else " (unchanged)"))
|
|
|
|
print("\n" + "-" * 100)
|
|
for case, skus_changed in deltas.items():
|
|
print(f"{case:24} verdict changed on {len(skus_changed)}/{len(rows)} "
|
|
f"{skus_changed}")
|
|
|
|
print("\n" + "=" * 100)
|
|
real_changed = [r[0] for r in rows if r[3]["REAL"] != r[3]["BLIND"]]
|
|
cf_u_changed = [r[0] for r in rows if r[3]["CF_UNDERCUT"] != r[3]["BLIND"]]
|
|
cf_s_changed = [r[0] for r in rows if r[3]["CF_SUPPRESSED"] != r[3]["BLIND"]]
|
|
print(f"SKUs analysed : {len(rows)}")
|
|
print(f"Verdicts changed by REAL competitor data : {len(real_changed)} "
|
|
f"{real_changed}")
|
|
print(f"Verdicts that WOULD change on a {args.undercut:.0%} undercut : "
|
|
f"{len(cf_u_changed)} {cf_u_changed}")
|
|
print(f"Verdicts that WOULD change if suppressed : {len(cf_s_changed)} "
|
|
f"{cf_s_changed}")
|
|
# The invariant that matters most: no arm, however cheap the counterfactual rival, may
|
|
# ship a price under break-even.
|
|
# Measured on the TARGET, not the step-capped first move. A SKU already selling under its
|
|
# own ad-inclusive floor cannot be lifted over it in one 5% step, and reporting that
|
|
# deliberate multi-cycle climb as a breach buries any real one. Both are counted, and the
|
|
# climb is reported separately so it stays visible.
|
|
violations, climbing = [], []
|
|
for sku, cur, _comp, arms, policy, floor, _rival in rows:
|
|
if not floor:
|
|
continue
|
|
checks = dict(arms)
|
|
# The policy arms ship prices too, so they are held to the same invariant.
|
|
for case, (old, new) in policy.items():
|
|
checks[f"{case}/before"], checks[f"{case}/after"] = old, new
|
|
for arm, (_action, _reasons, _obj, shipped, target) in checks.items():
|
|
if target is not None and target < floor - 0.005:
|
|
violations.append(f"{sku}/{arm} targets ${target:.2f} < ${floor:.2f}")
|
|
elif shipped < floor - 0.005:
|
|
climbing.append(f"{sku}/{arm} ${shipped:.2f} -> ${floor:.2f}")
|
|
print(f"Targets BELOW break-even (real violations) : {len(violations)} {violations}")
|
|
print(f"Below floor but stepping UP toward it (by design): {len(set(climbing))} "
|
|
f"{sorted({c.split('/')[0] for c in climbing})}")
|
|
print("\nA REAL delta of 0 is the fail-safe working, not the feature missing: with no "
|
|
"usable\ncompetitor state every verdict is byte-identical to the competitor-blind "
|
|
"one.\nThe counterfactual arms show the rules do fire once state IS usable.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|