"""Verdict backtest: what do the competitor rules actually change, on real SKUs? Same shape as the storage-fix backtest (61.9% -> 45.2%), but the quantity under test is not an error percentage -- it is the VERDICT. So this reports, per SKU, the action and the reason code with the competitor rules blind vs. live, and names every SKU that moved and why. Method ------ The real pipeline runs ONCE per SKU. `dashboard.live_data._decide` is wrapped so the exact arguments it was handed in production -- the real AnalysisResult, the real 180-day history, the real fee stack, the real scenario grid, the real inventory outlook -- are captured. Every arm below then re-runs that same captured input through the same cascade, varying ONLY the competitor state. Nothing is reconstructed by hand, so no arm can drift from what the live dashboard would actually compute. Four arms: BLIND comp=None -- the behaviour before this change REAL the actual competitor state -- what production would do right now CF_UNDERCUT counterfactual: a rival holds the Buy Box `--undercut` below us CF_SUPPRESSED counterfactual: our Buy Box is suppressed The REAL arm is built from the Apify data ALREADY ON DISK (`data/apify_cache.json`) rather than by scraping. The actor is pay-per-run, and a backtest is not a reason to spend: the cached payloads are real responses for real ASINs, and reading them exercises the same `from_apify` adapter production uses. Pass `--scrape` to fetch fresh data instead, which does cost money -- it is opt-in for that reason, never the default. The counterfactuals exist because the real arm can only exercise the states our data happens to be in today. They are labelled as counterfactual everywhere and are never presented as observed fact. python scripts/backtest_competitor_rules.py --skus SKU1,SKU2 --undercut 0.08 """ from __future__ import annotations import argparse import sys from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] for p in (ROOT / "src", ROOT): if str(p) not in sys.path: sys.path.insert(0, str(p)) import dashboard.live_data as L # noqa: E402 from pricing_agent.competitive_state import CompetitiveState # noqa: E402 from pricing_agent.schemas import BuyBoxStatus # noqa: E402 NOW = datetime.now(timezone.utc) # Defaults: the two ASINs with real cached competitive data, plus a slice of the duvet line # the workbook tool was last run against. DEFAULT_SKUS = [ "UBMICROFIBERGUSSETPILLOWWHITEQUEEN", "UBCFKFITTEDSHEETWHITECALKING", "UBMICROFIBERDUVETQUEENWHITE", ] def capture_decide(): """Wrap _decide so we keep the real arguments it was called with. Signature-agnostic on purpose. This spy was pinned to `_decide`'s exact parameter list and silently rotted when the cascade gained `cover_days` — the wrapper then raised TypeError on the first SKU and the whole backtest was unrunnable. Capturing *args/**kw and replaying them verbatim means the harness cannot drift from the function under test. """ calls: list[dict] = [] original = L._decide def spy(*a, **kw): calls.append({"args": a, "kwargs": dict(kw)}) return original(*a, **kw) L._decide = spy return calls, original # Positional slots in `_decide(r, cur_price, hist, fp, scen, outlook, comp, cover_days)`. _SLOTS = ("r", "cur_price", "hist", "fp", "scen", "outlook", "comp", "cover_days") def bound(call: dict) -> dict: """A captured call as a name->value dict, whichever way the args were passed.""" out = dict(zip(_SLOTS, call["args"])) out.update(call["kwargs"]) return out def with_comp_match(call: dict, rival: float | None): """The captured scenario grid, plus the `comp_match` candidate build_live_sku would add. Without this the counterfactual arms are untestable: the undercut branch requires a priced `comp_match` candidate to move toward, and build_live_sku only adds one when competitor state was usable AT PIPELINE TIME. Reusing the captured grid therefore holds the rule permanently off and the arm would report "no change" for the wrong reason. The row is computed the same way `_scenarios` computes every other row -- same elasticity, same fee stack -- so the candidate is priced consistently with its neighbours rather than pasted in. """ scen = call["scen"] if not rival or rival <= 0: return scen cur = call["cur_price"] row_cur = scen.set_index("scenario").loc["current"] el = ((call["r"].elasticity or {}).get("elasticity")) or L.FALLBACK_ELASTICITY units_day = float(row_cur["units_day"]) inventory = float(row_cur["cover_days"]) * max(units_day, 0.1) units = units_day * (rival / cur) ** el if cur > 0 else units_day th = L._take_home(rival, call["fp"]) extra = { "scenario": "comp_match", "price": round(rival, 2), "units_day": round(units, 1), "revenue_30d": round(units * 30 * rival), "profit_30d": round(units * 30 * th), "margin_pct": round(th / rival * 100, 1) if rival else 0.0, "cover_days": round(inventory / max(units, 0.1)), } import pandas as pd # REPLACE, never append. build_live_sku already adds a `comp_match` row whenever competitor # state was usable at pipeline time; concatenating a second one gave the frame a duplicate # index, and `.loc["comp_match", "price"]` then returned a two-row Series instead of a # price — which crashed the whole backtest on exactly the SKUs that had real competitor # data, i.e. the ones it most needed to report on. scen = scen[scen["scenario"] != "comp_match"] return pd.concat([scen, pd.DataFrame([extra])], ignore_index=True) def rerun(call: dict, comp, *, rival: float | None = None ) -> tuple[str, str, str, float, float]: """Re-run the captured input through the real cascade with a different comp state. Returns the SHIPPABLE price too, i.e. after the same guardrails build_live_sku applies: lifted to the safe floor, then capped at one MAX_STEP_PCT move. Reporting the raw target would let a "match the rival at $8" recommendation look like it shipped when the floor would in fact have stopped it. """ scen = with_comp_match(call, rival) if rival else call["scen"] action, rec, _cons, _aggr, reasons, _root, objective = L._decide( call["r"], call["cur_price"], call["hist"], call["fp"], scen, call["outlook"], comp, call.get("cover_days"), ) cur = call["cur_price"] # `.loc[key]` on a duplicated index yields a Series, not a scalar. Take the first match # explicitly so a malformed grid is a wrong number rather than a TypeError deep in a run. hit = scen.loc[scen["scenario"] == rec, "price"] if hit.empty: raise KeyError(f"scenario {rec!r} missing from the grid: " f"{sorted(scen['scenario'])}") target = float(hit.iloc[0]) floor = L.price_floor(call["r"], cur).get("floor") or 0.0 if action == "Investigate": # Investigate HOLDS — it proposes no price, so it has no target to test against the # floor. `None` says that explicitly. (The live engine behaves the same way and # reports the breach in words: "selling $X BELOW the floor — but price is not the # lever here".) Scoring the held price as a "target below break-even" turned every # already-underpriced SKU into a fake invariant violation. return action, ",".join(reasons), objective, round(cur, 2), None else: floored = max(target, floor) if floor else target shipped = floored step = cur * L.MAX_STEP_PCT if abs(shipped - cur) > step + 0.005: shipped = cur + (step if shipped > cur else -step) # `floored` is the TARGET after the floor is applied but BEFORE the step cap. That is the # number the break-even invariant is about. `shipped` can legitimately sit under the floor # when a SKU is already priced below it: the cap only permits a 5% move, so the engine # walks up over several cycles and says so ("moving $X now; on the way to $Y"). Judging # the invariant on `shipped` reports that intended climb as a violation. return action, ",".join(reasons), objective, round(shipped, 2), round(floored, 2) def state_from_disk(asin: str | None, our_price: float): """The real competitor state for this ASIN from the Apify cache — no network, no spend. Uses the same `item_to_competitive` -> `from_apify` path production uses, so the state is gated (age, unknown ownership) exactly as it would be live. Returns None when the cache holds nothing for this ASIN, which is itself a real and common condition. """ import json from config.settings import get_rules, get_settings from pricing_agent.competitive_state import from_apify from pricing_agent.tools.amazon.apify import item_to_competitive s, rules = get_settings(), get_rules() try: raw = json.loads(Path(s.apify_cache_path).read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None entry = raw.get((asin or "").upper()) if not isinstance(entry, dict) or "item" not in entry: return None snap = item_to_competitive(entry["item"], asin=asin, our_price=our_price, our_seller_id=s.apify_seller_id) return from_apify( snap, our_price=our_price, as_of=datetime.fromtimestamp(float(entry["ts"]), tz=timezone.utc), max_age_hours=rules.competitor_state_max_age_hours, now=NOW) def cf(status: BuyBoxStatus, our_price: float, rival: float | None, basis: str | None = None) -> CompetitiveState: """A COUNTERFACTUAL state — fresh by construction, so the age gate cannot mask the rule.""" from pricing_agent.competitive_state import BASIS_SAME_ASIN return CompetitiveState( status=status, our_price=our_price, buy_box_price=rival, competitor_min=rival, competitor_median=rival, rivals=1 if rival else 0, source="counterfactual", basis=basis or BASIS_SAME_ASIN, as_of=NOW, reason="counterfactual injected by the verdict backtest", ) @contextmanager def old_policy(): """Restore the cascade's PRE-CHANGE behaviour for the two policy fixes under test. Both are re-expressed through the switches the change itself introduced, rather than by keeping a second copy of the old code around — so the "before" arm cannot drift from what the "after" arm actually turned off. * sheet undercuts: `competitor_sheet_requires_corroboration` off -> any material like-for-like undercut cuts, even on a healthy Buy-Box-winning SKU. * ad spiral: the yield set narrowed back to BUYBOX_SUPPRESSED only -> AD_SPIRAL again overrides NO_COST_DATA and LOW_STOCK. """ from config.settings import get_rules rules = get_rules() prev_corr = rules.competitor_sheet_requires_corroboration prev_yield = L.AD_SPIRAL_YIELDS_TO rules.competitor_sheet_requires_corroboration = False L.AD_SPIRAL_YIELDS_TO = frozenset({"BUYBOX_SUPPRESSED"}) try: yield finally: rules.competitor_sheet_requires_corroboration = prev_corr L.AD_SPIRAL_YIELDS_TO = prev_yield def spiral(call: dict, cover: int | None = None) -> dict: """The captured call with an unrecoverable ad curve (and optionally a cover override). `ad_curve_unrecoverable` is a property of the SKU's own fitted ad model, so most real SKUs do not have it and the AD_SPIRAL ordering could not otherwise be exercised on live data. Everything else about the call stays real. """ r = call["r"].model_copy(update={"ad_curve_unrecoverable": True, "contribution_per_dollar": 0.05, "break_even_ad_curve": None}) out = dict(call, r=r) if cover is not None: out["cover_days"] = cover return out def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--skus", default=",".join(DEFAULT_SKUS)) ap.add_argument("--undercut", type=float, default=0.08, help="counterfactual rival price, as a fraction below ours") ap.add_argument("--scrape", action="store_true", help="fetch FRESH competitor data (COSTS MONEY — the Apify actor is " "pay-per-run). Off by default; the cached payloads on disk are real.") args = ap.parse_args() skus = tuple(s.strip() for s in args.skus.split(",") if s.strip()) calls, original = capture_decide() try: print(f"running the real pipeline for {len(skus)} SKU(s) — this fetches live COSMOS " f"data and may take a minute each") print("competitor data: " + ("FRESH SCRAPE (paid)" if args.scrape else "read from data/apify_cache.json (free)") + "\n") data = L.get_live_data(skus, with_competitive=args.scrape) finally: L._decide = original details, errors = data.get("details") or {}, data.get("errors") or {} for sku, err in errors.items(): print(f"!! {sku}: {err}") # Pair each captured call with its SKU by current price — build_live_sku calls _decide # exactly once per SKU. by_price = {round(c["cur_price"], 4): c for c in map(bound, calls)} rows = [] for sku in skus: d = details.get(sku) if not d: continue cur = round(float(d.get("current_price") or 0), 4) call = by_price.get(cur) if call is None: print(f"!! {sku}: could not pair a captured cascade call (price {cur})") continue # With --scrape the pipeline already built a real state; otherwise read the real # cached payload off disk. Either way the REAL arm is real Apify data. comp = call["comp"] if not args.scrape or comp is None or not comp.usable: from_disk = state_from_disk(d.get("asin"), cur) if from_disk is not None: comp = from_disk rival = round(cur * (1 - args.undercut), 2) arms = { "BLIND": rerun(call, None), "REAL": rerun(call, comp), "CF_UNDERCUT": rerun(call, cf(BuyBoxStatus.LOST_PRICE, cur, rival), rival=rival), "CF_SUPPRESSED": rerun(call, cf(BuyBoxStatus.SUPPRESSED, cur, None)), } # ── the two POLICY changes under test, each old-vs-new on this SKU's real data ── from pricing_agent.competitive_state import BASIS_SHEET sheet_won = cf(BuyBoxStatus.WON, cur, rival, basis=BASIS_SHEET) low_stock = spiral(call, cover=20) cases = { # A rival BRAND is `--undercut` cheaper while we hold our own Buy Box. "SHEET_UNDERCUT_WE_WIN": (call, sheet_won, rival), # An unrecoverable ad curve on a SKU that is about to run out of stock. "AD_SPIRAL_LOW_STOCK": (low_stock, None, None), # ...and with the COGS fields empty, which is what makes the curve untrustworthy. "AD_SPIRAL_NO_COST": (dict(spiral(call), fp={**call["fp"], "cost": 0.0, "fba": 0.0}), None, None), } policy = {} for name, (c, state_, riv) in cases.items(): new = rerun(c, state_, rival=riv) with old_policy(): old = rerun(c, state_, rival=riv) policy[name] = (old, new) # Carry THIS SKU's floor and counterfactual rival on the row. They used to be read # inside the reporting loop from `call` / `rival`, which by then were whatever the # last iteration of THIS loop left behind — so every SKU printed the final SKU's # break-even. On the run that found this, one SKU's floor showed as $14.12 while its # real floor was $22.80. floor = L.price_floor(call["r"], cur).get("floor") or 0.0 rows.append((sku, cur, comp, arms, policy, floor, rival)) print("\n" + "=" * 100) print("VERDICT BACKTEST — competitor rules blind vs live, on real SKUs") print("=" * 100) for sku, cur, comp, arms, _policy, floor, rival in rows: print(f"\n{sku} current ${cur:.2f}") st = (f"{comp.status.value} via {comp.source}" if comp is not None else "none") usable = "usable" if (comp is not None and comp.usable) else ( f"NOT usable — {comp.unusable_because}" if comp is not None else "n/a") print(f" real competitor state : {st}") print(f" : {usable}") if comp is not None and comp.competitor_min: print(f" : cheapest rival ${comp.competitor_min:.2f} " f"({comp.rivals} rival offer(s))") print(f" break-even floor : ${floor:.2f}" + (" — ALREADY PRICED BELOW IT; the 5% cap means several cycles to climb out" if floor and cur < floor - 0.005 else "") + f" (counterfactual rival ${rival:.2f})") for arm, (action, reasons, obj, shipped, target) in arms.items(): moved = "" if arms[arm] == arms["BLIND"] else " <<< CHANGED" tag = " (counterfactual)" if arm.startswith("CF_") else "" flag = "" if floor and target is not None and target < floor - 0.005: flag = " *** TARGETS BELOW BREAK-EVEN — INVARIANT VIOLATED ***" elif floor and shipped < floor - 0.005: flag = (" (holding below the floor — price is not the lever)" if target is None else " (step-capped, still climbing to the floor)") print(f" {arm:14} {action:12} ${shipped:>7.2f} {reasons:42} " f"{obj}{moved}{tag}{flag}") # ── POLICY DELTA: what the two cascade changes do, old vs new, on these SKUs ── print("\n" + "=" * 100) print("POLICY DELTA — cascade BEFORE vs AFTER the two changes") print("=" * 100) CASE_NOTE = { "SHEET_UNDERCUT_WE_WIN": f"a rival BRAND is {args.undercut:.0%} cheaper, we HOLD our own Buy Box, and this " f"SKU's real demand decides whether that is corroborated", "AD_SPIRAL_LOW_STOCK": "unrecoverable ad curve on a SKU with 20 days of cover (counterfactual curve)", "AD_SPIRAL_NO_COST": "unrecoverable ad curve with costPerUnit/fbaFee empty (counterfactual curve)", } deltas = {k: [] for k in CASE_NOTE} for sku, cur, _comp, _arms, policy, _floor, _rival in rows: print(f"\n{sku} current ${cur:.2f}") for case, (old, new) in policy.items(): changed = old != new if changed: deltas[case].append(sku) print(f" {case}") print(f" {CASE_NOTE[case]}") print(f" before {old[0]:12} ${old[3]:>7.2f} {old[1]}") print(f" after {new[0]:12} ${new[3]:>7.2f} {new[1]}" + (" <<< CHANGED" if changed else " (unchanged)")) print("\n" + "-" * 100) for case, skus_changed in deltas.items(): print(f"{case:24} verdict changed on {len(skus_changed)}/{len(rows)} " f"{skus_changed}") print("\n" + "=" * 100) real_changed = [r[0] for r in rows if r[3]["REAL"] != r[3]["BLIND"]] cf_u_changed = [r[0] for r in rows if r[3]["CF_UNDERCUT"] != r[3]["BLIND"]] cf_s_changed = [r[0] for r in rows if r[3]["CF_SUPPRESSED"] != r[3]["BLIND"]] print(f"SKUs analysed : {len(rows)}") print(f"Verdicts changed by REAL competitor data : {len(real_changed)} " f"{real_changed}") print(f"Verdicts that WOULD change on a {args.undercut:.0%} undercut : " f"{len(cf_u_changed)} {cf_u_changed}") print(f"Verdicts that WOULD change if suppressed : {len(cf_s_changed)} " f"{cf_s_changed}") # The invariant that matters most: no arm, however cheap the counterfactual rival, may # ship a price under break-even. # Measured on the TARGET, not the step-capped first move. A SKU already selling under its # own ad-inclusive floor cannot be lifted over it in one 5% step, and reporting that # deliberate multi-cycle climb as a breach buries any real one. Both are counted, and the # climb is reported separately so it stays visible. violations, climbing = [], [] for sku, cur, _comp, arms, policy, floor, _rival in rows: if not floor: continue checks = dict(arms) # The policy arms ship prices too, so they are held to the same invariant. for case, (old, new) in policy.items(): checks[f"{case}/before"], checks[f"{case}/after"] = old, new for arm, (_action, _reasons, _obj, shipped, target) in checks.items(): if target is not None and target < floor - 0.005: violations.append(f"{sku}/{arm} targets ${target:.2f} < ${floor:.2f}") elif shipped < floor - 0.005: climbing.append(f"{sku}/{arm} ${shipped:.2f} -> ${floor:.2f}") print(f"Targets BELOW break-even (real violations) : {len(violations)} {violations}") print(f"Below floor but stepping UP toward it (by design): {len(set(climbing))} " f"{sorted({c.split('/')[0] for c in climbing})}") print("\nA REAL delta of 0 is the fail-safe working, not the feature missing: with no " "usable\ncompetitor state every verdict is byte-identical to the competitor-blind " "one.\nThe counterfactual arms show the rules do fire once state IS usable.") return 0 if __name__ == "__main__": raise SystemExit(main())