""" The two readers must be interchangeable. `AR_USE_CALAMINE` selects between the Rust-backed CalamineReader (default) and the pure-Python streaming TransactionReader (low-memory fallback). An auditor re-performing a close must get the same number either way, so any behavioural difference between them is a defect — not a performance trade-off. Two real divergences lived here before this file existed: * sheet selection tie-break — score-only/first-wins vs (score, height)/tallest-wins, so the two readers could pick DIFFERENT worksheets out of the same workbook; * row emptiness — one tested the CONVERTED value (empty amount cells become 0.0, so every row qualified) while the other tested the source cell, so they emitted different row sets. """ from __future__ import annotations import os import tempfile import pytest from app.core.calamine_reader import CalamineReader from app.core.xlsx_reader import TransactionReader from .test_excel_export import make_amazon_xlsx from .test_per_market import _make_dutch_file _TMP = tempfile.mkdtemp() def _make_blank_row_file(path: str) -> None: """ Real data rows with empty rows INTERLEAVED between them. This is the exact shape the two readers disagreed on. CalamineReader tested the CONVERTED value, and every amount field converts an empty cell to 0.0, so a row of empty strings still qualified — it emitted a phantom row 11 that TransactionReader dropped. Row counts, and the duplicate count (identical blank rows hash alike), therefore depended on which reader ran. The blanks must be INTERLEAVED, not trailing: calamine trims trailing empty rows itself, so a trailing-blanks fixture passes even against the unfixed reader. """ from openpyxl import Workbook wb = Workbook() ws = wb.active ws.title = "USA Amazon Transactions" for i in range(7): ws.append([f"preamble {i}"]) ws.append(["date/time", "settlement id", "type", "order id", "sku", "account type", "marketplace", "product sales", "total"]) def row(i): return [f"Jan {i + 1}, 2026 1:00:00 PM PST", 500, "Order", f"o{i}", "sku", "Standard Orders", "amazon.com", 10.0, 10.0] ws.append(row(0)) ws.append([None] * 9) # openpyxl writes no cells for these ws.append([""] * 9) # …but empty strings ARE materialized — the divergent case ws.append(row(1)) ws.append(row(2)) wb.save(path) def _fixtures() -> list[str]: usa = os.path.join(_TMP, "USA equivalence.xlsx") nl = os.path.join(_TMP, "Netherlands Amazon Transactions January, 2026.xlsx") blanks = os.path.join(_TMP, "USA blank rows.xlsx") make_amazon_xlsx(usa, order_rows=10) _make_dutch_file(nl) _make_blank_row_file(blanks) return [usa, nl, blanks] def _read(reader) -> dict: """The FINANCIAL output — everything that could change a reported number.""" reader.detect() rows = list(reader.iter_records()) m = reader.file_meta out = { "sheet": m.data_sheet, "header_row": m.header_row, "rows": len(rows), "sum_total": round(sum(float(r.get("total") or 0.0) for r in rows), 6), "helper_rows_skipped": m.helper_rows_skipped, "mapped_fields": sorted(reader.column_mapping.field_to_col), "source_rows": [r.get("_source_row") for r in rows], } reader.close() return out @pytest.mark.parametrize("path", _fixtures()) def test_readers_produce_identical_output(path): cal = _read(CalamineReader(path)) itp = _read(TransactionReader(path)) assert cal == itp, ( f"{os.path.basename(path)}: the two readers disagree.\n" f" calamine : {cal}\n iterparse: {itp}" ) @pytest.mark.parametrize("path", _fixtures()) def test_each_reader_accounts_for_every_row_it_declares(path): """ Control C1's invariant, checked per reader. The declared extent itself is deliberately NOT compared across readers: calamine trims trailing blank rows from `total_height` while `` counts them, so on the trailing-blanks fixture one sees 11 rows and the other 15. Both are right, and each stays internally consistent — which is all C1 needs. """ for reader in (CalamineReader(path), TransactionReader(path)): reader.detect() list(reader.iter_records()) m = reader.file_meta assert m.sheet_last_row > 0, f"{type(reader).__name__} declared no extent" assert m.rows_accounted_for == m.expected_data_rows, ( f"{os.path.basename(path)} via {type(reader).__name__}: " f"declared {m.expected_data_rows}, accounted for {m.rows_accounted_for}" ) reader.close() def test_blank_rows_are_dropped_by_both_readers(): """ Guards the specific divergence: 3 real rows with 2 blanks between them must read as exactly 3 rows, from source rows 9/12/13, under either reader. Verified non-vacuous — with the pre-fix `has_value` logic CalamineReader returns 4 rows (source rows 9/11/12/13) and this fails. """ path = _fixtures()[2] for R in (CalamineReader, TransactionReader): r = R(path) r.detect() rows = list(r.iter_records()) assert [x["_source_row"] for x in rows] == [9, 12, 13], ( f"{R.__name__} emitted rows {[x['_source_row'] for x in rows]}, expected [9, 12, 13]" ) assert round(sum(float(x.get("total") or 0.0) for x in rows), 2) == 30.0 r.close()