143 lines
5.4 KiB
Python
143 lines
5.4 KiB
Python
"""
|
|
The two readers must be interchangeable.
|
|
|
|
`AR_USE_CALAMINE` selects between the Rust-backed CalamineReader (default) and the
|
|
pure-Python streaming TransactionReader (low-memory fallback). An auditor re-performing a
|
|
close must get the same number either way, so any behavioural difference between them is a
|
|
defect — not a performance trade-off.
|
|
|
|
Two real divergences lived here before this file existed:
|
|
* sheet selection tie-break — score-only/first-wins vs (score, height)/tallest-wins, so the
|
|
two readers could pick DIFFERENT worksheets out of the same workbook;
|
|
* row emptiness — one tested the CONVERTED value (empty amount cells become 0.0, so every
|
|
row qualified) while the other tested the source cell, so they emitted different row sets.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
|
|
import pytest
|
|
|
|
from app.core.calamine_reader import CalamineReader
|
|
from app.core.xlsx_reader import TransactionReader
|
|
|
|
from .test_excel_export import make_amazon_xlsx
|
|
from .test_per_market import _make_dutch_file
|
|
|
|
_TMP = tempfile.mkdtemp()
|
|
|
|
|
|
def _make_blank_row_file(path: str) -> None:
|
|
"""
|
|
Real data rows with empty rows INTERLEAVED between them.
|
|
|
|
This is the exact shape the two readers disagreed on. CalamineReader tested the CONVERTED
|
|
value, and every amount field converts an empty cell to 0.0, so a row of empty strings
|
|
still qualified — it emitted a phantom row 11 that TransactionReader dropped. Row counts,
|
|
and the duplicate count (identical blank rows hash alike), therefore depended on which
|
|
reader ran.
|
|
|
|
The blanks must be INTERLEAVED, not trailing: calamine trims trailing empty rows itself,
|
|
so a trailing-blanks fixture passes even against the unfixed reader.
|
|
"""
|
|
from openpyxl import Workbook
|
|
wb = Workbook()
|
|
ws = wb.active
|
|
ws.title = "USA Amazon Transactions"
|
|
for i in range(7):
|
|
ws.append([f"preamble {i}"])
|
|
ws.append(["date/time", "settlement id", "type", "order id", "sku", "account type",
|
|
"marketplace", "product sales", "total"])
|
|
|
|
def row(i):
|
|
return [f"Jan {i + 1}, 2026 1:00:00 PM PST", 500, "Order", f"o{i}", "sku",
|
|
"Standard Orders", "amazon.com", 10.0, 10.0]
|
|
|
|
ws.append(row(0))
|
|
ws.append([None] * 9) # openpyxl writes no cells for these
|
|
ws.append([""] * 9) # …but empty strings ARE materialized — the divergent case
|
|
ws.append(row(1))
|
|
ws.append(row(2))
|
|
wb.save(path)
|
|
|
|
|
|
def _fixtures() -> list[str]:
|
|
usa = os.path.join(_TMP, "USA equivalence.xlsx")
|
|
nl = os.path.join(_TMP, "Netherlands Amazon Transactions January, 2026.xlsx")
|
|
blanks = os.path.join(_TMP, "USA blank rows.xlsx")
|
|
make_amazon_xlsx(usa, order_rows=10)
|
|
_make_dutch_file(nl)
|
|
_make_blank_row_file(blanks)
|
|
return [usa, nl, blanks]
|
|
|
|
|
|
def _read(reader) -> dict:
|
|
"""The FINANCIAL output — everything that could change a reported number."""
|
|
reader.detect()
|
|
rows = list(reader.iter_records())
|
|
m = reader.file_meta
|
|
out = {
|
|
"sheet": m.data_sheet,
|
|
"header_row": m.header_row,
|
|
"rows": len(rows),
|
|
"sum_total": round(sum(float(r.get("total") or 0.0) for r in rows), 6),
|
|
"helper_rows_skipped": m.helper_rows_skipped,
|
|
"mapped_fields": sorted(reader.column_mapping.field_to_col),
|
|
"source_rows": [r.get("_source_row") for r in rows],
|
|
}
|
|
reader.close()
|
|
return out
|
|
|
|
|
|
@pytest.mark.parametrize("path", _fixtures())
|
|
def test_readers_produce_identical_output(path):
|
|
cal = _read(CalamineReader(path))
|
|
itp = _read(TransactionReader(path))
|
|
assert cal == itp, (
|
|
f"{os.path.basename(path)}: the two readers disagree.\n"
|
|
f" calamine : {cal}\n iterparse: {itp}"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("path", _fixtures())
|
|
def test_each_reader_accounts_for_every_row_it_declares(path):
|
|
"""
|
|
Control C1's invariant, checked per reader.
|
|
|
|
The declared extent itself is deliberately NOT compared across readers: calamine trims
|
|
trailing blank rows from `total_height` while `<dimension>` counts them, so on the
|
|
trailing-blanks fixture one sees 11 rows and the other 15. Both are right, and each stays
|
|
internally consistent — which is all C1 needs.
|
|
"""
|
|
for reader in (CalamineReader(path), TransactionReader(path)):
|
|
reader.detect()
|
|
list(reader.iter_records())
|
|
m = reader.file_meta
|
|
assert m.sheet_last_row > 0, f"{type(reader).__name__} declared no extent"
|
|
assert m.rows_accounted_for == m.expected_data_rows, (
|
|
f"{os.path.basename(path)} via {type(reader).__name__}: "
|
|
f"declared {m.expected_data_rows}, accounted for {m.rows_accounted_for}"
|
|
)
|
|
reader.close()
|
|
|
|
|
|
def test_blank_rows_are_dropped_by_both_readers():
|
|
"""
|
|
Guards the specific divergence: 3 real rows with 2 blanks between them must read as
|
|
exactly 3 rows, from source rows 9/12/13, under either reader.
|
|
|
|
Verified non-vacuous — with the pre-fix `has_value` logic CalamineReader returns 4 rows
|
|
(source rows 9/11/12/13) and this fails.
|
|
"""
|
|
path = _fixtures()[2]
|
|
for R in (CalamineReader, TransactionReader):
|
|
r = R(path)
|
|
r.detect()
|
|
rows = list(r.iter_records())
|
|
assert [x["_source_row"] for x in rows] == [9, 12, 13], (
|
|
f"{R.__name__} emitted rows {[x['_source_row'] for x in rows]}, expected [9, 12, 13]"
|
|
)
|
|
assert round(sum(float(x.get("total") or 0.0) for x in rows), 2) == 30.0
|
|
r.close()
|