170 lines
6.5 KiB
Python
170 lines
6.5 KiB
Python
"""File upload, listing, and removal for a session."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import os
|
|
|
|
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
|
from sqlalchemy.orm import Session as OrmSession
|
|
|
|
from ...config import ALLOWED_EXTENSIONS, MAX_UPLOAD_BYTES, UPLOAD_DIR
|
|
from ...core.readers import make_reader
|
|
from ...core.xlsx_reader import ParseError
|
|
from ...db import models
|
|
from ..deps import db_dep, ensure_editable, file_dict, get_session_or_404, sanitize_filename
|
|
|
|
router = APIRouter(prefix="/api/sessions", tags=["files"])
|
|
|
|
|
|
@router.get("/{session_id}/files")
|
|
def list_files(session_id: int, db: OrmSession = Depends(db_dep)) -> list[dict]:
|
|
get_session_or_404(session_id, db)
|
|
rows = db.query(models.SessionFile).filter(models.SessionFile.session_id == session_id).all()
|
|
return [file_dict(f) for f in rows]
|
|
|
|
|
|
def _validate(rec: models.SessionFile, path: str) -> None:
|
|
"""Light validation: detect sheet/header + required columns (no full row scan)."""
|
|
try:
|
|
reader = make_reader(path)
|
|
reader.detect()
|
|
rec.data_sheet = reader.sheet_name
|
|
rec.status = "invalid" if reader.column_mapping.missing_required else "parsed"
|
|
rec.message = (f"missing required columns: {reader.column_mapping.missing_required}"
|
|
if reader.column_mapping.missing_required else "")
|
|
# Surface marketplace / date span when cheap (CSV already has rows in memory;
|
|
# for xlsx this stays blank until processing).
|
|
meta = reader.file_meta
|
|
if getattr(meta, "currency", None):
|
|
rec.currency = meta.currency
|
|
reader.close()
|
|
except ParseError as e:
|
|
rec.status = "invalid"
|
|
rec.message = str(e)
|
|
|
|
|
|
@router.post("/{session_id}/files")
|
|
async def upload_files(session_id: int, files: list[UploadFile] = File(...),
|
|
db: OrmSession = Depends(db_dep)) -> dict:
|
|
"""
|
|
Upload one or more source files into a closing.
|
|
|
|
Duplicate protection (both were real double-count bugs):
|
|
* same filename again -> the existing row is UPDATED in place (the file is replaced),
|
|
never a second row pointing at the same path — two rows would make the pipeline
|
|
parse and sum the file twice.
|
|
* same content under a different name -> skipped and reported, for the same reason.
|
|
|
|
Returns {"files": [...saved/replaced...], "skipped": [{"filename", "reason"}]} so one
|
|
duplicate in a 13-file batch doesn't fail the other twelve.
|
|
"""
|
|
session = get_session_or_404(session_id, db)
|
|
ensure_editable(session)
|
|
dest_dir = UPLOAD_DIR / f"session_{session_id}"
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
existing = db.query(models.SessionFile).filter(
|
|
models.SessionFile.session_id == session_id).all()
|
|
by_name = {f.filename: f for f in existing}
|
|
by_sha = {f.sha256: f for f in existing if f.sha256}
|
|
|
|
was_processed = session.status in ("processed", "blocked")
|
|
out: list[models.SessionFile] = []
|
|
skipped: list[dict] = []
|
|
changed = False
|
|
|
|
for uf in files:
|
|
safe = sanitize_filename(uf.filename or "upload.xlsx")
|
|
ext = os.path.splitext(safe)[1].lower()
|
|
if ext not in ALLOWED_EXTENSIONS:
|
|
raise HTTPException(400, f"Unsupported file type: {safe} ({ext})")
|
|
path = dest_dir / safe
|
|
# Stream to a temp name first: the hash decides whether this upload is kept, and a
|
|
# failed/oversized upload must never clobber a good file already on disk.
|
|
tmp = dest_dir / (safe + ".part")
|
|
h = hashlib.sha256()
|
|
size = 0
|
|
with open(tmp, "wb") as fh:
|
|
while True:
|
|
chunk = await uf.read(1 << 20)
|
|
if not chunk:
|
|
break
|
|
size += len(chunk)
|
|
if size > MAX_UPLOAD_BYTES:
|
|
fh.close()
|
|
os.remove(tmp)
|
|
raise HTTPException(413, f"File too large: {safe}")
|
|
h.update(chunk)
|
|
fh.write(chunk)
|
|
sha = h.hexdigest()
|
|
|
|
same_name = by_name.get(safe)
|
|
same_content = by_sha.get(sha)
|
|
|
|
if same_name is not None and same_name.sha256 == sha:
|
|
os.remove(tmp)
|
|
skipped.append({"filename": safe,
|
|
"reason": "identical file already uploaded — unchanged"})
|
|
continue
|
|
if same_content is not None and (same_name is None or same_content.id != same_name.id):
|
|
os.remove(tmp)
|
|
skipped.append({"filename": safe,
|
|
"reason": f"identical content already uploaded as "
|
|
f"'{same_content.filename}'"})
|
|
continue
|
|
|
|
os.replace(tmp, path)
|
|
changed = True
|
|
if same_name is not None:
|
|
# Replace in place: update the existing row rather than adding a second one.
|
|
rec = same_name
|
|
rec.stored_path = str(path)
|
|
rec.size_bytes = size
|
|
rec.sha256 = sha
|
|
rec.status = "uploaded"
|
|
rec.message = ""
|
|
rec.imported_rows = 0
|
|
rec.min_date = None
|
|
rec.max_date = None
|
|
rec.marketplace = None
|
|
rec.sheet_last_row = 0
|
|
rec.blank_rows_skipped = 0
|
|
rec.helper_rows_skipped = 0
|
|
else:
|
|
rec = models.SessionFile(
|
|
session_id=session_id, filename=safe, stored_path=str(path),
|
|
size_bytes=size, sha256=sha, status="uploaded",
|
|
)
|
|
db.add(rec)
|
|
_validate(rec, str(path))
|
|
by_name[safe] = rec
|
|
by_sha[sha] = rec
|
|
out.append(rec)
|
|
|
|
if changed:
|
|
session.status = "draft"
|
|
if was_processed:
|
|
# The stored results no longer reflect the files on disk.
|
|
session.needs_reprocess = True
|
|
db.commit()
|
|
return {"files": [file_dict(f) for f in out], "skipped": skipped}
|
|
|
|
|
|
@router.delete("/{session_id}/files/{file_id}")
|
|
def delete_file(session_id: int, file_id: int, db: OrmSession = Depends(db_dep)) -> dict:
|
|
s = get_session_or_404(session_id, db)
|
|
ensure_editable(s)
|
|
f = db.get(models.SessionFile, file_id)
|
|
if not f or f.session_id != session_id:
|
|
raise HTTPException(404, "File not found")
|
|
try:
|
|
if f.stored_path and os.path.exists(f.stored_path):
|
|
os.remove(f.stored_path)
|
|
except OSError:
|
|
pass
|
|
db.delete(f)
|
|
if s.status in ("processed", "blocked"):
|
|
s.needs_reprocess = True
|
|
db.commit()
|
|
return {"deleted": file_id}
|