"""HTML reduction, signal extraction, and triage column builders. Pure module: no FastAPI imports and no HTTPException. Plain functions despite the file name, following agent/decorators.py. Stdlib only (html.parser + re). requirements.txt is deliberately untouched: a dependency on an HTML library for one classifier prompt is not worth the pin. """ from __future__ import annotations import re from html.parser import HTMLParser from inbox_classifier.enums import Block_Tags, Drop_Tags _TAG=re.compile(r"<[^>]+>") # \xa0 is listed explicitly:   unescapes to a NO-BREAK SPACE, which a plain \s # collapse does not match, so an HTML mail would otherwise reach the prompt full of # stray non-breaking spaces. Written as an escape, not the literal character, so it # stays visible in a diff. _SPACES=re.compile(r"[ \t\xa0\r\f\v]+") _BLANK_LINES=re.compile(r"\n{3,}") # Quoted-history markers, in the order Outlook and Gmail actually emit them. _QUOTE_MARKERS=( re.compile(r"^-{2,}\s*original message\s*-{2,}", re.IGNORECASE | re.MULTILINE), re.compile(r"^-{2,}\s*forwarded message\s*-{2,}", re.IGNORECASE | re.MULTILINE), re.compile(r"^\s*on .{0,200}? wrote:\s*$", re.IGNORECASE | re.MULTILINE), re.compile(r"^\s*from:\s.+$", re.IGNORECASE | re.MULTILINE), re.compile(r"^\s*>", re.MULTILINE), ) # Below this many characters of new text, a "quoted" reply is really a bare forward # with nothing above the line. Load-bearing: the prompt says to judge the quoted text # in exactly that case, so it must not be trimmed away. _MIN_NEW_TEXT=40 MANUAL_UPLOAD_PREFIX="manual-cv:" class _TextExtractor(HTMLParser): """Visible text only, block tags collapsed to newlines. convert_charrefs (default True) means handle_data already receives unescaped text, so & /   / ' never reach the prompt as entities. handle_startendtag dispatches to start+end by default, so
needs no special case. """ def __init__(self): super().__init__(convert_charrefs=True) self._parts=[] self._suppress=0 def _break(self): """One line break per boundary, however many tags meet there. `

` is a single break, not two: closing and opening tags both mark a boundary, and emitting a newline for each would turn every paragraph gap into a blank line. Genuine blank lines in the source survive as data parts. """ if self._parts and self._parts[-1]=="\n": return self._parts.append("\n") def handle_starttag(self, tag, attrs): if Drop_Tags.has(tag): self._suppress+=1 elif Block_Tags.has(tag): self._break() def handle_endtag(self, tag): if Drop_Tags.has(tag): self._suppress=max(self._suppress-1,0) elif Block_Tags.has(tag): self._break() def handle_data(self, data): if not self._suppress: self._parts.append(data) def text(self) -> str: return "".join(self._parts) def _tidy(text, limit=None) -> str: """Collapse runs of whitespace without destroying meaningful line breaks.""" text=text.replace("\x00","") text=_SPACES.sub(" ",text) text="\n".join(line.strip() for line in text.split("\n")) text=_BLANK_LINES.sub("\n\n",text).strip() if limit is not None and len(text)>limit: text=text[:limit].rstrip()+"\n[truncated]" return text def html_to_text(value, limit=None) -> str: """Graph body HTML -> plain text. Empty in, empty out. message_body is stored as raw Graph HTML (inbox/models.py:353-360) and there is no other html-to-text helper in backend/, so the reduction happens here. """ if not value or not isinstance(value,str): return "" if "<" not in value: # Already plain text (Graph sends contentType "text" for some senders). return _tidy(value,limit) parser=_TextExtractor() try: parser.feed(value) parser.close() text=parser.text() except Exception: # Malformed markup should degrade, never fail a whole fetch round. text="" if not text.strip(): text=_TAG.sub(" ",value) return _tidy(text,limit) def strip_quoted_reply(text) -> str: """Trim at the first quoted-history marker, keeping only the newest message. Only trims when at least _MIN_NEW_TEXT characters precede the marker: a bare forward whose new text is empty must reach the model whole. """ if not text: return "" cut=len(text) for marker in _QUOTE_MARKERS: match=marker.search(text) if match is not None and match.start()=len(text): return text head=text[:cut].strip() return head if len(head)>=_MIN_NEW_TEXT else text def _raw_body(email_data) -> str: """body dict -> body str -> bodyPreview, mirroring Inbox_Messages._body_text. The bodyPreview fallback matters: an image-only or malformed mail still carries its preview line, which is often the only signal available. """ body=email_data.get("body") if isinstance(body,dict): return body.get("content") or "" if isinstance(body,str): return body return email_data.get("bodyPreview") or "" def email_signals(email_data, subject_limit, body_limit) -> tuple[str,str]: """(subject, body_text) for the prompt. Subject and body only, by design.""" subject=_tidy(str(email_data.get("subject") or ""),subject_limit) body=html_to_text(_raw_body(email_data)) body=_tidy(strip_quoted_reply(body),body_limit) return subject,body def is_manual_upload(email_data) -> bool: """Recruiter CV upload (id "manual-cv:...") — an application by construction. Defence in depth: the gate lives in inbox.views.Email.get_email_by_id, which FileRead.ingest_upload never calls, so the manual path already bypasses it. This keeps the invariant testable and stops a future caller from re-introducing the empty-body false negative (that path always sends body content ""). """ return str(email_data.get("id") or "").startswith(MANUAL_UPLOAD_PREFIX) def triage_fields(email_data, verdict, status, reason_code, error="", model_name="", ingested=False) -> dict: """The inbox_message_triage column dict. No body key, ever: the body is what this feature keeps out of the database, and the override route re-reads the mail from upstream by message_id. The subject is kept (capped) because a review screen without it is unusable. """ attachments=email_data.get("attachments") or [] file_names=",".join(str(a.get("name") or "") for a in attachments if a.get("name")) return { "message_id":str(email_data.get("id") or ""), "is_application":bool(getattr(verdict,"is_application",False)), "reason_code":str(reason_code or "")[:60], "confidence":getattr(verdict,"confidence",None), "evidence":(getattr(verdict,"evidence","") or "")[:200], "status":str(status or "classified")[:30], "error":(error or None), "model_name":str(model_name or "")[:120], "message_subject":str(email_data.get("subject") or "")[:300], "message_from":( email_data.get("from",{}).get("emailAddress",{}).get("address","") or "" )[:320], "message_received_time":str(email_data.get("receivedDateTime") or "")[:64], "file_name":file_names[:1000], "attachment":bool(email_data.get("hasAttachments")), "ingested":bool(ingested), }