"""HTML reduction, signal extraction, and triage column builders.
Pure module: no FastAPI imports and no HTTPException. Plain functions despite the file
name, following agent/decorators.py.
Stdlib only (html.parser + re). requirements.txt is deliberately untouched: a
dependency on an HTML library for one classifier prompt is not worth the pin.
"""
from __future__ import annotations
import re
from html.parser import HTMLParser
from inbox_classifier.enums import Block_Tags, Drop_Tags
_TAG=re.compile(r"<[^>]+>")
# \xa0 is listed explicitly: unescapes to a NO-BREAK SPACE, which a plain \s
# collapse does not match, so an HTML mail would otherwise reach the prompt full of
# stray non-breaking spaces. Written as an escape, not the literal character, so it
# stays visible in a diff.
_SPACES=re.compile(r"[ \t\xa0\r\f\v]+")
_BLANK_LINES=re.compile(r"\n{3,}")
# Quoted-history markers, in the order Outlook and Gmail actually emit them.
_QUOTE_MARKERS=(
re.compile(r"^-{2,}\s*original message\s*-{2,}", re.IGNORECASE | re.MULTILINE),
re.compile(r"^-{2,}\s*forwarded message\s*-{2,}", re.IGNORECASE | re.MULTILINE),
re.compile(r"^\s*on .{0,200}? wrote:\s*$", re.IGNORECASE | re.MULTILINE),
re.compile(r"^\s*from:\s.+$", re.IGNORECASE | re.MULTILINE),
re.compile(r"^\s*>", re.MULTILINE),
)
# Below this many characters of new text, a "quoted" reply is really a bare forward
# with nothing above the line. Load-bearing: the prompt says to judge the quoted text
# in exactly that case, so it must not be trimmed away.
_MIN_NEW_TEXT=40
MANUAL_UPLOAD_PREFIX="manual-cv:"
class _TextExtractor(HTMLParser):
"""Visible text only, block tags collapsed to newlines.
convert_charrefs (default True) means handle_data already receives unescaped text,
so & / / ' never reach the prompt as entities. handle_startendtag
dispatches to start+end by default, so
needs no special case.
"""
def __init__(self):
super().__init__(convert_charrefs=True)
self._parts=[]
self._suppress=0
def _break(self):
"""One line break per boundary, however many tags meet there.
`
` is a single break, not two: closing and opening tags both mark a
boundary, and emitting a newline for each would turn every paragraph gap into a
blank line. Genuine blank lines in the source survive as data parts.
"""
if self._parts and self._parts[-1]=="\n":
return
self._parts.append("\n")
def handle_starttag(self, tag, attrs):
if Drop_Tags.has(tag):
self._suppress+=1
elif Block_Tags.has(tag):
self._break()
def handle_endtag(self, tag):
if Drop_Tags.has(tag):
self._suppress=max(self._suppress-1,0)
elif Block_Tags.has(tag):
self._break()
def handle_data(self, data):
if not self._suppress:
self._parts.append(data)
def text(self) -> str:
return "".join(self._parts)
def _tidy(text, limit=None) -> str:
"""Collapse runs of whitespace without destroying meaningful line breaks."""
text=text.replace("\x00","")
text=_SPACES.sub(" ",text)
text="\n".join(line.strip() for line in text.split("\n"))
text=_BLANK_LINES.sub("\n\n",text).strip()
if limit is not None and len(text)>limit:
text=text[:limit].rstrip()+"\n[truncated]"
return text
def html_to_text(value, limit=None) -> str:
"""Graph body HTML -> plain text. Empty in, empty out.
message_body is stored as raw Graph HTML (inbox/models.py:353-360) and there is no
other html-to-text helper in backend/, so the reduction happens here.
"""
if not value or not isinstance(value,str):
return ""
if "<" not in value:
# Already plain text (Graph sends contentType "text" for some senders).
return _tidy(value,limit)
parser=_TextExtractor()
try:
parser.feed(value)
parser.close()
text=parser.text()
except Exception:
# Malformed markup should degrade, never fail a whole fetch round.
text=""
if not text.strip():
text=_TAG.sub(" ",value)
return _tidy(text,limit)
def strip_quoted_reply(text) -> str:
"""Trim at the first quoted-history marker, keeping only the newest message.
Only trims when at least _MIN_NEW_TEXT characters precede the marker: a bare
forward whose new text is empty must reach the model whole.
"""
if not text:
return ""
cut=len(text)
for marker in _QUOTE_MARKERS:
match=marker.search(text)
if match is not None and match.start()=len(text):
return text
head=text[:cut].strip()
return head if len(head)>=_MIN_NEW_TEXT else text
def _raw_body(email_data) -> str:
"""body dict -> body str -> bodyPreview, mirroring Inbox_Messages._body_text.
The bodyPreview fallback matters: an image-only or malformed mail still carries its
preview line, which is often the only signal available.
"""
body=email_data.get("body")
if isinstance(body,dict):
return body.get("content") or ""
if isinstance(body,str):
return body
return email_data.get("bodyPreview") or ""
def email_signals(email_data, subject_limit, body_limit) -> tuple[str,str]:
"""(subject, body_text) for the prompt. Subject and body only, by design."""
subject=_tidy(str(email_data.get("subject") or ""),subject_limit)
body=html_to_text(_raw_body(email_data))
body=_tidy(strip_quoted_reply(body),body_limit)
return subject,body
def is_manual_upload(email_data) -> bool:
"""Recruiter CV upload (id "manual-cv:...") — an application by construction.
Defence in depth: the gate lives in inbox.views.Email.get_email_by_id, which
FileRead.ingest_upload never calls, so the manual path already bypasses it. This
keeps the invariant testable and stops a future caller from re-introducing the
empty-body false negative (that path always sends body content "").
"""
return str(email_data.get("id") or "").startswith(MANUAL_UPLOAD_PREFIX)
def triage_fields(email_data, verdict, status, reason_code, error="", model_name="",
ingested=False) -> dict:
"""The inbox_message_triage column dict.
No body key, ever: the body is what this feature keeps out of the database, and the
override route re-reads the mail from upstream by message_id. The subject is kept
(capped) because a review screen without it is unusable.
"""
attachments=email_data.get("attachments") or []
file_names=",".join(str(a.get("name") or "") for a in attachments if a.get("name"))
return {
"message_id":str(email_data.get("id") or ""),
"is_application":bool(getattr(verdict,"is_application",False)),
"reason_code":str(reason_code or "")[:60],
"confidence":getattr(verdict,"confidence",None),
"evidence":(getattr(verdict,"evidence","") or "")[:200],
"status":str(status or "classified")[:30],
"error":(error or None),
"model_name":str(model_name or "")[:120],
"message_subject":str(email_data.get("subject") or "")[:300],
"message_from":(
email_data.get("from",{}).get("emailAddress",{}).get("address","") or ""
)[:320],
"message_received_time":str(email_data.get("receivedDateTime") or "")[:64],
"file_name":file_names[:1000],
"attachment":bool(email_data.get("hasAttachments")),
"ingested":bool(ingested),
}