Add page-marker grouping and party-aware naming

Consecutive captures whose printed PAGE X OF Y markers advance under the same total are grouped into one multi-page PDF, with duplicate pages keeping the better read. Naming gains doc-title/party components and a scan-date fallback so nothing lands in UNSORTED. Vision LLM default switches to minicpm-v (faster, fewer false blanks on dense mono pages).
This commit is contained in:
2026-07-26 15:37:28 -04:00
parent b11f46b00f
commit 12e5d1aeb1
10 changed files with 986 additions and 135 deletions
+46 -18
View File
@@ -1,10 +1,11 @@
"""Build the final filename/PDF title from OCR (and, later, speech) metadata.
Pattern: YYYY-MM-DD_vendor.pdf, falling back progressively as pieces of
information go missing, down to UNSORTED_<timestamp>.pdf when nothing usable
was extracted at all. Module 4 (speech) isn't built yet, so "description" in
the original spec's pattern is currently always the OCR vendor slug; once
speech transcription lands, a spoken description will take priority here.
Pattern: YYYY-MM-DD_vendor[_doc-title].pdf. When the document's own date
can't be read, the scan date (capture timestamp from the video) is used
instead so files still sort chronologically and never get an opaque
"UNSORTED" prefix; export_summary.csv records that the date came from the
scan, not the document. Module 4 (speech) isn't built yet; once speech
transcription lands, a spoken description will take priority here.
"""
from __future__ import annotations
@@ -24,7 +25,7 @@ def slugify(text: str, max_len: int = 40) -> str:
@dataclass
class NamingResult:
stem: str # filename without extension
source: str # "ocr" | "none" (will include "speech" once module 4 exists)
source: str # "ocr" | "scan_date" | "none" ("speech" once module 4 exists)
_TIME_RE = re.compile(r"^([01]\d|2[0-3]):([0-5]\d)$")
@@ -39,21 +40,23 @@ def build_name(
tax_year: str | None = None,
org_name: str | None = None,
time: str | None = None,
doc_title: str | None = None,
party: str | None = None,
) -> NamingResult:
"""date must already be normalized to YYYY-MM-DD, time to HH:MM, or None.
Time is appended to the date (e.g. 2023-03-20_1525) whenever it was
extracted, so two same-day visits to the same vendor (Walmart in the
morning and again in the evening) don't collide or silently overwrite.
doc_title is a recognized printed title slug ("transaction_history",
"invoice"), appended after the vendor unless it would be redundant.
party is an account holder / SHORTNAME when printed on the document.
"""
vendor_slug = slugify(vendor, max_vendor_len) if vendor else None
ts = fallback_dt.strftime("%Y%m%d_%H%M%S")
party_slug = slugify(party, max_vendor_len) if party else None
date_part = date
if date and time and _TIME_RE.match(time):
date_part = f"{date}_{time.replace(':', '')}"
# Tax slips: prefer 2023_T4_York_University over UNSORTED_t4_york_university_<timestamp>.
# Tax slips: prefer 2023_T4_York_University over a date-based name.
if form_code:
org_slug = slugify(org_name, max_vendor_len) if org_name else None
if not org_slug and vendor:
@@ -71,13 +74,38 @@ def build_name(
return NamingResult(stem=f"{code}_{org_slug}", source="ocr")
return NamingResult(stem=code.lower(), source="ocr")
if date_part and vendor_slug:
return NamingResult(stem=f"{date_part}_{vendor_slug}", source="ocr")
if date_part:
return NamingResult(stem=f"{date_part}_UNSORTED", source="ocr")
if date:
date_part = date
if time and _TIME_RE.match(time):
date_part = f"{date}_{time.replace(':', '')}"
source = "ocr"
else:
# Scan date beats an opaque "UNSORTED" prefix: files still sort,
# and the CSV records that the date is when it was scanned.
date_part = fallback_dt.strftime("%Y-%m-%d")
source = "scan_date"
title_slug = slugify(doc_title, max_vendor_len) if doc_title else None
parts = [date_part]
if vendor_slug:
return NamingResult(stem=f"UNSORTED_{vendor_slug}_{ts}", source="ocr")
return NamingResult(stem=f"UNSORTED_{ts}", source="none")
parts.append(vendor_slug)
# Skip the title when the vendor line already says the same thing
# (Tesseract often picks the printed title as the "vendor").
if title_slug and not (vendor_slug and title_slug in vendor_slug):
parts.append(title_slug)
# Account-holder shortname after the doc type — disambiguates family
# members who share a bank (LEVIT I vs LEVIT M).
if party_slug and party_slug not in parts and (
not vendor_slug or party_slug not in vendor_slug
):
parts.append(party_slug)
if len(parts) == 1:
parts.append("document" if date else "scan")
if not date:
source = "none"
return NamingResult(stem="_".join(parts), source=source)
def dedupe_stem(stem: str, used: set[str]) -> str: