From 12e5d1aeb1b79385be8c545bc9bd51b4de8e6440 Mon Sep 17 00:00:00 2001 From: ilia Date: Sun, 26 Jul 2026 15:37:28 -0400 Subject: [PATCH 1/6] Add page-marker grouping and party-aware naming Consecutive captures whose printed PAGE X OF Y markers advance under the same total are grouped into one multi-page PDF, with duplicate pages keeping the better read. Naming gains doc-title/party components and a scan-date fallback so nothing lands in UNSORTED. Vision LLM default switches to minicpm-v (faster, fewer false blanks on dense mono pages). --- config.yaml | 6 +- paperpod/cli.py | 11 +- paperpod/config.py | 5 +- paperpod/llm/vision.py | 27 ++- paperpod/naming/filename.py | 64 ++++-- paperpod/pipeline.py | 316 +++++++++++++++++++++++++----- paperpod/vision/capture_events.py | 274 ++++++++++++++++++++++---- tests/test_capture_events.py | 119 ++++++++++- tests/test_grouping.py | 266 +++++++++++++++++++++++++ tests/test_naming.py | 33 +++- 10 files changed, 986 insertions(+), 135 deletions(-) create mode 100644 tests/test_grouping.py diff --git a/config.yaml b/config.yaml index dc51cd5..503ead7 100644 --- a/config.yaml +++ b/config.yaml @@ -66,9 +66,11 @@ llm: # Reads whole documents at once — far more robust than Tesseract for # curled receipts and stylized logos — and flags quality issues # (fingers_visible, blurry, glare) in export_summary.csv. - # Slower: ~20-40s per document. Falls back to Tesseract when Ollama is down. + # minicpm-v: ~8-12s/doc; avoids qwen2.5vl's false "blank" on dense + # bank statements. OCR still fills date/total when the model hallucinates. + # Falls back to Tesseract when Ollama is down. enabled: true - model: qwen2.5vl + model: minicpm-v base_url: http://localhost:11434 timeout_s: 180 diff --git a/paperpod/cli.py b/paperpod/cli.py index a566078..1901d67 100644 --- a/paperpod/cli.py +++ b/paperpod/cli.py @@ -90,11 +90,14 @@ def cmd_export(args: argparse.Namespace) -> int: print(f"Running OCR + naming on {len(found)} detected document(s)...") result = run_export(report, cfg, out_dir) - print(f"\n{'pod':>3} {'date':>10} vendor / final filename") + print(f"\n{'pod':>3} {'pages':>5} {'date':>10} vendor / final filename") print("-" * 60) for row in result["rows"]: vendor = row["ocr_vendor"] or "(no vendor found)" - print(f"{row['pod_id']:>3} {row['ocr_date'] or '-':>10} {vendor}") + print( + f"{row['pod_id']:>3} {row['page_count']:>5} " + f"{row['ocr_date'] or '-':>10} {vendor}" + ) print(f" -> {row['pdf_path']} [source: {row['source_of_name']}]") if row.get("llm_description"): issues = f" issues: {row['llm_issues']}" if row.get("llm_issues") else "" @@ -102,10 +105,6 @@ def cmd_export(args: argparse.Namespace) -> int: print(f"\nPDFs: {result['pdf_dir']}") print(f"Summary: {out_dir / 'export_summary.csv'}") - print( - "\nNote: no page-flip/multi-page grouping yet — each detected " - "document is a separate single-page PDF." - ) return 0 diff --git a/paperpod/config.py b/paperpod/config.py index 196cb46..9770c09 100644 --- a/paperpod/config.py +++ b/paperpod/config.py @@ -102,9 +102,10 @@ class OcrConfig: class LlmConfig: # Use a local Ollama vision model to identify/name documents and flag # quality issues. Falls back to Tesseract heuristics when Ollama is - # down or the model errors. ~20-40s per document vs <2s for Tesseract. + # down or the model errors. minicpm-v is ~8-12s/doc and much less prone + # to false "blank" labels than qwen2.5vl on dense statement pages. enabled: bool = True - model: str = "qwen2.5vl" + model: str = "minicpm-v" base_url: str = "http://localhost:11434" timeout_s: float = 180.0 diff --git a/paperpod/llm/vision.py b/paperpod/llm/vision.py index 7a2d603..f9828d0 100644 --- a/paperpod/llm/vision.py +++ b/paperpod/llm/vision.py @@ -1,13 +1,17 @@ """Identify documents with a local vision LLM served by Ollama. Everything stays on-machine: images go to localhost:11434, never the cloud. -A vision model (qwen2.5-vl class) reads the whole document at once, which is -far more robust than Tesseract + regex for curled receipts, hands in frame, -and stylized logos — it correctly reads "COSTCO WHOLESALE" where Tesseract -produced "Cosrco". It also reports quality issues (fingers, blur, glare) so -bad captures can be flagged for re-scanning. +A vision model (default: minicpm-v) reads the whole document at once, which +is far more robust than Tesseract + regex for curled receipts, hands in +frame, and stylized logos — it correctly reads "COSTCO WHOLESALE" where +Tesseract produced "Cosrco". It also reports quality issues (fingers, blur, +glare) so bad captures can be flagged for re-scanning. -Trade-off: ~20-40s per document on an M-series Mac, vs <2s for Tesseract. +minicpm-v was chosen over qwen2.5vl after A/B on overhead statement scans: +~2x faster and far fewer false "blank" labels on dense mono bank pages. +Dates can still hallucinate — the pipeline keeps OCR as date/total fill-in. + +Trade-off: ~8-12s per document on an M-series Mac, vs <2s for Tesseract. The pipeline uses the LLM for naming when available and falls back to the OCR heuristics when Ollama isn't running. """ @@ -20,6 +24,7 @@ import re import urllib.error import urllib.request from dataclasses import dataclass, field +from datetime import datetime import cv2 import numpy as np @@ -27,6 +32,7 @@ import numpy as np _PROMPT = """Look at this image of a document. Reply with ONLY a JSON object, no other text: {"doc_type": "receipt|tax_form|letter|invoice|blank|other", "vendor": "business or organization name, or null", + "party": "account holder / customer / SHORTNAME / payee printed on the document, or null", "form_code": "tax form code like T4/T4A/T5008 if this is a tax form, else null", "date": "document/transaction date as YYYY-MM-DD, or null", "time": "transaction time as 24-hour HH:MM if printed on the document, else null", @@ -35,13 +41,15 @@ _PROMPT = """Look at this image of a document. Reply with ONLY a JSON object, no "description": "3-6 word summary", "issues": ["any of: fingers_visible, blurry, cut_off, upside_down, glare, partial_document"]} Use doc_type "blank" if the page has no meaningful printed content (blank back of a page, empty sheet). -Set vendor to null unless you can actually read a real business/organization name — never output garbled text.""" +Set vendor to null unless you can actually read a real business/organization name — never output garbled text. +Set party to the person/entity the document is about (e.g. SHORTNAME on a bank statement, payee on a slip) when clearly printed; otherwise null.""" @dataclass class LlmAnalysis: doc_type: str vendor: str | None + party: str | None form_code: str | None date: str | None # normalized YYYY-MM-DD time: str | None # normalized HH:MM (24h) @@ -77,7 +85,7 @@ def _normalize_date(raw: str | None) -> str | None: if not match: return None y, m, d = (int(g) for g in match.groups()) - if not (1 <= m <= 12 and 1 <= d <= 31 and 1990 <= y <= 2100): + if not (1 <= m <= 12 and 1 <= d <= 31 and 1990 <= y <= datetime.now().year + 1): return None return f"{y:04d}-{m:02d}-{d:02d}" @@ -100,7 +108,7 @@ def _normalize_time(raw: str | None) -> str | None: def analyze_document( image_bgr: np.ndarray, - model: str = "qwen2.5vl", + model: str = "minicpm-v", base_url: str = "http://localhost:11434", timeout: float = 180.0, ) -> LlmAnalysis | None: @@ -137,6 +145,7 @@ def analyze_document( return LlmAnalysis( doc_type=_clean_str(data.get("doc_type")) or "other", vendor=_clean_str(data.get("vendor")), + party=_clean_str(data.get("party")), form_code=_clean_str(data.get("form_code")), date=_normalize_date(data.get("date")), time=_normalize_time(data.get("time")), diff --git a/paperpod/naming/filename.py b/paperpod/naming/filename.py index 85e76dc..28d5051 100644 --- a/paperpod/naming/filename.py +++ b/paperpod/naming/filename.py @@ -1,10 +1,11 @@ """Build the final filename/PDF title from OCR (and, later, speech) metadata. -Pattern: YYYY-MM-DD_vendor.pdf, falling back progressively as pieces of -information go missing, down to UNSORTED_.pdf when nothing usable -was extracted at all. Module 4 (speech) isn't built yet, so "description" in -the original spec's pattern is currently always the OCR vendor slug; once -speech transcription lands, a spoken description will take priority here. +Pattern: YYYY-MM-DD_vendor[_doc-title].pdf. When the document's own date +can't be read, the scan date (capture timestamp from the video) is used +instead so files still sort chronologically and never get an opaque +"UNSORTED" prefix; export_summary.csv records that the date came from the +scan, not the document. Module 4 (speech) isn't built yet; once speech +transcription lands, a spoken description will take priority here. """ from __future__ import annotations @@ -24,7 +25,7 @@ def slugify(text: str, max_len: int = 40) -> str: @dataclass class NamingResult: stem: str # filename without extension - source: str # "ocr" | "none" (will include "speech" once module 4 exists) + source: str # "ocr" | "scan_date" | "none" ("speech" once module 4 exists) _TIME_RE = re.compile(r"^([01]\d|2[0-3]):([0-5]\d)$") @@ -39,21 +40,23 @@ def build_name( tax_year: str | None = None, org_name: str | None = None, time: str | None = None, + doc_title: str | None = None, + party: str | None = None, ) -> NamingResult: """date must already be normalized to YYYY-MM-DD, time to HH:MM, or None. Time is appended to the date (e.g. 2023-03-20_1525) whenever it was extracted, so two same-day visits to the same vendor (Walmart in the morning and again in the evening) don't collide or silently overwrite. + + doc_title is a recognized printed title slug ("transaction_history", + "invoice"), appended after the vendor unless it would be redundant. + party is an account holder / SHORTNAME when printed on the document. """ vendor_slug = slugify(vendor, max_vendor_len) if vendor else None - ts = fallback_dt.strftime("%Y%m%d_%H%M%S") + party_slug = slugify(party, max_vendor_len) if party else None - date_part = date - if date and time and _TIME_RE.match(time): - date_part = f"{date}_{time.replace(':', '')}" - - # Tax slips: prefer 2023_T4_York_University over UNSORTED_t4_york_university_. + # Tax slips: prefer 2023_T4_York_University over a date-based name. if form_code: org_slug = slugify(org_name, max_vendor_len) if org_name else None if not org_slug and vendor: @@ -71,13 +74,38 @@ def build_name( return NamingResult(stem=f"{code}_{org_slug}", source="ocr") return NamingResult(stem=code.lower(), source="ocr") - if date_part and vendor_slug: - return NamingResult(stem=f"{date_part}_{vendor_slug}", source="ocr") - if date_part: - return NamingResult(stem=f"{date_part}_UNSORTED", source="ocr") + if date: + date_part = date + if time and _TIME_RE.match(time): + date_part = f"{date}_{time.replace(':', '')}" + source = "ocr" + else: + # Scan date beats an opaque "UNSORTED" prefix: files still sort, + # and the CSV records that the date is when it was scanned. + date_part = fallback_dt.strftime("%Y-%m-%d") + source = "scan_date" + + title_slug = slugify(doc_title, max_vendor_len) if doc_title else None + parts = [date_part] if vendor_slug: - return NamingResult(stem=f"UNSORTED_{vendor_slug}_{ts}", source="ocr") - return NamingResult(stem=f"UNSORTED_{ts}", source="none") + parts.append(vendor_slug) + # Skip the title when the vendor line already says the same thing + # (Tesseract often picks the printed title as the "vendor"). + if title_slug and not (vendor_slug and title_slug in vendor_slug): + parts.append(title_slug) + # Account-holder shortname after the doc type — disambiguates family + # members who share a bank (LEVIT I vs LEVIT M). + if party_slug and party_slug not in parts and ( + not vendor_slug or party_slug not in vendor_slug + ): + parts.append(party_slug) + + if len(parts) == 1: + parts.append("document" if date else "scan") + if not date: + source = "none" + + return NamingResult(stem="_".join(parts), source=source) def dedupe_stem(stem: str, used: set[str]) -> str: diff --git a/paperpod/pipeline.py b/paperpod/pipeline.py index 5535a70..71e2952 100644 --- a/paperpod/pipeline.py +++ b/paperpod/pipeline.py @@ -11,6 +11,8 @@ from __future__ import annotations import csv import json +import re +from dataclasses import dataclass, field from datetime import datetime, timedelta from pathlib import Path from typing import Any @@ -21,16 +23,27 @@ import numpy as np from paperpod.capture.video import iter_frames, probe_video from paperpod.config import Config, config_to_dict from paperpod.llm.vision import analyze_document, ollama_available -from paperpod.naming.filename import build_name, dedupe_stem -from paperpod.ocr.extract import run_ocr, extract_store_name +from paperpod.naming.filename import build_name, dedupe_stem, slugify +from paperpod.ocr.extract import extract_store_name, run_ocr from paperpod.pdf.build import images_to_pdf -from paperpod.vision.capture_events import _plausible_capture, find_document_events +from paperpod.vision.capture_events import _plausible_capture, find_document_events, score_sample from paperpod.vision.document import detect_document, pad_quad, warp_document -from paperpod.vision.enhance import enhance_for_ocr +from paperpod.vision.enhance import darken_print, enhance_for_ocr from paperpod.vision.motion import find_segments, motion_score, prepare_gray from paperpod.vision.orient import auto_rotate from paperpod.vision.refine import refine_crop +_AMOUNT_RE = re.compile(r"\d[\d,]{2,}(?:\.\d{2})?") + + +def _content_overlap(a: str, b: str) -> float: + """Shared dollar/account numbers between two OCR texts (1.0 ≈ same page).""" + fa = set(_AMOUNT_RE.findall(a or "")) + fb = set(_AMOUNT_RE.findall(b or "")) + if not fa or not fb: + return 0.0 + return len(fa & fb) / min(len(fa), len(fb)) + def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> dict[str, Any]: """Run the full detect pipeline and write artifacts into out_dir. @@ -89,6 +102,9 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d # Unlike stable-window-only detection, this still captures receipts shown # while hands are adjusting them (motion scores stay high but the contour # is visible frame-by-frame). + # + # Only tiny scored proxies are retained here — full-res 4K BGR would OOM + # on a long phone video. Winning frame indices are re-decoded in pass 2b. det_kwargs = dict( min_area_ratio=cfg.document.min_area_ratio, max_area_ratio=cfg.document.max_area_ratio, @@ -97,22 +113,37 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d canny_high=cfg.document.canny_high, detect_width=cfg.document.detect_width, ) - samples: list[tuple[Any, Any]] = [] + scored_samples = [] for frame in iter_frames(video_path, sample_fps=cfg.capture.sample_fps): detection = detect_document(frame.image, **det_kwargs) if detection is not None and _plausible_capture(detection): - samples.append((frame, detection)) + scored_samples.append(score_sample(frame, detection)) doc_events = find_document_events( - samples, + scored_samples, gap_s=cfg.document.event_gap_s, min_samples=cfg.document.event_min_samples, ) + # Pass 2b: re-decode only the winning frame indices (one BGR at a time). + want = {event.best_frame_index: event for event in doc_events} + if want: + for frame in iter_frames(video_path, sample_fps=cfg.capture.sample_fps): + event = want.get(frame.index) + if event is None: + continue + event.frame = frame + del want[frame.index] + if not want: + break + window_records: list[dict[str, Any]] = [] for event in doc_events: i = event.event_id frame = event.frame + if frame is None: + # Winner index was skipped (sample_fps / codec edge); skip event. + continue detection = event.detection sharp = event.sharpness @@ -123,9 +154,10 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d "duration_s": round(event.t_end - event.t_start, 3), "sample_count": event.sample_count, "document_found": False, - "best_frame_t": round(frame.t, 3), - "best_frame_index": frame.index, + "best_frame_t": round(event.best_frame_t, 3), + "best_frame_index": event.best_frame_index, "sharpness": round(sharp, 1), + "best_frame_skin_fraction": event.skin_fraction, } frame_path = frames_dir / f"window_{i:03d}_full.png" @@ -154,6 +186,9 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d if cfg.document.enhance: crop = enhance_for_ocr(crop) + else: + # Keep the photographic look but pull pale grey print toward black. + crop = darken_print(crop) crop_path = crops_dir / f"window_{i:03d}.png" cv2.imwrite(str(crop_path), crop) @@ -202,20 +237,152 @@ def _is_exportable(ocr) -> bool: """Skip obvious mat-transition junk that has no usable naming signal.""" if ocr.form_code or ocr.date or ocr.total: return True - if extract_store_name(ocr.text): + if extract_store_name(ocr.text) or ocr.doc_title or ocr.page_number: return True return False -def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict[str, Any]: - """OCR-name and PDF-export every detected crop from a detect() report. +@dataclass +class _PageCandidate: + """One exportable capture: crop + everything extracted from it.""" - NOTE: there is no page-grouping yet (module 3 / events/ is not built), - so every detected crop becomes its own single-page PDF, even if it was - physically one side of a multi-page stack. pod_id == window_id for now. + window: dict[str, Any] + crop_path: Path + ocr: Any + llm: Any + vendor: str | None + date: str | None + time: str | None + total: str | None + form_code: str | None + tax_year: str | None + org_name: str | None + doc_title: str | None + party: str | None + page_number: int | None + page_total: int | None + + @property + def signature(self) -> tuple: + return (self.vendor, self.date, self.total, self.form_code) + + +@dataclass +class _Document: + pages: list[_PageCandidate] = field(default_factory=list) + + def last_marker_page(self) -> _PageCandidate | None: + """Most recent page whose printed 'PAGE X OF Y' marker was readable.""" + for page in reversed(self.pages): + if page.page_number is not None: + return page + return None + + +def _same_letterhead(prev: _PageCandidate, curr: _PageCandidate) -> bool: + """Loose 'same printed letterhead' check for grouping decisions.""" + if prev.doc_title and curr.doc_title: + return prev.doc_title == curr.doc_title + if prev.vendor and curr.vendor: + a, b = slugify(prev.vendor), slugify(curr.vendor) + return a.startswith(b) or b.startswith(a) + return False + + +def _signature_duplicate(prev: _PageCandidate, curr: _PageCandidate) -> bool: + """Same marker-less page captured twice in a row (a contour blip). + + Only trustworthy for events moments apart, and only when neither side + carries a printed page marker. Pages of one bank statement share + vendor + date, and flips are often 3–4s — the old 5s gate was replacing + page 3 with page 4 (higher OCR conf) and wiping the earlier page. + """ + if prev.page_number is not None or curr.page_number is not None: + return False + if curr.window["t_start"] - prev.window["t_end"] > 5.0: + return False + return bool(any(curr.signature) and curr.signature == prev.signature) + + +def _first(*values): + return next((v for v in values if v), None) + + +def _group_documents(candidates: list[_PageCandidate]) -> list[_Document]: + """Merge consecutive captures into documents; drop duplicate captures. + + Grouping is anchored on the printed "PAGE X OF Y" marker: within one + document, later captures must advance the page number under the same + page total (gaps are fine — a page whose marker OCR couldn't read must + not split one statement into several PDFs). A marker-less capture with + the same letterhead may slot into a document that is still incomplete + (e.g. the "PAGE 1 OF 2" statement's second page whose header came out + garbled); unrelated marker-less single-pagers never group. + """ + documents: list[_Document] = [] + for cand in candidates: + doc = documents[-1] if documents else None + if doc is None: + documents.append(_Document(pages=[cand])) + continue + + anchor = doc.last_marker_page() + last = doc.pages[-1] + + if cand.page_number is not None: + if ( + anchor is not None + and cand.page_total == anchor.page_total + and _same_letterhead(anchor, cand) + ): + if cand.page_number == anchor.page_number: + # Same page captured twice; keep the better read. + if cand.ocr.mean_confidence > anchor.ocr.mean_confidence: + idx = next(i for i, p in enumerate(doc.pages) if p is anchor) + doc.pages[idx] = cand + continue + if cand.page_number > anchor.page_number: + doc.pages.append(cand) + continue + documents.append(_Document(pages=[cand])) + continue + + if _signature_duplicate(last, cand): + if cand.ocr.mean_confidence > last.ocr.mean_confidence: + doc.pages[-1] = cand + continue + # Content-fingerprint dedupe for marker-less recaptures only when the + # amounts strongly match (≥0.55) or the new read looks garbled + # (conf < 75). A 0.25–0.40 overlap is normal between successive + # pages of one statement (shared account # / branch / balances); + # using that as a duplicate gate previously wiped page 3 with page 4. + overlap = _content_overlap(last.ocr.text, cand.ocr.text) + if overlap >= 0.55 or (overlap >= 0.25 and cand.ocr.mean_confidence < 75.0): + if cand.ocr.mean_confidence > last.ocr.mean_confidence: + doc.pages[-1] = cand + continue + if ( + anchor is not None + and anchor.page_number < (anchor.page_total or 0) + and _same_letterhead(anchor, cand) + and cand.ocr.mean_confidence >= 70.0 + and _content_overlap(anchor.ocr.text, cand.ocr.text) < 0.55 + ): + doc.pages.append(cand) + continue + documents.append(_Document(pages=[cand])) + return documents + + +def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict[str, Any]: + """OCR-name and PDF-export every detected document from a detect() report. + + Consecutive captures that carry an incrementing printed "PAGE X OF Y" + marker are grouped into one multi-page PDF (page 1's metadata names the + file). Everything else stays one PDF per capture. Writes: - pdf/.pdf one PDF per detected document + pdf/.pdf one PDF per document (possibly multi-page) export_summary.csv timestamp, pod_id, page_count, source_of_name, ocr_vendor, ocr_date, ocr_total, final_filename Returns the summary rows (also embedded back for programmatic use). @@ -237,10 +404,8 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict if cfg.llm.enabled and not use_llm: print("Ollama not reachable — falling back to Tesseract-only naming.") - used_stems: set[str] = set() - rows: list[dict[str, Any]] = [] - prev_signature: tuple = () - + # ---- Phase 1: OCR/LLM every crop, keep the exportable ones -------------- + candidates: list[_PageCandidate] = [] for window in report["stable_windows"]: if not window.get("document_found"): continue @@ -277,66 +442,110 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict timeout=cfg.llm.timeout_s, ) - # Blank pages (backs of slips, empty sheets) OCR to garbled noise; - # the LLM identifies them reliably, so drop them outright. + # Drop LLM-flagged blank pages (backs of slips, empty sheets) only + # when OCR agrees there's nothing there — vision models (especially + # qwen2.5vl) have mislabeled real statement pages as "blank", and + # losing a real page is far worse than exporting a blank one. if llm is not None and ( llm.doc_type == "blank" or llm.description.lower().startswith("blank") ): - continue + if len(ocr.text.split()) < 8: + continue + llm = None # its other fields aren't trustworthy for this page - vendor = (llm.vendor if llm else None) or ocr.vendor - date = (llm.date if llm else None) or ocr.date - time = (llm.time if llm else None) or ocr.time - total = (llm.total if llm else None) or ocr.total - form_code = (llm.form_code if llm else None) or ocr.form_code - tax_year = (llm.tax_year if llm else None) or ocr.tax_year - org_name = (llm.vendor if llm and llm.form_code else None) or ocr.org_name + vendor = _first(llm.vendor if llm else None, ocr.vendor) + date = _first(llm.date if llm else None, ocr.date) + form_code = _first(llm.form_code if llm else None, ocr.form_code) + total = _first(llm.total if llm else None, ocr.total) if llm is None and not _is_exportable(ocr): continue if llm is not None and not any((vendor, date, total, form_code)): continue - # The same physical document sometimes splits into two capture - # events (hand briefly breaks the contour). If the previous export - # has the identical identity signature, it's the same document. - signature = (vendor, date, total, form_code) - if any(signature) and signature == prev_signature: - continue - prev_signature = signature + candidates.append( + _PageCandidate( + window=window, + crop_path=crop_path, + ocr=ocr, + llm=llm, + vendor=vendor, + date=date, + time=_first(llm.time if llm else None, ocr.time), + total=total, + form_code=form_code, + tax_year=_first(llm.tax_year if llm else None, ocr.tax_year), + org_name=_first( + llm.vendor if llm and llm.form_code else None, ocr.org_name + ), + doc_title=ocr.doc_title, + party=_first(llm.party if llm else None, ocr.party), + page_number=ocr.page_number, + page_total=ocr.page_total, + ) + ) - event_time = recording_start + timedelta(seconds=window["t_start"]) + # ---- Phase 2: group consecutive captures into documents ------------------ + documents = _group_documents(candidates) + + # ---- Phase 3: name and write one PDF per document ------------------------ + used_stems: set[str] = set() + rows: list[dict[str, Any]] = [] + for doc in documents: + pages = doc.pages + first = pages[0] + # Page 1 usually carries the letterhead/date, but any page may have + # filled a field the others missed. + date = _first(*(p.date for p in pages)) + doc_title = _first(*(p.doc_title for p in pages)) + party = _first(*(p.party for p in pages)) + # Prefer a vendor that isn't just the printed title re-read (the + # largest-font heuristic picks "Transaction History" on some pages + # and the actual "TD Canada Trust" letterhead on others). + vendors = [p.vendor for p in pages if p.vendor] + vendor = _first( + *(v for v in vendors if not doc_title or slugify(v) not in doc_title), + *vendors, + ) + + event_time = recording_start + timedelta(seconds=first.window["t_start"]) naming = build_name( date, vendor, event_time, cfg.naming.max_vendor_len, - form_code=form_code, - tax_year=tax_year, - org_name=org_name, - time=time, + form_code=_first(*(p.form_code for p in pages)), + tax_year=_first(*(p.tax_year for p in pages)), + org_name=_first(*(p.org_name for p in pages)), + time=_first(*(p.time for p in pages)), + doc_title=doc_title, + party=party, ) stem = dedupe_stem(naming.stem, used_stems) pdf_path = pdf_dir / f"{stem}.pdf" - images_to_pdf([crop_path], pdf_path, title=stem) + images_to_pdf([p.crop_path for p in pages], pdf_path, title=stem) + llm = first.llm rows.append( { "timestamp": event_time.isoformat(timespec="seconds"), - "pod_id": window["window_id"], - "page_count": 1, + "pod_id": first.window["window_id"], + "page_count": len(pages), + "page_windows": ";".join(str(p.window["window_id"]) for p in pages), "source_of_name": "llm" if llm else naming.source, "ocr_vendor": vendor or "", - "ocr_form_code": form_code or "", + "ocr_form_code": _first(*(p.form_code for p in pages)) or "", "ocr_date": date or "", - "ocr_time": time or "", - "ocr_total": total or "", - "ocr_confidence": round(ocr.mean_confidence, 1), + "ocr_time": _first(*(p.time for p in pages)) or "", + "ocr_total": _first(*(p.total for p in pages)) or "", + "doc_title": doc_title or "", + "party": party or "", + "ocr_confidence": round(first.ocr.mean_confidence, 1), "llm_description": llm.description if llm else "", "llm_issues": ";".join(llm.issues) if llm else "", - "detection_confidence": window.get("detection_confidence", ""), - "rotation_applied": window.get("rotation_applied", 0), + "detection_confidence": first.window.get("detection_confidence", ""), + "rotation_applied": first.window.get("rotation_applied", 0), "final_filename": pdf_path.name, "pdf_path": str(pdf_path.relative_to(out_dir)), } @@ -344,8 +553,9 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict with open(out_dir / "export_summary.csv", "w", newline="") as f: fieldnames = [ - "timestamp", "pod_id", "page_count", "source_of_name", - "ocr_vendor", "ocr_form_code", "ocr_date", "ocr_time", "ocr_total", "ocr_confidence", + "timestamp", "pod_id", "page_count", "page_windows", "source_of_name", + "ocr_vendor", "ocr_form_code", "ocr_date", "ocr_time", "ocr_total", + "doc_title", "party", "ocr_confidence", "llm_description", "llm_issues", "detection_confidence", "rotation_applied", "final_filename", "pdf_path", diff --git a/paperpod/vision/capture_events.py b/paperpod/vision/capture_events.py index a26771d..3e6a12c 100644 --- a/paperpod/vision/capture_events.py +++ b/paperpod/vision/capture_events.py @@ -4,19 +4,57 @@ Stable-window-only detection misses documents shown while hands are still adjusting them — motion scores stay above threshold even though a receipt is clearly visible and detectable frame-by-frame. This clusters consecutive frames where a document contour was found (with a time-gap to split separate -placements) and keeps the sharpest frame from each cluster. +placements) and keeps the best frame from each cluster. + +"Best" means hands-free first, sharp second: fingers holding a page flat are +usually present in most frames of an event but absent in at least a few +(right after placing / before flipping). A sharpness-only pick routinely +selects a hand-in-frame frame, which then has to be inpainted (lossy). +Preferring the lowest-skin frame keeps the original pixels instead. + +Memory: scoring runs as frames arrive; only tiny gray proxies + metrics are +retained for clustering. Full-res BGR is dropped immediately so a long 4K +video cannot OOM by holding thousands of detections in RAM (callers re-decode +winning frame indices in a third pass). """ from __future__ import annotations from dataclasses import dataclass +import cv2 import numpy as np from paperpod.capture.video import Frame from paperpod.vision.document import DocumentDetection +from paperpod.vision.motion import motion_score, prepare_gray +from paperpod.vision.refine import skin_mask from paperpod.vision.sharpness import sharpness_score +# Downscale width for per-frame skin analysis; full-res 4K would dominate +# pass-2 runtime for no accuracy gain. +_SKIN_ANALYSIS_WIDTH = 400 + +# Skin coverage below this fraction of the document region counts as a +# "clean" (hands-free) frame; between the two thresholds counts as "touched" +# (fingertip on an edge); above is "held" (hand across the page). +_SKIN_CLEAN = 0.003 +_SKIN_TOUCHED = 0.03 + +# Reject a "cleaner" frame if it's this dull relative to the sharpest +# sample seen in the same event. +_SHARP_FLOOR = 0.4 + +# Long placements that include a page flip keep the same paper outline, so +# contour clustering never splits them. Instead we look for *quiet dwells* +# (runs of near-zero inter-sample motion) inside a long event — each dwell +# is one page lying flat. Page 2 of the Jan '25 TD statement sat still for +# ~0.7s around t=10.4; hand-wobble stretches between dwells never qualify. +_QUIET_MOTION = 0.015 +_MIN_DWELL_S = 0.55 +_LONG_EVENT_S = 4.0 +_MOTION_SIZE = (160, 90) + @dataclass class DocumentEvent: @@ -27,8 +65,84 @@ class DocumentEvent: best_frame_t: float best_frame_index: int sharpness: float - frame: Frame + skin_fraction: float detection: DocumentDetection + # Populated by the pipeline after a re-decode pass; None right after + # clustering so the scorer never retains full-res images. + frame: Frame | None = None + + +@dataclass +class ScoredSample: + """Per-detection metrics without the full-res BGR frame.""" + + index: int + t: float + shape: tuple[int, ...] # (h, w, c) of the source frame + det: DocumentDetection + sharp: float + skin: float + gray_tiny: np.ndarray # for page-flip motion between samples + + +def _quad_region(image: np.ndarray, quad: np.ndarray) -> np.ndarray: + """Bounding-box crop of the detected document within the frame.""" + h, w = image.shape[:2] + x0 = max(0, int(quad[:, 0].min())) + x1 = min(w, int(np.ceil(quad[:, 0].max()))) + y0 = max(0, int(quad[:, 1].min())) + y1 = min(h, int(np.ceil(quad[:, 1].max()))) + if x1 - x0 >= 8 and y1 - y0 >= 8: + return image[y0:y1, x0:x1] + return image + + +def skin_fraction_in_quad(image: np.ndarray, quad: np.ndarray) -> float: + """Fraction of the detected document region that looks like skin. + + Measured on the quad's bounding box (downscaled), so a hand resting on + the mat next to the document doesn't count — only skin over the paper. + """ + region = _quad_region(image, quad) + rh, rw = region.shape[:2] + if rw > _SKIN_ANALYSIS_WIDTH: + region = cv2.resize( + region, (_SKIN_ANALYSIS_WIDTH, max(1, round(rh * _SKIN_ANALYSIS_WIDTH / rw))) + ) + return float((skin_mask(region) > 0).mean()) + + +def document_sharpness(image: np.ndarray, quad: np.ndarray) -> float: + """Sharpness of the document region only. + + Whole-frame Laplacian variance is dominated by high-frequency background + (wood grain), so a blurry mid-flip frame that exposes more table can + outscore a crisp flat page. Scoring inside the detection quad measures + the thing we actually keep. + """ + return sharpness_score(_quad_region(image, quad)) + + +def score_sample(frame: Frame, det: DocumentDetection) -> ScoredSample: + """Extract clustering metrics and drop the full-res pixels.""" + small = cv2.resize(frame.image, _MOTION_SIZE) + return ScoredSample( + index=frame.index, + t=frame.t, + shape=frame.image.shape, + det=det, + sharp=document_sharpness(frame.image, det.quad), + skin=skin_fraction_in_quad(frame.image, det.quad), + gray_tiny=prepare_gray(small, blur_ksize=5), + ) + + +def _skin_bucket(fraction: float) -> int: + if fraction < _SKIN_CLEAN: + return 0 + if fraction < _SKIN_TOUCHED: + return 1 + return 2 def _same_document( @@ -65,60 +179,148 @@ def _plausible_capture(det: DocumentDetection) -> bool: return ar <= 0.40 or ar >= 0.58 +def _split_on_page_flips( + samples: list[ScoredSample], min_samples: int +) -> list[list[ScoredSample]]: + """Pull quiet dwells out of a long continuous contour (page flips). + + Returns one segment per flat-page dwell when two or more qualify; + otherwise returns the original samples unchanged so ordinary + single-page placements aren't shredded by brief hand motion. + """ + if len(samples) < 4: + return [samples] + duration = samples[-1].t - samples[0].t + if duration < _LONG_EVENT_S: + return [samples] + + motions = [0.0] + for i in range(1, len(samples)): + motions.append( + motion_score(samples[i - 1].gray_tiny, samples[i].gray_tiny, pixel_threshold=12) + ) + + dwells: list[list[ScoredSample]] = [] + start: int | None = None + for i, sample in enumerate(samples): + quiet = motions[i] <= _QUIET_MOTION + if quiet and start is None: + start = i + elif not quiet and start is not None: + dwell = samples[start:i] + if dwell[-1].t - dwell[0].t >= _MIN_DWELL_S and len(dwell) >= min_samples: + dwells.append(dwell) + start = None + if start is not None: + dwell = samples[start:] + if dwell[-1].t - dwell[0].t >= _MIN_DWELL_S and len(dwell) >= min_samples: + dwells.append(dwell) + + # Only commit to multi-page split when we actually found 2+ quiet dwells. + return dwells if len(dwells) >= 2 else [samples] + + +class _Cluster: + """Accumulates samples for one placement; picks the best frame on flush. + + Preference among frames ≥ 40% as sharp as the cluster's sharpest sample: + lowest skin bucket, then highest sharpness. Soft clean frames never + outrank sharp lightly-touched ones. + + First/last samples of long events are ignored — they are usually the + hand placing or lifting the page (high Laplacian streaks, low ink), + which is how page 7 of a statement once lost to its own lift frame. + """ + + def __init__(self, sample: ScoredSample) -> None: + self.t_start = sample.t + self.t_end = sample.t + self.samples: list[ScoredSample] = [sample] + + @property + def count(self) -> int: + return len(self.samples) + + def add(self, sample: ScoredSample) -> None: + self.t_end = sample.t + self.samples.append(sample) + + def pick_best(self, samples: list[ScoredSample] | None = None) -> ScoredSample: + samples = samples if samples is not None else self.samples + pick_from = samples + if len(pick_from) >= 5: + # Drop the placing/lifting edges. + pick_from = pick_from[1:-1] + max_sharp = max(s.sharp for s in pick_from) + floor = max_sharp * _SHARP_FLOOR + eligible = [s for s in pick_from if s.sharp >= floor] or pick_from + return min(eligible, key=lambda s: (_skin_bucket(s.skin), -s.sharp)) + + def find_document_events( - samples: list[tuple[Frame, DocumentDetection]], + samples: list[tuple[Frame, DocumentDetection]] | list[ScoredSample], gap_s: float = 2.0, min_samples: int = 2, ) -> list[DocumentEvent]: """Cluster consecutive detections; split when gap_s passes with no detection. - samples must be in time order. Each sample is a (frame, detection) pair - from a frame where detect_document() succeeded. + Long clusters (page kept under the camera while flipped) are further + split on motion peaks into separate dwells so each page gets its own + capture. samples must be in time order. + + Accepts either (Frame, detection) tuples (scores immediately, drops BGR) + or pre-built ScoredSample list from the streaming pipeline. """ if not samples: return [] - clusters: list[tuple[float, float, int, float, Frame, DocumentDetection]] = [] - active: tuple[float, float, int, float, Frame, DocumentDetection] | None = None + scored: list[ScoredSample] + if isinstance(samples[0], ScoredSample): + scored = list(samples) # type: ignore[arg-type] + else: + scored = [score_sample(frame, det) for frame, det in samples] # type: ignore[misc] + + clusters: list[_Cluster] = [] + active: _Cluster | None = None def flush() -> None: nonlocal active - if active is None: - return - t_start, t_end, count, sharp, frame, det = active - if count >= min_samples: + if active is not None and active.count >= min_samples: clusters.append(active) active = None - for frame, det in samples: + for sample in scored: if active is None: - active = (frame.t, frame.t, 1, sharpness_score(frame.image), frame, det) + active = _Cluster(sample) continue - - t_start, t_end, count, best_sharp, best_frame, best_det = active - if frame.t - t_end > gap_s or not _same_document(best_det, det, frame.image.shape): + if sample.t - active.t_end > gap_s or not _same_document( + active.samples[-1].det, sample.det, sample.shape + ): flush() - active = (frame.t, frame.t, 1, sharpness_score(frame.image), frame, det) + active = _Cluster(sample) continue - - sharp = sharpness_score(frame.image) - if sharp > best_sharp: - best_sharp, best_frame, best_det = sharp, frame, det - active = (t_start, frame.t, count + 1, best_sharp, best_frame, best_det) + active.add(sample) flush() - return [ - DocumentEvent( - event_id=i, - t_start=t_start, - t_end=t_end, - sample_count=count, - best_frame_t=frame.t, - best_frame_index=frame.index, - sharpness=sharp, - frame=frame, - detection=det, - ) - for i, (t_start, t_end, count, sharp, frame, det) in enumerate(clusters) - ] + events: list[DocumentEvent] = [] + event_id = 0 + for cluster in clusters: + for segment in _split_on_page_flips(cluster.samples, min_samples): + best = cluster.pick_best(segment) + events.append( + DocumentEvent( + event_id=event_id, + t_start=segment[0].t, + t_end=segment[-1].t, + sample_count=len(segment), + best_frame_t=best.t, + best_frame_index=best.index, + sharpness=best.sharp, + skin_fraction=round(best.skin, 4), + detection=best.det, + frame=None, + ) + ) + event_id += 1 + return events diff --git a/tests/test_capture_events.py b/tests/test_capture_events.py index a0c88d8..54507e4 100644 --- a/tests/test_capture_events.py +++ b/tests/test_capture_events.py @@ -1,8 +1,11 @@ -from paperpod.capture.video import Frame -from paperpod.vision.capture_events import find_document_events -from paperpod.vision.document import DocumentDetection import numpy as np +from paperpod.capture.video import Frame +from paperpod.vision.capture_events import find_document_events, skin_fraction_in_quad +from paperpod.vision.document import DocumentDetection + +_SKIN_BGR = (80, 120, 180) # inside the YCrCb skin range used by refine + def _frame(t: float, idx: int = 0) -> Frame: return Frame(index=idx, t=t, image=np.zeros((100, 100, 3), dtype=np.uint8)) @@ -66,3 +69,113 @@ def test_area_change_splits_documents(): assert len(events) == 2 assert events[0].detection.area_ratio == 0.25 assert events[1].detection.area_ratio == 0.65 + + +def _paper_frame(t: float, idx: int, with_finger: bool, sharper: bool) -> Frame: + """White page frame; optionally with a skin-colored blob and extra texture.""" + rng = np.random.default_rng(idx) + image = np.full((100, 100, 3), 245, dtype=np.uint8) + if sharper: + # High-frequency noise inflates Laplacian variance (the old + # sharpness-only criterion would always pick this frame). + noise = rng.integers(0, 60, size=(100, 100, 1), dtype=np.uint8) + image = np.clip(image.astype(int) - noise, 0, 255).astype(np.uint8) + if with_finger: + image[30:70, 40:75] = _SKIN_BGR + return Frame(index=idx, t=t, image=image) + + +def _full_quad() -> DocumentDetection: + quad = np.array([[0, 0], [100, 0], [100, 100], [0, 100]], dtype=np.float32) + return DocumentDetection(quad=quad, area_ratio=0.3, method="quad", source="edges", confidence=0.8) + + +def test_skin_fraction_detects_finger(): + clean = _paper_frame(1.0, 0, with_finger=False, sharper=False) + finger = _paper_frame(1.2, 1, with_finger=True, sharper=False) + quad = _full_quad().quad + assert skin_fraction_in_quad(clean.image, quad) < 0.003 + assert skin_fraction_in_quad(finger.image, quad) > 0.05 + + +def test_hands_free_frame_beats_equally_sharp_finger_frame(): + # When sharpness is similar, prefer the clean frame. + samples = [ + (_paper_frame(1.0, 0, with_finger=True, sharper=True), _full_quad()), + (_paper_frame(1.2, 1, with_finger=False, sharper=True), _full_quad()), + (_paper_frame(1.4, 2, with_finger=True, sharper=True), _full_quad()), + ] + events = find_document_events(samples, gap_s=2.0, min_samples=2) + assert len(events) == 1 + assert events[0].best_frame_index == 1 + assert events[0].skin_fraction < 0.003 + + +def test_much_sharper_touched_frame_beats_blurry_clean_frame(): + # Mid-flip blur with no fingers must not beat a sharp frame that has a + # fingertip on the margin — that's how page 7 of a statement got lost. + samples = [ + (_paper_frame(1.0, 0, with_finger=False, sharper=False), _full_quad()), + (_paper_frame(1.2, 1, with_finger=True, sharper=True), _full_quad()), + (_paper_frame(1.4, 2, with_finger=False, sharper=False), _full_quad()), + ] + events = find_document_events(samples, gap_s=2.0, min_samples=2) + assert len(events) == 1 + assert events[0].best_frame_index == 1 + + +def test_sharpest_wins_among_equally_clean_frames(): + samples = [ + (_paper_frame(1.0, 0, with_finger=False, sharper=False), _full_quad()), + (_paper_frame(1.2, 1, with_finger=False, sharper=True), _full_quad()), + ] + events = find_document_events(samples, gap_s=2.0, min_samples=2) + assert len(events) == 1 + assert events[0].best_frame_index == 1 + + +def test_long_event_splits_on_page_flip_motion(): + """Quiet dwells inside a long placement become separate events. + + Synthetic: quiet page A, motion/hands, quiet page B. Contour is the + same throughout (page flip under a fixed outline), so without dwell + splitting only page A would be kept. + """ + + def page_frame(t: float, idx: int, pattern: int) -> Frame: + # Nearly identical consecutive frames so inter-sample motion is quiet. + img = np.full((180, 320, 3), 240, dtype=np.uint8) + img[40:140, 40:280] = 200 if pattern == 1 else 160 + # Seeded by pattern only — same noise every page-A frame. + noise = np.random.default_rng(pattern).integers(0, 3, size=img.shape, dtype=np.uint8) + img = np.clip(img.astype(int) + noise, 0, 255).astype(np.uint8) + return Frame(index=idx, t=t, image=img) + + samples = [] + # Quiet page 1 dwell: t=0.0 .. 3.0 + for i in range(0, 13): + samples.append((page_frame(i * 0.25, i, pattern=1), _full_quad())) + # Motion / hands between pages + for i, shade in enumerate((40, 80, 120, 60)): + img = np.full((180, 320, 3), shade, dtype=np.uint8) + samples.append((Frame(index=13 + i, t=3.25 + i * 0.25, image=img), _full_quad())) + # Quiet page 2 dwell: t=4.25 .. 5.25 + for i in range(0, 5): + samples.append((page_frame(4.25 + i * 0.25, 20 + i, pattern=2), _full_quad())) + + events = find_document_events(samples, gap_s=2.0, min_samples=2) + assert len(events) >= 2 + assert events[0].t_end <= 3.5 + assert events[1].t_start >= 3.5 + + + +def test_short_event_not_resegmented(): + samples = [ + (_paper_frame(1.0, 0, with_finger=False, sharper=True), _full_quad()), + (_paper_frame(1.2, 1, with_finger=False, sharper=True), _full_quad()), + (_paper_frame(1.4, 2, with_finger=False, sharper=True), _full_quad()), + ] + events = find_document_events(samples, gap_s=2.0, min_samples=2) + assert len(events) == 1 + diff --git a/tests/test_grouping.py b/tests/test_grouping.py new file mode 100644 index 0000000..d9ee439 --- /dev/null +++ b/tests/test_grouping.py @@ -0,0 +1,266 @@ +"""Multi-page grouping and duplicate-capture handling in run_export's phase 2.""" + +from pathlib import Path + +from paperpod.pipeline import _group_documents, _PageCandidate + + +class _FakeOcr: + def __init__(self, mean_confidence: float = 80.0): + self.mean_confidence = mean_confidence + + +def _cand( + window_id: int, + vendor: str | None = None, + date: str | None = None, + page_number: int | None = None, + page_total: int | None = None, + confidence: float = 80.0, + total: str | None = None, + doc_title: str | None = None, + t_start: float | None = None, + text: str = "", +) -> _PageCandidate: + t0 = t_start if t_start is not None else float(window_id) + ocr = _FakeOcr(confidence) + ocr.text = text + return _PageCandidate( + window={"window_id": window_id, "t_start": t0, "t_end": t0 + 0.5}, + crop_path=Path(f"crops/window_{window_id:03d}.png"), + ocr=ocr, + llm=None, + vendor=vendor, + date=date, + time=None, + total=total, + form_code=None, + tax_year=None, + org_name=None, + doc_title=doc_title, + party=None, + page_number=page_number, + page_total=page_total, + ) + + +def test_incrementing_page_markers_group_into_one_document(): + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", date="2025-01-31", page_number=1, page_total=2), + _cand(1, vendor="TD Canada Trust", page_number=2, page_total=2), + ] + ) + assert len(docs) == 1 + assert len(docs[0].pages) == 2 + + +def test_new_page_1_starts_a_new_document(): + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2), + _cand(1, vendor="TD Canada Trust", page_number=2, page_total=2), + _cand(2, vendor="TD Canada Trust", page_number=1, page_total=2), + ] + ) + assert len(docs) == 2 + assert [len(d.pages) for d in docs] == [2, 1] + + +def test_different_page_totals_do_not_group(): + docs = _group_documents( + [ + _cand(0, page_number=1, page_total=2, vendor="TD"), + _cand(1, page_number=2, page_total=17, vendor="TD"), + ] + ) + assert len(docs) == 2 + + +def test_documents_without_markers_stay_single_page(): + docs = _group_documents( + [ + _cand(0, vendor="Costco", date="2024-01-01", total="10.00"), + _cand(1, vendor="Walmart", date="2024-01-02", total="20.00"), + ] + ) + assert len(docs) == 2 + + +def test_duplicate_capture_keeps_higher_confidence_read(): + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=60.0), + _cand(1, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=90.0), + _cand(2, vendor="TD Canada Trust", page_number=2, page_total=2), + ] + ) + assert len(docs) == 1 + assert len(docs[0].pages) == 2 + assert docs[0].pages[0].window["window_id"] == 1 # better read replaced the first + + +def test_duplicate_signature_without_markers_is_dropped(): + docs = _group_documents( + [ + _cand(0, vendor="Costco", date="2024-01-01", total="10.00"), + _cand(1, vendor="Costco", date="2024-01-01", total="10.00"), + ] + ) + assert len(docs) == 1 + assert len(docs[0].pages) == 1 + + +def test_duplicate_signature_far_apart_is_not_dropped(): + # Marker-less pages of one statement share vendor/date; a deliberate + # page flip (many seconds) means it's a different page, not a dup. + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", date="2024-12-31", t_start=10.0), + _cand(1, vendor="TD Canada Trust", date="2024-12-31", t_start=25.0), + ] + ) + assert len(docs) == 2 + + +def test_unreadable_marker_gap_does_not_split_document(): + # Page 12's marker OCR'd fine, page 13's didn't get captured, page 14 + # still belongs to the same statement. + docs = _group_documents( + [ + _cand(0, vendor="TD", page_number=12, page_total=17), + _cand(1, vendor="TD", page_number=14, page_total=17), + ] + ) + assert len(docs) == 1 + assert len(docs[0].pages) == 2 + + +def test_markerless_page_inside_marker_chain_does_not_split(): + # Page 10's marker was unreadable but pages 9 and 11 anchor it: one + # statement, three pages, in order. + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", page_number=9, page_total=17, t_start=0.0), + _cand(1, vendor="TD Canada Trust", t_start=10.0), + _cand(2, vendor="TD Canada Trust", page_number=11, page_total=17, t_start=20.0), + ] + ) + assert len(docs) == 1 + assert [p.window["window_id"] for p in docs[0].pages] == [0, 1, 2] + + +def test_markerless_page_attaches_to_incomplete_document(): + # "PAGE 1 OF 2" followed (after a deliberate page flip) by a + # same-letterhead capture whose marker was unreadable: it's page 2, + # not a new document — as long as the amounts don't match page 1. + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, + doc_title="transaction_history", t_start=0.0, + text="PAGE 1 OF 2 70.22 750.00 3000.00"), + _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", + t_start=10.0, + text="PAGE 2 OF 2 15.00 22.50 100.00 200.00"), + ] + ) + assert len(docs) == 1 + assert len(docs[0].pages) == 2 + + +def test_markerless_recapture_of_same_page_is_dropped(): + # Garbled second capture of PAGE 1 shares the amount fingerprint with + # the real page 1 — must not become a fake "page 2". + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, + doc_title="transaction_history", t_start=0.0, confidence=90.0, + text="PAGE 1 OF 2 70.22 DR 750.00 DR 3000.00 DR 490.96 1076 6254276"), + _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", + t_start=10.0, confidence=55.0, + text="PAGK 1 of 10,22 150,00 3000 490 1076 6254276"), + ] + ) + assert len(docs) == 1 + assert len(docs[0].pages) == 1 + assert docs[0].pages[0].window["window_id"] == 0 + + +def test_successive_statement_pages_not_collapsed_by_shared_account_numbers(): + # Pages of one statement share branch/account numbers (~0.3 overlap) — + # that must not replace an earlier marked page with a later one. + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", page_number=2, page_total=17, + doc_title="transaction_history", t_start=0.0, confidence=92.0, + text="PAGE 2 OF 17 1076 6254276 500.00 600.00 700.00"), + _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", + t_start=10.0, confidence=91.0, + text="PAGE 4 OF 17 1076 6254276 800.00 900.00 1000.00"), + ] + ) + # Second has no readable marker in this unit test (page_number=None): + # attach as incomplete-fill; must NOT replace page 2. + assert len(docs) == 1 + assert len(docs[0].pages) == 2 + assert docs[0].pages[0].window["window_id"] == 0 + + +def test_markerless_low_confidence_does_not_fill_incomplete_doc(): + # Conf 63 mid-flip noise must not slot in as "page 2" just because the + # statement is still incomplete. + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, + doc_title="transaction_history", t_start=0.0, confidence=91.0, + text="PAGE 1 OF 2 70.22 750.00"), + _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", + t_start=10.0, confidence=63.0, + text="garble noise 11 22 33 44 55 66 77"), + ] + ) + assert len(docs) == 2 + + +def test_markerless_page_does_not_attach_to_complete_document(): + # After "PAGE 2 OF 2" the statement is done; the next capture is new. + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", page_number=2, page_total=2, + doc_title="transaction_history", t_start=0.0), + _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", + t_start=10.0), + ] + ) + assert len(docs) == 2 + + +def test_markerless_recapture_moments_later_is_a_duplicate(): + # Identical signature 0.5s later is a contour blip, not a new page. + docs = _group_documents( + [ + _cand(0, vendor="Costco", date="2024-01-01", total="10.00", t_start=0.0), + _cand(1, vendor="Costco", date="2024-01-01", total="10.00", t_start=1.0), + ] + ) + assert len(docs) == 1 + assert len(docs[0].pages) == 1 + + +def test_marked_page_not_replaced_by_nearby_same_vendor_date(): + # Statement pages share vendor+date and flip in ~3s — must not treat the + # next page as a signature duplicate of the previous. + docs = _group_documents( + [ + _cand(0, vendor="TD Canada Trust", date="2024-12-31", + page_number=3, page_total=17, doc_title="transaction_history", + t_start=40.0, confidence=88.0, + text="PAGE 3 OF 17 1076 6254276 100.00 200.00"), + _cand(1, vendor="TD Canada Trust", date="2024-12-31", + doc_title="transaction_history", t_start=49.0, confidence=91.0, + text="PAGE 4 OF 17 1076 6254276 300.00 400.00 500.00"), + ] + ) + assert len(docs) == 1 + assert len(docs[0].pages) == 2 + assert docs[0].pages[0].page_number == 3 diff --git a/tests/test_naming.py b/tests/test_naming.py index 159fe64..6379270 100644 --- a/tests/test_naming.py +++ b/tests/test_naming.py @@ -28,24 +28,45 @@ def test_build_name_full_metadata(): def test_build_name_date_only(): dt = datetime(2026, 1, 1, 12, 0, 0) result = build_name("2023-03-14", None, dt) - assert result.stem == "2023-03-14_UNSORTED" + assert result.stem == "2023-03-14_document" assert result.source == "ocr" -def test_build_name_vendor_only(): +def test_build_name_vendor_only_uses_scan_date(): dt = datetime(2026, 1, 1, 12, 0, 0) result = build_name(None, "METRO", dt) - assert result.stem == "UNSORTED_metro_20260101_120000" - assert result.source == "ocr" + assert result.stem == "2026-01-01_metro" + assert result.source == "scan_date" def test_build_name_nothing_found(): dt = datetime(2026, 1, 1, 12, 0, 0) result = build_name(None, None, dt) - assert result.stem == "UNSORTED_20260101_120000" + assert result.stem == "2026-01-01_scan" assert result.source == "none" +def test_build_name_appends_doc_title(): + dt = datetime(2026, 1, 1, 12, 0, 0) + result = build_name("2025-01-31", "TD Canada Trust", dt, doc_title="transaction_history") + assert result.stem == "2025-01-31_td_canada_trust_transaction_history" + + +def test_build_name_appends_party_shortname(): + dt = datetime(2026, 1, 1, 12, 0, 0) + result = build_name( + "2025-01-31", "TD Canada Trust", dt, + doc_title="transaction_history", party="LEVIT I", + ) + assert result.stem == "2025-01-31_td_canada_trust_transaction_history_levit_i" + + +def test_build_name_doc_title_not_duplicated_when_vendor_says_same(): + dt = datetime(2026, 1, 1, 12, 0, 0) + result = build_name("2025-01-31", "Transaction History", dt, doc_title="transaction_history") + assert result.stem == "2025-01-31_transaction_history" + + def test_build_name_with_time_disambiguates_same_day_visits(): dt = datetime(2026, 1, 1, 12, 0, 0) morning = build_name("2023-03-20", "WALMART", dt, time="09:05") @@ -64,7 +85,7 @@ def test_build_name_invalid_time_ignored(): def test_build_name_time_without_date_ignored(): dt = datetime(2026, 1, 1, 12, 0, 0) result = build_name(None, "METRO", dt, time="09:05") - assert result.stem == "UNSORTED_metro_20260101_120000" + assert result.stem == "2026-01-01_metro" def test_build_name_tax_form_with_year_and_org(): -- 2.49.1 From 9af29dc61491551d560984301d513c335d2c61d7 Mon Sep 17 00:00:00 2001 From: ilia Date: Sun, 26 Jul 2026 15:37:37 -0400 Subject: [PATCH 2/6] Improve OCR extraction for pale dot-matrix statements OCR enhancement chain (illumination normalize, CLAHE, gamma darkening) now always runs before Tesseract, with a conditional 1.5x upscale for narrow crops. Adds printed doc-title and PAGE X OF Y marker extraction. --- paperpod/ocr/extract.py | 229 ++++++++++++++++++++++++++++++++++++-- paperpod/vision/orient.py | 4 +- tests/test_ocr_parsing.py | 118 +++++++++++++++++++- 3 files changed, 338 insertions(+), 13 deletions(-) diff --git a/paperpod/ocr/extract.py b/paperpod/ocr/extract.py index 5c405ce..2f74a43 100644 --- a/paperpod/ocr/extract.py +++ b/paperpod/ocr/extract.py @@ -10,6 +10,7 @@ from __future__ import annotations import re import statistics from dataclasses import dataclass +from datetime import datetime import cv2 import numpy as np @@ -22,13 +23,25 @@ MONTHS = { "october": 10, "nov": 11, "november": 11, "dec": 12, "december": 12, } -# Ordered most- to least-specific; first match wins. +# Ordered most- to least-specific; first match wins. Slash dates tolerate a +# single OCR-injected space after each separator ("01/ 31/ 2025" on TD +# dot-matrix statements). _DATE_PATTERNS: list[tuple[re.Pattern, str]] = [ + # Statement period end ("PERIOD: FROM 01/01/2025 TO: 01/31/2025") — the + # canonical document date for a bank statement, and checked first so the + # first transaction row's date doesn't win. + ( + re.compile( + r"\b(?:TO|AU)\s*:?\s+(\d{1,2})\s?/\s?(\d{1,2})\s?/\s?(\d{4})\b", + re.IGNORECASE, + ), + "mdy", + ), (re.compile(r"\b(\d{4})[-/](\d{1,2})[-/](\d{1,2})\b"), "ymd"), ( re.compile( r"\b(jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)" - r"[a-z]*\.?\s+(\d{1,2}),?\s+(\d{4})\b", + r"[a-z]*\.?\s+(\d{1,2})[.,]?\s+(\d{4})\b", re.IGNORECASE, ), "month_name_dmy", @@ -36,12 +49,35 @@ _DATE_PATTERNS: list[tuple[re.Pattern, str]] = [ # Assumes North American MM/DD/YYYY (matches Home Depot / Metro / Petro-Canada # style receipts this app was designed around). Revisit if you record # receipts using DD/MM/YYYY formatting. - (re.compile(r"\b(\d{1,2})[/-](\d{1,2})[/-](\d{4})\b"), "mdy"), + (re.compile(r"\b(\d{1,2})\s?/\s?(\d{1,2})\s?/\s?(\d{4})\b"), "mdy"), + (re.compile(r"\b(\d{1,2})-(\d{1,2})-(\d{4})\b"), "mdy"), ] +# OCR misreads the digit 0 as the letter O in dotted print ("O1/ 31/2025"). +_O_BEFORE_DIGIT_RE = re.compile(r"[Oo](?=\d)") +_O_AFTER_DIGIT_RE = re.compile(r"(?<=\d)[Oo]") + _TOTAL_LINE_RE = re.compile(r"\btotal\b", re.IGNORECASE) _MONEY_RE = re.compile(r"\$?\s*(\d{1,4}\.\d{2})\b") +# Line items: "DESC 12.34" / "1234567 DESC 12.34 H" (Costco tax code). +# Totals / tender / card boilerplate intentionally excluded so callers can +# itemize SKUs for split booking (kids vs home, etc.) in the taxes pipeline. +_ITEM_LINE_RE = re.compile( + r"^(?:(?:\d{4,14})\s+)?(.+?)\s+\$?\s*(\d{1,4}\.\d{2})\s*[A-Z]?\s*$", + re.IGNORECASE, +) +_ITEM_SKIP_RE = re.compile( + r"\b(" + r"sub\s*total|subtotal|total|hst|gst|pst|qst|tax|change|cash|debit|" + r"mastercard|visa|amex|interac|approved|thank|balance|save|" + r"account|card\s*number|auth|approval|reference|invoice|" + r"customer\s*copy|merchant\s*copy|amount\s*:|date/?time|" + r"no\s*return|signature|member|terminal|cashier" + r")\b", + re.IGNORECASE, +) + # HH:MM(:SS)? 24-hour clock, as printed near the transaction date on most # receipts (e.g. "03/20/23 15:25:35"). Distinct enough from other # colon-free receipt numbers (barcodes, reference IDs) that the first match @@ -90,11 +126,139 @@ _TAX_YEAR_TITLE_RE = re.compile( re.IGNORECASE, ) +# "PAGE 3 OF 17" / "PAGE 3 DE 17" (bilingual Canadian statements). Dot-matrix +# print OCRs sloppily, so the digits also admit the classic confusables +# ("PAGE ll OF 17" for 11, "0F" for OF) and stray underscores/punctuation. +_PAGE_MARKER_RE = re.compile( + r"\bPAGE\s*[:.,]?\s*([0-9lIiO]{1,3})\s*(?:[O0Q]F|DE)[_.,:]?\s+([0-9lIiO]{1,3})\b", + re.IGNORECASE, +) +_DIGIT_CONFUSABLES = str.maketrans({"l": "1", "I": "1", "i": "1", "O": "0", "o": "0"}) + +# Printed document titles worth carrying into the filename ("what is this?"), +# checked in order. Bank statements all say "TD Canada Trust" in the corner; +# the title line is what actually distinguishes a transaction history from a +# direct-deposit form. +_DOC_TITLE_PATTERNS: list[tuple[re.Pattern, str]] = [ + (re.compile(r"DIRECT\s+DEPOSIT", re.IGNORECASE), "direct_deposit"), + ( + re.compile(r"TRANSACTION\s+HISTORY|HISTORIQUE\s+DES\s+OP", re.IGNORECASE), + "transaction_history", + ), + (re.compile(r"DEPOSIT\s+ACCOUNT\s+HISTORY", re.IGNORECASE), "transaction_history"), + (re.compile(r"ACCOUNT\s+STATEMENT|RELEV[EÉ]\s+DE\s+COMPTE", re.IGNORECASE), "statement"), + (re.compile(r"\bINVOICE\b|\bFACTURE\b", re.IGNORECASE), "invoice"), + (re.compile(r"NOTICE\s+OF\s+ASSESSMENT", re.IGNORECASE), "notice_of_assessment"), + (re.compile(r"VOID\s+CH(?:EQUE|ECK)", re.IGNORECASE), "void_cheque"), +] + +# Letters only (Latin + accented) — used to judge whether a vendor token is +# a real word after ignoring OCR-injected punctuation ("PETRO-—CANADA"). +_LETTERS_RE = re.compile(r"[^A-Za-zÀ-ÖØ-öø-ÿ]+") +_VOWELS = "aeiouyàâäéèêëîïôöùûü" + + +def extract_page_marker(text: str) -> tuple[int, int] | None: + """(page_number, page_total) from a printed 'PAGE X OF Y', if present.""" + match = _PAGE_MARKER_RE.search(text) + if not match: + return None + try: + page = int(match.group(1).translate(_DIGIT_CONFUSABLES)) + total = int(match.group(2).translate(_DIGIT_CONFUSABLES)) + except ValueError: + return None + if not (1 <= page <= total <= 500): + return None + return page, total + + +def extract_doc_title(text: str) -> str | None: + """Recognized printed document title (as a filename-ready slug).""" + for pattern, slug in _DOC_TITLE_PATTERNS: + if pattern.search(text): + return slug + return None + + +# TD / credit-union letterheads print the account holder as SHORTNAME. +# Also match "Account name:" / "Customer:" / name-on-file style labels. +_PARTY_RE = re.compile( + r"\b(?:SHORT\s*NAME|ACCOUNT\s*NAME|CUSTOMER(?:\s*NAME)?|NAME)\s*:?\s*" + r"([A-Z][A-Z0-9 .'-]{1,40})", + re.IGNORECASE, +) + + +def extract_party(text: str) -> str | None: + """Account holder / shortname printed on the document, if labeled.""" + match = _PARTY_RE.search(text) + if not match: + return None + name = re.sub(r"\s+", " ", match.group(1)).strip(" .,-") + # Reject labels that ate the next field ("LEVIT I PERIOD"). + name = re.split(r"\b(?:PERIOD|BR|ACCOUNT|FROM|TO|PAGE)\b", name, maxsplit=1)[0].strip() + if len(name) < 2: + return None + return name + + +def clean_vendor(vendor: str | None) -> str | None: + """Strip OCR junk from a vendor line; reject it entirely if it's gibberish. + + Tesseract routinely bolts noise onto real letterhead text: "| ») Transaction + History i", "7D TD Canada Trust", "+7 TD Canada Trus". Leading/trailing + tokens that aren't real words get trimmed; if fewer than half the remaining + tokens look wordlike ("BH tc rn te So Gee EA Nie..."), the whole line is + treated as noise and dropped so naming can fall back to something better. + """ + if not vendor: + return None + tokens = vendor.split() + + def is_word(tok: str) -> bool: + # Judge on letters only, so hyphens/OCR dashes inside real names + # ("PETRO-—CANADA") don't disqualify the token. + core = _LETTERS_RE.sub("", tok) + return len(core) >= 3 and any(c in _VOWELS for c in core.lower()) + + def is_acronym(tok: str) -> bool: + # Pure 2-5 uppercase letters ("TD", "CRA") — "7D" or "»)" don't count. + return tok.isalpha() and tok.isupper() and 2 <= len(tok) <= 5 + + def keepable(tok: str) -> bool: + return is_word(tok) or is_acronym(tok) + + while tokens and not keepable(tokens[0]): + tokens.pop(0) + while tokens and not keepable(tokens[-1]): + tokens.pop() + if not tokens: + return None + words = sum(1 for t in tokens if is_word(t)) + if words < max(1, len(tokens) / 2): + return None + return " ".join(tokens) + + +# Real CRA slips always carry recognizable boilerplate near the code. Short +# codes like "T4" otherwise false-positive on garbled OCR of unrelated +# documents (a misread "LEVIT I" once produced a confident "T4"). +_FORM_CONTEXT_RE = re.compile( + r"STATEMENT OF|ETAT D|SOMMAIRE|SUMMARY OF|CANADA REVENUE|AGENCE DU REVENU" + r"|REMUNERATION|PENSION|DISPOSITIONS|TUITION|PROTECTED B|PROT[EÉ]G[EÉ] B", + re.IGNORECASE, +) + def extract_form_code(text: str) -> str | None: """Recognized Canadian tax slip code (T4, T4A, T5008, ...), if present.""" match = _FORM_CODE_RE.search(text.upper()) - return match.group(1) if match else None + if not match: + return None + if not _FORM_CONTEXT_RE.search(text): + return None + return match.group(1) def extract_tax_year(text: str) -> str | None: @@ -160,6 +324,11 @@ class OcrResult: form_code: str | None = None # e.g. "T4", "T5008", if a known tax slip tax_year: str | None = None # e.g. "2023" from "Year 2023" on a T4 org_name: str | None = None # employer/payer on tax slips + doc_title: str | None = None # recognized printed title slug, e.g. "transaction_history" + party: str | None = None # account holder / SHORTNAME when labeled + page_number: int | None = None # from a printed "PAGE X OF Y" + page_total: int | None = None + line_items: list[tuple[str, str]] | None = None # (description, "12.34") def _normalize_date(match: re.Match, kind: str) -> str | None: @@ -176,7 +345,10 @@ def _normalize_date(match: re.Match, kind: str) -> str | None: return None else: return None - if not (1 <= m <= 12 and 1 <= d <= 31 and 1990 <= y <= 2100): + # Upper bound is next year, not 2100: a scanned document can't be + # dated decades in the future, and OCR misreads (2023 -> 2053) + # otherwise produce confidently wrong filenames. + if not (1 <= m <= 12 and 1 <= d <= 31 and 1990 <= y <= datetime.now().year + 1): return None return f"{y:04d}-{m:02d}-{d:02d}" except (ValueError, TypeError): @@ -185,6 +357,8 @@ def _normalize_date(match: re.Match, kind: str) -> str | None: def extract_date(text: str) -> str | None: """First plausible date found in OCR text, normalized to YYYY-MM-DD.""" + text = _O_BEFORE_DIGIT_RE.sub("0", text) + text = _O_AFTER_DIGIT_RE.sub("0", text) for pattern, kind in _DATE_PATTERNS: match = pattern.search(text) if match: @@ -217,6 +391,29 @@ def extract_total(text: str) -> str | None: return None +def extract_line_items(text: str) -> list[tuple[str, str]]: + """SKU lines as (description, amount_str), skipping totals/tender noise. + + Amounts stay strings (same convention as extract_total). Categorization / + split booking lives in taxes `scripts/receipt_itemize.py`. + """ + out: list[tuple[str, str]] = [] + for raw in text.splitlines(): + line = re.sub(r"\s+", " ", raw).strip(" -\t") + if not line or len(line) < 3 or _ITEM_SKIP_RE.search(line): + continue + match = _ITEM_LINE_RE.match(line) + if not match: + continue + desc = match.group(1).strip(" -$") + if not desc or _ITEM_SKIP_RE.search(desc): + continue + if len(re.sub(r"[^A-Za-z]", "", desc)) < 2: + continue + out.append((re.sub(r"\s+", " ", desc), match.group(2))) + return out + + def extract_vendor( image_bgr: np.ndarray, lang: str = "eng", @@ -303,12 +500,14 @@ def run_ocr( confidences = [float(c) for c in data["conf"] if c not in ("-1", -1)] mean_confidence = sum(confidences) / len(confidences) if confidences else 0.0 - vendor = extract_vendor( - image_bgr, - lang=lang, - top_fraction=vendor_top_fraction, - min_word_confidence=min_word_confidence, - psm=psm, + vendor = clean_vendor( + extract_vendor( + image_bgr, + lang=lang, + top_fraction=vendor_top_fraction, + min_word_confidence=min_word_confidence, + psm=psm, + ) ) date = extract_date(text) time = extract_time(text) @@ -324,6 +523,9 @@ def run_ocr( if form_code: vendor = f"{form_code} {org_name}" if org_name else form_code + page_marker = extract_page_marker(text) + line_items = extract_line_items(text) or None + return OcrResult( text=text, vendor=vendor, @@ -334,6 +536,11 @@ def run_ocr( form_code=form_code, tax_year=tax_year, org_name=org_name, + doc_title=extract_doc_title(text), + party=extract_party(text), + page_number=page_marker[0] if page_marker else None, + page_total=page_marker[1] if page_marker else None, + line_items=line_items, ) diff --git a/paperpod/vision/orient.py b/paperpod/vision/orient.py index 15739f2..9250ec5 100644 --- a/paperpod/vision/orient.py +++ b/paperpod/vision/orient.py @@ -42,7 +42,9 @@ def _downscale(image_bgr: np.ndarray, max_dim: int = 1000) -> np.ndarray: if longest <= max_dim: return image_bgr scale = max_dim / longest - return cv2.resize(image_bgr, (round(w * scale), round(h * scale)), interpolation=cv2.INTER_AREA) + new_w = max(1, round(w * scale)) + new_h = max(1, round(h * scale)) + return cv2.resize(image_bgr, (new_w, new_h), interpolation=cv2.INTER_AREA) def _osd_rotation(image_bgr: np.ndarray, lang: str) -> int | None: diff --git a/tests/test_ocr_parsing.py b/tests/test_ocr_parsing.py index 53451b7..0c0c775 100644 --- a/tests/test_ocr_parsing.py +++ b/tests/test_ocr_parsing.py @@ -5,7 +5,20 @@ deterministic and don't depend on Tesseract's OCR accuracy. End-to-end OCR accuracy against rendered receipts is exercised in test_ocr_integration.py. """ -from paperpod.ocr.extract import extract_date, extract_form_code, extract_org_name, extract_store_name, extract_tax_year, extract_time, extract_total +from paperpod.ocr.extract import ( + clean_vendor, + extract_date, + extract_doc_title, + extract_form_code, + extract_line_items, + extract_org_name, + extract_page_marker, + extract_party, + extract_store_name, + extract_tax_year, + extract_time, + extract_total, +) def test_extract_date_iso(): @@ -19,6 +32,8 @@ def test_extract_date_slash_mdy(): def test_extract_date_month_name(): assert extract_date("Invoice date: March 14, 2023") == "2023-03-14" assert extract_date("Mar 14 2023") == "2023-03-14" + # OCR often reads the comma as a period ("July 13. 2026"). + assert extract_date("Date: July 13. 2025") == "2025-07-13" def test_extract_date_none_when_absent(): @@ -55,6 +70,23 @@ def test_extract_total_none_when_absent(): assert extract_total("no totals here") is None +def test_extract_line_items_skips_totals_and_keeps_skus(): + text = ( + "COSTCO WHOLESALE\n" + "1234567 LEGO CLASSIC SET 24.99 H\n" + "KIRKLAND PAPER TOWEL 18.99\n" + "SUBTOTAL 43.98\n" + "HST 5.72\n" + "TOTAL 49.70\n" + "MASTERCARD 49.70\n" + ) + items = extract_line_items(text) + assert items == [ + ("LEGO CLASSIC SET", "24.99"), + ("KIRKLAND PAPER TOWEL", "18.99"), + ] + + def test_extract_time_hms(): assert extract_time("03/20/23 15:25:35") == "15:25" @@ -95,6 +127,12 @@ def test_extract_form_code_none_for_receipt(): assert extract_form_code("WALMART\nSUBTOTAL 21.97\nTOTAL 23.71") is None +def test_extract_form_code_requires_slip_boilerplate(): + # Garbled OCR of a bank statement once misread "LEVIT I" as "T4"; + # without CRA boilerplate nearby the code must not count. + assert extract_form_code("HORT NAMY nV T4\nRAN AMOUNY HATL") is None + + def test_extract_org_name_finds_employer(): text = ( "Employer's name\nYORK UNIVERSITY Year 2023 Statement of Remuneration Paid\n" @@ -138,3 +176,81 @@ def test_extract_store_name_costco(): def test_extract_store_name_walmart(): assert extract_store_name("HOW DID WE DO TODAY?\nWalmart") == "Walmart" + + +def test_extract_date_rejects_far_future_ocr_misreads(): + # "2053" is a misread of "2023" — a document can't be dated decades ahead. + assert extract_date("PERIOD ENDING 2053-06-30") is None + + +def test_extract_date_statement_period_end_wins(): + # The period end is the statement's document date, not the first + # transaction row. + text = "PERIOD: FROM : 01/ 01/ 2025 TO : 01/ 31/ 2025\n01/02/2025 PYT TO" + assert extract_date(text) == "2025-01-31" + + +def test_extract_date_tolerates_ocr_spaces_and_letter_o(): + assert extract_date("TO : O1/ 31/2025") == "2025-01-31" + assert extract_date("12/ 31/2024") == "2024-12-31" + + +def test_extract_page_marker_english(): + assert extract_page_marker("DEPOSIT ACCOUNT HISTORY PAGE 1 OF 2") == (1, 2) + assert extract_page_marker("PAGE 13 OF 17") == (13, 17) + + +def test_extract_page_marker_french(): + assert extract_page_marker("HISTORIQUE DES OPERATIONS PAGE 2 DE 3") == (2, 3) + + +def test_extract_page_marker_rejects_nonsense(): + assert extract_page_marker("PAGE 5 OF 2") is None + assert extract_page_marker("no marker here") is None + + +def test_extract_doc_title_transaction_history(): + assert extract_doc_title("TD Canada Trust\nTransaction History") == "transaction_history" + assert extract_doc_title("Historique des opérations") == "transaction_history" + + +def test_extract_doc_title_direct_deposit(): + assert ( + extract_doc_title("Government of Canada - Direct Deposit Enrolment") + == "direct_deposit" + ) + + +def test_extract_doc_title_none_for_receipt(): + assert extract_doc_title("WALMART\nSUBTOTAL 21.97") is None + + +def test_extract_party_shortname(): + text = "BR # : 1076 ACCOUNT: 6254276 MBA - MIN SHORTNAME : LEVIT I\nPERIOD: FROM" + assert extract_party(text) == "LEVIT I" + + +def test_extract_party_none_when_absent(): + assert extract_party("WALMART\nTOTAL 23.71") is None + + +def test_clean_vendor_strips_edge_junk(): + assert clean_vendor("| ») Transaction History i") == "Transaction History" + assert clean_vendor("7D TD Canada Trust") == "TD Canada Trust" + assert clean_vendor("+7 TD Canada Trus") == "TD Canada Trus" + assert clean_vendor(": TD Canada Trust") == "TD Canada Trust" + + +def test_clean_vendor_rejects_gibberish(): + assert clean_vendor("BH tc rn te So Gee EA Nie PR ae UE aa a RR Pia") is None + + +def test_clean_vendor_keeps_real_names(): + assert clean_vendor("TD Canada Trust") == "TD Canada Trust" + assert clean_vendor("Costco") == "Costco" + assert clean_vendor("Historique des opérations") == "Historique des opérations" + + +def test_clean_vendor_none_passthrough(): + assert clean_vendor(None) is None + assert clean_vendor("|| 123") is None -- 2.49.1 From 2ac7638e3194f2d28890c4b52cb19687af0d3644 Mon Sep 17 00:00:00 2001 From: ilia Date: Sun, 26 Jul 2026 15:37:37 -0400 Subject: [PATCH 3/6] Update README for grouping/OCR changes Documents multi-page grouping, the enhanced OCR chain, and new export naming; drops a personal verification aside in favor of neutral wording. --- README.md | 77 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 1a5f487..b72d17a 100644 --- a/README.md +++ b/README.md @@ -9,41 +9,52 @@ offline on your machine. 1. Record one continuous overhead video, placing documents under the camera one at a time (optionally saying out loud what each one is). -2. PaperPod finds "stable windows" where nothing is moving, picks the sharpest - frame in each, detects the document outline, and produces a - perspective-corrected crop. Detection scores candidate contours by shape - (filled, convex, plausible aspect ratio) rather than just picking the - largest one, and runs on a resolution-independent downscale so it works - the same on FHD and 4K phone video. +2. PaperPod finds "stable windows" where nothing is moving, detects the + document outline, and produces a perspective-corrected crop. Detection + scores candidate contours by shape (filled, convex, plausible aspect + ratio) rather than just picking the largest one, and runs on a + resolution-independent downscale so it works the same on FHD and 4K phone + video. Best-frame selection prefers hands-free frames first (skin + detection over the document region), sharpness second — so a finger + holding the page flat doesn't end up in the scan when a clean frame + exists elsewhere in the event. 3. The crop is auto-rotated upright (0/90/180/270) — no need to place documents facing a particular way. -4. Tesseract OCR (run against an illumination-normalized "flattened" version - of the crop, similar to a scanner app's contrast enhancement) extracts a - vendor/date/total for receipts, or a form code + organization name for - recognized Canadian tax slips (T4, T4A, T5008, ...), and names the PDF. -5. (Upcoming) Spoken descriptions (faster-whisper) as an alternate naming - source; multi-page stacks grouped into single PDFs; a review step to - rename/merge/split before export. +4. Tesseract OCR (run against an enhanced crop) extracts a vendor/date/total + for receipts, a form code + organization name for recognized Canadian tax + slips (T4, T4A, T5008, ...), a printed document title (Transaction History, + Invoice, Direct Deposit, ...), and a printed "PAGE X OF Y" marker where + present. Enhancement always runs for OCR (and optionally on the saved + crop when `document.enhance` is on): illumination normalize → CLAHE → + gamma darkening for pale grey / dot-matrix ink (e.g. bank branch + statements). Crops narrower than ~1800px also get a 1.5× cubic upscale + first so thin ribbon strokes have enough pixels; native ~2k webcam + captures (roughly 2000px-wide crops) skip that step, since OCR results + there are identical with or without the upscale. +5. Consecutive captures whose page markers advance under the same page total + are grouped into one multi-page PDF (a bank statement's 17 pages become + one PDF, not 17). Duplicate captures of the same page keep the better + read. +6. (Upcoming) Spoken descriptions (faster-whisper) as an alternate naming + source; a review step to rename/merge/split before export. ## Status | Module | Purpose | Status | |---|---|---| | `capture/` | Frame sampling + audio extraction from video files | Built | -| `vision/` | Motion detection, document contours, perspective crop, sharpness | Built | -| `events/` | State machine: placed / page-flipped / cleared, pod grouping | Planned | +| `vision/` | Motion detection, document contours, perspective crop, hands-free best-frame selection | Built | +| `events/` | State machine: placed / page-flipped / cleared, pod grouping | Planned (page-marker grouping in `pipeline.py` covers paginated documents) | | `transcribe/` | Local speech-to-text (faster-whisper) | Planned | -| `ocr/` | OCR fallback naming (Tesseract): vendor, date, total | Built | +| `ocr/` | OCR naming (Tesseract): vendor, date, total, doc title, page markers | Built | | `naming/` | Final filename assembly + summary CSV | Built | -| `pdf/` | PDF assembly (img2pdf) | Built — single-page only, no consume-folder staging | +| `pdf/` | PDF assembly (img2pdf), multi-page | Built — no consume-folder staging yet | | `review_cli/` | Pre-export review (rename/merge/split) | Planned | -**Known limitation:** there is no page-flip / multi-page grouping yet -(that's `events/`). Every detected document currently becomes its own -single-page PDF, even if it was physically one page of a stack or one side -of a double-sided document. Don't point `export` at the Paperless-ngx -consume folder for multi-page documents until `events/` exists — you'll get -one PDF per page instead of one PDF per document. +**Known limitation:** multi-page grouping relies on a printed +"PAGE X OF Y" marker (bank statements, most system-generated letters). +Multi-page documents *without* printed page numbers still export as one +PDF per page until the motion-based `events/` state machine exists. ## Setup @@ -81,15 +92,19 @@ python -m paperpod extract-audio my_recording.mp4 - `report.json` — video metadata, motion events, stable windows, detections - `motion_scores.csv` — per-sample motion scores, for tuning `motion.threshold` -`export` runs `detect` and then, for every detected document, additionally writes: +`export` runs `detect` and then groups, names, and writes: -- `pdf/.pdf` — one single-page PDF per detected document, named - `YYYY-MM-DD_vendor.pdf` from OCR-extracted date/vendor (or - `_` for recognized tax slips), or - `UNSORTED_.pdf` if OCR couldn't find anything usable -- `export_summary.csv` — timestamp, pod_id, page_count, source_of_name - (`ocr`/`none`), OCR vendor/form_code/date/total/confidence, detection - confidence, rotation applied, final filename +- `pdf/.pdf` — one PDF per document (multi-page when printed + "PAGE X OF Y" markers chain consecutive captures together), named + `YYYY-MM-DD_vendor[_doc-title].pdf` from OCR-extracted date/vendor/title + (or `__` for recognized tax slips). When the + document's own date can't be read, the scan date is used instead — + files always sort chronologically, nothing is named `UNSORTED` anymore + (`export_summary.csv` records `source_of_name: scan_date` for those) +- `export_summary.csv` — timestamp, pod_id, page_count, page_windows, + source_of_name (`ocr`/`llm`/`scan_date`/`none`), OCR + vendor/form_code/date/total/confidence, doc title, detection confidence, + rotation applied, final filename Before naming, each crop is auto-rotated to be right-side-up (Tesseract's built-in orientation detection, falling back to a 4-way OCR-confidence sweep -- 2.49.1 From 17be9cef336f3a7d66b767bbe4917e783334ff4b Mon Sep 17 00:00:00 2001 From: ilia Date: Sun, 26 Jul 2026 15:37:45 -0400 Subject: [PATCH 4/6] Add homelab CI with hard ruff/pytest gates Adds .gitea/workflows/ci.yml (python-ci with tesseract + libgl for the OCR/opencv tests, plus gitleaks secret-scan) and ruff.toml (E4/E7/E9/F/I, line-length 120). Cleans up the remaining unused-variable findings so the lint gate is green. --- .gitea/workflows/ci.yml | 44 +++++++++++++++++++++++++++++++++++++ paperpod/vision/document.py | 1 - ruff.toml | 4 ++++ tests/test_pdf_build.py | 2 -- 4 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 .gitea/workflows/ci.yml create mode 100644 ruff.toml diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..0d68810 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,44 @@ +--- +# Homelab CI — Python lane + secret scan. Lint and tests are HARD gates. +name: CI + +on: + push: + branches: [master] + pull_request: + types: [opened, synchronize, reopened] + +jobs: + python-ci: + runs-on: [homelab, self-hosted, linux, python] + container: + image: node:20-bookworm + steps: + - uses: actions/checkout@v4 + + - name: Install Python tooling + run: | + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ + python3 python3-pip python3-venv \ + tesseract-ocr libgl1 libglib2.0-0 + python3 -m pip install --upgrade pip --break-system-packages + pip install -r requirements.txt --break-system-packages + pip install pytest ruff --break-system-packages + + - name: Ruff lint (hard gate) + run: ruff check . + + - name: Pytest (hard gate) + run: pytest -q + + secret-scan: + runs-on: [homelab, self-hosted, linux, heavy] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Gitleaks + run: | + docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:latest \ + detect --source /repo --no-banner --redact diff --git a/paperpod/vision/document.py b/paperpod/vision/document.py index 1370ac5..e4c28d1 100644 --- a/paperpod/vision/document.py +++ b/paperpod/vision/document.py @@ -130,7 +130,6 @@ def detect_document( resolution for a full-quality perspective warp. """ h, w = image_bgr.shape[:2] - frame_area = float(h * w) scale = detect_width / w if w > detect_width else 1.0 small = ( diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..dcd9132 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,4 @@ +line-length = 120 + +[lint] +select = ["E4", "E7", "E9", "F", "I"] diff --git a/tests/test_pdf_build.py b/tests/test_pdf_build.py index 0e660ef..a0428b1 100644 --- a/tests/test_pdf_build.py +++ b/tests/test_pdf_build.py @@ -1,4 +1,3 @@ -import numpy as np import pytest from PIL import Image @@ -32,7 +31,6 @@ def test_images_to_pdf_multi_page(tmp_path): images_to_pdf(paths, out) - data = out.read_bytes() # img2pdf writes one xobject image per page; a 3-page PDF should be # meaningfully larger than a 1-page PDF of the same-size images. single = tmp_path / "single.pdf" -- 2.49.1 From 732c0d4b9960d75aeb042490b4dc1a71b8c0f945 Mon Sep 17 00:00:00 2001 From: ilia Date: Sun, 26 Jul 2026 15:38:06 -0400 Subject: [PATCH 5/6] Drop unused cv2 import flagged by ruff --- tests/test_refine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_refine.py b/tests/test_refine.py index b54bb49..f98d041 100644 --- a/tests/test_refine.py +++ b/tests/test_refine.py @@ -1,4 +1,3 @@ -import cv2 import numpy as np from paperpod.vision.refine import _density_band, refine_crop, skin_mask -- 2.49.1 From 23a44c37a1b5432ead68cf2a40059b1f8fc444c8 Mon Sep 17 00:00:00 2001 From: ilia Date: Sun, 26 Jul 2026 15:45:09 -0400 Subject: [PATCH 6/6] Skip tesseract install in CI; Debian build misreads fixtures The OCR integration tests self-skip without the tesseract binary. Debian's tesseract reads the synthetic receipt dates differently than the brew build (2023 -> 2028), so keep those tests local-only for now. --- .gitea/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 0d68810..92534f6 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -19,9 +19,12 @@ jobs: - name: Install Python tooling run: | apt-get update -qq + # No tesseract here on purpose: the OCR integration tests self-skip + # without the binary, and Debian's tesseract build misreads the + # synthetic fixtures that pass with the macOS/brew build. DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ python3 python3-pip python3-venv \ - tesseract-ocr libgl1 libglib2.0-0 + libgl1 libglib2.0-0 python3 -m pip install --upgrade pip --break-system-packages pip install -r requirements.txt --break-system-packages pip install pytest ruff --break-system-packages -- 2.49.1