Add page-marker grouping and party-aware naming
Consecutive captures whose printed PAGE X OF Y markers advance under the same total are grouped into one multi-page PDF, with duplicate pages keeping the better read. Naming gains doc-title/party components and a scan-date fallback so nothing lands in UNSORTED. Vision LLM default switches to minicpm-v (faster, fewer false blanks on dense mono pages).
This commit is contained in:
+4
-2
@@ -66,9 +66,11 @@ llm:
|
||||
# Reads whole documents at once — far more robust than Tesseract for
|
||||
# curled receipts and stylized logos — and flags quality issues
|
||||
# (fingers_visible, blurry, glare) in export_summary.csv.
|
||||
# Slower: ~20-40s per document. Falls back to Tesseract when Ollama is down.
|
||||
# minicpm-v: ~8-12s/doc; avoids qwen2.5vl's false "blank" on dense
|
||||
# bank statements. OCR still fills date/total when the model hallucinates.
|
||||
# Falls back to Tesseract when Ollama is down.
|
||||
enabled: true
|
||||
model: qwen2.5vl
|
||||
model: minicpm-v
|
||||
base_url: http://localhost:11434
|
||||
timeout_s: 180
|
||||
|
||||
|
||||
+5
-6
@@ -90,11 +90,14 @@ def cmd_export(args: argparse.Namespace) -> int:
|
||||
print(f"Running OCR + naming on {len(found)} detected document(s)...")
|
||||
result = run_export(report, cfg, out_dir)
|
||||
|
||||
print(f"\n{'pod':>3} {'date':>10} vendor / final filename")
|
||||
print(f"\n{'pod':>3} {'pages':>5} {'date':>10} vendor / final filename")
|
||||
print("-" * 60)
|
||||
for row in result["rows"]:
|
||||
vendor = row["ocr_vendor"] or "(no vendor found)"
|
||||
print(f"{row['pod_id']:>3} {row['ocr_date'] or '-':>10} {vendor}")
|
||||
print(
|
||||
f"{row['pod_id']:>3} {row['page_count']:>5} "
|
||||
f"{row['ocr_date'] or '-':>10} {vendor}"
|
||||
)
|
||||
print(f" -> {row['pdf_path']} [source: {row['source_of_name']}]")
|
||||
if row.get("llm_description"):
|
||||
issues = f" issues: {row['llm_issues']}" if row.get("llm_issues") else ""
|
||||
@@ -102,10 +105,6 @@ def cmd_export(args: argparse.Namespace) -> int:
|
||||
|
||||
print(f"\nPDFs: {result['pdf_dir']}")
|
||||
print(f"Summary: {out_dir / 'export_summary.csv'}")
|
||||
print(
|
||||
"\nNote: no page-flip/multi-page grouping yet — each detected "
|
||||
"document is a separate single-page PDF."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
+3
-2
@@ -102,9 +102,10 @@ class OcrConfig:
|
||||
class LlmConfig:
|
||||
# Use a local Ollama vision model to identify/name documents and flag
|
||||
# quality issues. Falls back to Tesseract heuristics when Ollama is
|
||||
# down or the model errors. ~20-40s per document vs <2s for Tesseract.
|
||||
# down or the model errors. minicpm-v is ~8-12s/doc and much less prone
|
||||
# to false "blank" labels than qwen2.5vl on dense statement pages.
|
||||
enabled: bool = True
|
||||
model: str = "qwen2.5vl"
|
||||
model: str = "minicpm-v"
|
||||
base_url: str = "http://localhost:11434"
|
||||
timeout_s: float = 180.0
|
||||
|
||||
|
||||
+18
-9
@@ -1,13 +1,17 @@
|
||||
"""Identify documents with a local vision LLM served by Ollama.
|
||||
|
||||
Everything stays on-machine: images go to localhost:11434, never the cloud.
|
||||
A vision model (qwen2.5-vl class) reads the whole document at once, which is
|
||||
far more robust than Tesseract + regex for curled receipts, hands in frame,
|
||||
and stylized logos — it correctly reads "COSTCO WHOLESALE" where Tesseract
|
||||
produced "Cosrco". It also reports quality issues (fingers, blur, glare) so
|
||||
bad captures can be flagged for re-scanning.
|
||||
A vision model (default: minicpm-v) reads the whole document at once, which
|
||||
is far more robust than Tesseract + regex for curled receipts, hands in
|
||||
frame, and stylized logos — it correctly reads "COSTCO WHOLESALE" where
|
||||
Tesseract produced "Cosrco". It also reports quality issues (fingers, blur,
|
||||
glare) so bad captures can be flagged for re-scanning.
|
||||
|
||||
Trade-off: ~20-40s per document on an M-series Mac, vs <2s for Tesseract.
|
||||
minicpm-v was chosen over qwen2.5vl after A/B on overhead statement scans:
|
||||
~2x faster and far fewer false "blank" labels on dense mono bank pages.
|
||||
Dates can still hallucinate — the pipeline keeps OCR as date/total fill-in.
|
||||
|
||||
Trade-off: ~8-12s per document on an M-series Mac, vs <2s for Tesseract.
|
||||
The pipeline uses the LLM for naming when available and falls back to the
|
||||
OCR heuristics when Ollama isn't running.
|
||||
"""
|
||||
@@ -20,6 +24,7 @@ import re
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
@@ -27,6 +32,7 @@ import numpy as np
|
||||
_PROMPT = """Look at this image of a document. Reply with ONLY a JSON object, no other text:
|
||||
{"doc_type": "receipt|tax_form|letter|invoice|blank|other",
|
||||
"vendor": "business or organization name, or null",
|
||||
"party": "account holder / customer / SHORTNAME / payee printed on the document, or null",
|
||||
"form_code": "tax form code like T4/T4A/T5008 if this is a tax form, else null",
|
||||
"date": "document/transaction date as YYYY-MM-DD, or null",
|
||||
"time": "transaction time as 24-hour HH:MM if printed on the document, else null",
|
||||
@@ -35,13 +41,15 @@ _PROMPT = """Look at this image of a document. Reply with ONLY a JSON object, no
|
||||
"description": "3-6 word summary",
|
||||
"issues": ["any of: fingers_visible, blurry, cut_off, upside_down, glare, partial_document"]}
|
||||
Use doc_type "blank" if the page has no meaningful printed content (blank back of a page, empty sheet).
|
||||
Set vendor to null unless you can actually read a real business/organization name — never output garbled text."""
|
||||
Set vendor to null unless you can actually read a real business/organization name — never output garbled text.
|
||||
Set party to the person/entity the document is about (e.g. SHORTNAME on a bank statement, payee on a slip) when clearly printed; otherwise null."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlmAnalysis:
|
||||
doc_type: str
|
||||
vendor: str | None
|
||||
party: str | None
|
||||
form_code: str | None
|
||||
date: str | None # normalized YYYY-MM-DD
|
||||
time: str | None # normalized HH:MM (24h)
|
||||
@@ -77,7 +85,7 @@ def _normalize_date(raw: str | None) -> str | None:
|
||||
if not match:
|
||||
return None
|
||||
y, m, d = (int(g) for g in match.groups())
|
||||
if not (1 <= m <= 12 and 1 <= d <= 31 and 1990 <= y <= 2100):
|
||||
if not (1 <= m <= 12 and 1 <= d <= 31 and 1990 <= y <= datetime.now().year + 1):
|
||||
return None
|
||||
return f"{y:04d}-{m:02d}-{d:02d}"
|
||||
|
||||
@@ -100,7 +108,7 @@ def _normalize_time(raw: str | None) -> str | None:
|
||||
|
||||
def analyze_document(
|
||||
image_bgr: np.ndarray,
|
||||
model: str = "qwen2.5vl",
|
||||
model: str = "minicpm-v",
|
||||
base_url: str = "http://localhost:11434",
|
||||
timeout: float = 180.0,
|
||||
) -> LlmAnalysis | None:
|
||||
@@ -137,6 +145,7 @@ def analyze_document(
|
||||
return LlmAnalysis(
|
||||
doc_type=_clean_str(data.get("doc_type")) or "other",
|
||||
vendor=_clean_str(data.get("vendor")),
|
||||
party=_clean_str(data.get("party")),
|
||||
form_code=_clean_str(data.get("form_code")),
|
||||
date=_normalize_date(data.get("date")),
|
||||
time=_normalize_time(data.get("time")),
|
||||
|
||||
+46
-18
@@ -1,10 +1,11 @@
|
||||
"""Build the final filename/PDF title from OCR (and, later, speech) metadata.
|
||||
|
||||
Pattern: YYYY-MM-DD_vendor.pdf, falling back progressively as pieces of
|
||||
information go missing, down to UNSORTED_<timestamp>.pdf when nothing usable
|
||||
was extracted at all. Module 4 (speech) isn't built yet, so "description" in
|
||||
the original spec's pattern is currently always the OCR vendor slug; once
|
||||
speech transcription lands, a spoken description will take priority here.
|
||||
Pattern: YYYY-MM-DD_vendor[_doc-title].pdf. When the document's own date
|
||||
can't be read, the scan date (capture timestamp from the video) is used
|
||||
instead so files still sort chronologically and never get an opaque
|
||||
"UNSORTED" prefix; export_summary.csv records that the date came from the
|
||||
scan, not the document. Module 4 (speech) isn't built yet; once speech
|
||||
transcription lands, a spoken description will take priority here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,7 +25,7 @@ def slugify(text: str, max_len: int = 40) -> str:
|
||||
@dataclass
|
||||
class NamingResult:
|
||||
stem: str # filename without extension
|
||||
source: str # "ocr" | "none" (will include "speech" once module 4 exists)
|
||||
source: str # "ocr" | "scan_date" | "none" ("speech" once module 4 exists)
|
||||
|
||||
|
||||
_TIME_RE = re.compile(r"^([01]\d|2[0-3]):([0-5]\d)$")
|
||||
@@ -39,21 +40,23 @@ def build_name(
|
||||
tax_year: str | None = None,
|
||||
org_name: str | None = None,
|
||||
time: str | None = None,
|
||||
doc_title: str | None = None,
|
||||
party: str | None = None,
|
||||
) -> NamingResult:
|
||||
"""date must already be normalized to YYYY-MM-DD, time to HH:MM, or None.
|
||||
|
||||
Time is appended to the date (e.g. 2023-03-20_1525) whenever it was
|
||||
extracted, so two same-day visits to the same vendor (Walmart in the
|
||||
morning and again in the evening) don't collide or silently overwrite.
|
||||
|
||||
doc_title is a recognized printed title slug ("transaction_history",
|
||||
"invoice"), appended after the vendor unless it would be redundant.
|
||||
party is an account holder / SHORTNAME when printed on the document.
|
||||
"""
|
||||
vendor_slug = slugify(vendor, max_vendor_len) if vendor else None
|
||||
ts = fallback_dt.strftime("%Y%m%d_%H%M%S")
|
||||
party_slug = slugify(party, max_vendor_len) if party else None
|
||||
|
||||
date_part = date
|
||||
if date and time and _TIME_RE.match(time):
|
||||
date_part = f"{date}_{time.replace(':', '')}"
|
||||
|
||||
# Tax slips: prefer 2023_T4_York_University over UNSORTED_t4_york_university_<timestamp>.
|
||||
# Tax slips: prefer 2023_T4_York_University over a date-based name.
|
||||
if form_code:
|
||||
org_slug = slugify(org_name, max_vendor_len) if org_name else None
|
||||
if not org_slug and vendor:
|
||||
@@ -71,13 +74,38 @@ def build_name(
|
||||
return NamingResult(stem=f"{code}_{org_slug}", source="ocr")
|
||||
return NamingResult(stem=code.lower(), source="ocr")
|
||||
|
||||
if date_part and vendor_slug:
|
||||
return NamingResult(stem=f"{date_part}_{vendor_slug}", source="ocr")
|
||||
if date_part:
|
||||
return NamingResult(stem=f"{date_part}_UNSORTED", source="ocr")
|
||||
if date:
|
||||
date_part = date
|
||||
if time and _TIME_RE.match(time):
|
||||
date_part = f"{date}_{time.replace(':', '')}"
|
||||
source = "ocr"
|
||||
else:
|
||||
# Scan date beats an opaque "UNSORTED" prefix: files still sort,
|
||||
# and the CSV records that the date is when it was scanned.
|
||||
date_part = fallback_dt.strftime("%Y-%m-%d")
|
||||
source = "scan_date"
|
||||
|
||||
title_slug = slugify(doc_title, max_vendor_len) if doc_title else None
|
||||
parts = [date_part]
|
||||
if vendor_slug:
|
||||
return NamingResult(stem=f"UNSORTED_{vendor_slug}_{ts}", source="ocr")
|
||||
return NamingResult(stem=f"UNSORTED_{ts}", source="none")
|
||||
parts.append(vendor_slug)
|
||||
# Skip the title when the vendor line already says the same thing
|
||||
# (Tesseract often picks the printed title as the "vendor").
|
||||
if title_slug and not (vendor_slug and title_slug in vendor_slug):
|
||||
parts.append(title_slug)
|
||||
# Account-holder shortname after the doc type — disambiguates family
|
||||
# members who share a bank (LEVIT I vs LEVIT M).
|
||||
if party_slug and party_slug not in parts and (
|
||||
not vendor_slug or party_slug not in vendor_slug
|
||||
):
|
||||
parts.append(party_slug)
|
||||
|
||||
if len(parts) == 1:
|
||||
parts.append("document" if date else "scan")
|
||||
if not date:
|
||||
source = "none"
|
||||
|
||||
return NamingResult(stem="_".join(parts), source=source)
|
||||
|
||||
|
||||
def dedupe_stem(stem: str, used: set[str]) -> str:
|
||||
|
||||
+263
-53
@@ -11,6 +11,8 @@ from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -21,16 +23,27 @@ import numpy as np
|
||||
from paperpod.capture.video import iter_frames, probe_video
|
||||
from paperpod.config import Config, config_to_dict
|
||||
from paperpod.llm.vision import analyze_document, ollama_available
|
||||
from paperpod.naming.filename import build_name, dedupe_stem
|
||||
from paperpod.ocr.extract import run_ocr, extract_store_name
|
||||
from paperpod.naming.filename import build_name, dedupe_stem, slugify
|
||||
from paperpod.ocr.extract import extract_store_name, run_ocr
|
||||
from paperpod.pdf.build import images_to_pdf
|
||||
from paperpod.vision.capture_events import _plausible_capture, find_document_events
|
||||
from paperpod.vision.capture_events import _plausible_capture, find_document_events, score_sample
|
||||
from paperpod.vision.document import detect_document, pad_quad, warp_document
|
||||
from paperpod.vision.enhance import enhance_for_ocr
|
||||
from paperpod.vision.enhance import darken_print, enhance_for_ocr
|
||||
from paperpod.vision.motion import find_segments, motion_score, prepare_gray
|
||||
from paperpod.vision.orient import auto_rotate
|
||||
from paperpod.vision.refine import refine_crop
|
||||
|
||||
_AMOUNT_RE = re.compile(r"\d[\d,]{2,}(?:\.\d{2})?")
|
||||
|
||||
|
||||
def _content_overlap(a: str, b: str) -> float:
|
||||
"""Shared dollar/account numbers between two OCR texts (1.0 ≈ same page)."""
|
||||
fa = set(_AMOUNT_RE.findall(a or ""))
|
||||
fb = set(_AMOUNT_RE.findall(b or ""))
|
||||
if not fa or not fb:
|
||||
return 0.0
|
||||
return len(fa & fb) / min(len(fa), len(fb))
|
||||
|
||||
|
||||
def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> dict[str, Any]:
|
||||
"""Run the full detect pipeline and write artifacts into out_dir.
|
||||
@@ -89,6 +102,9 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d
|
||||
# Unlike stable-window-only detection, this still captures receipts shown
|
||||
# while hands are adjusting them (motion scores stay high but the contour
|
||||
# is visible frame-by-frame).
|
||||
#
|
||||
# Only tiny scored proxies are retained here — full-res 4K BGR would OOM
|
||||
# on a long phone video. Winning frame indices are re-decoded in pass 2b.
|
||||
det_kwargs = dict(
|
||||
min_area_ratio=cfg.document.min_area_ratio,
|
||||
max_area_ratio=cfg.document.max_area_ratio,
|
||||
@@ -97,22 +113,37 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d
|
||||
canny_high=cfg.document.canny_high,
|
||||
detect_width=cfg.document.detect_width,
|
||||
)
|
||||
samples: list[tuple[Any, Any]] = []
|
||||
scored_samples = []
|
||||
for frame in iter_frames(video_path, sample_fps=cfg.capture.sample_fps):
|
||||
detection = detect_document(frame.image, **det_kwargs)
|
||||
if detection is not None and _plausible_capture(detection):
|
||||
samples.append((frame, detection))
|
||||
scored_samples.append(score_sample(frame, detection))
|
||||
|
||||
doc_events = find_document_events(
|
||||
samples,
|
||||
scored_samples,
|
||||
gap_s=cfg.document.event_gap_s,
|
||||
min_samples=cfg.document.event_min_samples,
|
||||
)
|
||||
|
||||
# Pass 2b: re-decode only the winning frame indices (one BGR at a time).
|
||||
want = {event.best_frame_index: event for event in doc_events}
|
||||
if want:
|
||||
for frame in iter_frames(video_path, sample_fps=cfg.capture.sample_fps):
|
||||
event = want.get(frame.index)
|
||||
if event is None:
|
||||
continue
|
||||
event.frame = frame
|
||||
del want[frame.index]
|
||||
if not want:
|
||||
break
|
||||
|
||||
window_records: list[dict[str, Any]] = []
|
||||
for event in doc_events:
|
||||
i = event.event_id
|
||||
frame = event.frame
|
||||
if frame is None:
|
||||
# Winner index was skipped (sample_fps / codec edge); skip event.
|
||||
continue
|
||||
detection = event.detection
|
||||
sharp = event.sharpness
|
||||
|
||||
@@ -123,9 +154,10 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d
|
||||
"duration_s": round(event.t_end - event.t_start, 3),
|
||||
"sample_count": event.sample_count,
|
||||
"document_found": False,
|
||||
"best_frame_t": round(frame.t, 3),
|
||||
"best_frame_index": frame.index,
|
||||
"best_frame_t": round(event.best_frame_t, 3),
|
||||
"best_frame_index": event.best_frame_index,
|
||||
"sharpness": round(sharp, 1),
|
||||
"best_frame_skin_fraction": event.skin_fraction,
|
||||
}
|
||||
|
||||
frame_path = frames_dir / f"window_{i:03d}_full.png"
|
||||
@@ -154,6 +186,9 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d
|
||||
|
||||
if cfg.document.enhance:
|
||||
crop = enhance_for_ocr(crop)
|
||||
else:
|
||||
# Keep the photographic look but pull pale grey print toward black.
|
||||
crop = darken_print(crop)
|
||||
|
||||
crop_path = crops_dir / f"window_{i:03d}.png"
|
||||
cv2.imwrite(str(crop_path), crop)
|
||||
@@ -202,20 +237,152 @@ def _is_exportable(ocr) -> bool:
|
||||
"""Skip obvious mat-transition junk that has no usable naming signal."""
|
||||
if ocr.form_code or ocr.date or ocr.total:
|
||||
return True
|
||||
if extract_store_name(ocr.text):
|
||||
if extract_store_name(ocr.text) or ocr.doc_title or ocr.page_number:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict[str, Any]:
|
||||
"""OCR-name and PDF-export every detected crop from a detect() report.
|
||||
@dataclass
|
||||
class _PageCandidate:
|
||||
"""One exportable capture: crop + everything extracted from it."""
|
||||
|
||||
NOTE: there is no page-grouping yet (module 3 / events/ is not built),
|
||||
so every detected crop becomes its own single-page PDF, even if it was
|
||||
physically one side of a multi-page stack. pod_id == window_id for now.
|
||||
window: dict[str, Any]
|
||||
crop_path: Path
|
||||
ocr: Any
|
||||
llm: Any
|
||||
vendor: str | None
|
||||
date: str | None
|
||||
time: str | None
|
||||
total: str | None
|
||||
form_code: str | None
|
||||
tax_year: str | None
|
||||
org_name: str | None
|
||||
doc_title: str | None
|
||||
party: str | None
|
||||
page_number: int | None
|
||||
page_total: int | None
|
||||
|
||||
@property
|
||||
def signature(self) -> tuple:
|
||||
return (self.vendor, self.date, self.total, self.form_code)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Document:
|
||||
pages: list[_PageCandidate] = field(default_factory=list)
|
||||
|
||||
def last_marker_page(self) -> _PageCandidate | None:
|
||||
"""Most recent page whose printed 'PAGE X OF Y' marker was readable."""
|
||||
for page in reversed(self.pages):
|
||||
if page.page_number is not None:
|
||||
return page
|
||||
return None
|
||||
|
||||
|
||||
def _same_letterhead(prev: _PageCandidate, curr: _PageCandidate) -> bool:
|
||||
"""Loose 'same printed letterhead' check for grouping decisions."""
|
||||
if prev.doc_title and curr.doc_title:
|
||||
return prev.doc_title == curr.doc_title
|
||||
if prev.vendor and curr.vendor:
|
||||
a, b = slugify(prev.vendor), slugify(curr.vendor)
|
||||
return a.startswith(b) or b.startswith(a)
|
||||
return False
|
||||
|
||||
|
||||
def _signature_duplicate(prev: _PageCandidate, curr: _PageCandidate) -> bool:
|
||||
"""Same marker-less page captured twice in a row (a contour blip).
|
||||
|
||||
Only trustworthy for events moments apart, and only when neither side
|
||||
carries a printed page marker. Pages of one bank statement share
|
||||
vendor + date, and flips are often 3–4s — the old 5s gate was replacing
|
||||
page 3 with page 4 (higher OCR conf) and wiping the earlier page.
|
||||
"""
|
||||
if prev.page_number is not None or curr.page_number is not None:
|
||||
return False
|
||||
if curr.window["t_start"] - prev.window["t_end"] > 5.0:
|
||||
return False
|
||||
return bool(any(curr.signature) and curr.signature == prev.signature)
|
||||
|
||||
|
||||
def _first(*values):
|
||||
return next((v for v in values if v), None)
|
||||
|
||||
|
||||
def _group_documents(candidates: list[_PageCandidate]) -> list[_Document]:
|
||||
"""Merge consecutive captures into documents; drop duplicate captures.
|
||||
|
||||
Grouping is anchored on the printed "PAGE X OF Y" marker: within one
|
||||
document, later captures must advance the page number under the same
|
||||
page total (gaps are fine — a page whose marker OCR couldn't read must
|
||||
not split one statement into several PDFs). A marker-less capture with
|
||||
the same letterhead may slot into a document that is still incomplete
|
||||
(e.g. the "PAGE 1 OF 2" statement's second page whose header came out
|
||||
garbled); unrelated marker-less single-pagers never group.
|
||||
"""
|
||||
documents: list[_Document] = []
|
||||
for cand in candidates:
|
||||
doc = documents[-1] if documents else None
|
||||
if doc is None:
|
||||
documents.append(_Document(pages=[cand]))
|
||||
continue
|
||||
|
||||
anchor = doc.last_marker_page()
|
||||
last = doc.pages[-1]
|
||||
|
||||
if cand.page_number is not None:
|
||||
if (
|
||||
anchor is not None
|
||||
and cand.page_total == anchor.page_total
|
||||
and _same_letterhead(anchor, cand)
|
||||
):
|
||||
if cand.page_number == anchor.page_number:
|
||||
# Same page captured twice; keep the better read.
|
||||
if cand.ocr.mean_confidence > anchor.ocr.mean_confidence:
|
||||
idx = next(i for i, p in enumerate(doc.pages) if p is anchor)
|
||||
doc.pages[idx] = cand
|
||||
continue
|
||||
if cand.page_number > anchor.page_number:
|
||||
doc.pages.append(cand)
|
||||
continue
|
||||
documents.append(_Document(pages=[cand]))
|
||||
continue
|
||||
|
||||
if _signature_duplicate(last, cand):
|
||||
if cand.ocr.mean_confidence > last.ocr.mean_confidence:
|
||||
doc.pages[-1] = cand
|
||||
continue
|
||||
# Content-fingerprint dedupe for marker-less recaptures only when the
|
||||
# amounts strongly match (≥0.55) or the new read looks garbled
|
||||
# (conf < 75). A 0.25–0.40 overlap is normal between successive
|
||||
# pages of one statement (shared account # / branch / balances);
|
||||
# using that as a duplicate gate previously wiped page 3 with page 4.
|
||||
overlap = _content_overlap(last.ocr.text, cand.ocr.text)
|
||||
if overlap >= 0.55 or (overlap >= 0.25 and cand.ocr.mean_confidence < 75.0):
|
||||
if cand.ocr.mean_confidence > last.ocr.mean_confidence:
|
||||
doc.pages[-1] = cand
|
||||
continue
|
||||
if (
|
||||
anchor is not None
|
||||
and anchor.page_number < (anchor.page_total or 0)
|
||||
and _same_letterhead(anchor, cand)
|
||||
and cand.ocr.mean_confidence >= 70.0
|
||||
and _content_overlap(anchor.ocr.text, cand.ocr.text) < 0.55
|
||||
):
|
||||
doc.pages.append(cand)
|
||||
continue
|
||||
documents.append(_Document(pages=[cand]))
|
||||
return documents
|
||||
|
||||
|
||||
def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict[str, Any]:
|
||||
"""OCR-name and PDF-export every detected document from a detect() report.
|
||||
|
||||
Consecutive captures that carry an incrementing printed "PAGE X OF Y"
|
||||
marker are grouped into one multi-page PDF (page 1's metadata names the
|
||||
file). Everything else stays one PDF per capture.
|
||||
|
||||
Writes:
|
||||
pdf/<name>.pdf one PDF per detected document
|
||||
pdf/<name>.pdf one PDF per document (possibly multi-page)
|
||||
export_summary.csv timestamp, pod_id, page_count, source_of_name,
|
||||
ocr_vendor, ocr_date, ocr_total, final_filename
|
||||
Returns the summary rows (also embedded back for programmatic use).
|
||||
@@ -237,10 +404,8 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict
|
||||
if cfg.llm.enabled and not use_llm:
|
||||
print("Ollama not reachable — falling back to Tesseract-only naming.")
|
||||
|
||||
used_stems: set[str] = set()
|
||||
rows: list[dict[str, Any]] = []
|
||||
prev_signature: tuple = ()
|
||||
|
||||
# ---- Phase 1: OCR/LLM every crop, keep the exportable ones --------------
|
||||
candidates: list[_PageCandidate] = []
|
||||
for window in report["stable_windows"]:
|
||||
if not window.get("document_found"):
|
||||
continue
|
||||
@@ -277,66 +442,110 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict
|
||||
timeout=cfg.llm.timeout_s,
|
||||
)
|
||||
|
||||
# Blank pages (backs of slips, empty sheets) OCR to garbled noise;
|
||||
# the LLM identifies them reliably, so drop them outright.
|
||||
# Drop LLM-flagged blank pages (backs of slips, empty sheets) only
|
||||
# when OCR agrees there's nothing there — vision models (especially
|
||||
# qwen2.5vl) have mislabeled real statement pages as "blank", and
|
||||
# losing a real page is far worse than exporting a blank one.
|
||||
if llm is not None and (
|
||||
llm.doc_type == "blank" or llm.description.lower().startswith("blank")
|
||||
):
|
||||
continue
|
||||
if len(ocr.text.split()) < 8:
|
||||
continue
|
||||
llm = None # its other fields aren't trustworthy for this page
|
||||
|
||||
vendor = (llm.vendor if llm else None) or ocr.vendor
|
||||
date = (llm.date if llm else None) or ocr.date
|
||||
time = (llm.time if llm else None) or ocr.time
|
||||
total = (llm.total if llm else None) or ocr.total
|
||||
form_code = (llm.form_code if llm else None) or ocr.form_code
|
||||
tax_year = (llm.tax_year if llm else None) or ocr.tax_year
|
||||
org_name = (llm.vendor if llm and llm.form_code else None) or ocr.org_name
|
||||
vendor = _first(llm.vendor if llm else None, ocr.vendor)
|
||||
date = _first(llm.date if llm else None, ocr.date)
|
||||
form_code = _first(llm.form_code if llm else None, ocr.form_code)
|
||||
total = _first(llm.total if llm else None, ocr.total)
|
||||
|
||||
if llm is None and not _is_exportable(ocr):
|
||||
continue
|
||||
if llm is not None and not any((vendor, date, total, form_code)):
|
||||
continue
|
||||
|
||||
# The same physical document sometimes splits into two capture
|
||||
# events (hand briefly breaks the contour). If the previous export
|
||||
# has the identical identity signature, it's the same document.
|
||||
signature = (vendor, date, total, form_code)
|
||||
if any(signature) and signature == prev_signature:
|
||||
continue
|
||||
prev_signature = signature
|
||||
candidates.append(
|
||||
_PageCandidate(
|
||||
window=window,
|
||||
crop_path=crop_path,
|
||||
ocr=ocr,
|
||||
llm=llm,
|
||||
vendor=vendor,
|
||||
date=date,
|
||||
time=_first(llm.time if llm else None, ocr.time),
|
||||
total=total,
|
||||
form_code=form_code,
|
||||
tax_year=_first(llm.tax_year if llm else None, ocr.tax_year),
|
||||
org_name=_first(
|
||||
llm.vendor if llm and llm.form_code else None, ocr.org_name
|
||||
),
|
||||
doc_title=ocr.doc_title,
|
||||
party=_first(llm.party if llm else None, ocr.party),
|
||||
page_number=ocr.page_number,
|
||||
page_total=ocr.page_total,
|
||||
)
|
||||
)
|
||||
|
||||
event_time = recording_start + timedelta(seconds=window["t_start"])
|
||||
# ---- Phase 2: group consecutive captures into documents ------------------
|
||||
documents = _group_documents(candidates)
|
||||
|
||||
# ---- Phase 3: name and write one PDF per document ------------------------
|
||||
used_stems: set[str] = set()
|
||||
rows: list[dict[str, Any]] = []
|
||||
for doc in documents:
|
||||
pages = doc.pages
|
||||
first = pages[0]
|
||||
# Page 1 usually carries the letterhead/date, but any page may have
|
||||
# filled a field the others missed.
|
||||
date = _first(*(p.date for p in pages))
|
||||
doc_title = _first(*(p.doc_title for p in pages))
|
||||
party = _first(*(p.party for p in pages))
|
||||
# Prefer a vendor that isn't just the printed title re-read (the
|
||||
# largest-font heuristic picks "Transaction History" on some pages
|
||||
# and the actual "TD Canada Trust" letterhead on others).
|
||||
vendors = [p.vendor for p in pages if p.vendor]
|
||||
vendor = _first(
|
||||
*(v for v in vendors if not doc_title or slugify(v) not in doc_title),
|
||||
*vendors,
|
||||
)
|
||||
|
||||
event_time = recording_start + timedelta(seconds=first.window["t_start"])
|
||||
naming = build_name(
|
||||
date,
|
||||
vendor,
|
||||
event_time,
|
||||
cfg.naming.max_vendor_len,
|
||||
form_code=form_code,
|
||||
tax_year=tax_year,
|
||||
org_name=org_name,
|
||||
time=time,
|
||||
form_code=_first(*(p.form_code for p in pages)),
|
||||
tax_year=_first(*(p.tax_year for p in pages)),
|
||||
org_name=_first(*(p.org_name for p in pages)),
|
||||
time=_first(*(p.time for p in pages)),
|
||||
doc_title=doc_title,
|
||||
party=party,
|
||||
)
|
||||
stem = dedupe_stem(naming.stem, used_stems)
|
||||
|
||||
pdf_path = pdf_dir / f"{stem}.pdf"
|
||||
images_to_pdf([crop_path], pdf_path, title=stem)
|
||||
images_to_pdf([p.crop_path for p in pages], pdf_path, title=stem)
|
||||
|
||||
llm = first.llm
|
||||
rows.append(
|
||||
{
|
||||
"timestamp": event_time.isoformat(timespec="seconds"),
|
||||
"pod_id": window["window_id"],
|
||||
"page_count": 1,
|
||||
"pod_id": first.window["window_id"],
|
||||
"page_count": len(pages),
|
||||
"page_windows": ";".join(str(p.window["window_id"]) for p in pages),
|
||||
"source_of_name": "llm" if llm else naming.source,
|
||||
"ocr_vendor": vendor or "",
|
||||
"ocr_form_code": form_code or "",
|
||||
"ocr_form_code": _first(*(p.form_code for p in pages)) or "",
|
||||
"ocr_date": date or "",
|
||||
"ocr_time": time or "",
|
||||
"ocr_total": total or "",
|
||||
"ocr_confidence": round(ocr.mean_confidence, 1),
|
||||
"ocr_time": _first(*(p.time for p in pages)) or "",
|
||||
"ocr_total": _first(*(p.total for p in pages)) or "",
|
||||
"doc_title": doc_title or "",
|
||||
"party": party or "",
|
||||
"ocr_confidence": round(first.ocr.mean_confidence, 1),
|
||||
"llm_description": llm.description if llm else "",
|
||||
"llm_issues": ";".join(llm.issues) if llm else "",
|
||||
"detection_confidence": window.get("detection_confidence", ""),
|
||||
"rotation_applied": window.get("rotation_applied", 0),
|
||||
"detection_confidence": first.window.get("detection_confidence", ""),
|
||||
"rotation_applied": first.window.get("rotation_applied", 0),
|
||||
"final_filename": pdf_path.name,
|
||||
"pdf_path": str(pdf_path.relative_to(out_dir)),
|
||||
}
|
||||
@@ -344,8 +553,9 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict
|
||||
|
||||
with open(out_dir / "export_summary.csv", "w", newline="") as f:
|
||||
fieldnames = [
|
||||
"timestamp", "pod_id", "page_count", "source_of_name",
|
||||
"ocr_vendor", "ocr_form_code", "ocr_date", "ocr_time", "ocr_total", "ocr_confidence",
|
||||
"timestamp", "pod_id", "page_count", "page_windows", "source_of_name",
|
||||
"ocr_vendor", "ocr_form_code", "ocr_date", "ocr_time", "ocr_total",
|
||||
"doc_title", "party", "ocr_confidence",
|
||||
"llm_description", "llm_issues",
|
||||
"detection_confidence", "rotation_applied",
|
||||
"final_filename", "pdf_path",
|
||||
|
||||
@@ -4,19 +4,57 @@ Stable-window-only detection misses documents shown while hands are still
|
||||
adjusting them — motion scores stay above threshold even though a receipt
|
||||
is clearly visible and detectable frame-by-frame. This clusters consecutive
|
||||
frames where a document contour was found (with a time-gap to split separate
|
||||
placements) and keeps the sharpest frame from each cluster.
|
||||
placements) and keeps the best frame from each cluster.
|
||||
|
||||
"Best" means hands-free first, sharp second: fingers holding a page flat are
|
||||
usually present in most frames of an event but absent in at least a few
|
||||
(right after placing / before flipping). A sharpness-only pick routinely
|
||||
selects a hand-in-frame frame, which then has to be inpainted (lossy).
|
||||
Preferring the lowest-skin frame keeps the original pixels instead.
|
||||
|
||||
Memory: scoring runs as frames arrive; only tiny gray proxies + metrics are
|
||||
retained for clustering. Full-res BGR is dropped immediately so a long 4K
|
||||
video cannot OOM by holding thousands of detections in RAM (callers re-decode
|
||||
winning frame indices in a third pass).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from paperpod.capture.video import Frame
|
||||
from paperpod.vision.document import DocumentDetection
|
||||
from paperpod.vision.motion import motion_score, prepare_gray
|
||||
from paperpod.vision.refine import skin_mask
|
||||
from paperpod.vision.sharpness import sharpness_score
|
||||
|
||||
# Downscale width for per-frame skin analysis; full-res 4K would dominate
|
||||
# pass-2 runtime for no accuracy gain.
|
||||
_SKIN_ANALYSIS_WIDTH = 400
|
||||
|
||||
# Skin coverage below this fraction of the document region counts as a
|
||||
# "clean" (hands-free) frame; between the two thresholds counts as "touched"
|
||||
# (fingertip on an edge); above is "held" (hand across the page).
|
||||
_SKIN_CLEAN = 0.003
|
||||
_SKIN_TOUCHED = 0.03
|
||||
|
||||
# Reject a "cleaner" frame if it's this dull relative to the sharpest
|
||||
# sample seen in the same event.
|
||||
_SHARP_FLOOR = 0.4
|
||||
|
||||
# Long placements that include a page flip keep the same paper outline, so
|
||||
# contour clustering never splits them. Instead we look for *quiet dwells*
|
||||
# (runs of near-zero inter-sample motion) inside a long event — each dwell
|
||||
# is one page lying flat. Page 2 of the Jan '25 TD statement sat still for
|
||||
# ~0.7s around t=10.4; hand-wobble stretches between dwells never qualify.
|
||||
_QUIET_MOTION = 0.015
|
||||
_MIN_DWELL_S = 0.55
|
||||
_LONG_EVENT_S = 4.0
|
||||
_MOTION_SIZE = (160, 90)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DocumentEvent:
|
||||
@@ -27,8 +65,84 @@ class DocumentEvent:
|
||||
best_frame_t: float
|
||||
best_frame_index: int
|
||||
sharpness: float
|
||||
frame: Frame
|
||||
skin_fraction: float
|
||||
detection: DocumentDetection
|
||||
# Populated by the pipeline after a re-decode pass; None right after
|
||||
# clustering so the scorer never retains full-res images.
|
||||
frame: Frame | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScoredSample:
|
||||
"""Per-detection metrics without the full-res BGR frame."""
|
||||
|
||||
index: int
|
||||
t: float
|
||||
shape: tuple[int, ...] # (h, w, c) of the source frame
|
||||
det: DocumentDetection
|
||||
sharp: float
|
||||
skin: float
|
||||
gray_tiny: np.ndarray # for page-flip motion between samples
|
||||
|
||||
|
||||
def _quad_region(image: np.ndarray, quad: np.ndarray) -> np.ndarray:
|
||||
"""Bounding-box crop of the detected document within the frame."""
|
||||
h, w = image.shape[:2]
|
||||
x0 = max(0, int(quad[:, 0].min()))
|
||||
x1 = min(w, int(np.ceil(quad[:, 0].max())))
|
||||
y0 = max(0, int(quad[:, 1].min()))
|
||||
y1 = min(h, int(np.ceil(quad[:, 1].max())))
|
||||
if x1 - x0 >= 8 and y1 - y0 >= 8:
|
||||
return image[y0:y1, x0:x1]
|
||||
return image
|
||||
|
||||
|
||||
def skin_fraction_in_quad(image: np.ndarray, quad: np.ndarray) -> float:
|
||||
"""Fraction of the detected document region that looks like skin.
|
||||
|
||||
Measured on the quad's bounding box (downscaled), so a hand resting on
|
||||
the mat next to the document doesn't count — only skin over the paper.
|
||||
"""
|
||||
region = _quad_region(image, quad)
|
||||
rh, rw = region.shape[:2]
|
||||
if rw > _SKIN_ANALYSIS_WIDTH:
|
||||
region = cv2.resize(
|
||||
region, (_SKIN_ANALYSIS_WIDTH, max(1, round(rh * _SKIN_ANALYSIS_WIDTH / rw)))
|
||||
)
|
||||
return float((skin_mask(region) > 0).mean())
|
||||
|
||||
|
||||
def document_sharpness(image: np.ndarray, quad: np.ndarray) -> float:
|
||||
"""Sharpness of the document region only.
|
||||
|
||||
Whole-frame Laplacian variance is dominated by high-frequency background
|
||||
(wood grain), so a blurry mid-flip frame that exposes more table can
|
||||
outscore a crisp flat page. Scoring inside the detection quad measures
|
||||
the thing we actually keep.
|
||||
"""
|
||||
return sharpness_score(_quad_region(image, quad))
|
||||
|
||||
|
||||
def score_sample(frame: Frame, det: DocumentDetection) -> ScoredSample:
|
||||
"""Extract clustering metrics and drop the full-res pixels."""
|
||||
small = cv2.resize(frame.image, _MOTION_SIZE)
|
||||
return ScoredSample(
|
||||
index=frame.index,
|
||||
t=frame.t,
|
||||
shape=frame.image.shape,
|
||||
det=det,
|
||||
sharp=document_sharpness(frame.image, det.quad),
|
||||
skin=skin_fraction_in_quad(frame.image, det.quad),
|
||||
gray_tiny=prepare_gray(small, blur_ksize=5),
|
||||
)
|
||||
|
||||
|
||||
def _skin_bucket(fraction: float) -> int:
|
||||
if fraction < _SKIN_CLEAN:
|
||||
return 0
|
||||
if fraction < _SKIN_TOUCHED:
|
||||
return 1
|
||||
return 2
|
||||
|
||||
|
||||
def _same_document(
|
||||
@@ -65,60 +179,148 @@ def _plausible_capture(det: DocumentDetection) -> bool:
|
||||
return ar <= 0.40 or ar >= 0.58
|
||||
|
||||
|
||||
def _split_on_page_flips(
|
||||
samples: list[ScoredSample], min_samples: int
|
||||
) -> list[list[ScoredSample]]:
|
||||
"""Pull quiet dwells out of a long continuous contour (page flips).
|
||||
|
||||
Returns one segment per flat-page dwell when two or more qualify;
|
||||
otherwise returns the original samples unchanged so ordinary
|
||||
single-page placements aren't shredded by brief hand motion.
|
||||
"""
|
||||
if len(samples) < 4:
|
||||
return [samples]
|
||||
duration = samples[-1].t - samples[0].t
|
||||
if duration < _LONG_EVENT_S:
|
||||
return [samples]
|
||||
|
||||
motions = [0.0]
|
||||
for i in range(1, len(samples)):
|
||||
motions.append(
|
||||
motion_score(samples[i - 1].gray_tiny, samples[i].gray_tiny, pixel_threshold=12)
|
||||
)
|
||||
|
||||
dwells: list[list[ScoredSample]] = []
|
||||
start: int | None = None
|
||||
for i, sample in enumerate(samples):
|
||||
quiet = motions[i] <= _QUIET_MOTION
|
||||
if quiet and start is None:
|
||||
start = i
|
||||
elif not quiet and start is not None:
|
||||
dwell = samples[start:i]
|
||||
if dwell[-1].t - dwell[0].t >= _MIN_DWELL_S and len(dwell) >= min_samples:
|
||||
dwells.append(dwell)
|
||||
start = None
|
||||
if start is not None:
|
||||
dwell = samples[start:]
|
||||
if dwell[-1].t - dwell[0].t >= _MIN_DWELL_S and len(dwell) >= min_samples:
|
||||
dwells.append(dwell)
|
||||
|
||||
# Only commit to multi-page split when we actually found 2+ quiet dwells.
|
||||
return dwells if len(dwells) >= 2 else [samples]
|
||||
|
||||
|
||||
class _Cluster:
|
||||
"""Accumulates samples for one placement; picks the best frame on flush.
|
||||
|
||||
Preference among frames ≥ 40% as sharp as the cluster's sharpest sample:
|
||||
lowest skin bucket, then highest sharpness. Soft clean frames never
|
||||
outrank sharp lightly-touched ones.
|
||||
|
||||
First/last samples of long events are ignored — they are usually the
|
||||
hand placing or lifting the page (high Laplacian streaks, low ink),
|
||||
which is how page 7 of a statement once lost to its own lift frame.
|
||||
"""
|
||||
|
||||
def __init__(self, sample: ScoredSample) -> None:
|
||||
self.t_start = sample.t
|
||||
self.t_end = sample.t
|
||||
self.samples: list[ScoredSample] = [sample]
|
||||
|
||||
@property
|
||||
def count(self) -> int:
|
||||
return len(self.samples)
|
||||
|
||||
def add(self, sample: ScoredSample) -> None:
|
||||
self.t_end = sample.t
|
||||
self.samples.append(sample)
|
||||
|
||||
def pick_best(self, samples: list[ScoredSample] | None = None) -> ScoredSample:
|
||||
samples = samples if samples is not None else self.samples
|
||||
pick_from = samples
|
||||
if len(pick_from) >= 5:
|
||||
# Drop the placing/lifting edges.
|
||||
pick_from = pick_from[1:-1]
|
||||
max_sharp = max(s.sharp for s in pick_from)
|
||||
floor = max_sharp * _SHARP_FLOOR
|
||||
eligible = [s for s in pick_from if s.sharp >= floor] or pick_from
|
||||
return min(eligible, key=lambda s: (_skin_bucket(s.skin), -s.sharp))
|
||||
|
||||
|
||||
def find_document_events(
|
||||
samples: list[tuple[Frame, DocumentDetection]],
|
||||
samples: list[tuple[Frame, DocumentDetection]] | list[ScoredSample],
|
||||
gap_s: float = 2.0,
|
||||
min_samples: int = 2,
|
||||
) -> list[DocumentEvent]:
|
||||
"""Cluster consecutive detections; split when gap_s passes with no detection.
|
||||
|
||||
samples must be in time order. Each sample is a (frame, detection) pair
|
||||
from a frame where detect_document() succeeded.
|
||||
Long clusters (page kept under the camera while flipped) are further
|
||||
split on motion peaks into separate dwells so each page gets its own
|
||||
capture. samples must be in time order.
|
||||
|
||||
Accepts either (Frame, detection) tuples (scores immediately, drops BGR)
|
||||
or pre-built ScoredSample list from the streaming pipeline.
|
||||
"""
|
||||
if not samples:
|
||||
return []
|
||||
|
||||
clusters: list[tuple[float, float, int, float, Frame, DocumentDetection]] = []
|
||||
active: tuple[float, float, int, float, Frame, DocumentDetection] | None = None
|
||||
scored: list[ScoredSample]
|
||||
if isinstance(samples[0], ScoredSample):
|
||||
scored = list(samples) # type: ignore[arg-type]
|
||||
else:
|
||||
scored = [score_sample(frame, det) for frame, det in samples] # type: ignore[misc]
|
||||
|
||||
clusters: list[_Cluster] = []
|
||||
active: _Cluster | None = None
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal active
|
||||
if active is None:
|
||||
return
|
||||
t_start, t_end, count, sharp, frame, det = active
|
||||
if count >= min_samples:
|
||||
if active is not None and active.count >= min_samples:
|
||||
clusters.append(active)
|
||||
active = None
|
||||
|
||||
for frame, det in samples:
|
||||
for sample in scored:
|
||||
if active is None:
|
||||
active = (frame.t, frame.t, 1, sharpness_score(frame.image), frame, det)
|
||||
active = _Cluster(sample)
|
||||
continue
|
||||
|
||||
t_start, t_end, count, best_sharp, best_frame, best_det = active
|
||||
if frame.t - t_end > gap_s or not _same_document(best_det, det, frame.image.shape):
|
||||
if sample.t - active.t_end > gap_s or not _same_document(
|
||||
active.samples[-1].det, sample.det, sample.shape
|
||||
):
|
||||
flush()
|
||||
active = (frame.t, frame.t, 1, sharpness_score(frame.image), frame, det)
|
||||
active = _Cluster(sample)
|
||||
continue
|
||||
|
||||
sharp = sharpness_score(frame.image)
|
||||
if sharp > best_sharp:
|
||||
best_sharp, best_frame, best_det = sharp, frame, det
|
||||
active = (t_start, frame.t, count + 1, best_sharp, best_frame, best_det)
|
||||
active.add(sample)
|
||||
|
||||
flush()
|
||||
|
||||
return [
|
||||
DocumentEvent(
|
||||
event_id=i,
|
||||
t_start=t_start,
|
||||
t_end=t_end,
|
||||
sample_count=count,
|
||||
best_frame_t=frame.t,
|
||||
best_frame_index=frame.index,
|
||||
sharpness=sharp,
|
||||
frame=frame,
|
||||
detection=det,
|
||||
)
|
||||
for i, (t_start, t_end, count, sharp, frame, det) in enumerate(clusters)
|
||||
]
|
||||
events: list[DocumentEvent] = []
|
||||
event_id = 0
|
||||
for cluster in clusters:
|
||||
for segment in _split_on_page_flips(cluster.samples, min_samples):
|
||||
best = cluster.pick_best(segment)
|
||||
events.append(
|
||||
DocumentEvent(
|
||||
event_id=event_id,
|
||||
t_start=segment[0].t,
|
||||
t_end=segment[-1].t,
|
||||
sample_count=len(segment),
|
||||
best_frame_t=best.t,
|
||||
best_frame_index=best.index,
|
||||
sharpness=best.sharp,
|
||||
skin_fraction=round(best.skin, 4),
|
||||
detection=best.det,
|
||||
frame=None,
|
||||
)
|
||||
)
|
||||
event_id += 1
|
||||
return events
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from paperpod.capture.video import Frame
|
||||
from paperpod.vision.capture_events import find_document_events
|
||||
from paperpod.vision.document import DocumentDetection
|
||||
import numpy as np
|
||||
|
||||
from paperpod.capture.video import Frame
|
||||
from paperpod.vision.capture_events import find_document_events, skin_fraction_in_quad
|
||||
from paperpod.vision.document import DocumentDetection
|
||||
|
||||
_SKIN_BGR = (80, 120, 180) # inside the YCrCb skin range used by refine
|
||||
|
||||
|
||||
def _frame(t: float, idx: int = 0) -> Frame:
|
||||
return Frame(index=idx, t=t, image=np.zeros((100, 100, 3), dtype=np.uint8))
|
||||
@@ -66,3 +69,113 @@ def test_area_change_splits_documents():
|
||||
assert len(events) == 2
|
||||
assert events[0].detection.area_ratio == 0.25
|
||||
assert events[1].detection.area_ratio == 0.65
|
||||
|
||||
|
||||
def _paper_frame(t: float, idx: int, with_finger: bool, sharper: bool) -> Frame:
|
||||
"""White page frame; optionally with a skin-colored blob and extra texture."""
|
||||
rng = np.random.default_rng(idx)
|
||||
image = np.full((100, 100, 3), 245, dtype=np.uint8)
|
||||
if sharper:
|
||||
# High-frequency noise inflates Laplacian variance (the old
|
||||
# sharpness-only criterion would always pick this frame).
|
||||
noise = rng.integers(0, 60, size=(100, 100, 1), dtype=np.uint8)
|
||||
image = np.clip(image.astype(int) - noise, 0, 255).astype(np.uint8)
|
||||
if with_finger:
|
||||
image[30:70, 40:75] = _SKIN_BGR
|
||||
return Frame(index=idx, t=t, image=image)
|
||||
|
||||
|
||||
def _full_quad() -> DocumentDetection:
|
||||
quad = np.array([[0, 0], [100, 0], [100, 100], [0, 100]], dtype=np.float32)
|
||||
return DocumentDetection(quad=quad, area_ratio=0.3, method="quad", source="edges", confidence=0.8)
|
||||
|
||||
|
||||
def test_skin_fraction_detects_finger():
|
||||
clean = _paper_frame(1.0, 0, with_finger=False, sharper=False)
|
||||
finger = _paper_frame(1.2, 1, with_finger=True, sharper=False)
|
||||
quad = _full_quad().quad
|
||||
assert skin_fraction_in_quad(clean.image, quad) < 0.003
|
||||
assert skin_fraction_in_quad(finger.image, quad) > 0.05
|
||||
|
||||
|
||||
def test_hands_free_frame_beats_equally_sharp_finger_frame():
|
||||
# When sharpness is similar, prefer the clean frame.
|
||||
samples = [
|
||||
(_paper_frame(1.0, 0, with_finger=True, sharper=True), _full_quad()),
|
||||
(_paper_frame(1.2, 1, with_finger=False, sharper=True), _full_quad()),
|
||||
(_paper_frame(1.4, 2, with_finger=True, sharper=True), _full_quad()),
|
||||
]
|
||||
events = find_document_events(samples, gap_s=2.0, min_samples=2)
|
||||
assert len(events) == 1
|
||||
assert events[0].best_frame_index == 1
|
||||
assert events[0].skin_fraction < 0.003
|
||||
|
||||
|
||||
def test_much_sharper_touched_frame_beats_blurry_clean_frame():
|
||||
# Mid-flip blur with no fingers must not beat a sharp frame that has a
|
||||
# fingertip on the margin — that's how page 7 of a statement got lost.
|
||||
samples = [
|
||||
(_paper_frame(1.0, 0, with_finger=False, sharper=False), _full_quad()),
|
||||
(_paper_frame(1.2, 1, with_finger=True, sharper=True), _full_quad()),
|
||||
(_paper_frame(1.4, 2, with_finger=False, sharper=False), _full_quad()),
|
||||
]
|
||||
events = find_document_events(samples, gap_s=2.0, min_samples=2)
|
||||
assert len(events) == 1
|
||||
assert events[0].best_frame_index == 1
|
||||
|
||||
|
||||
def test_sharpest_wins_among_equally_clean_frames():
|
||||
samples = [
|
||||
(_paper_frame(1.0, 0, with_finger=False, sharper=False), _full_quad()),
|
||||
(_paper_frame(1.2, 1, with_finger=False, sharper=True), _full_quad()),
|
||||
]
|
||||
events = find_document_events(samples, gap_s=2.0, min_samples=2)
|
||||
assert len(events) == 1
|
||||
assert events[0].best_frame_index == 1
|
||||
|
||||
|
||||
def test_long_event_splits_on_page_flip_motion():
|
||||
"""Quiet dwells inside a long placement become separate events.
|
||||
|
||||
Synthetic: quiet page A, motion/hands, quiet page B. Contour is the
|
||||
same throughout (page flip under a fixed outline), so without dwell
|
||||
splitting only page A would be kept.
|
||||
"""
|
||||
|
||||
def page_frame(t: float, idx: int, pattern: int) -> Frame:
|
||||
# Nearly identical consecutive frames so inter-sample motion is quiet.
|
||||
img = np.full((180, 320, 3), 240, dtype=np.uint8)
|
||||
img[40:140, 40:280] = 200 if pattern == 1 else 160
|
||||
# Seeded by pattern only — same noise every page-A frame.
|
||||
noise = np.random.default_rng(pattern).integers(0, 3, size=img.shape, dtype=np.uint8)
|
||||
img = np.clip(img.astype(int) + noise, 0, 255).astype(np.uint8)
|
||||
return Frame(index=idx, t=t, image=img)
|
||||
|
||||
samples = []
|
||||
# Quiet page 1 dwell: t=0.0 .. 3.0
|
||||
for i in range(0, 13):
|
||||
samples.append((page_frame(i * 0.25, i, pattern=1), _full_quad()))
|
||||
# Motion / hands between pages
|
||||
for i, shade in enumerate((40, 80, 120, 60)):
|
||||
img = np.full((180, 320, 3), shade, dtype=np.uint8)
|
||||
samples.append((Frame(index=13 + i, t=3.25 + i * 0.25, image=img), _full_quad()))
|
||||
# Quiet page 2 dwell: t=4.25 .. 5.25
|
||||
for i in range(0, 5):
|
||||
samples.append((page_frame(4.25 + i * 0.25, 20 + i, pattern=2), _full_quad()))
|
||||
|
||||
events = find_document_events(samples, gap_s=2.0, min_samples=2)
|
||||
assert len(events) >= 2
|
||||
assert events[0].t_end <= 3.5
|
||||
assert events[1].t_start >= 3.5
|
||||
|
||||
|
||||
|
||||
def test_short_event_not_resegmented():
|
||||
samples = [
|
||||
(_paper_frame(1.0, 0, with_finger=False, sharper=True), _full_quad()),
|
||||
(_paper_frame(1.2, 1, with_finger=False, sharper=True), _full_quad()),
|
||||
(_paper_frame(1.4, 2, with_finger=False, sharper=True), _full_quad()),
|
||||
]
|
||||
events = find_document_events(samples, gap_s=2.0, min_samples=2)
|
||||
assert len(events) == 1
|
||||
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Multi-page grouping and duplicate-capture handling in run_export's phase 2."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from paperpod.pipeline import _group_documents, _PageCandidate
|
||||
|
||||
|
||||
class _FakeOcr:
|
||||
def __init__(self, mean_confidence: float = 80.0):
|
||||
self.mean_confidence = mean_confidence
|
||||
|
||||
|
||||
def _cand(
|
||||
window_id: int,
|
||||
vendor: str | None = None,
|
||||
date: str | None = None,
|
||||
page_number: int | None = None,
|
||||
page_total: int | None = None,
|
||||
confidence: float = 80.0,
|
||||
total: str | None = None,
|
||||
doc_title: str | None = None,
|
||||
t_start: float | None = None,
|
||||
text: str = "",
|
||||
) -> _PageCandidate:
|
||||
t0 = t_start if t_start is not None else float(window_id)
|
||||
ocr = _FakeOcr(confidence)
|
||||
ocr.text = text
|
||||
return _PageCandidate(
|
||||
window={"window_id": window_id, "t_start": t0, "t_end": t0 + 0.5},
|
||||
crop_path=Path(f"crops/window_{window_id:03d}.png"),
|
||||
ocr=ocr,
|
||||
llm=None,
|
||||
vendor=vendor,
|
||||
date=date,
|
||||
time=None,
|
||||
total=total,
|
||||
form_code=None,
|
||||
tax_year=None,
|
||||
org_name=None,
|
||||
doc_title=doc_title,
|
||||
party=None,
|
||||
page_number=page_number,
|
||||
page_total=page_total,
|
||||
)
|
||||
|
||||
|
||||
def test_incrementing_page_markers_group_into_one_document():
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", date="2025-01-31", page_number=1, page_total=2),
|
||||
_cand(1, vendor="TD Canada Trust", page_number=2, page_total=2),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 2
|
||||
|
||||
|
||||
def test_new_page_1_starts_a_new_document():
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2),
|
||||
_cand(1, vendor="TD Canada Trust", page_number=2, page_total=2),
|
||||
_cand(2, vendor="TD Canada Trust", page_number=1, page_total=2),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 2
|
||||
assert [len(d.pages) for d in docs] == [2, 1]
|
||||
|
||||
|
||||
def test_different_page_totals_do_not_group():
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, page_number=1, page_total=2, vendor="TD"),
|
||||
_cand(1, page_number=2, page_total=17, vendor="TD"),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 2
|
||||
|
||||
|
||||
def test_documents_without_markers_stay_single_page():
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="Costco", date="2024-01-01", total="10.00"),
|
||||
_cand(1, vendor="Walmart", date="2024-01-02", total="20.00"),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 2
|
||||
|
||||
|
||||
def test_duplicate_capture_keeps_higher_confidence_read():
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=60.0),
|
||||
_cand(1, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=90.0),
|
||||
_cand(2, vendor="TD Canada Trust", page_number=2, page_total=2),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 2
|
||||
assert docs[0].pages[0].window["window_id"] == 1 # better read replaced the first
|
||||
|
||||
|
||||
def test_duplicate_signature_without_markers_is_dropped():
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="Costco", date="2024-01-01", total="10.00"),
|
||||
_cand(1, vendor="Costco", date="2024-01-01", total="10.00"),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 1
|
||||
|
||||
|
||||
def test_duplicate_signature_far_apart_is_not_dropped():
|
||||
# Marker-less pages of one statement share vendor/date; a deliberate
|
||||
# page flip (many seconds) means it's a different page, not a dup.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", date="2024-12-31", t_start=10.0),
|
||||
_cand(1, vendor="TD Canada Trust", date="2024-12-31", t_start=25.0),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 2
|
||||
|
||||
|
||||
def test_unreadable_marker_gap_does_not_split_document():
|
||||
# Page 12's marker OCR'd fine, page 13's didn't get captured, page 14
|
||||
# still belongs to the same statement.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD", page_number=12, page_total=17),
|
||||
_cand(1, vendor="TD", page_number=14, page_total=17),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 2
|
||||
|
||||
|
||||
def test_markerless_page_inside_marker_chain_does_not_split():
|
||||
# Page 10's marker was unreadable but pages 9 and 11 anchor it: one
|
||||
# statement, three pages, in order.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", page_number=9, page_total=17, t_start=0.0),
|
||||
_cand(1, vendor="TD Canada Trust", t_start=10.0),
|
||||
_cand(2, vendor="TD Canada Trust", page_number=11, page_total=17, t_start=20.0),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert [p.window["window_id"] for p in docs[0].pages] == [0, 1, 2]
|
||||
|
||||
|
||||
def test_markerless_page_attaches_to_incomplete_document():
|
||||
# "PAGE 1 OF 2" followed (after a deliberate page flip) by a
|
||||
# same-letterhead capture whose marker was unreadable: it's page 2,
|
||||
# not a new document — as long as the amounts don't match page 1.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
|
||||
doc_title="transaction_history", t_start=0.0,
|
||||
text="PAGE 1 OF 2 70.22 750.00 3000.00"),
|
||||
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
||||
t_start=10.0,
|
||||
text="PAGE 2 OF 2 15.00 22.50 100.00 200.00"),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 2
|
||||
|
||||
|
||||
def test_markerless_recapture_of_same_page_is_dropped():
|
||||
# Garbled second capture of PAGE 1 shares the amount fingerprint with
|
||||
# the real page 1 — must not become a fake "page 2".
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
|
||||
doc_title="transaction_history", t_start=0.0, confidence=90.0,
|
||||
text="PAGE 1 OF 2 70.22 DR 750.00 DR 3000.00 DR 490.96 1076 6254276"),
|
||||
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
||||
t_start=10.0, confidence=55.0,
|
||||
text="PAGK 1 of 10,22 150,00 3000 490 1076 6254276"),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 1
|
||||
assert docs[0].pages[0].window["window_id"] == 0
|
||||
|
||||
|
||||
def test_successive_statement_pages_not_collapsed_by_shared_account_numbers():
|
||||
# Pages of one statement share branch/account numbers (~0.3 overlap) —
|
||||
# that must not replace an earlier marked page with a later one.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", page_number=2, page_total=17,
|
||||
doc_title="transaction_history", t_start=0.0, confidence=92.0,
|
||||
text="PAGE 2 OF 17 1076 6254276 500.00 600.00 700.00"),
|
||||
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
||||
t_start=10.0, confidence=91.0,
|
||||
text="PAGE 4 OF 17 1076 6254276 800.00 900.00 1000.00"),
|
||||
]
|
||||
)
|
||||
# Second has no readable marker in this unit test (page_number=None):
|
||||
# attach as incomplete-fill; must NOT replace page 2.
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 2
|
||||
assert docs[0].pages[0].window["window_id"] == 0
|
||||
|
||||
|
||||
def test_markerless_low_confidence_does_not_fill_incomplete_doc():
|
||||
# Conf 63 mid-flip noise must not slot in as "page 2" just because the
|
||||
# statement is still incomplete.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
|
||||
doc_title="transaction_history", t_start=0.0, confidence=91.0,
|
||||
text="PAGE 1 OF 2 70.22 750.00"),
|
||||
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
||||
t_start=10.0, confidence=63.0,
|
||||
text="garble noise 11 22 33 44 55 66 77"),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 2
|
||||
|
||||
|
||||
def test_markerless_page_does_not_attach_to_complete_document():
|
||||
# After "PAGE 2 OF 2" the statement is done; the next capture is new.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", page_number=2, page_total=2,
|
||||
doc_title="transaction_history", t_start=0.0),
|
||||
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
||||
t_start=10.0),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 2
|
||||
|
||||
|
||||
def test_markerless_recapture_moments_later_is_a_duplicate():
|
||||
# Identical signature 0.5s later is a contour blip, not a new page.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="Costco", date="2024-01-01", total="10.00", t_start=0.0),
|
||||
_cand(1, vendor="Costco", date="2024-01-01", total="10.00", t_start=1.0),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 1
|
||||
|
||||
|
||||
def test_marked_page_not_replaced_by_nearby_same_vendor_date():
|
||||
# Statement pages share vendor+date and flip in ~3s — must not treat the
|
||||
# next page as a signature duplicate of the previous.
|
||||
docs = _group_documents(
|
||||
[
|
||||
_cand(0, vendor="TD Canada Trust", date="2024-12-31",
|
||||
page_number=3, page_total=17, doc_title="transaction_history",
|
||||
t_start=40.0, confidence=88.0,
|
||||
text="PAGE 3 OF 17 1076 6254276 100.00 200.00"),
|
||||
_cand(1, vendor="TD Canada Trust", date="2024-12-31",
|
||||
doc_title="transaction_history", t_start=49.0, confidence=91.0,
|
||||
text="PAGE 4 OF 17 1076 6254276 300.00 400.00 500.00"),
|
||||
]
|
||||
)
|
||||
assert len(docs) == 1
|
||||
assert len(docs[0].pages) == 2
|
||||
assert docs[0].pages[0].page_number == 3
|
||||
+27
-6
@@ -28,24 +28,45 @@ def test_build_name_full_metadata():
|
||||
def test_build_name_date_only():
|
||||
dt = datetime(2026, 1, 1, 12, 0, 0)
|
||||
result = build_name("2023-03-14", None, dt)
|
||||
assert result.stem == "2023-03-14_UNSORTED"
|
||||
assert result.stem == "2023-03-14_document"
|
||||
assert result.source == "ocr"
|
||||
|
||||
|
||||
def test_build_name_vendor_only():
|
||||
def test_build_name_vendor_only_uses_scan_date():
|
||||
dt = datetime(2026, 1, 1, 12, 0, 0)
|
||||
result = build_name(None, "METRO", dt)
|
||||
assert result.stem == "UNSORTED_metro_20260101_120000"
|
||||
assert result.source == "ocr"
|
||||
assert result.stem == "2026-01-01_metro"
|
||||
assert result.source == "scan_date"
|
||||
|
||||
|
||||
def test_build_name_nothing_found():
|
||||
dt = datetime(2026, 1, 1, 12, 0, 0)
|
||||
result = build_name(None, None, dt)
|
||||
assert result.stem == "UNSORTED_20260101_120000"
|
||||
assert result.stem == "2026-01-01_scan"
|
||||
assert result.source == "none"
|
||||
|
||||
|
||||
def test_build_name_appends_doc_title():
|
||||
dt = datetime(2026, 1, 1, 12, 0, 0)
|
||||
result = build_name("2025-01-31", "TD Canada Trust", dt, doc_title="transaction_history")
|
||||
assert result.stem == "2025-01-31_td_canada_trust_transaction_history"
|
||||
|
||||
|
||||
def test_build_name_appends_party_shortname():
|
||||
dt = datetime(2026, 1, 1, 12, 0, 0)
|
||||
result = build_name(
|
||||
"2025-01-31", "TD Canada Trust", dt,
|
||||
doc_title="transaction_history", party="LEVIT I",
|
||||
)
|
||||
assert result.stem == "2025-01-31_td_canada_trust_transaction_history_levit_i"
|
||||
|
||||
|
||||
def test_build_name_doc_title_not_duplicated_when_vendor_says_same():
|
||||
dt = datetime(2026, 1, 1, 12, 0, 0)
|
||||
result = build_name("2025-01-31", "Transaction History", dt, doc_title="transaction_history")
|
||||
assert result.stem == "2025-01-31_transaction_history"
|
||||
|
||||
|
||||
def test_build_name_with_time_disambiguates_same_day_visits():
|
||||
dt = datetime(2026, 1, 1, 12, 0, 0)
|
||||
morning = build_name("2023-03-20", "WALMART", dt, time="09:05")
|
||||
@@ -64,7 +85,7 @@ def test_build_name_invalid_time_ignored():
|
||||
def test_build_name_time_without_date_ignored():
|
||||
dt = datetime(2026, 1, 1, 12, 0, 0)
|
||||
result = build_name(None, "METRO", dt, time="09:05")
|
||||
assert result.stem == "UNSORTED_metro_20260101_120000"
|
||||
assert result.stem == "2026-01-01_metro"
|
||||
|
||||
|
||||
def test_build_name_tax_form_with_year_and_org():
|
||||
|
||||
Reference in New Issue
Block a user