Add page-marker grouping and party-aware naming

Consecutive captures whose printed PAGE X OF Y markers advance under the same total are grouped into one multi-page PDF, with duplicate pages keeping the better read. Naming gains doc-title/party components and a scan-date fallback so nothing lands in UNSORTED. Vision LLM default switches to minicpm-v (faster, fewer false blanks on dense mono pages).
2026-07-26 15:37:28 -04:00
parent b11f46b00f
commit 12e5d1aeb1
10 changed files with 986 additions and 135 deletions
@@ -0,0 +1,266 @@
+"""Multi-page grouping and duplicate-capture handling in run_export's phase 2."""
+
+from pathlib import Path
+
+from paperpod.pipeline import _group_documents, _PageCandidate
+
+
+class _FakeOcr:
+    def __init__(self, mean_confidence: float = 80.0):
+        self.mean_confidence = mean_confidence
+
+
+def _cand(
+    window_id: int,
+    vendor: str | None = None,
+    date: str | None = None,
+    page_number: int | None = None,
+    page_total: int | None = None,
+    confidence: float = 80.0,
+    total: str | None = None,
+    doc_title: str | None = None,
+    t_start: float | None = None,
+    text: str = "",
+) -> _PageCandidate:
+    t0 = t_start if t_start is not None else float(window_id)
+    ocr = _FakeOcr(confidence)
+    ocr.text = text
+    return _PageCandidate(
+        window={"window_id": window_id, "t_start": t0, "t_end": t0 + 0.5},
+        crop_path=Path(f"crops/window_{window_id:03d}.png"),
+        ocr=ocr,
+        llm=None,
+        vendor=vendor,
+        date=date,
+        time=None,
+        total=total,
+        form_code=None,
+        tax_year=None,
+        org_name=None,
+        doc_title=doc_title,
+        party=None,
+        page_number=page_number,
+        page_total=page_total,
+    )
+
+
+def test_incrementing_page_markers_group_into_one_document():
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", date="2025-01-31", page_number=1, page_total=2),
+            _cand(1, vendor="TD Canada Trust", page_number=2, page_total=2),
+        ]
+    )
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 2
+
+
+def test_new_page_1_starts_a_new_document():
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2),
+            _cand(1, vendor="TD Canada Trust", page_number=2, page_total=2),
+            _cand(2, vendor="TD Canada Trust", page_number=1, page_total=2),
+        ]
+    )
+    assert len(docs) == 2
+    assert [len(d.pages) for d in docs] == [2, 1]
+
+
+def test_different_page_totals_do_not_group():
+    docs = _group_documents(
+        [
+            _cand(0, page_number=1, page_total=2, vendor="TD"),
+            _cand(1, page_number=2, page_total=17, vendor="TD"),
+        ]
+    )
+    assert len(docs) == 2
+
+
+def test_documents_without_markers_stay_single_page():
+    docs = _group_documents(
+        [
+            _cand(0, vendor="Costco", date="2024-01-01", total="10.00"),
+            _cand(1, vendor="Walmart", date="2024-01-02", total="20.00"),
+        ]
+    )
+    assert len(docs) == 2
+
+
+def test_duplicate_capture_keeps_higher_confidence_read():
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=60.0),
+            _cand(1, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=90.0),
+            _cand(2, vendor="TD Canada Trust", page_number=2, page_total=2),
+        ]
+    )
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 2
+    assert docs[0].pages[0].window["window_id"] == 1  # better read replaced the first
+
+
+def test_duplicate_signature_without_markers_is_dropped():
+    docs = _group_documents(
+        [
+            _cand(0, vendor="Costco", date="2024-01-01", total="10.00"),
+            _cand(1, vendor="Costco", date="2024-01-01", total="10.00"),
+        ]
+    )
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 1
+
+
+def test_duplicate_signature_far_apart_is_not_dropped():
+    # Marker-less pages of one statement share vendor/date; a deliberate
+    # page flip (many seconds) means it's a different page, not a dup.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", date="2024-12-31", t_start=10.0),
+            _cand(1, vendor="TD Canada Trust", date="2024-12-31", t_start=25.0),
+        ]
+    )
+    assert len(docs) == 2
+
+
+def test_unreadable_marker_gap_does_not_split_document():
+    # Page 12's marker OCR'd fine, page 13's didn't get captured, page 14
+    # still belongs to the same statement.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD", page_number=12, page_total=17),
+            _cand(1, vendor="TD", page_number=14, page_total=17),
+        ]
+    )
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 2
+
+
+def test_markerless_page_inside_marker_chain_does_not_split():
+    # Page 10's marker was unreadable but pages 9 and 11 anchor it: one
+    # statement, three pages, in order.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", page_number=9, page_total=17, t_start=0.0),
+            _cand(1, vendor="TD Canada Trust", t_start=10.0),
+            _cand(2, vendor="TD Canada Trust", page_number=11, page_total=17, t_start=20.0),
+        ]
+    )
+    assert len(docs) == 1
+    assert [p.window["window_id"] for p in docs[0].pages] == [0, 1, 2]
+
+
+def test_markerless_page_attaches_to_incomplete_document():
+    # "PAGE 1 OF 2" followed (after a deliberate page flip) by a
+    # same-letterhead capture whose marker was unreadable: it's page 2,
+    # not a new document — as long as the amounts don't match page 1.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
+                  doc_title="transaction_history", t_start=0.0,
+                  text="PAGE 1 OF 2 70.22 750.00 3000.00"),
+            _cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
+                  t_start=10.0,
+                  text="PAGE 2 OF 2 15.00 22.50 100.00 200.00"),
+        ]
+    )
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 2
+
+
+def test_markerless_recapture_of_same_page_is_dropped():
+    # Garbled second capture of PAGE 1 shares the amount fingerprint with
+    # the real page 1 — must not become a fake "page 2".
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
+                  doc_title="transaction_history", t_start=0.0, confidence=90.0,
+                  text="PAGE 1 OF 2 70.22 DR 750.00 DR 3000.00 DR 490.96 1076 6254276"),
+            _cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
+                  t_start=10.0, confidence=55.0,
+                  text="PAGK 1 of 10,22 150,00 3000 490 1076 6254276"),
+        ]
+    )
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 1
+    assert docs[0].pages[0].window["window_id"] == 0
+
+
+def test_successive_statement_pages_not_collapsed_by_shared_account_numbers():
+    # Pages of one statement share branch/account numbers (~0.3 overlap) —
+    # that must not replace an earlier marked page with a later one.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", page_number=2, page_total=17,
+                  doc_title="transaction_history", t_start=0.0, confidence=92.0,
+                  text="PAGE 2 OF 17 1076 6254276 500.00 600.00 700.00"),
+            _cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
+                  t_start=10.0, confidence=91.0,
+                  text="PAGE 4 OF 17 1076 6254276 800.00 900.00 1000.00"),
+        ]
+    )
+    # Second has no readable marker in this unit test (page_number=None):
+    # attach as incomplete-fill; must NOT replace page 2.
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 2
+    assert docs[0].pages[0].window["window_id"] == 0
+
+
+def test_markerless_low_confidence_does_not_fill_incomplete_doc():
+    # Conf 63 mid-flip noise must not slot in as "page 2" just because the
+    # statement is still incomplete.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
+                  doc_title="transaction_history", t_start=0.0, confidence=91.0,
+                  text="PAGE 1 OF 2 70.22 750.00"),
+            _cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
+                  t_start=10.0, confidence=63.0,
+                  text="garble noise 11 22 33 44 55 66 77"),
+        ]
+    )
+    assert len(docs) == 2
+
+
+def test_markerless_page_does_not_attach_to_complete_document():
+    # After "PAGE 2 OF 2" the statement is done; the next capture is new.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", page_number=2, page_total=2,
+                  doc_title="transaction_history", t_start=0.0),
+            _cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
+                  t_start=10.0),
+        ]
+    )
+    assert len(docs) == 2
+
+
+def test_markerless_recapture_moments_later_is_a_duplicate():
+    # Identical signature 0.5s later is a contour blip, not a new page.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="Costco", date="2024-01-01", total="10.00", t_start=0.0),
+            _cand(1, vendor="Costco", date="2024-01-01", total="10.00", t_start=1.0),
+        ]
+    )
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 1
+
+
+def test_marked_page_not_replaced_by_nearby_same_vendor_date():
+    # Statement pages share vendor+date and flip in ~3s — must not treat the
+    # next page as a signature duplicate of the previous.
+    docs = _group_documents(
+        [
+            _cand(0, vendor="TD Canada Trust", date="2024-12-31",
+                  page_number=3, page_total=17, doc_title="transaction_history",
+                  t_start=40.0, confidence=88.0,
+                  text="PAGE 3 OF 17 1076 6254276 100.00 200.00"),
+            _cand(1, vendor="TD Canada Trust", date="2024-12-31",
+                  doc_title="transaction_history", t_start=49.0, confidence=91.0,
+                  text="PAGE 4 OF 17 1076 6254276 300.00 400.00 500.00"),
+        ]
+    )
+    assert len(docs) == 1
+    assert len(docs[0].pages) == 2
+    assert docs[0].pages[0].page_number == 3