"""Multi-page grouping and duplicate-capture handling in run_export's phase 2.""" from pathlib import Path from paperpod.pipeline import _group_documents, _PageCandidate class _FakeOcr: def __init__(self, mean_confidence: float = 80.0): self.mean_confidence = mean_confidence def _cand( window_id: int, vendor: str | None = None, date: str | None = None, page_number: int | None = None, page_total: int | None = None, confidence: float = 80.0, total: str | None = None, doc_title: str | None = None, t_start: float | None = None, text: str = "", ) -> _PageCandidate: t0 = t_start if t_start is not None else float(window_id) ocr = _FakeOcr(confidence) ocr.text = text return _PageCandidate( window={"window_id": window_id, "t_start": t0, "t_end": t0 + 0.5}, crop_path=Path(f"crops/window_{window_id:03d}.png"), ocr=ocr, llm=None, vendor=vendor, date=date, time=None, total=total, form_code=None, tax_year=None, org_name=None, doc_title=doc_title, party=None, page_number=page_number, page_total=page_total, ) def test_incrementing_page_markers_group_into_one_document(): docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", date="2025-01-31", page_number=1, page_total=2), _cand(1, vendor="TD Canada Trust", page_number=2, page_total=2), ] ) assert len(docs) == 1 assert len(docs[0].pages) == 2 def test_new_page_1_starts_a_new_document(): docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2), _cand(1, vendor="TD Canada Trust", page_number=2, page_total=2), _cand(2, vendor="TD Canada Trust", page_number=1, page_total=2), ] ) assert len(docs) == 2 assert [len(d.pages) for d in docs] == [2, 1] def test_different_page_totals_do_not_group(): docs = _group_documents( [ _cand(0, page_number=1, page_total=2, vendor="TD"), _cand(1, page_number=2, page_total=17, vendor="TD"), ] ) assert len(docs) == 2 def test_documents_without_markers_stay_single_page(): docs = _group_documents( [ _cand(0, vendor="Costco", date="2024-01-01", total="10.00"), _cand(1, vendor="Walmart", date="2024-01-02", total="20.00"), ] ) assert len(docs) == 2 def test_duplicate_capture_keeps_higher_confidence_read(): docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=60.0), _cand(1, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=90.0), _cand(2, vendor="TD Canada Trust", page_number=2, page_total=2), ] ) assert len(docs) == 1 assert len(docs[0].pages) == 2 assert docs[0].pages[0].window["window_id"] == 1 # better read replaced the first def test_duplicate_signature_without_markers_is_dropped(): docs = _group_documents( [ _cand(0, vendor="Costco", date="2024-01-01", total="10.00"), _cand(1, vendor="Costco", date="2024-01-01", total="10.00"), ] ) assert len(docs) == 1 assert len(docs[0].pages) == 1 def test_duplicate_signature_far_apart_is_not_dropped(): # Marker-less pages of one statement share vendor/date; a deliberate # page flip (many seconds) means it's a different page, not a dup. docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", date="2024-12-31", t_start=10.0), _cand(1, vendor="TD Canada Trust", date="2024-12-31", t_start=25.0), ] ) assert len(docs) == 2 def test_unreadable_marker_gap_does_not_split_document(): # Page 12's marker OCR'd fine, page 13's didn't get captured, page 14 # still belongs to the same statement. docs = _group_documents( [ _cand(0, vendor="TD", page_number=12, page_total=17), _cand(1, vendor="TD", page_number=14, page_total=17), ] ) assert len(docs) == 1 assert len(docs[0].pages) == 2 def test_markerless_page_inside_marker_chain_does_not_split(): # Page 10's marker was unreadable but pages 9 and 11 anchor it: one # statement, three pages, in order. docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", page_number=9, page_total=17, t_start=0.0), _cand(1, vendor="TD Canada Trust", t_start=10.0), _cand(2, vendor="TD Canada Trust", page_number=11, page_total=17, t_start=20.0), ] ) assert len(docs) == 1 assert [p.window["window_id"] for p in docs[0].pages] == [0, 1, 2] def test_markerless_page_attaches_to_incomplete_document(): # "PAGE 1 OF 2" followed (after a deliberate page flip) by a # same-letterhead capture whose marker was unreadable: it's page 2, # not a new document — as long as the amounts don't match page 1. docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, doc_title="transaction_history", t_start=0.0, text="PAGE 1 OF 2 70.22 750.00 3000.00"), _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", t_start=10.0, text="PAGE 2 OF 2 15.00 22.50 100.00 200.00"), ] ) assert len(docs) == 1 assert len(docs[0].pages) == 2 def test_markerless_recapture_of_same_page_is_dropped(): # Garbled second capture of PAGE 1 shares the amount fingerprint with # the real page 1 — must not become a fake "page 2". docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, doc_title="transaction_history", t_start=0.0, confidence=90.0, text="PAGE 1 OF 2 70.22 DR 750.00 DR 3000.00 DR 490.96 1076 6254276"), _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", t_start=10.0, confidence=55.0, text="PAGK 1 of 10,22 150,00 3000 490 1076 6254276"), ] ) assert len(docs) == 1 assert len(docs[0].pages) == 1 assert docs[0].pages[0].window["window_id"] == 0 def test_successive_statement_pages_not_collapsed_by_shared_account_numbers(): # Pages of one statement share branch/account numbers (~0.3 overlap) — # that must not replace an earlier marked page with a later one. docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", page_number=2, page_total=17, doc_title="transaction_history", t_start=0.0, confidence=92.0, text="PAGE 2 OF 17 1076 6254276 500.00 600.00 700.00"), _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", t_start=10.0, confidence=91.0, text="PAGE 4 OF 17 1076 6254276 800.00 900.00 1000.00"), ] ) # Second has no readable marker in this unit test (page_number=None): # attach as incomplete-fill; must NOT replace page 2. assert len(docs) == 1 assert len(docs[0].pages) == 2 assert docs[0].pages[0].window["window_id"] == 0 def test_markerless_low_confidence_does_not_fill_incomplete_doc(): # Conf 63 mid-flip noise must not slot in as "page 2" just because the # statement is still incomplete. docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, doc_title="transaction_history", t_start=0.0, confidence=91.0, text="PAGE 1 OF 2 70.22 750.00"), _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", t_start=10.0, confidence=63.0, text="garble noise 11 22 33 44 55 66 77"), ] ) assert len(docs) == 2 def test_markerless_page_does_not_attach_to_complete_document(): # After "PAGE 2 OF 2" the statement is done; the next capture is new. docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", page_number=2, page_total=2, doc_title="transaction_history", t_start=0.0), _cand(1, vendor="TD Canada Trust", doc_title="transaction_history", t_start=10.0), ] ) assert len(docs) == 2 def test_markerless_recapture_moments_later_is_a_duplicate(): # Identical signature 0.5s later is a contour blip, not a new page. docs = _group_documents( [ _cand(0, vendor="Costco", date="2024-01-01", total="10.00", t_start=0.0), _cand(1, vendor="Costco", date="2024-01-01", total="10.00", t_start=1.0), ] ) assert len(docs) == 1 assert len(docs[0].pages) == 1 def test_marked_page_not_replaced_by_nearby_same_vendor_date(): # Statement pages share vendor+date and flip in ~3s — must not treat the # next page as a signature duplicate of the previous. docs = _group_documents( [ _cand(0, vendor="TD Canada Trust", date="2024-12-31", page_number=3, page_total=17, doc_title="transaction_history", t_start=40.0, confidence=88.0, text="PAGE 3 OF 17 1076 6254276 100.00 200.00"), _cand(1, vendor="TD Canada Trust", date="2024-12-31", doc_title="transaction_history", t_start=49.0, confidence=91.0, text="PAGE 4 OF 17 1076 6254276 300.00 400.00 500.00"), ] ) assert len(docs) == 1 assert len(docs[0].pages) == 2 assert docs[0].pages[0].page_number == 3