ilia
12e5d1aeb1
Consecutive captures whose printed PAGE X OF Y markers advance under the same total are grouped into one multi-page PDF, with duplicate pages keeping the better read. Naming gains doc-title/party components and a scan-date fallback so nothing lands in UNSORTED. Vision LLM default switches to minicpm-v (faster, fewer false blanks on dense mono pages).
267 lines
9.5 KiB
Python
267 lines
9.5 KiB
Python
"""Multi-page grouping and duplicate-capture handling in run_export's phase 2."""
|
|
|
|
from pathlib import Path
|
|
|
|
from paperpod.pipeline import _group_documents, _PageCandidate
|
|
|
|
|
|
class _FakeOcr:
|
|
def __init__(self, mean_confidence: float = 80.0):
|
|
self.mean_confidence = mean_confidence
|
|
|
|
|
|
def _cand(
|
|
window_id: int,
|
|
vendor: str | None = None,
|
|
date: str | None = None,
|
|
page_number: int | None = None,
|
|
page_total: int | None = None,
|
|
confidence: float = 80.0,
|
|
total: str | None = None,
|
|
doc_title: str | None = None,
|
|
t_start: float | None = None,
|
|
text: str = "",
|
|
) -> _PageCandidate:
|
|
t0 = t_start if t_start is not None else float(window_id)
|
|
ocr = _FakeOcr(confidence)
|
|
ocr.text = text
|
|
return _PageCandidate(
|
|
window={"window_id": window_id, "t_start": t0, "t_end": t0 + 0.5},
|
|
crop_path=Path(f"crops/window_{window_id:03d}.png"),
|
|
ocr=ocr,
|
|
llm=None,
|
|
vendor=vendor,
|
|
date=date,
|
|
time=None,
|
|
total=total,
|
|
form_code=None,
|
|
tax_year=None,
|
|
org_name=None,
|
|
doc_title=doc_title,
|
|
party=None,
|
|
page_number=page_number,
|
|
page_total=page_total,
|
|
)
|
|
|
|
|
|
def test_incrementing_page_markers_group_into_one_document():
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", date="2025-01-31", page_number=1, page_total=2),
|
|
_cand(1, vendor="TD Canada Trust", page_number=2, page_total=2),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 2
|
|
|
|
|
|
def test_new_page_1_starts_a_new_document():
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2),
|
|
_cand(1, vendor="TD Canada Trust", page_number=2, page_total=2),
|
|
_cand(2, vendor="TD Canada Trust", page_number=1, page_total=2),
|
|
]
|
|
)
|
|
assert len(docs) == 2
|
|
assert [len(d.pages) for d in docs] == [2, 1]
|
|
|
|
|
|
def test_different_page_totals_do_not_group():
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, page_number=1, page_total=2, vendor="TD"),
|
|
_cand(1, page_number=2, page_total=17, vendor="TD"),
|
|
]
|
|
)
|
|
assert len(docs) == 2
|
|
|
|
|
|
def test_documents_without_markers_stay_single_page():
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="Costco", date="2024-01-01", total="10.00"),
|
|
_cand(1, vendor="Walmart", date="2024-01-02", total="20.00"),
|
|
]
|
|
)
|
|
assert len(docs) == 2
|
|
|
|
|
|
def test_duplicate_capture_keeps_higher_confidence_read():
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=60.0),
|
|
_cand(1, vendor="TD Canada Trust", page_number=1, page_total=2, confidence=90.0),
|
|
_cand(2, vendor="TD Canada Trust", page_number=2, page_total=2),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 2
|
|
assert docs[0].pages[0].window["window_id"] == 1 # better read replaced the first
|
|
|
|
|
|
def test_duplicate_signature_without_markers_is_dropped():
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="Costco", date="2024-01-01", total="10.00"),
|
|
_cand(1, vendor="Costco", date="2024-01-01", total="10.00"),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 1
|
|
|
|
|
|
def test_duplicate_signature_far_apart_is_not_dropped():
|
|
# Marker-less pages of one statement share vendor/date; a deliberate
|
|
# page flip (many seconds) means it's a different page, not a dup.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", date="2024-12-31", t_start=10.0),
|
|
_cand(1, vendor="TD Canada Trust", date="2024-12-31", t_start=25.0),
|
|
]
|
|
)
|
|
assert len(docs) == 2
|
|
|
|
|
|
def test_unreadable_marker_gap_does_not_split_document():
|
|
# Page 12's marker OCR'd fine, page 13's didn't get captured, page 14
|
|
# still belongs to the same statement.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD", page_number=12, page_total=17),
|
|
_cand(1, vendor="TD", page_number=14, page_total=17),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 2
|
|
|
|
|
|
def test_markerless_page_inside_marker_chain_does_not_split():
|
|
# Page 10's marker was unreadable but pages 9 and 11 anchor it: one
|
|
# statement, three pages, in order.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", page_number=9, page_total=17, t_start=0.0),
|
|
_cand(1, vendor="TD Canada Trust", t_start=10.0),
|
|
_cand(2, vendor="TD Canada Trust", page_number=11, page_total=17, t_start=20.0),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert [p.window["window_id"] for p in docs[0].pages] == [0, 1, 2]
|
|
|
|
|
|
def test_markerless_page_attaches_to_incomplete_document():
|
|
# "PAGE 1 OF 2" followed (after a deliberate page flip) by a
|
|
# same-letterhead capture whose marker was unreadable: it's page 2,
|
|
# not a new document — as long as the amounts don't match page 1.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
|
|
doc_title="transaction_history", t_start=0.0,
|
|
text="PAGE 1 OF 2 70.22 750.00 3000.00"),
|
|
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
|
t_start=10.0,
|
|
text="PAGE 2 OF 2 15.00 22.50 100.00 200.00"),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 2
|
|
|
|
|
|
def test_markerless_recapture_of_same_page_is_dropped():
|
|
# Garbled second capture of PAGE 1 shares the amount fingerprint with
|
|
# the real page 1 — must not become a fake "page 2".
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
|
|
doc_title="transaction_history", t_start=0.0, confidence=90.0,
|
|
text="PAGE 1 OF 2 70.22 DR 750.00 DR 3000.00 DR 490.96 1076 6254276"),
|
|
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
|
t_start=10.0, confidence=55.0,
|
|
text="PAGK 1 of 10,22 150,00 3000 490 1076 6254276"),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 1
|
|
assert docs[0].pages[0].window["window_id"] == 0
|
|
|
|
|
|
def test_successive_statement_pages_not_collapsed_by_shared_account_numbers():
|
|
# Pages of one statement share branch/account numbers (~0.3 overlap) —
|
|
# that must not replace an earlier marked page with a later one.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", page_number=2, page_total=17,
|
|
doc_title="transaction_history", t_start=0.0, confidence=92.0,
|
|
text="PAGE 2 OF 17 1076 6254276 500.00 600.00 700.00"),
|
|
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
|
t_start=10.0, confidence=91.0,
|
|
text="PAGE 4 OF 17 1076 6254276 800.00 900.00 1000.00"),
|
|
]
|
|
)
|
|
# Second has no readable marker in this unit test (page_number=None):
|
|
# attach as incomplete-fill; must NOT replace page 2.
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 2
|
|
assert docs[0].pages[0].window["window_id"] == 0
|
|
|
|
|
|
def test_markerless_low_confidence_does_not_fill_incomplete_doc():
|
|
# Conf 63 mid-flip noise must not slot in as "page 2" just because the
|
|
# statement is still incomplete.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", page_number=1, page_total=2,
|
|
doc_title="transaction_history", t_start=0.0, confidence=91.0,
|
|
text="PAGE 1 OF 2 70.22 750.00"),
|
|
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
|
t_start=10.0, confidence=63.0,
|
|
text="garble noise 11 22 33 44 55 66 77"),
|
|
]
|
|
)
|
|
assert len(docs) == 2
|
|
|
|
|
|
def test_markerless_page_does_not_attach_to_complete_document():
|
|
# After "PAGE 2 OF 2" the statement is done; the next capture is new.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", page_number=2, page_total=2,
|
|
doc_title="transaction_history", t_start=0.0),
|
|
_cand(1, vendor="TD Canada Trust", doc_title="transaction_history",
|
|
t_start=10.0),
|
|
]
|
|
)
|
|
assert len(docs) == 2
|
|
|
|
|
|
def test_markerless_recapture_moments_later_is_a_duplicate():
|
|
# Identical signature 0.5s later is a contour blip, not a new page.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="Costco", date="2024-01-01", total="10.00", t_start=0.0),
|
|
_cand(1, vendor="Costco", date="2024-01-01", total="10.00", t_start=1.0),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 1
|
|
|
|
|
|
def test_marked_page_not_replaced_by_nearby_same_vendor_date():
|
|
# Statement pages share vendor+date and flip in ~3s — must not treat the
|
|
# next page as a signature duplicate of the previous.
|
|
docs = _group_documents(
|
|
[
|
|
_cand(0, vendor="TD Canada Trust", date="2024-12-31",
|
|
page_number=3, page_total=17, doc_title="transaction_history",
|
|
t_start=40.0, confidence=88.0,
|
|
text="PAGE 3 OF 17 1076 6254276 100.00 200.00"),
|
|
_cand(1, vendor="TD Canada Trust", date="2024-12-31",
|
|
doc_title="transaction_history", t_start=49.0, confidence=91.0,
|
|
text="PAGE 4 OF 17 1076 6254276 300.00 400.00 500.00"),
|
|
]
|
|
)
|
|
assert len(docs) == 1
|
|
assert len(docs[0].pages) == 2
|
|
assert docs[0].pages[0].page_number == 3
|