ilia
5be9d5935d
- vision/document.py: pad_quad() expands the detected quad outward before warping so a straight-line contour fit doesn't clip the first character of every line on a slightly curled/wrinkled receipt - ocr/extract.py + llm/vision.py: extract transaction time (HH:MM) so same-day repeat visits to a vendor get distinct filenames (2023-03-20_1525_walmart.pdf vs. a later same-day trip) - vision/orient.py: downscale before Tesseract OSD/confidence-sweep rotation detection — this was the dominant cost in the detection phase (7x speedup: 472s -> 67s on a 36s test video)
141 lines
4.3 KiB
Python
141 lines
4.3 KiB
Python
"""Unit tests for the regex/heuristic parsing logic in paperpod.ocr.extract.
|
|
|
|
These test the parsing functions directly on strings so they stay
|
|
deterministic and don't depend on Tesseract's OCR accuracy. End-to-end OCR
|
|
accuracy against rendered receipts is exercised in test_ocr_integration.py.
|
|
"""
|
|
|
|
from paperpod.ocr.extract import extract_date, extract_form_code, extract_org_name, extract_store_name, extract_tax_year, extract_time, extract_total
|
|
|
|
|
|
def test_extract_date_iso():
|
|
assert extract_date("2023-03-14 09:41 #4821") == "2023-03-14"
|
|
|
|
|
|
def test_extract_date_slash_mdy():
|
|
assert extract_date("Purchased on 03/14/2023 at noon") == "2023-03-14"
|
|
|
|
|
|
def test_extract_date_month_name():
|
|
assert extract_date("Invoice date: March 14, 2023") == "2023-03-14"
|
|
assert extract_date("Mar 14 2023") == "2023-03-14"
|
|
|
|
|
|
def test_extract_date_none_when_absent():
|
|
assert extract_date("no date here, just text") is None
|
|
|
|
|
|
def test_extract_date_rejects_implausible_values():
|
|
# Not a real month/day -> not treated as a date.
|
|
assert extract_date("13/45/2023") is None
|
|
|
|
|
|
def test_extract_total_basic():
|
|
text = "SUBTOTAL 72.14\nHST 13% 9.38\nTOTAL 81.52\n"
|
|
assert extract_total(text) == "81.52"
|
|
|
|
|
|
def test_extract_total_ignores_subtotal():
|
|
text = "SUBTOTAL 30.63\n"
|
|
assert extract_total(text) is None
|
|
|
|
|
|
def test_extract_total_word_boundary_with_prefix():
|
|
# "FUEL TOTAL" should still match "total" as its own word.
|
|
text = "FUEL TOTAL 65.10\n"
|
|
assert extract_total(text) == "65.10"
|
|
|
|
|
|
def test_extract_total_amount_on_next_line():
|
|
text = "TOTAL\n81.52\n"
|
|
assert extract_total(text) == "81.52"
|
|
|
|
|
|
def test_extract_total_none_when_absent():
|
|
assert extract_total("no totals here") is None
|
|
|
|
|
|
def test_extract_time_hms():
|
|
assert extract_time("03/20/23 15:25:35") == "15:25"
|
|
|
|
|
|
def test_extract_time_hm():
|
|
assert extract_time("Transaction at 09:05 today") == "09:05"
|
|
|
|
|
|
def test_extract_time_zero_pads_single_digit_hour():
|
|
assert extract_time("open at 9:05am") == "09:05"
|
|
|
|
|
|
def test_extract_time_none_when_absent():
|
|
assert extract_time("no time here, just $103.60 total") is None
|
|
|
|
|
|
def test_extract_time_ignores_colonless_numbers():
|
|
assert extract_time("TC# 3821 2498 6394 0609 7307") is None
|
|
|
|
|
|
def test_extract_form_code_t4():
|
|
text = "Employer's name\nYORK UNIVERSITY\nT4\nStatement of Remuneration Paid"
|
|
assert extract_form_code(text) == "T4"
|
|
|
|
|
|
def test_extract_form_code_prefers_more_specific_match():
|
|
# T4A should win over the more generic T4 substring it contains.
|
|
text = "Payer's name\nT4A\nStatement of Pension, Retirement, Annuity"
|
|
assert extract_form_code(text) == "T4A"
|
|
|
|
|
|
def test_extract_form_code_t5008():
|
|
text = "BOX 14:\nT5008\nSUMMARY OF SECURITY DISPOSITIONS 2023"
|
|
assert extract_form_code(text) == "T5008"
|
|
|
|
|
|
def test_extract_form_code_none_for_receipt():
|
|
assert extract_form_code("WALMART\nSUBTOTAL 21.97\nTOTAL 23.71") is None
|
|
|
|
|
|
def test_extract_org_name_finds_employer():
|
|
text = (
|
|
"Employer's name\nYORK UNIVERSITY Year 2023 Statement of Remuneration Paid\n"
|
|
"4700 KEELE STREET\nTORONTO ON M3J 1P3"
|
|
)
|
|
assert extract_org_name(text) == "YORK UNIVERSITY"
|
|
|
|
|
|
def test_extract_org_name_skips_addresses():
|
|
text = "MR ILIA DOBKIN\n153 NIAGARA DR\nOSHAWA ON L1G 8A6"
|
|
assert extract_org_name(text) == "MR ILIA DOBKIN"
|
|
|
|
|
|
def test_extract_org_name_skips_boilerplate_and_barcodes():
|
|
text = (
|
|
"JTA9500902-0301613-26631-0004-0004-00-\n"
|
|
"Canada Revenue Agence du revenu\n"
|
|
"Agency du Canada\n"
|
|
"AMC ENTERTAINMENT HOLDINGS INC AMC PRFRD EQT"
|
|
)
|
|
assert extract_org_name(text) == "AMC ENTERTAINMENT HOLDINGS INC AMC"
|
|
|
|
|
|
def test_extract_org_name_none_when_nothing_plausible():
|
|
assert extract_org_name("Canada Revenue Agency\nStatement of Remuneration Paid") is None
|
|
|
|
|
|
def test_extract_tax_year_from_t4():
|
|
text = "YORK UNIVERSITY Year 2023 Statement of Remuneration Paid"
|
|
assert extract_tax_year(text) == "2023"
|
|
|
|
|
|
def test_extract_tax_year_from_t5008_title():
|
|
text = "T5008\nSUMMARY OF SECURITY DISPOSITIONS 2023"
|
|
assert extract_tax_year(text) == "2023"
|
|
|
|
|
|
def test_extract_store_name_costco():
|
|
assert extract_store_name("COSTCO WHOLESALE\nVaughan #547") == "Costco"
|
|
|
|
|
|
def test_extract_store_name_walmart():
|
|
assert extract_store_name("HOW DID WE DO TODAY?\nWalmart") == "Walmart"
|