ilia/PaperPod
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add crop refinement, finger removal, and local LLM naming

Browse Source 
- vision/refine.py: tighten crops to the paper band (removes mat
  margins and hands beside receipts) and inpaint border-connected
  skin regions so fingers disappear from output
- llm/vision.py: identify documents with a local Ollama vision model
  (qwen2.5vl); extracts vendor/date/total/form code and flags quality
  issues (fingers, blur, glare); falls back to Tesseract when down
- pipeline: drop blank pages, dedupe consecutive captures of the same
  document, record refine/LLM fields in report and export summary
This commit is contained in:
ilia
2026-07-08 18:23:50 -04:00
parent 2c28a623fb
commit a43ab20df6
22 changed files with 1497 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
paperpod/pipeline.py
+170 -63
View File
@@ -2,8 +2,9 @@
Two sequential passes over the video (no seeking, works on any codec):
Pass 1: sample downscaled frames, score motion, find stable/moving segments.
Pass 2: within each stable window, pick the sharpest full-resolution frame,
detect the document contour, perspective-correct, save the crop.
Pass 2: scan every sampled frame for document contours, cluster consecutive
detections into capture events (works even while hands are moving),
pick the sharpest frame per event, perspective-correct, save the crop.
"""
from __future__ import annotations
@@ -19,13 +20,16 @@ import numpy as np
from paperpod.capture.video import iter_frames, probe_video
from paperpod.config import Config, config_to_dict
from paperpod.llm.vision import analyze_document, ollama_available
from paperpod.naming.filename import build_name, dedupe_stem
from paperpod.ocr.extract import run_ocr
from paperpod.ocr.extract import run_ocr, extract_store_name
from paperpod.pdf.build import images_to_pdf
from paperpod.vision.capture_events import _plausible_capture, find_document_events
from paperpod.vision.document import detect_document, warp_document
from paperpod.vision.enhance import enhance_for_ocr
from paperpod.vision.motion import Segment, find_segments, motion_score, prepare_gray
from paperpod.vision.sharpness import sharpness_score
from paperpod.vision.motion import find_segments, motion_score, prepare_gray
from paperpod.vision.orient import auto_rotate
from paperpod.vision.refine import refine_crop
def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> dict[str, Any]:
@@ -73,71 +77,98 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d
segments = find_segments(
timestamps, scores, cfg.motion.threshold, cfg.motion.stable_min_duration_s
)
stable_windows = [s for s in segments if s.kind == "stable"]
with open(out_dir / "motion_scores.csv", "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(["t", "score"])
writer.writerows(zip(timestamps, scores))
# ---- Pass 2: best frame per stable window, document detection -----------
# Map each sampled timestamp range to its stable window for quick lookup.
# ---- Pass 2: frame-based document capture events ------------------------
# Scan every sampled frame for a document contour, cluster consecutive
# detections into separate placements, keep the sharpest frame per cluster.
# Unlike stable-window-only detection, this still captures receipts shown
# while hands are adjusting them (motion scores stay high but the contour
# is visible frame-by-frame).
det_kwargs = dict(
min_area_ratio=cfg.document.min_area_ratio,
max_area_ratio=cfg.document.max_area_ratio,
max_aspect=cfg.document.max_aspect,
canny_low=cfg.document.canny_low,
canny_high=cfg.document.canny_high,
detect_width=cfg.document.detect_width,
)
samples: list[tuple[Any, Any]] = []
for frame in iter_frames(video_path, sample_fps=cfg.capture.sample_fps):
detection = detect_document(frame.image, **det_kwargs)
if detection is not None and _plausible_capture(detection):
samples.append((frame, detection))
doc_events = find_document_events(
samples,
gap_s=cfg.document.event_gap_s,
min_samples=cfg.document.event_min_samples,
)
window_records: list[dict[str, Any]] = []
best_frames: dict[int, tuple[float, Any]] = {} # window_idx -> (sharpness, frame)
for event in doc_events:
i = event.event_id
frame = event.frame
detection = event.detection
sharp = event.sharpness
def window_for(t: float) -> int | None:
for i, w in enumerate(stable_windows):
if w.t_start <= t <= w.t_end:
return i
return None
if stable_windows:
for frame in iter_frames(video_path, sample_fps=cfg.capture.sample_fps):
idx = window_for(frame.t)
if idx is None:
continue
score = sharpness_score(frame.image)
if idx not in best_frames or score > best_frames[idx][0]:
best_frames[idx] = (score, frame)
for i, window in enumerate(stable_windows):
record: dict[str, Any] = {
"window_id": i,
"t_start": round(window.t_start, 3),
"t_end": round(window.t_end, 3),
"duration_s": round(window.duration, 3),
"t_start": round(event.t_start, 3),
"t_end": round(event.t_end, 3),
"duration_s": round(event.t_end - event.t_start, 3),
"sample_count": event.sample_count,
"document_found": False,
"best_frame_t": round(frame.t, 3),
"best_frame_index": frame.index,
"sharpness": round(sharp, 1),
}
if i in best_frames:
sharp, frame = best_frames[i]
record["best_frame_t"] = round(frame.t, 3)
record["best_frame_index"] = frame.index
record["sharpness"] = round(sharp, 1)
frame_path = frames_dir / f"window_{i:03d}_full.png"
cv2.imwrite(str(frame_path), frame.image)
record["frame_path"] = str(frame_path.relative_to(out_dir))
frame_path = frames_dir / f"window_{i:03d}_full.png"
cv2.imwrite(str(frame_path), frame.image)
record["frame_path"] = str(frame_path.relative_to(out_dir))
detection = detect_document(
frame.image,
min_area_ratio=cfg.document.min_area_ratio,
canny_low=cfg.document.canny_low,
canny_high=cfg.document.canny_high,
)
if detection is not None:
crop = warp_document(frame.image, detection.quad)
if cfg.document.enhance:
crop = enhance_for_ocr(crop)
crop_path = crops_dir / f"window_{i:03d}.png"
cv2.imwrite(str(crop_path), crop)
record.update(
document_found=True,
detection_method=detection.method,
area_ratio=round(detection.area_ratio, 4),
quad=[[round(float(x), 1), round(float(y), 1)] for x, y in detection.quad],
crop_path=str(crop_path.relative_to(out_dir)),
crop_size=[crop.shape[1], crop.shape[0]],
)
crop = warp_document(frame.image, detection.quad)
refine_info: dict[str, Any] = {}
if cfg.document.refine:
refined = refine_crop(crop, remove_fingers=cfg.document.remove_fingers)
crop = refined.image
refine_info = {
"crop_tightened": refined.tightened,
"fingers_removed": refined.fingers_removed,
"skin_fraction": refined.skin_fraction,
}
rotation_applied = 0
rotation_method = "none"
if cfg.document.auto_rotate:
crop, orientation = auto_rotate(crop, lang=cfg.ocr.lang, psm=cfg.ocr.psm)
rotation_applied = orientation.rotation
rotation_method = orientation.method
if cfg.document.enhance:
crop = enhance_for_ocr(crop)
crop_path = crops_dir / f"window_{i:03d}.png"
cv2.imwrite(str(crop_path), crop)
record.update(
document_found=True,
detection_method=detection.method,
detection_source=detection.source,
detection_confidence=round(detection.confidence, 3),
area_ratio=round(detection.area_ratio, 4),
quad=[[round(float(x), 1), round(float(y), 1)] for x, y in detection.quad],
rotation_applied=rotation_applied,
rotation_method=rotation_method,
crop_path=str(crop_path.relative_to(out_dir)),
crop_size=[crop.shape[1], crop.shape[0]],
**refine_info,
)
window_records.append(record)
report = {
@@ -166,6 +197,15 @@ def run_detection(video_path: str | Path, cfg: Config, out_dir: str | Path) -> d
return report
def _is_exportable(ocr) -> bool:
"""Skip obvious mat-transition junk that has no usable naming signal."""
if ocr.form_code or ocr.date or ocr.total:
return True
if extract_store_name(ocr.text):
return True
return False
def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict[str, Any]:
"""OCR-name and PDF-export every detected crop from a detect() report.
@@ -192,8 +232,13 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict
except FileNotFoundError:
recording_start = datetime.now() - timedelta(seconds=duration_s)
use_llm = cfg.llm.enabled and ollama_available(cfg.llm.base_url)
if cfg.llm.enabled and not use_llm:
print("Ollama not reachable — falling back to Tesseract-only naming.")
used_stems: set[str] = set()
rows: list[dict[str, Any]] = []
prev_signature: tuple = ()
for window in report["stable_windows"]:
if not window.get("document_found"):
@@ -204,15 +249,70 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict
if image is None:
continue
# Always OCR the illumination-normalized version, even if the saved
# crop/PDF itself wasn't enhanced (document.enhance only controls the
# saved image's appearance) — this is pure naming-accuracy upside.
# Skip re-normalizing if the crop was already enhanced at detect time
# (cfg.document.enhance) to avoid dividing an already-flat image by
# its own blurred background twice.
ocr_image = image if cfg.document.enhance else enhance_for_ocr(image)
ocr = run_ocr(
image,
ocr_image,
lang=cfg.ocr.lang,
vendor_top_fraction=cfg.ocr.vendor_top_fraction,
min_word_confidence=cfg.ocr.min_word_confidence,
psm=cfg.ocr.psm,
)
# Local vision LLM (Ollama): reads the whole document at once, so it
# names curled receipts and hand-occluded captures that defeat
# Tesseract. Its fields take priority; OCR heuristics fill gaps.
llm = None
if use_llm:
llm = analyze_document(
image,
model=cfg.llm.model,
base_url=cfg.llm.base_url,
timeout=cfg.llm.timeout_s,
)
# Blank pages (backs of slips, empty sheets) OCR to garbled noise;
# the LLM identifies them reliably, so drop them outright.
if llm is not None and (
llm.doc_type == "blank" or llm.description.lower().startswith("blank")
):
continue
vendor = (llm.vendor if llm else None) or ocr.vendor
date = (llm.date if llm else None) or ocr.date
total = (llm.total if llm else None) or ocr.total
form_code = (llm.form_code if llm else None) or ocr.form_code
tax_year = (llm.tax_year if llm else None) or ocr.tax_year
org_name = (llm.vendor if llm and llm.form_code else None) or ocr.org_name
if llm is None and not _is_exportable(ocr):
continue
if llm is not None and not any((vendor, date, total, form_code)):
continue
# The same physical document sometimes splits into two capture
# events (hand briefly breaks the contour). If the previous export
# has the identical identity signature, it's the same document.
signature = (vendor, date, total, form_code)
if any(signature) and signature == prev_signature:
continue
prev_signature = signature
event_time = recording_start + timedelta(seconds=window["t_start"])
naming = build_name(ocr.date, ocr.vendor, event_time, cfg.naming.max_vendor_len)
naming = build_name(
date,
vendor,
event_time,
cfg.naming.max_vendor_len,
form_code=form_code,
tax_year=tax_year,
org_name=org_name,
)
stem = dedupe_stem(naming.stem, used_stems)
pdf_path = pdf_dir / f"{stem}.pdf"
@@ -223,11 +323,16 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict
"timestamp": event_time.isoformat(timespec="seconds"),
"pod_id": window["window_id"],
"page_count": 1,
"source_of_name": naming.source,
"ocr_vendor": ocr.vendor or "",
"ocr_date": ocr.date or "",
"ocr_total": ocr.total or "",
"source_of_name": "llm" if llm else naming.source,
"ocr_vendor": vendor or "",
"ocr_form_code": form_code or "",
"ocr_date": date or "",
"ocr_total": total or "",
"ocr_confidence": round(ocr.mean_confidence, 1),
"llm_description": llm.description if llm else "",
"llm_issues": ";".join(llm.issues) if llm else "",
"detection_confidence": window.get("detection_confidence", ""),
"rotation_applied": window.get("rotation_applied", 0),
"final_filename": pdf_path.name,
"pdf_path": str(pdf_path.relative_to(out_dir)),
}
@@ -236,7 +341,9 @@ def run_export(report: dict[str, Any], cfg: Config, out_dir: str | Path) -> dict
with open(out_dir / "export_summary.csv", "w", newline="") as f:
fieldnames = [
"timestamp", "pod_id", "page_count", "source_of_name",
"ocr_vendor", "ocr_date", "ocr_total", "ocr_confidence",
"ocr_vendor", "ocr_form_code", "ocr_date", "ocr_total", "ocr_confidence",
"llm_description", "llm_issues",
"detection_confidence", "rotation_applied",
"final_filename", "pdf_path",
]
writer = csv.DictWriter(f, fieldnames=fieldnames)