Initial scaffold: capture + vision modules with detect CLI
- capture/: frame sampling at configurable fps, ffmpeg audio extraction (WAV, whisper-ready) - vision/: changed-pixel motion scoring, stable/moving segmentation, contour + perspective document detection, Laplacian sharpness scoring, optional CLAHE enhancement - pipeline: two-pass detect (motion timeline, then best-frame crop per stable window) writing crops, debug frames, report.json, and motion_scores.csv - CLI: probe / detect / extract-audio subcommands - config.yaml with tunable thresholds; placeholder packages for events, transcribe, ocr, naming, pdf, review_cli - synthetic sample video generator + 16 unit tests
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
"""Module 2: motion analysis, document detection, perspective correction."""
|
||||
|
||||
from paperpod.vision.document import DocumentDetection, detect_document, warp_document
|
||||
from paperpod.vision.enhance import enhance_for_ocr
|
||||
from paperpod.vision.motion import (
|
||||
Segment,
|
||||
find_segments,
|
||||
motion_score,
|
||||
prepare_gray,
|
||||
)
|
||||
from paperpod.vision.sharpness import sharpness_score
|
||||
|
||||
__all__ = [
|
||||
"Segment",
|
||||
"find_segments",
|
||||
"motion_score",
|
||||
"prepare_gray",
|
||||
"DocumentDetection",
|
||||
"detect_document",
|
||||
"warp_document",
|
||||
"sharpness_score",
|
||||
"enhance_for_ocr",
|
||||
]
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Document/receipt contour detection and perspective correction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class DocumentDetection:
|
||||
quad: np.ndarray # 4x2 float32, corner points in source-image coordinates
|
||||
area_ratio: float # contour area / frame area
|
||||
method: str # "quad" (clean 4-corner polygon) | "min_rect" (fallback)
|
||||
|
||||
|
||||
def _order_points(pts: np.ndarray) -> np.ndarray:
|
||||
"""Order 4 points as top-left, top-right, bottom-right, bottom-left."""
|
||||
pts = pts.astype(np.float32)
|
||||
s = pts.sum(axis=1)
|
||||
d = np.diff(pts, axis=1).ravel()
|
||||
return np.array(
|
||||
[
|
||||
pts[np.argmin(s)], # top-left: smallest x+y
|
||||
pts[np.argmin(d)], # top-right: smallest y-x
|
||||
pts[np.argmax(s)], # bottom-right: largest x+y
|
||||
pts[np.argmax(d)], # bottom-left: largest y-x
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
|
||||
def detect_document(
|
||||
image_bgr: np.ndarray,
|
||||
min_area_ratio: float = 0.04,
|
||||
canny_low: int = 50,
|
||||
canny_high: int = 150,
|
||||
) -> DocumentDetection | None:
|
||||
"""Find the largest document-like contour in the frame.
|
||||
|
||||
Pipeline: grayscale -> blur -> Canny -> dilate -> contours -> largest
|
||||
contour above min_area_ratio. If its polygon approximation has 4 corners
|
||||
we use it directly; otherwise we fall back to the minimum-area rectangle.
|
||||
"""
|
||||
h, w = image_bgr.shape[:2]
|
||||
frame_area = float(h * w)
|
||||
|
||||
gray = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2GRAY)
|
||||
gray = cv2.GaussianBlur(gray, (5, 5), 0)
|
||||
edges = cv2.Canny(gray, canny_low, canny_high)
|
||||
# Close gaps in the document outline so it forms one contour.
|
||||
edges = cv2.dilate(edges, np.ones((5, 5), np.uint8), iterations=2)
|
||||
|
||||
contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||||
if not contours:
|
||||
return None
|
||||
|
||||
best = max(contours, key=cv2.contourArea)
|
||||
area_ratio = cv2.contourArea(best) / frame_area
|
||||
if area_ratio < min_area_ratio:
|
||||
return None
|
||||
|
||||
peri = cv2.arcLength(best, True)
|
||||
approx = cv2.approxPolyDP(best, 0.02 * peri, True)
|
||||
if len(approx) == 4:
|
||||
quad = _order_points(approx.reshape(4, 2))
|
||||
method = "quad"
|
||||
else:
|
||||
rect = cv2.minAreaRect(best)
|
||||
quad = _order_points(cv2.boxPoints(rect))
|
||||
method = "min_rect"
|
||||
|
||||
return DocumentDetection(quad=quad, area_ratio=float(area_ratio), method=method)
|
||||
|
||||
|
||||
def warp_document(image_bgr: np.ndarray, quad: np.ndarray) -> np.ndarray:
|
||||
"""Perspective-correct the quad region into a clean top-down crop."""
|
||||
quad = _order_points(np.asarray(quad, dtype=np.float32))
|
||||
tl, tr, br, bl = quad
|
||||
|
||||
width = int(max(np.linalg.norm(br - bl), np.linalg.norm(tr - tl)))
|
||||
height = int(max(np.linalg.norm(tr - br), np.linalg.norm(tl - bl)))
|
||||
width, height = max(width, 1), max(height, 1)
|
||||
|
||||
dst = np.array(
|
||||
[[0, 0], [width - 1, 0], [width - 1, height - 1], [0, height - 1]],
|
||||
dtype=np.float32,
|
||||
)
|
||||
matrix = cv2.getPerspectiveTransform(quad, dst)
|
||||
return cv2.warpPerspective(image_bgr, matrix, (width, height))
|
||||
|
||||
|
||||
def scale_quad(quad: np.ndarray, scale_x: float, scale_y: float) -> np.ndarray:
|
||||
"""Rescale a quad detected on a downscaled frame back to full resolution."""
|
||||
quad = np.asarray(quad, dtype=np.float32).copy()
|
||||
quad[:, 0] *= scale_x
|
||||
quad[:, 1] *= scale_y
|
||||
return quad
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Optional image enhancement for OCR-readiness."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
|
||||
def enhance_for_ocr(image_bgr: np.ndarray) -> np.ndarray:
|
||||
"""Boost local contrast (CLAHE on the luminance channel).
|
||||
|
||||
Deliberately conservative: no binarization, so the crop stays pleasant to
|
||||
read in the PDF while giving OCR more to work with. Thresholding can be
|
||||
added behind a config flag later if OCR needs it.
|
||||
"""
|
||||
lab = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2LAB)
|
||||
l_chan, a_chan, b_chan = cv2.split(lab)
|
||||
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
|
||||
lab = cv2.merge((clahe.apply(l_chan), a_chan, b_chan))
|
||||
return cv2.cvtColor(lab, cv2.COLOR_LAB2BGR)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Frame-to-frame motion detection and stable/moving segmentation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
|
||||
def prepare_gray(image_bgr: np.ndarray, blur_ksize: int = 21) -> np.ndarray:
|
||||
"""Grayscale + Gaussian blur, the representation used for differencing."""
|
||||
if blur_ksize % 2 == 0:
|
||||
blur_ksize += 1
|
||||
gray = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2GRAY)
|
||||
return cv2.GaussianBlur(gray, (blur_ksize, blur_ksize), 0)
|
||||
|
||||
|
||||
def motion_score(
|
||||
prev_gray: np.ndarray, gray: np.ndarray, pixel_threshold: int = 12
|
||||
) -> float:
|
||||
"""Fraction of pixels (0.0-1.0) whose gray level changed by more than
|
||||
pixel_threshold between consecutive frames.
|
||||
|
||||
A changed-pixel ratio separates localized motion (a hand flipping a page)
|
||||
from sensor noise far better than the mean absolute difference, which
|
||||
dilutes small moving regions across the whole frame.
|
||||
"""
|
||||
diff = cv2.absdiff(prev_gray, gray)
|
||||
return float(np.count_nonzero(diff > pixel_threshold)) / diff.size
|
||||
|
||||
|
||||
@dataclass
|
||||
class Segment:
|
||||
kind: str # "stable" | "moving"
|
||||
t_start: float
|
||||
t_end: float
|
||||
# Indices into the sampled score series (inclusive range).
|
||||
first_sample: int
|
||||
last_sample: int
|
||||
|
||||
@property
|
||||
def duration(self) -> float:
|
||||
return self.t_end - self.t_start
|
||||
|
||||
|
||||
def find_segments(
|
||||
timestamps: list[float],
|
||||
scores: list[float],
|
||||
threshold: float,
|
||||
stable_min_duration_s: float,
|
||||
) -> list[Segment]:
|
||||
"""Classify each sample as stable/moving and group into segments.
|
||||
|
||||
Stable segments shorter than stable_min_duration_s are reclassified as
|
||||
moving (they are usually mid-gesture pauses), then adjacent segments of
|
||||
the same kind are merged. scores[i] describes the transition into
|
||||
timestamps[i], so both lists must be the same length.
|
||||
"""
|
||||
if len(timestamps) != len(scores):
|
||||
raise ValueError("timestamps and scores must have the same length")
|
||||
if not timestamps:
|
||||
return []
|
||||
|
||||
labels = ["stable" if s < threshold else "moving" for s in scores]
|
||||
|
||||
segments: list[Segment] = []
|
||||
start = 0
|
||||
for i in range(1, len(labels) + 1):
|
||||
if i == len(labels) or labels[i] != labels[start]:
|
||||
segments.append(
|
||||
Segment(
|
||||
kind=labels[start],
|
||||
t_start=timestamps[start],
|
||||
t_end=timestamps[i - 1],
|
||||
first_sample=start,
|
||||
last_sample=i - 1,
|
||||
)
|
||||
)
|
||||
start = i
|
||||
|
||||
# Demote too-short stable segments, then merge same-kind neighbors.
|
||||
for seg in segments:
|
||||
if seg.kind == "stable" and seg.duration < stable_min_duration_s:
|
||||
seg.kind = "moving"
|
||||
|
||||
merged: list[Segment] = []
|
||||
for seg in segments:
|
||||
if merged and merged[-1].kind == seg.kind:
|
||||
merged[-1].t_end = seg.t_end
|
||||
merged[-1].last_sample = seg.last_sample
|
||||
else:
|
||||
merged.append(seg)
|
||||
return merged
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Blur/sharpness scoring for best-frame selection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
|
||||
def sharpness_score(image: np.ndarray) -> float:
|
||||
"""Variance of the Laplacian; higher = sharper."""
|
||||
if image.ndim == 3:
|
||||
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
||||
return float(cv2.Laplacian(image, cv2.CV_64F).var())
|
||||
Reference in New Issue
Block a user