diff --git a/.gitignore b/.gitignore index 9f4e4fa..789074d 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,4 @@ viewer-frontend/.data/ !admin-frontend/public/brand/ !admin-frontend/public/brand/**/*.png backend/data/confidence_calibration.json +.dev-admin-password.tmp diff --git a/ROADMAP.md b/ROADMAP.md index d93a4cd..15615c0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -113,10 +113,11 @@ Living plan for product quality, auth/email reliability, and automation. - [x] **Immich precision gates** — max recognition distance, next-best person margin, detection floor 0.55, auto-accept distance cap; admin Chabad Blue theme (shell + login) - [x] **Person merge** — `POST /people/{id}/merge` + Modify People UI (#95) - [x] **Admin brand parity** — JRCC logos tracked; page tokens; navy sidebar + light/dark toggle (#96 + chrome follow-up) -- [ ] **Phase 5 — harder reject of junk detections (tiny/blur/pose)** — partial via detection floor; blur/pose still open -- [ ] **Phase 6 — cluster-name Identify** (merge done; naming unnamed clusters still open) -- [ ] **Admin People hub IA** — tabs for Identify / Auto-Match / Modify (see `docs/ADMIN_UX_REVIEW.md`) -- [ ] **a11y smoke** — axe on admin login + one workflow; viewer skip-link +- [x] **Admin People hub IA** — `/people/{identify,auto-match,modify}` tabs + redirects from old paths +- [x] **a11y smoke** — axe on admin login + viewer home; viewer skip-link already present +- [x] **Phase 5 — harder reject of junk detections** — blur (laplacian) + extreme yaw/pitch at Process time +- [x] **Cluster-name Identify** — “Name cluster” selects similarity group into Identify payload +- [ ] Soak `match_decisions` on DEV then revisit calibration `--apply` (needs ≥20 same + ≥20 diff pairs) ## Later diff --git a/admin-frontend/src/App.tsx b/admin-frontend/src/App.tsx index 7cdccc4..13bde9f 100644 --- a/admin-frontend/src/App.tsx +++ b/admin-frontend/src/App.tsx @@ -11,6 +11,7 @@ import Process from './pages/Process' import Identify from './pages/Identify' import AutoMatch from './pages/AutoMatch' import Modify from './pages/Modify' +import PeopleHub from './pages/PeopleHub' import Tags from './pages/Tags' import FacesMaintenance from './pages/FacesMaintenance' import ApproveIdentified from './pages/ApproveIdentified' @@ -117,9 +118,15 @@ function AppRoutes() { } /> } /> } /> - } /> - } /> - } /> + }> + } /> + } /> + } /> + } /> + + } /> + } /> + } /> } /> } /> } /> diff --git a/admin-frontend/src/components/Layout.tsx b/admin-frontend/src/components/Layout.tsx index 2c434c2..f7ca9a2 100644 --- a/admin-frontend/src/components/Layout.tsx +++ b/admin-frontend/src/components/Layout.tsx @@ -24,6 +24,10 @@ const PAGE_TITLES: Record = { '/scan': 'Scan photos', '/process': 'Process faces', '/search': 'Search photos', + '/people': 'People', + '/people/identify': 'Identify people', + '/people/auto-match': 'Auto-Match', + '/people/modify': 'Modify people', '/identify': 'Identify people', '/auto-match': 'Auto-Match', '/modify': 'Modify people', @@ -60,9 +64,7 @@ export default function Layout() { { path: '/scan', label: 'Scan', featureKey: 'scan' }, { path: '/process', label: 'Process', featureKey: 'process' }, { path: '/search', label: 'Search photos', featureKey: 'search_photos' }, - { path: '/identify', label: 'Identify people', featureKey: 'identify_people' }, - { path: '/auto-match', label: 'Auto-Match', featureKey: 'auto_match' }, - { path: '/modify', label: 'Modify people', featureKey: 'modify_people' }, + { path: '/people', label: 'People', featureKey: 'identify_people' }, { path: '/tags', label: 'Tag photos', featureKey: 'tag_photos' }, ] @@ -81,11 +83,14 @@ export default function Layout() { items.filter((item) => !item.featureKey || hasPermission(item.featureKey)) const renderNavLink = (item: NavItem, extraClasses = '') => { - const isActive = location.pathname === item.path + const isActive = + item.path === '/people' + ? location.pathname.startsWith('/people') + : location.pathname === item.path return ( { if (isIOSDevice) setSidebarOpen(false) }} @@ -103,7 +108,9 @@ export default function Layout() { const visiblePrimary = filterNavItems(primaryNavItems) const visibleMaintenance = filterNavItems(maintenanceNavItems) const visibleFooter = filterNavItems(footerNavItems) - const pageTitle = PAGE_TITLES[location.pathname] || 'JRCC Photos Admin' + const pageTitle = + PAGE_TITLES[location.pathname] || + (location.pathname.startsWith('/people') ? 'People' : 'JRCC Photos Admin') const railWidth = isIOSDevice ? 'w-16' : 'w-64' const contentOffset = isIOSDevice ? 'ml-16' : 'ml-64' diff --git a/admin-frontend/src/pages/Identify.tsx b/admin-frontend/src/pages/Identify.tsx index 62c3bbd..43abe7b 100644 --- a/admin-frontend/src/pages/Identify.tsx +++ b/admin-frontend/src/pages/Identify.tsx @@ -91,6 +91,8 @@ export default function Identify() { const [compareEnabled, setCompareEnabled] = useState(true) const [selectedSimilar, setSelectedSimilar] = useState>({}) const [uniqueFacesOnly, setUniqueFacesOnly] = useState(true) + /** face id → other face ids in the same similarity cluster (excludes self) */ + const [clusterMembersByRep, setClusterMembersByRep] = useState>({}) // Excluded faces filter const [includeExcludedFaces, setIncludeExcludedFaces] = useState(false) @@ -222,6 +224,8 @@ export default function Identify() { let filtered = res.items if (uniqueFacesOnly) { filtered = await filterUniqueFaces(filtered) + } else { + setClusterMembersByRep({}) } setFaces(filtered) @@ -329,23 +333,28 @@ export default function Identify() { // Track which faces should be excluded (duplicates in groups) const excludeSet = new Set() - + const clusterMap: Record = {} + // For each group, keep only the first face and mark others for exclusion for (const group of groups) { - let firstFaceFound = false + let representative: number | null = null + const members: number[] = [] for (const face of faces) { if (group.has(face.id)) { - if (!firstFaceFound) { - // Keep this face (first representative) - firstFaceFound = true + if (representative === null) { + representative = face.id } else { - // Exclude this face (duplicate) excludeSet.add(face.id) + members.push(face.id) } } } + if (representative !== null && members.length > 0) { + clusterMap[representative] = members + } } - + + setClusterMembersByRep(clusterMap) // Return faces that are not excluded return faces.filter(face => !excludeSet.has(face.id)) } @@ -1557,7 +1566,27 @@ export default function Identify() { readOnly={!!personId} disabled={!!personId} /> - + + {currentFace && (clusterMembersByRep[currentFace.id]?.length ?? 0) > 0 && ( + { + const members = clusterMembersByRep[currentFace.id] || [] + const next: Record = { ...selectedSimilar } + for (const id of members) next[id] = true + setSelectedSimilar(next) + setCompareEnabled(true) + showToast( + `Selected ${members.length} similar face(s) in this cluster — then Identify`, + 'success', + ) + }} + className="px-3 py-2 rounded border border-primary/40 bg-primary/10 text-primary text-sm font-medium hover:bg-primary/15" + > + Name cluster ({(clusterMembersByRep[currentFace.id]?.length ?? 0) + 1}) + + )} = [ + { to: '/people/identify', label: 'Identify', featureKey: 'identify_people' }, + { to: '/people/auto-match', label: 'Auto-Match', featureKey: 'auto_match' }, + { to: '/people/modify', label: 'Modify', featureKey: 'modify_people' }, +] + +export default function PeopleHub() { + const { hasPermission } = useAuth() + const location = useLocation() + const tabs = TABS.filter((t) => !t.featureKey || hasPermission(t.featureKey)) + + return ( + + + {tabs.map((tab) => { + const active = location.pathname.startsWith(tab.to) + return ( + + {tab.label} + + ) + })} + + + + ) +} diff --git a/admin-frontend/src/pages/Tags.tsx b/admin-frontend/src/pages/Tags.tsx index d81bbd2..e473477 100644 --- a/admin-frontend/src/pages/Tags.tsx +++ b/admin-frontend/src/pages/Tags.tsx @@ -335,7 +335,7 @@ export default function Tags() { // Navigate to Identify page with photo IDs in new tab const photoIdsStr = processedPhotos.map(p => p.id).join(',') - window.open(`/identify?photo_ids=${photoIdsStr}`, '_blank') + window.open(`/people/identify?photo_ids=${photoIdsStr}`, '_blank') } // Add tag to photo (with optional immediate save) diff --git a/backend/config.py b/backend/config.py index fea5c96..ea9fd7e 100644 --- a/backend/config.py +++ b/backend/config.py @@ -52,4 +52,11 @@ DEFAULT_RUN_TOLERANCE = 0.4 # Phase 4: match each person against up to N trusted refs; keep best (min) distance AUTO_MATCH_MAX_REFERENCE_FACES = 3 +# Phase 5: reject junk detections at Process time (Immich-style precision) +# Laplacian variance on grayscale face crop; below this = too blurry to keep +MIN_LAPLACIAN_VARIANCE = 80.0 +# Extreme pose (degrees) — mirrors PoseDetector EXTREME_* thresholds +MAX_FACE_YAW_DEGREES = 60.0 +MAX_FACE_PITCH_DEGREES = 45.0 + diff --git a/backend/services/face_service.py b/backend/services/face_service.py index 2720f4c..c13ce39 100644 --- a/backend/services/face_service.py +++ b/backend/services/face_service.py @@ -33,12 +33,15 @@ from backend.config import ( DEEPFACE_ALIGN_FACES, DEEPFACE_ENFORCE_DETECTION, DEFAULT_FACE_TOLERANCE, + MAX_FACE_PITCH_DEGREES, MAX_FACE_SIZE, + MAX_FACE_YAW_DEGREES, MAX_RECOGNITION_DISTANCE, MIN_AUTO_MATCH_FACE_SIZE_RATIO, MIN_AUTO_MATCH_REFERENCE_QUALITY, MIN_FACE_CONFIDENCE, MIN_FACE_SIZE, + MIN_LAPLACIAN_VARIANCE, MIN_NEXT_BEST_PERSON_MARGIN, USE_CALIBRATED_CONFIDENCE, ) @@ -326,6 +329,53 @@ def is_valid_face_detection_with_reason( return True, "" +def is_junk_face_reject( + image_np: np.ndarray, + face_location: dict, + yaw_angle: Optional[float], + pitch_angle: Optional[float], +) -> Tuple[bool, str]: + """Phase 5: reject blurry / extreme-pose faces before they enter the DB. + + Returns (ok, reason). ok=True means keep the face. + """ + try: + if yaw_angle is not None and abs(float(yaw_angle)) >= MAX_FACE_YAW_DEGREES: + return False, f"yaw too extreme (got {float(yaw_angle):.1f}°, need |yaw| < {MAX_FACE_YAW_DEGREES})" + if pitch_angle is not None and abs(float(pitch_angle)) >= MAX_FACE_PITCH_DEGREES: + return False, ( + f"pitch too extreme (got {float(pitch_angle):.1f}°, need |pitch| < {MAX_FACE_PITCH_DEGREES})" + ) + + x = int(face_location.get("x", 0)) + y = int(face_location.get("y", 0)) + w = int(face_location.get("w", 0)) + h = int(face_location.get("h", 0)) + if w < 1 or h < 1: + return False, "invalid crop size for sharpness check" + + img_h, img_w = image_np.shape[:2] + x1, y1 = max(0, x), max(0, y) + x2, y2 = min(img_w, x + w), min(img_h, y + h) + crop = image_np[y1:y2, x1:x2] + if crop.size == 0: + return False, "empty crop for sharpness check" + + if crop.ndim == 3: + gray = np.mean(crop.astype(np.float32), axis=2) + else: + gray = crop.astype(np.float32) + lap = face_laplacian_variance(gray) + if lap < MIN_LAPLACIAN_VARIANCE: + return False, ( + f"too blurry (laplacian={lap:.1f}, need >= {MIN_LAPLACIAN_VARIANCE})" + ) + return True, "" + except Exception as e: + print(f"[FaceService] ⚠️ Error in junk face reject: {e}") + return True, "" + + def process_photo_faces( db: Session, photo: Photo, @@ -741,6 +791,19 @@ def process_photo_faces( yaw_str = f"{yaw_angle:.2f}°" if yaw_angle is not None else "None" _print_with_stderr(f"[FaceService] Face {idx+1}/{faces_detected} in {photo.filename}: " f"face_width=None, pose_mode={pose_mode}, yaw={yaw_str}") + + # Phase 5: reject blur / extreme pose junk + junk_ok, junk_reason = is_junk_face_reject( + image_np, location, yaw_angle, pitch_angle + ) + if not junk_ok: + failure_type = junk_reason.split(":")[0].strip() if ":" in junk_reason else junk_reason + validation_failures[failure_type] = validation_failures.get(failure_type, 0) + 1 + _print_with_stderr( + f"[FaceService] Face {idx+1}/{faces_detected} in {photo.filename} " + f"rejected as junk: {junk_reason}" + ) + continue # Store face in database - match desktop schema exactly # Desktop: confidence REAL DEFAULT 0.0 (legacy), face_confidence REAL (actual) diff --git a/docs/ADMIN_UX_REVIEW.md b/docs/ADMIN_UX_REVIEW.md index d6b6d34..5a1fd9b 100644 --- a/docs/ADMIN_UX_REVIEW.md +++ b/docs/ADMIN_UX_REVIEW.md @@ -28,9 +28,9 @@ Snapshot after Chabad Blue chrome parity (navy sidebar, theme toggle, shared tok | Area | Admin | Viewer | |------|-------|--------| | Light/dark | Yes (this work) | Yes | -| Skip link | Yes | No (gap) | +| Skip link | Yes | Yes (`app/layout.tsx`) | | Theme control a11y | `aria-label` / `aria-pressed` | Same | -| Systematic axe CI | No | No | +| Systematic axe CI | Smoke (`e2e/tests/a11y.smoke.spec.ts`) | Smoke (viewer home) | | Form labels / live regions | Partial (login, toasts, some dialogs) | Stronger on gallery controls | ## What’s next (product) diff --git a/docs/FACE_ACCURACY_STATUS.md b/docs/FACE_ACCURACY_STATUS.md index ea9c4d0..b3442fd 100644 --- a/docs/FACE_ACCURACY_STATUS.md +++ b/docs/FACE_ACCURACY_STATUS.md @@ -27,11 +27,9 @@ plus Immich-inspired precision gates. ## What's next -1. Run Auto-Match Saves on DEV so `match_decisions` fills; then revisit calibration `--apply`. -2. Phase 5 blur/pose junk reject. -3. Cluster-name Identify (Phase 6 remainder). -4. Admin People hub IA + a11y axe smoke — `docs/ADMIN_UX_REVIEW.md`. -5. Rotate DEV `ADMIN_PASSWORD` if still the default. +1. Run Auto-Match Saves on DEV so `match_decisions` fills; then revisit calibration `--apply` (needs ≥20 same + ≥20 diff). +2. People hub + Phase 5 junk reject + cluster-name Identify shipped (this release). +3. Rotate DEV `ADMIN_PASSWORD` — done via `scripts/rotate-dev-admin-password.py` (update Vaultwarden). ## Immich review — steal vs skip diff --git a/e2e/package-lock.json b/e2e/package-lock.json index 9464c7a..8ee4937 100644 --- a/e2e/package-lock.json +++ b/e2e/package-lock.json @@ -8,6 +8,7 @@ "name": "punimtag-e2e", "version": "1.0.0", "devDependencies": { + "@axe-core/playwright": "^4.10.1", "@levkin/playkit": "git+https://git.levkin.ca/ilia/playkit.git#v0.3.1", "@playwright/test": "^1.52.0", "@types/node": "^22.15.0", @@ -18,6 +19,19 @@ "node": ">=20" } }, + "node_modules/@axe-core/playwright": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.10.1.tgz", + "integrity": "sha512-EV5t39VV68kuAfMKqb/RL+YjYKhfuGim9rgIaQ6Vntb2HgaCaau0h98Y3WEUqW1+PbdzxDtDNjFAipbtZuBmEA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.10.2" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@levkin/playkit": { "version": "0.3.1", "resolved": "git+https://git.levkin.ca/ilia/playkit.git#3380ba95115423950b4ba0855cc9367475d7f7cf", @@ -60,6 +74,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/axe-core": { + "version": "4.10.3", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.3.tgz", + "integrity": "sha512-Xm7bpRXnDSX2YE2YFfBk2FnF0ep6tmG7xPh8iHee8MIcrgq762Nkce856dYtJYLkuIoYZvGfTs/PbZhideTcEg==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/fsevents": { "version": "2.3.2", "dev": true, diff --git a/e2e/package.json b/e2e/package.json index e6a11ea..76504d3 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -11,6 +11,7 @@ "node": ">=20" }, "devDependencies": { + "@axe-core/playwright": "^4.10.1", "@levkin/playkit": "git+https://git.levkin.ca/ilia/playkit.git#v0.3.1", "@playwright/test": "^1.52.0", "@types/node": "^22.15.0", diff --git a/e2e/tests/a11y.smoke.spec.ts b/e2e/tests/a11y.smoke.spec.ts new file mode 100644 index 0000000..a40b57c --- /dev/null +++ b/e2e/tests/a11y.smoke.spec.ts @@ -0,0 +1,30 @@ +import AxeBuilder from '@axe-core/playwright'; +import { test, expect } from '../fixtures'; +import { DEFAULT_ADMIN_BASE_URL } from '../env-defaults'; + +/** + * Lightweight a11y smoke — critical violations only (axe impact: critical|serious). + * Does not block on moderate/minor noise. + */ +test.describe('a11y smoke @smoke', () => { + test('viewer home has no critical axe violations', async ({ page }) => { + await page.goto('/'); + await expect(page.getByRole('link', { name: /skip to main content/i })).toBeAttached(); + const results = await new AxeBuilder({ page }) + .withTags(['wcag2a', 'wcag2aa']) + .analyze(); + const bad = results.violations.filter((v) => v.impact === 'critical' || v.impact === 'serious'); + expect(bad, JSON.stringify(bad, null, 2)).toEqual([]); + }); + + test('admin login has no critical axe violations', async ({ page }) => { + const adminBaseUrl = process.env.PLAYKIT_ADMIN_BASE_URL || DEFAULT_ADMIN_BASE_URL; + await page.goto(`${adminBaseUrl}/login`); + await expect(page.getByRole('button', { name: /sign in/i })).toBeVisible({ timeout: 20_000 }); + const results = await new AxeBuilder({ page }) + .withTags(['wcag2a', 'wcag2aa']) + .analyze(); + const bad = results.violations.filter((v) => v.impact === 'critical' || v.impact === 'serious'); + expect(bad, JSON.stringify(bad, null, 2)).toEqual([]); + }); +}); diff --git a/e2e/tests/admin.review-pages.spec.ts b/e2e/tests/admin.review-pages.spec.ts index abb9f0d..af2d9b2 100644 --- a/e2e/tests/admin.review-pages.spec.ts +++ b/e2e/tests/admin.review-pages.spec.ts @@ -52,7 +52,7 @@ test.describe('admin review pages @smoke', () => { expectWithinBudget(timings, 'admin_login', BUDGET_MS.uiLogin); await timings.measure('admin_identify', async () => { - await page.goto(`${adminBaseUrl}/identify`); + await page.goto(`${adminBaseUrl}/people/identify`); await expect(page.getByRole('heading', { name: /Identify/i })).toBeVisible({ timeout: 20_000, }); @@ -61,7 +61,7 @@ test.describe('admin review pages @smoke', () => { expectWithinBudget(timings, 'admin_identify', BUDGET_MS.uiAction); await timings.measure('admin_auto_match', async () => { - await page.goto(`${adminBaseUrl}/auto-match`); + await page.goto(`${adminBaseUrl}/people/auto-match`); await expect(page.getByRole('heading', { name: /Auto-Match/i })).toBeVisible({ timeout: 20_000, }); diff --git a/scripts/rotate-dev-admin-password.py b/scripts/rotate-dev-admin-password.py new file mode 100644 index 0000000..e5f9649 --- /dev/null +++ b/scripts/rotate-dev-admin-password.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Rotate DEV admin login: DB password_hash + .env ADMIN_PASSWORD.""" +from __future__ import annotations + +import base64 +import secrets +import string +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TMP = ROOT / ".dev-admin-password.tmp" +HOST = "root@10.0.10.201" +LXC = "9101" + + +def main() -> int: + alphabet = string.ascii_letters + string.digits + pw = "".join(secrets.choice(alphabet) for _ in range(28)) + TMP.write_text(pw + "\n") + TMP.chmod(0o600) + gi = ROOT / ".gitignore" + if gi.exists() and ".dev-admin-password.tmp" not in gi.read_text(): + gi.write_text(gi.read_text().rstrip() + "\n.dev-admin-password.tmp\n") + + remote_py = f""" +from pathlib import Path +import os, re, sys +sys.path.insert(0, '/opt/punimtag') +os.chdir('/opt/punimtag') +# load .env into os.environ for DATABASE_URL +env_path = Path('/opt/punimtag/.env') +text = env_path.read_text() +for line in text.splitlines(): + if not line.strip() or line.strip().startswith('#') or '=' not in line: + continue + k, v = line.split('=', 1) + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + +pw = {pw!r} +# update .env ADMIN_PASSWORD +if re.search(r'^ADMIN_PASSWORD=.*$', text, re.M): + text = re.sub(r'^ADMIN_PASSWORD=.*$', 'ADMIN_PASSWORD=' + pw, text, count=1, flags=re.M) +else: + text = text.rstrip() + '\\nADMIN_PASSWORD=' + pw + '\\n' +env_path.write_text(text) + +from backend.utils.password import hash_password, verify_password +from backend.db.session import SessionLocal +from backend.db.models import User + +db = SessionLocal() +try: + users = db.query(User).filter(User.username == 'admin').all() + if not users: + # also try is_admin + users = db.query(User).filter(User.is_admin == True).all() + print('users_found', [(u.id, u.username, bool(u.password_hash)) for u in users]) + if not users: + raise SystemExit('no admin user in DB') + h = hash_password(pw) + for u in users: + if u.username == 'admin' or u.is_admin: + u.password_hash = h + u.password_change_required = False + db.add(u) + db.commit() + # verify hash + u = db.query(User).filter(User.username == 'admin').first() + assert u and verify_password(pw, u.password_hash) + assert not verify_password('admin', u.password_hash) + print('db_rotated_ok') +finally: + db.close() +""" + b64 = base64.b64encode(remote_py.encode()).decode() + update = f"""set -euo pipefail +printf '%s' '{b64}' | base64 -d > /tmp/rotate_admin_db.py +sudo -u appuser bash -lc 'cd /opt/punimtag && ./venv/bin/python /tmp/rotate_admin_db.py' +sudo -u appuser bash -lc 'cd /opt/punimtag && pm2 restart punimtag-api --update-env' +sleep 5 +""" + subprocess.run( + ["ssh", "-o", "BatchMode=yes", HOST, f"pct exec {LXC} -- bash -s"], + input=update.encode(), + check=True, + ) + + verify = f"""set -euo pipefail +export PW=$(printf '%s' '{base64.b64encode(pw.encode()).decode()}' | base64 -d) +python3 - <<'PY' +import json, os, urllib.request, urllib.error +def login(pw): + req = urllib.request.Request( + 'http://127.0.0.1:8000/api/v1/auth/login', + data=json.dumps({{'username':'admin','password':pw}}).encode(), + headers={{'Content-Type':'application/json'}}, + ) + try: + with urllib.request.urlopen(req) as r: + return r.status + except urllib.error.HTTPError as e: + return e.code +print('old', login('admin')) +print('new', login(os.environ['PW'])) +PY +""" + out = subprocess.run( + ["ssh", "-o", "BatchMode=yes", HOST, f"pct exec {LXC} -- bash -s"], + input=verify.encode(), + check=True, + capture_output=True, + ) + print(out.stdout.decode()) + if b"old 401" not in out.stdout or b"new 200" not in out.stdout: + print("VERIFY FAILED", out.stderr.decode(), file=sys.stderr) + return 1 + print(f"OK — password in {TMP} (gitignored). Update Vaultwarden PunimTag admin entry.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/seed_match_decisions_from_identified.py b/scripts/seed_match_decisions_from_identified.py new file mode 100644 index 0000000..e0db486 --- /dev/null +++ b/scripts/seed_match_decisions_from_identified.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Seed match_decisions from identified faces on a running DEV DB, then dry-run calibration. + +Does NOT --apply calibration (prints readiness only). +Usage (on DEV or with DATABASE_URL): + cd /opt/punimtag && ./venv/bin/python scripts/seed_match_decisions_from_identified.py +""" +from __future__ import annotations + +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +os.chdir(ROOT) + +# load .env +env_path = ROOT / ".env" +if env_path.exists(): + for line in env_path.read_text().splitlines(): + if not line.strip() or line.strip().startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + +import numpy as np + +from backend.db.session import SessionLocal +from backend.db.models import Face, MatchDecision, Person +from backend.services.face_service import record_match_decisions + + +def _encoding(face: Face) -> np.ndarray | None: + if not face.encoding: + return None + arr = np.frombuffer(face.encoding, dtype=np.float32) + if arr.size == 0: + return None + return arr + + +def _cosine_distance(a: np.ndarray, b: np.ndarray) -> float: + na = np.linalg.norm(a) + nb = np.linalg.norm(b) + if na < 1e-9 or nb < 1e-9: + return 1.0 + sim = float(np.dot(a, b) / (na * nb)) + return float(1.0 - sim) + + +def main() -> int: + db = SessionLocal() + try: + existing = db.query(MatchDecision).count() + print(f"match_decisions before: {existing}") + + people = db.query(Person).all() + same_items: list[dict] = [] + diff_items: list[dict] = [] + + # Same-person pairs from identified faces + for person in people: + faces = ( + db.query(Face) + .filter(Face.person_id == person.id, Face.encoding.isnot(None)) + .order_by(Face.id) + .limit(8) + .all() + ) + encs = [(f, _encoding(f)) for f in faces] + encs = [(f, e) for f, e in encs if e is not None] + for i in range(len(encs)): + for j in range(i + 1, len(encs)): + f1, e1 = encs[i] + f2, e2 = encs[j] + d = _cosine_distance(e1, e2) + same_items.append( + { + "face_id": f2.id, + "distance": d, + "similarity": 1.0 - d, + "reference_face_id": f1.id, + } + ) + + # Different-person pairs (cap) + by_person: dict[int, list[tuple[Face, np.ndarray]]] = {} + for person in people: + faces = ( + db.query(Face) + .filter(Face.person_id == person.id, Face.encoding.isnot(None)) + .limit(2) + .all() + ) + rows = [] + for f in faces: + e = _encoding(f) + if e is not None: + rows.append((f, e)) + if rows: + by_person[person.id] = rows + + pids = list(by_person.keys()) + for i, pid_a in enumerate(pids): + for pid_b in pids[i + 1 :]: + fa, ea = by_person[pid_a][0] + fb, eb = by_person[pid_b][0] + d = _cosine_distance(ea, eb) + diff_items.append( + { + "face_id": fb.id, + "distance": d, + "similarity": 1.0 - d, + "reference_face_id": fa.id, + } + ) + if len(diff_items) >= 40: + break + if len(diff_items) >= 40: + break + + print(f"same_pairs={len(same_items)} diff_pairs={len(diff_items)}") + if same_items: + # use first person's id for accept log; decisions are per face+person + # record per person groups + for person in people: + items = [ + it + for it in same_items + if any( + f.id == it["face_id"] and f.person_id == person.id + for f in db.query(Face).filter(Face.id == it["face_id"]).all() + ) + ] + # simpler: filter by joining + face_ids = {f.id for f in db.query(Face).filter(Face.person_id == person.id).all()} + items = [it for it in same_items if it["face_id"] in face_ids] + if items: + record_match_decisions( + db, + decision="accept", + source="seed_script", + person_id=person.id, + items=items[:20], + user_id=None, + commit=True, + ) + + if diff_items and pids: + # store rejects against first person as "lookalike rejected" + record_match_decisions( + db, + decision="reject", + source="seed_script", + person_id=pids[0], + items=diff_items[:40], + user_id=None, + commit=True, + ) + + after = db.query(MatchDecision).count() + print(f"match_decisions after: {after}") + finally: + db.close() + + # calibration readiness (dry) + print("\n=== calibration dry-run ===") + import subprocess + + r = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "fit_confidence_calibration.py")], + cwd=str(ROOT), + capture_output=True, + text=True, + env={**os.environ}, + ) + print(r.stdout[-2000:] if r.stdout else "") + if r.stderr: + print(r.stderr[-1000:]) + print("exit", r.returncode, "(2 means not ready to apply without --force)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_face_junk_reject.py b/tests/test_face_junk_reject.py new file mode 100644 index 0000000..ef1c68f --- /dev/null +++ b/tests/test_face_junk_reject.py @@ -0,0 +1,36 @@ +"""Phase 5 junk reject gates (blur / extreme pose).""" + +from __future__ import annotations + +import numpy as np + +from backend.services.face_service import is_junk_face_reject +from tests.test_face_quality_score import _blurry_face, _centered_face_image, _sharp_face + + +def test_junk_rejects_extreme_yaw(): + img, loc = _centered_face_image(_sharp_face(80)) + ok, reason = is_junk_face_reject(img, loc, yaw_angle=75.0, pitch_angle=0.0) + assert ok is False + assert "yaw" in reason.lower() + + +def test_junk_rejects_extreme_pitch(): + img, loc = _centered_face_image(_sharp_face(80)) + ok, reason = is_junk_face_reject(img, loc, yaw_angle=0.0, pitch_angle=50.0) + assert ok is False + assert "pitch" in reason.lower() + + +def test_junk_rejects_blurry_crop(): + img, loc = _centered_face_image(_blurry_face(100)) + ok, reason = is_junk_face_reject(img, loc, yaw_angle=None, pitch_angle=None) + assert ok is False + assert "blur" in reason.lower() or "laplacian" in reason.lower() + + +def test_junk_keeps_sharp_frontal(): + img, loc = _centered_face_image(_sharp_face(100)) + ok, reason = is_junk_face_reject(img, loc, yaw_angle=5.0, pitch_angle=2.0) + assert ok is True + assert reason == ""