ilia/punimtag
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance API startup script and add file hash management for photos

Browse Source 
This commit improves the `run_api_with_worker.sh` script by ensuring the virtual environment is created if it doesn't exist and dependencies are installed. It also adds a check to ensure the database schema is up to date. Additionally, new functionality has been introduced to calculate and store file hashes for uploaded photos, preventing duplicates. The database schema has been updated to include a `file_hash` column in the `photos` table, along with an index for efficient querying. The frontend has been updated to handle warnings for duplicate photos during the review process. Documentation has been updated to reflect these changes.
This commit is contained in:
tanyar09
2025-11-24 13:16:41 -05:00
parent 93cb4eda5b
commit 661e812193
35 changed files with 593 additions and 5628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/__init__.py
-18
View File
@@ -1,18 +0,0 @@
"""
Core business logic modules for PunimTag
"""
from .database import DatabaseManager
from .face_processing import FaceProcessor
from .photo_management import PhotoManager
from .tag_management import TagManager
from .search_stats import SearchStats
__all__ = [
'DatabaseManager',
'FaceProcessor',
'PhotoManager',
'TagManager',
'SearchStats',
]
src/core/config.py
-63
View File
@@ -1,63 +0,0 @@
#!/usr/bin/env python3
"""
Configuration constants and settings for PunimTag
"""
import os
import warnings
# Suppress TensorFlow warnings (must be before DeepFace import)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
warnings.filterwarnings('ignore')
# Default file paths
DEFAULT_DB_PATH = "data/photos.db"
DEFAULT_CONFIG_FILE = "gui_config.json"
DEFAULT_WINDOW_SIZE = "600x500"
# DeepFace Settings
DEEPFACE_DETECTOR_BACKEND = "retinaface" # Options: retinaface, mtcnn, opencv, ssd
DEEPFACE_MODEL_NAME = "ArcFace" # Best accuracy model
DEEPFACE_DISTANCE_METRIC = "cosine" # For similarity calculation
DEEPFACE_ENFORCE_DETECTION = False # Don't fail if no faces found
DEEPFACE_ALIGN_FACES = True # Face alignment for better accuracy
# DeepFace Options for GUI
DEEPFACE_DETECTOR_OPTIONS = ["retinaface", "mtcnn", "opencv", "ssd"]
DEEPFACE_MODEL_OPTIONS = ["ArcFace", "Facenet", "Facenet512", "VGG-Face"]
# Face tolerance/threshold settings (adjusted for DeepFace)
DEFAULT_FACE_TOLERANCE = 0.6 # Default tolerance for face matching
DEEPFACE_SIMILARITY_THRESHOLD = 60 # Minimum similarity percentage (0-100)
# Confidence calibration settings
USE_CALIBRATED_CONFIDENCE = True # Use calibrated confidence instead of linear transformation
CONFIDENCE_CALIBRATION_METHOD = "empirical" # Method: "empirical", "linear", "sigmoid"
# Legacy settings (kept for compatibility until Phase 3 migration)
DEFAULT_FACE_DETECTION_MODEL = "hog" # Legacy - will be replaced by DEEPFACE_DETECTOR_BACKEND
DEFAULT_BATCH_SIZE = 20
DEFAULT_PROCESSING_LIMIT = 50
# Face quality settings
MIN_FACE_QUALITY = 0.3
DEFAULT_CONFIDENCE_THRESHOLD = 0.5
# Face detection filtering settings
MIN_FACE_CONFIDENCE = 0.4 # Minimum confidence from detector to accept face (lowered to allow more low-quality faces)
MIN_FACE_SIZE = 40 # Minimum face size in pixels (width or height) - lowered to allow smaller faces
MAX_FACE_SIZE = 1500 # Maximum face size in pixels (to avoid full-image false positives)
# GUI settings
FACE_CROP_SIZE = 100
ICON_SIZE = 20
MAX_SUGGESTIONS = 10
# Database settings
DB_TIMEOUT = 30.0
# Supported image formats
SUPPORTED_IMAGE_FORMATS = {'.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif'}
# Face crop temporary directory
TEMP_FACE_CROP_DIR = "temp_face_crops"
src/core/database.py
-592
View File
@@ -1,592 +0,0 @@
#!/usr/bin/env python3
"""
Database operations and schema management for PunimTag
"""
import sqlite3
import threading
from contextlib import contextmanager
from typing import Dict, List, Tuple, Optional
from src.core.config import DEFAULT_DB_PATH, DB_TIMEOUT
class DatabaseManager:
"""Handles all database operations for the photo tagger"""
def __init__(self, db_path: str = DEFAULT_DB_PATH, verbose: int = 0):
"""Initialize database manager"""
self.db_path = db_path
self.verbose = verbose
self._db_connection = None
self._db_lock = threading.Lock()
self.init_database()
@contextmanager
def get_db_connection(self):
"""Context manager for database connections with connection pooling"""
with self._db_lock:
if self._db_connection is None:
self._db_connection = sqlite3.connect(self.db_path, timeout=DB_TIMEOUT, check_same_thread=False)
self._db_connection.row_factory = sqlite3.Row
try:
yield self._db_connection
except Exception:
self._db_connection.rollback()
raise
else:
self._db_connection.commit()
def close_db_connection(self):
"""Close database connection"""
with self._db_lock:
if self._db_connection:
self._db_connection.close()
self._db_connection = None
def init_database(self):
"""Create database tables if they don't exist"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
# Photos table
cursor.execute('''
CREATE TABLE IF NOT EXISTS photos (
id INTEGER PRIMARY KEY AUTOINCREMENT,
path TEXT UNIQUE NOT NULL,
filename TEXT NOT NULL,
date_added DATETIME DEFAULT CURRENT_TIMESTAMP,
date_taken DATE,
processed BOOLEAN DEFAULT 0
)
''')
# People table
cursor.execute('''
CREATE TABLE IF NOT EXISTS people (
id INTEGER PRIMARY KEY AUTOINCREMENT,
first_name TEXT NOT NULL,
last_name TEXT NOT NULL,
middle_name TEXT,
maiden_name TEXT,
date_of_birth DATE,
created_date DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE(first_name, last_name, middle_name, maiden_name, date_of_birth)
)
''')
# Faces table (updated for DeepFace and pose detection)
cursor.execute('''
CREATE TABLE IF NOT EXISTS faces (
id INTEGER PRIMARY KEY AUTOINCREMENT,
photo_id INTEGER NOT NULL,
person_id INTEGER,
encoding BLOB NOT NULL,
location TEXT NOT NULL,
confidence REAL DEFAULT 0.0,
quality_score REAL DEFAULT 0.0,
is_primary_encoding BOOLEAN DEFAULT 0,
detector_backend TEXT DEFAULT 'retinaface',
model_name TEXT DEFAULT 'ArcFace',
face_confidence REAL DEFAULT 0.0,
exif_orientation INTEGER DEFAULT NULL,
pose_mode TEXT DEFAULT 'frontal',
yaw_angle REAL DEFAULT NULL,
pitch_angle REAL DEFAULT NULL,
roll_angle REAL DEFAULT NULL,
landmarks TEXT DEFAULT NULL,
FOREIGN KEY (photo_id) REFERENCES photos (id),
FOREIGN KEY (person_id) REFERENCES people (id)
)
''')
# Add pose fields if they don't exist (for existing databases)
try:
cursor.execute('ALTER TABLE faces ADD COLUMN pose_mode TEXT DEFAULT "frontal"')
except sqlite3.OperationalError:
pass # Column already exists
try:
cursor.execute('ALTER TABLE faces ADD COLUMN yaw_angle REAL DEFAULT NULL')
except sqlite3.OperationalError:
pass # Column already exists
try:
cursor.execute('ALTER TABLE faces ADD COLUMN pitch_angle REAL DEFAULT NULL')
except sqlite3.OperationalError:
pass # Column already exists
try:
cursor.execute('ALTER TABLE faces ADD COLUMN roll_angle REAL DEFAULT NULL')
except sqlite3.OperationalError:
pass # Column already exists
# Person encodings table for multiple encodings per person (updated for DeepFace)
cursor.execute('''
CREATE TABLE IF NOT EXISTS person_encodings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
person_id INTEGER NOT NULL,
face_id INTEGER NOT NULL,
encoding BLOB NOT NULL,
quality_score REAL DEFAULT 0.0,
detector_backend TEXT DEFAULT 'retinaface',
model_name TEXT DEFAULT 'ArcFace',
created_date DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (person_id) REFERENCES people (id),
FOREIGN KEY (face_id) REFERENCES faces (id)
)
''')
# Tags table - holds only tag information
cursor.execute('''
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tag_name TEXT UNIQUE NOT NULL,
created_date DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
# Photo-Tag linkage table
# linkage_type: INTEGER enum → 0 = single (per-photo add), 1 = bulk (folder-wide add)
cursor.execute('''
CREATE TABLE IF NOT EXISTS phototaglinkage (
linkage_id INTEGER PRIMARY KEY AUTOINCREMENT,
photo_id INTEGER NOT NULL,
tag_id INTEGER NOT NULL,
linkage_type INTEGER NOT NULL DEFAULT 0 CHECK(linkage_type IN (0,1)),
created_date DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (photo_id) REFERENCES photos (id),
FOREIGN KEY (tag_id) REFERENCES tags (id),
UNIQUE(photo_id, tag_id)
)
''')
# Add indexes for better performance
cursor.execute('CREATE INDEX IF NOT EXISTS idx_faces_person_id ON faces(person_id)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_faces_photo_id ON faces(photo_id)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_photos_processed ON photos(processed)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_faces_quality ON faces(quality_score)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_person_encodings_person_id ON person_encodings(person_id)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_person_encodings_quality ON person_encodings(quality_score)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_photos_date_taken ON photos(date_taken)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_photos_date_added ON photos(date_added)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_faces_pose_mode ON faces(pose_mode)')
if self.verbose >= 1:
print(f"✅ Database initialized: {self.db_path}")
def load_tag_mappings(self) -> Tuple[Dict[int, str], Dict[str, int]]:
"""Load tag name to ID and ID to name mappings from database (case-insensitive)"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('SELECT id, tag_name FROM tags ORDER BY LOWER(tag_name)')
tag_id_to_name = {}
tag_name_to_id = {}
for row in cursor.fetchall():
tag_id, tag_name = row
tag_id_to_name[tag_id] = tag_name
# Use lowercase for case-insensitive lookups
tag_name_to_id[tag_name.lower()] = tag_id
return tag_id_to_name, tag_name_to_id
def get_existing_tag_ids_for_photo(self, photo_id: int) -> List[int]:
"""Get list of tag IDs for a photo from database"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
SELECT ptl.tag_id
FROM phototaglinkage ptl
WHERE ptl.photo_id = ?
ORDER BY ptl.created_date
''', (photo_id,))
return [row[0] for row in cursor.fetchall()]
def get_tag_id_by_name(self, tag_name: str, tag_name_to_id_map: Dict[str, int]) -> Optional[int]:
"""Get tag ID by name, creating the tag if it doesn't exist"""
if tag_name in tag_name_to_id_map:
return tag_name_to_id_map[tag_name]
return None
def get_tag_name_by_id(self, tag_id: int, tag_id_to_name_map: Dict[int, str]) -> str:
"""Get tag name by ID"""
return tag_id_to_name_map.get(tag_id, f"Unknown Tag {tag_id}")
def show_people_list(self, cursor=None) -> List[Tuple]:
"""Show list of people in database"""
if cursor is None:
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
SELECT id, first_name, last_name, middle_name, maiden_name, date_of_birth, created_date
FROM people
ORDER BY last_name, first_name
''')
return cursor.fetchall()
def add_photo(self, photo_path: str, filename: str, date_taken: Optional[str] = None) -> int:
"""Add a photo to the database and return its ID if new, None if already exists"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
# Check if photo already exists
cursor.execute('SELECT id FROM photos WHERE path = ?', (photo_path,))
existing = cursor.fetchone()
if existing:
# Photo already exists, return None to indicate it wasn't added
return None
# Photo doesn't exist, insert it
cursor.execute('''
INSERT INTO photos (path, filename, date_taken)
VALUES (?, ?, ?)
''', (photo_path, filename, date_taken))
# Get the new photo ID
cursor.execute('SELECT id FROM photos WHERE path = ?', (photo_path,))
result = cursor.fetchone()
return result[0] if result else None
def mark_photo_processed(self, photo_id: int):
"""Mark a photo as processed"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('UPDATE photos SET processed = 1 WHERE id = ?', (photo_id,))
def add_face(self, photo_id: int, encoding: bytes, location: str, confidence: float = 0.0,
quality_score: float = 0.0, person_id: Optional[int] = None,
detector_backend: str = 'retinaface',
model_name: str = 'ArcFace',
face_confidence: float = 0.0,
exif_orientation: Optional[int] = None,
pose_mode: str = 'frontal',
yaw_angle: Optional[float] = None,
pitch_angle: Optional[float] = None,
roll_angle: Optional[float] = None,
landmarks: Optional[str] = None) -> int:
"""Add a face to the database and return its ID
Args:
photo_id: ID of the photo containing the face
encoding: Face encoding as bytes (512 floats for ArcFace = 4096 bytes)
location: Face location as string (DeepFace format: "{'x': x, 'y': y, 'w': w, 'h': h}")
confidence: Legacy confidence value (kept for compatibility)
quality_score: Quality score 0.0-1.0
person_id: ID of identified person (None if unidentified)
detector_backend: DeepFace detector used (retinaface, mtcnn, opencv, ssd)
model_name: DeepFace model used (ArcFace, Facenet, etc.)
face_confidence: Confidence from DeepFace detector
exif_orientation: EXIF orientation value (1-8) for coordinate transformation
pose_mode: Pose mode classification (e.g., 'frontal', 'profile_left', 'looking_up')
yaw_angle: Yaw angle in degrees (left/right rotation)
pitch_angle: Pitch angle in degrees (up/down tilt)
roll_angle: Roll angle in degrees (rotation around face axis)
landmarks: JSON string of facial landmarks (e.g., {'left_eye': [x, y], ...})
Returns:
Face ID
"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
INSERT INTO faces (photo_id, person_id, encoding, location, confidence,
quality_score, detector_backend, model_name, face_confidence,
exif_orientation, pose_mode, yaw_angle, pitch_angle, roll_angle, landmarks)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (photo_id, person_id, encoding, location, confidence, quality_score,
detector_backend, model_name, face_confidence, exif_orientation,
pose_mode, yaw_angle, pitch_angle, roll_angle, landmarks))
return cursor.lastrowid
def update_face_person(self, face_id: int, person_id: Optional[int]):
"""Update the person_id for a face"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('UPDATE faces SET person_id = ? WHERE id = ?', (person_id, face_id))
def add_person(self, first_name: str, last_name: str, middle_name: str = None,
maiden_name: str = None, date_of_birth: str = None) -> int:
"""Add a person to the database and return their ID (case-insensitive)"""
# Normalize names to title case for case-insensitive matching
normalized_first = first_name.strip().title()
normalized_last = last_name.strip().title()
normalized_middle = middle_name.strip().title() if middle_name else ''
normalized_maiden = maiden_name.strip().title() if maiden_name else ''
normalized_dob = date_of_birth.strip() if date_of_birth else ''
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
INSERT OR IGNORE INTO people (first_name, last_name, middle_name, maiden_name, date_of_birth)
VALUES (?, ?, ?, ?, ?)
''', (normalized_first, normalized_last, normalized_middle, normalized_maiden, normalized_dob))
# Get the person ID (case-insensitive lookup)
cursor.execute('''
SELECT id FROM people
WHERE LOWER(first_name) = LOWER(?) AND LOWER(last_name) = LOWER(?)
AND LOWER(COALESCE(middle_name, '')) = LOWER(?) AND LOWER(COALESCE(maiden_name, '')) = LOWER(?)
AND date_of_birth = ?
''', (normalized_first, normalized_last, normalized_middle, normalized_maiden, normalized_dob))
result = cursor.fetchone()
return result[0] if result else None
def add_tag(self, tag_name: str) -> int:
"""Add a tag to the database and return its ID (case-insensitive)"""
# Normalize tag name to lowercase for consistency
normalized_tag_name = tag_name.lower().strip()
with self.get_db_connection() as conn:
cursor = conn.cursor()
# Check if tag already exists (case-insensitive)
cursor.execute('SELECT id FROM tags WHERE LOWER(tag_name) = ?', (normalized_tag_name,))
existing = cursor.fetchone()
if existing:
return existing[0]
# Insert new tag with original case
cursor.execute('INSERT INTO tags (tag_name) VALUES (?)', (tag_name.strip(),))
# Get the tag ID
cursor.execute('SELECT id FROM tags WHERE LOWER(tag_name) = ?', (normalized_tag_name,))
result = cursor.fetchone()
return result[0] if result else None
def link_photo_tag(self, photo_id: int, tag_id: int):
"""Link a photo to a tag"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
INSERT OR IGNORE INTO phototaglinkage (photo_id, tag_id)
VALUES (?, ?)
''', (photo_id, tag_id))
def unlink_photo_tag(self, photo_id: int, tag_id: int):
"""Unlink a photo from a tag"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
DELETE FROM phototaglinkage
WHERE photo_id = ? AND tag_id = ?
''', (photo_id, tag_id))
def get_photos_by_pattern(self, pattern: str = None, limit: int = 10) -> List[Tuple]:
"""Get photos matching a pattern"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
if pattern:
cursor.execute('''
SELECT id, path, filename, date_taken, processed
FROM photos
WHERE filename LIKE ? OR path LIKE ?
ORDER BY date_added DESC
LIMIT ?
''', (f'%{pattern}%', f'%{pattern}%', limit))
else:
cursor.execute('''
SELECT id, path, filename, date_taken, processed
FROM photos
ORDER BY date_added DESC
LIMIT ?
''', (limit,))
return cursor.fetchall()
def get_unprocessed_photos(self, limit: Optional[int] = None) -> List[Tuple]:
"""Get unprocessed photos
Args:
limit: Maximum number of photos to return. If None, return all unprocessed photos.
"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
if limit is None:
cursor.execute('''
SELECT id, path, filename, date_taken
FROM photos
WHERE processed = 0
ORDER BY date_added ASC
''')
else:
cursor.execute('''
SELECT id, path, filename, date_taken
FROM photos
WHERE processed = 0
ORDER BY date_added ASC
LIMIT ?
''', (limit,))
return cursor.fetchall()
def get_unidentified_faces(self, limit: int = 20) -> List[Tuple]:
"""Get unidentified faces"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
SELECT f.id, f.photo_id, f.location, f.confidence, f.quality_score,
p.path, p.filename
FROM faces f
JOIN photos p ON f.photo_id = p.id
WHERE f.person_id IS NULL
ORDER BY f.quality_score DESC, f.confidence DESC
LIMIT ?
''', (limit,))
return cursor.fetchall()
def get_face_encodings(self, face_id: int) -> Optional[bytes]:
"""Get face encoding for a specific face"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('SELECT encoding FROM faces WHERE id = ?', (face_id,))
result = cursor.fetchone()
return result[0] if result else None
def get_face_photo_info(self, face_id: int) -> Optional[Tuple]:
"""Get photo information for a specific face including EXIF orientation"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
SELECT f.photo_id, p.filename, f.location, f.exif_orientation
FROM faces f
JOIN photos p ON f.photo_id = p.id
WHERE f.id = ?
''', (face_id,))
result = cursor.fetchone()
return result if result else None
def get_all_face_encodings(self) -> List[Tuple]:
"""Get all face encodings with their IDs"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('SELECT id, encoding, person_id, quality_score FROM faces')
return cursor.fetchall()
def get_person_encodings(self, person_id: int, min_quality: float = 0.3) -> List[Tuple]:
"""Get all encodings for a person above minimum quality"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
SELECT pe.encoding, pe.quality_score, pe.face_id
FROM person_encodings pe
WHERE pe.person_id = ? AND pe.quality_score >= ?
ORDER BY pe.quality_score DESC
''', (person_id, min_quality))
return cursor.fetchall()
def add_person_encoding(self, person_id: int, face_id: int, encoding: bytes,
quality_score: float,
detector_backend: str = 'retinaface',
model_name: str = 'ArcFace'):
"""Add a person encoding
Args:
person_id: ID of the person
face_id: ID of the face this encoding came from
encoding: Face encoding as bytes
quality_score: Quality score 0.0-1.0
detector_backend: DeepFace detector used
model_name: DeepFace model used
"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
cursor.execute('''
INSERT INTO person_encodings (person_id, face_id, encoding, quality_score,
detector_backend, model_name)
VALUES (?, ?, ?, ?, ?, ?)
''', (person_id, face_id, encoding, quality_score, detector_backend, model_name))
def update_person_encodings(self, person_id: int):
"""Update person encodings by removing old ones and adding current face encodings"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
# Remove old encodings
cursor.execute('DELETE FROM person_encodings WHERE person_id = ?', (person_id,))
# Add current face encodings
cursor.execute('''
INSERT INTO person_encodings (person_id, face_id, encoding, quality_score)
SELECT ?, id, encoding, quality_score
FROM faces
WHERE person_id = ? AND quality_score >= 0.3
''', (person_id, person_id))
def get_similar_faces(self, face_id: int, tolerance: float = 0.6,
include_same_photo: bool = False) -> List[Dict]:
"""Get faces similar to the given face ID"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
# Get the target face encoding and photo
cursor.execute('''
SELECT f.encoding, f.photo_id, p.path, p.filename
FROM faces f
JOIN photos p ON f.photo_id = p.id
WHERE f.id = ?
''', (face_id,))
target_result = cursor.fetchone()
if not target_result:
return []
target_encoding = target_result[0]
target_photo_id = target_result[1]
target_path = target_result[2]
target_filename = target_result[3]
# Get all other faces
if include_same_photo:
cursor.execute('''
SELECT f.id, f.encoding, f.person_id, f.quality_score, f.confidence,
p.path, p.filename, f.photo_id
FROM faces f
JOIN photos p ON f.photo_id = p.id
WHERE f.id != ?
''', (face_id,))
else:
cursor.execute('''
SELECT f.id, f.encoding, f.person_id, f.quality_score, f.confidence,
p.path, p.filename, f.photo_id
FROM faces f
JOIN photos p ON f.photo_id = p.id
WHERE f.id != ? AND f.photo_id != ?
''', (face_id, target_photo_id))
return cursor.fetchall()
def get_statistics(self) -> Dict:
"""Get database statistics"""
with self.get_db_connection() as conn:
cursor = conn.cursor()
stats = {}
# Photo statistics
cursor.execute('SELECT COUNT(*) FROM photos')
stats['total_photos'] = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(*) FROM photos WHERE processed = 1')
stats['processed_photos'] = cursor.fetchone()[0]
# Face statistics
cursor.execute('SELECT COUNT(*) FROM faces')
stats['total_faces'] = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(*) FROM faces WHERE person_id IS NOT NULL')
stats['identified_faces'] = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(*) FROM faces WHERE person_id IS NULL')
stats['unidentified_faces'] = cursor.fetchone()[0]
# People statistics
cursor.execute('SELECT COUNT(*) FROM people')
stats['total_people'] = cursor.fetchone()[0]
# Tag statistics
cursor.execute('SELECT COUNT(*) FROM tags')
stats['total_tags'] = cursor.fetchone()[0]
cursor.execute('SELECT COUNT(*) FROM phototaglinkage')
stats['total_photo_tags'] = cursor.fetchone()[0]
return stats
src/core/face_processing.py
-1452
View File
File diff suppressed because it is too large Load Diff
src/core/photo_management.py
-243
View File
@@ -1,243 +0,0 @@
#!/usr/bin/env python3
"""
Photo scanning, metadata extraction, and file operations for PunimTag
"""
import os
from pathlib import Path
from PIL import Image
from datetime import datetime
from typing import Optional, List, Tuple
from src.core.config import SUPPORTED_IMAGE_FORMATS
from src.core.database import DatabaseManager
from src.utils.path_utils import normalize_path, validate_path_exists
class PhotoManager:
"""Handles photo scanning, metadata extraction, and file operations"""
def __init__(self, db_manager: DatabaseManager, verbose: int = 0):
"""Initialize photo manager"""
self.db = db_manager
self.verbose = verbose
def extract_photo_date(self, photo_path: str) -> Optional[str]:
"""Extract date taken from photo with fallback to file modification time.
Tries in order:
1. EXIF date tags (DateTimeOriginal, DateTimeDigitized, DateTime)
2. File modification time (as fallback)
"""
# First try EXIF date extraction
try:
with Image.open(photo_path) as image:
exifdata = image.getexif()
# Look for date taken in EXIF tags
date_tags = [
36867, # DateTimeOriginal - when photo was actually taken (highest priority)
36868, # DateTimeDigitized - when photo was digitized
306, # DateTime - file modification date (lowest priority)
]
for tag_id in date_tags:
if tag_id in exifdata:
date_str = exifdata[tag_id]
if date_str:
# Parse EXIF date format (YYYY:MM:DD HH:MM:SS)
try:
date_obj = datetime.strptime(date_str, '%Y:%m:%d %H:%M:%S')
return date_obj.strftime('%Y-%m-%d')
except ValueError:
# Try alternative format
try:
date_obj = datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S')
return date_obj.strftime('%Y-%m-%d')
except ValueError:
continue
except Exception as e:
if self.verbose >= 2:
print(f" ⚠️ Could not extract EXIF date from {os.path.basename(photo_path)}: {e}")
# Fallback to file modification time
try:
if os.path.exists(photo_path):
mtime = os.path.getmtime(photo_path)
mtime_date = datetime.fromtimestamp(mtime)
return mtime_date.strftime('%Y-%m-%d')
except Exception as e:
if self.verbose >= 2:
print(f" ⚠️ Could not get file modification time from {os.path.basename(photo_path)}: {e}")
return None
def scan_folder(self, folder_path: str, recursive: bool = True) -> int:
"""Scan folder for photos and add to database"""
# Normalize path to absolute path
try:
folder_path = normalize_path(folder_path)
except ValueError as e:
print(f"❌ Invalid path: {e}")
return 0
if not validate_path_exists(folder_path):
print(f"❌ Folder not found or not accessible: {folder_path}")
return 0
found_photos = []
if recursive:
for root, dirs, files in os.walk(folder_path):
for file in files:
file_ext = Path(file).suffix.lower()
if file_ext in SUPPORTED_IMAGE_FORMATS:
photo_path = os.path.join(root, file)
found_photos.append((photo_path, file))
else:
for file in os.listdir(folder_path):
file_ext = Path(file).suffix.lower()
if file_ext in SUPPORTED_IMAGE_FORMATS:
photo_path = os.path.join(folder_path, file)
found_photos.append((photo_path, file))
if not found_photos:
print(f"📁 No photos found in {folder_path}")
return 0
# Add to database
added_count = 0
existing_count = 0
for photo_path, filename in found_photos:
try:
# Ensure photo path is absolute
photo_path = normalize_path(photo_path)
# Extract date taken from EXIF data
date_taken = self.extract_photo_date(photo_path)
# Add photo to database (with absolute path)
photo_id = self.db.add_photo(photo_path, filename, date_taken)
if photo_id:
# New photo was added
added_count += 1
if self.verbose >= 2:
date_info = f" (taken: {date_taken})" if date_taken else " (no date)"
print(f" 📸 Added: {filename}{date_info}")
else:
# Photo already exists
existing_count += 1
if self.verbose >= 2:
print(f" 📸 Already exists: {filename}")
except Exception as e:
print(f"⚠️ Error adding {filename}: {e}")
# Print summary
if added_count > 0 and existing_count > 0:
print(f"📁 Found {len(found_photos)} photos: {added_count} new, {existing_count} already in database")
elif added_count > 0:
print(f"📁 Found {len(found_photos)} photos, added {added_count} new photos")
elif existing_count > 0:
print(f"📁 Found {len(found_photos)} photos, all already in database")
else:
print(f"📁 Found {len(found_photos)} photos, none could be added")
return added_count
def get_photo_info(self, photo_id: int) -> Optional[Tuple]:
"""Get photo information by ID"""
photos = self.db.get_photos_by_pattern(limit=1000) # Get all photos
for photo in photos:
if photo[0] == photo_id: # photo[0] is the ID
return photo
return None
def get_photo_path(self, photo_id: int) -> Optional[str]:
"""Get photo path by ID"""
photo_info = self.get_photo_info(photo_id)
return photo_info[1] if photo_info else None # photo[1] is the path
def get_photo_filename(self, photo_id: int) -> Optional[str]:
"""Get photo filename by ID"""
photo_info = self.get_photo_info(photo_id)
return photo_info[2] if photo_info else None # photo[2] is the filename
def is_photo_processed(self, photo_id: int) -> bool:
"""Check if photo has been processed for faces"""
photo_info = self.get_photo_info(photo_id)
return photo_info[4] if photo_info else False # photo[4] is the processed flag
def mark_photo_processed(self, photo_id: int):
"""Mark a photo as processed"""
self.db.mark_photo_processed(photo_id)
def get_photos_by_date_range(self, date_from: str = None, date_to: str = None) -> List[Tuple]:
"""Get photos within a date range"""
# This would need to be implemented in the database module
# For now, return all photos
return self.db.get_photos_by_pattern()
def get_photos_by_pattern(self, pattern: str = None, limit: int = 10) -> List[Tuple]:
"""Get photos matching a pattern"""
return self.db.get_photos_by_pattern(pattern, limit)
def validate_photo_file(self, photo_path: str) -> bool:
"""Validate that a photo file exists and is readable"""
if not os.path.exists(photo_path):
return False
try:
with Image.open(photo_path) as image:
image.verify()
return True
except Exception:
return False
def get_photo_dimensions(self, photo_path: str) -> Optional[Tuple[int, int]]:
"""Get photo dimensions (width, height)"""
try:
with Image.open(photo_path) as image:
return image.size
except Exception:
return None
def get_photo_format(self, photo_path: str) -> Optional[str]:
"""Get photo format"""
try:
with Image.open(photo_path) as image:
return image.format
except Exception:
return None
def get_photo_exif_data(self, photo_path: str) -> dict:
"""Get EXIF data from photo"""
try:
with Image.open(photo_path) as image:
exifdata = image.getexif()
return dict(exifdata)
except Exception:
return {}
def get_photo_file_size(self, photo_path: str) -> Optional[int]:
"""Get photo file size in bytes"""
try:
return os.path.getsize(photo_path)
except Exception:
return None
def get_photo_creation_time(self, photo_path: str) -> Optional[datetime]:
"""Get photo file creation time"""
try:
timestamp = os.path.getctime(photo_path)
return datetime.fromtimestamp(timestamp)
except Exception:
return None
def get_photo_modification_time(self, photo_path: str) -> Optional[datetime]:
"""Get photo file modification time"""
try:
timestamp = os.path.getmtime(photo_path)
return datetime.fromtimestamp(timestamp)
except Exception:
return None
src/core/search_stats.py
-453
View File
@@ -1,453 +0,0 @@
#!/usr/bin/env python3
"""
Search functionality and statistics for PunimTag
"""
from typing import List, Dict, Tuple, Optional
from src.core.database import DatabaseManager
class SearchStats:
"""Handles search functionality and statistics generation"""
def __init__(self, db_manager: DatabaseManager, verbose: int = 0):
"""Initialize search and stats manager"""
self.db = db_manager
self.verbose = verbose
def search_faces(self, person_name: str) -> List[Tuple[str, str]]:
"""Search for photos containing a specific person by name (partial, case-insensitive).
Returns a list of tuples: (photo_path, person_full_name).
"""
# Get all people matching the name
people = self.db.show_people_list()
matching_people = []
search_name = (person_name or "").strip().lower()
if not search_name:
return []
for person in people:
person_id, first_name, last_name, middle_name, maiden_name, date_of_birth, created_date = person
full_name = f"{first_name or ''} {last_name or ''}".strip().lower()
# Check if search term matches any part of the name
if (
(full_name and search_name in full_name) or
(first_name and search_name in first_name.lower()) or
(last_name and search_name in last_name.lower()) or
(middle_name and search_name in middle_name.lower()) or
(maiden_name and search_name in maiden_name.lower())
):
matching_people.append(person_id)
if not matching_people:
return []
# Fetch photo paths for each matching person using database helper if available
results: List[Tuple[str, str]] = []
try:
with self.db.get_db_connection() as conn:
cursor = conn.cursor()
# faces.person_id links to photos via faces.photo_id
placeholders = ",".join(["?"] * len(matching_people))
cursor.execute(
f"""
SELECT DISTINCT p.path, pe.first_name, pe.last_name
FROM faces f
JOIN photos p ON p.id = f.photo_id
JOIN people pe ON pe.id = f.person_id
WHERE f.person_id IN ({placeholders})
ORDER BY pe.last_name, pe.first_name, p.path
""",
tuple(matching_people),
)
for row in cursor.fetchall():
if row and row[0]:
path = row[0]
first = (row[1] or "").strip()
last = (row[2] or "").strip()
full_name = (f"{first} {last}").strip() or "Unknown"
results.append((path, full_name))
except Exception:
# Fall back gracefully if schema differs
pass
return results
def get_statistics(self) -> Dict:
"""Get comprehensive database statistics"""
stats = self.db.get_statistics()
# Add calculated statistics
if stats['total_photos'] > 0:
stats['processing_percentage'] = (stats['processed_photos'] / stats['total_photos']) * 100
else:
stats['processing_percentage'] = 0
if stats['total_faces'] > 0:
stats['identification_percentage'] = (stats['identified_faces'] / stats['total_faces']) * 100
else:
stats['identification_percentage'] = 0
if stats['total_people'] > 0:
stats['faces_per_person'] = stats['identified_faces'] / stats['total_people']
else:
stats['faces_per_person'] = 0
if stats['total_photos'] > 0:
stats['faces_per_photo'] = stats['total_faces'] / stats['total_photos']
else:
stats['faces_per_photo'] = 0
if stats['total_photos'] > 0:
stats['tags_per_photo'] = stats['total_photo_tags'] / stats['total_photos']
else:
stats['tags_per_photo'] = 0
return stats
def print_statistics(self):
"""Print formatted statistics to console"""
stats = self.get_statistics()
print("\n📊 PunimTag Database Statistics")
print("=" * 50)
print(f"📸 Photos:")
print(f" Total photos: {stats['total_photos']}")
print(f" Processed: {stats['processed_photos']} ({stats['processing_percentage']:.1f}%)")
print(f" Unprocessed: {stats['total_photos'] - stats['processed_photos']}")
print(f"\n👤 Faces:")
print(f" Total faces: {stats['total_faces']}")
print(f" Identified: {stats['identified_faces']} ({stats['identification_percentage']:.1f}%)")
print(f" Unidentified: {stats['unidentified_faces']}")
print(f"\n👥 People:")
print(f" Total people: {stats['total_people']}")
print(f" Average faces per person: {stats['faces_per_person']:.1f}")
print(f"\n🏷️ Tags:")
print(f" Total tags: {stats['total_tags']}")
print(f" Total photo-tag links: {stats['total_photo_tags']}")
print(f" Average tags per photo: {stats['tags_per_photo']:.1f}")
print(f"\n📈 Averages:")
print(f" Faces per photo: {stats['faces_per_photo']:.1f}")
print(f" Tags per photo: {stats['tags_per_photo']:.1f}")
print("=" * 50)
def get_photo_statistics(self) -> Dict:
"""Get detailed photo statistics"""
stats = self.get_statistics()
# This could be expanded with more detailed photo analysis
return {
'total_photos': stats['total_photos'],
'processed_photos': stats['processed_photos'],
'unprocessed_photos': stats['total_photos'] - stats['processed_photos'],
'processing_percentage': stats['processing_percentage']
}
def get_face_statistics(self) -> Dict:
"""Get detailed face statistics"""
stats = self.get_statistics()
return {
'total_faces': stats['total_faces'],
'identified_faces': stats['identified_faces'],
'unidentified_faces': stats['unidentified_faces'],
'identification_percentage': stats['identification_percentage'],
'faces_per_photo': stats['faces_per_photo']
}
def get_people_statistics(self) -> Dict:
"""Get detailed people statistics"""
stats = self.get_statistics()
return {
'total_people': stats['total_people'],
'faces_per_person': stats['faces_per_person']
}
def get_tag_statistics(self) -> Dict:
"""Get detailed tag statistics"""
stats = self.get_statistics()
return {
'total_tags': stats['total_tags'],
'total_photo_tags': stats['total_photo_tags'],
'tags_per_photo': stats['tags_per_photo']
}
def search_photos_by_date(self, date_from: str = None, date_to: str = None) -> List[Tuple[str, str]]:
"""Search photos by date range.
Args:
date_from: Start date in YYYY-MM-DD format (inclusive)
date_to: End date in YYYY-MM-DD format (inclusive)
Returns:
List of tuples: (photo_path, date_taken)
"""
try:
with self.db.get_db_connection() as conn:
cursor = conn.cursor()
# Build the query based on provided date parameters
if date_from and date_to:
# Both dates provided - search within range
query = '''
SELECT path, date_taken
FROM photos
WHERE date_taken IS NOT NULL
AND date_taken >= ? AND date_taken <= ?
ORDER BY date_taken DESC, filename
'''
cursor.execute(query, (date_from, date_to))
elif date_from:
# Only start date provided - search from date onwards
query = '''
SELECT path, date_taken
FROM photos
WHERE date_taken IS NOT NULL
AND date_taken >= ?
ORDER BY date_taken DESC, filename
'''
cursor.execute(query, (date_from,))
elif date_to:
# Only end date provided - search up to date
query = '''
SELECT path, date_taken
FROM photos
WHERE date_taken IS NOT NULL
AND date_taken <= ?
ORDER BY date_taken DESC, filename
'''
cursor.execute(query, (date_to,))
else:
# No dates provided - return all photos with date_taken
query = '''
SELECT path, date_taken
FROM photos
WHERE date_taken IS NOT NULL
ORDER BY date_taken DESC, filename
'''
cursor.execute(query)
results = cursor.fetchall()
return [(row[0], row[1]) for row in results]
except Exception as e:
if self.verbose >= 1:
print(f"Error searching photos by date: {e}")
return []
def search_photos_by_tags(self, tags: List[str], match_all: bool = False) -> List[Tuple]:
"""Search photos by tags
Args:
tags: List of tag names to search for
match_all: If True, photos must have ALL tags. If False, photos with ANY tag.
Returns:
List of tuples: (photo_path, tag_info)
"""
if not tags:
return []
# Get tag IDs for the provided tag names (case-insensitive)
tag_id_to_name, tag_name_to_id = self.db.load_tag_mappings()
tag_ids = []
for tag_name in tags:
# Convert to lowercase for case-insensitive lookup
normalized_tag_name = tag_name.lower().strip()
if normalized_tag_name in tag_name_to_id:
tag_ids.append(tag_name_to_id[normalized_tag_name])
if not tag_ids:
return []
results = []
try:
with self.db.get_db_connection() as conn:
cursor = conn.cursor()
if match_all:
# Photos that have ALL specified tags
placeholders = ",".join(["?"] * len(tag_ids))
cursor.execute(f'''
SELECT p.path, GROUP_CONCAT(t.tag_name, ', ') as tag_names
FROM photos p
JOIN phototaglinkage ptl ON p.id = ptl.photo_id
JOIN tags t ON ptl.tag_id = t.id
WHERE ptl.tag_id IN ({placeholders})
GROUP BY p.id, p.path
HAVING COUNT(DISTINCT ptl.tag_id) = ?
ORDER BY p.path
''', tuple(tag_ids) + (len(tag_ids),))
else:
# Photos that have ANY of the specified tags
placeholders = ",".join(["?"] * len(tag_ids))
cursor.execute(f'''
SELECT DISTINCT p.path, GROUP_CONCAT(t.tag_name, ', ') as tag_names
FROM photos p
JOIN phototaglinkage ptl ON p.id = ptl.photo_id
JOIN tags t ON ptl.tag_id = t.id
WHERE ptl.tag_id IN ({placeholders})
GROUP BY p.id, p.path
ORDER BY p.path
''', tuple(tag_ids))
for row in cursor.fetchall():
if row and row[0]:
results.append((row[0], row[1] or ""))
except Exception as e:
if self.verbose > 0:
print(f"Error searching photos by tags: {e}")
return results
def search_photos_by_people(self, people: List[str]) -> List[Tuple]:
"""Search photos by people"""
# This would need to be implemented in the database module
# For now, return empty list
return []
def get_most_common_tags(self, limit: int = 10) -> List[Tuple[str, int]]:
"""Get most commonly used tags"""
# This would need to be implemented in the database module
# For now, return empty list
return []
def get_most_photographed_people(self, limit: int = 10) -> List[Tuple[str, int]]:
"""Get most photographed people"""
# This would need to be implemented in the database module
# For now, return empty list
return []
def get_photos_without_faces(self) -> List[Tuple]:
"""Get photos that have no detected faces
Only includes processed photos (photos that have been processed for face detection).
Returns:
List of tuples: (photo_path, filename)
"""
results = []
try:
with self.db.get_db_connection() as conn:
cursor = conn.cursor()
# Find photos that have no faces associated with them
# Only include processed photos
cursor.execute('''
SELECT p.path, p.filename
FROM photos p
LEFT JOIN faces f ON p.id = f.photo_id
WHERE f.photo_id IS NULL
AND p.processed = 1
ORDER BY p.filename
''')
for row in cursor.fetchall():
if row and row[0]:
results.append((row[0], row[1]))
except Exception as e:
if self.verbose > 0:
print(f"Error searching photos without faces: {e}")
return results
def get_photos_without_tags(self) -> List[Tuple]:
"""Get photos that have no tags
Returns:
List of tuples: (photo_path, filename)
"""
results = []
try:
with self.db.get_db_connection() as conn:
cursor = conn.cursor()
# Find photos that have no tags associated with them
cursor.execute('''
SELECT p.path, p.filename
FROM photos p
LEFT JOIN phototaglinkage ptl ON p.id = ptl.photo_id
WHERE ptl.photo_id IS NULL
ORDER BY p.filename
''')
for row in cursor.fetchall():
if row and row[0]:
results.append((row[0], row[1]))
except Exception as e:
if self.verbose > 0:
print(f"Error searching photos without tags: {e}")
return results
def get_duplicate_faces(self, tolerance: float = 0.6) -> List[Dict]:
"""Get potential duplicate faces (same person, different photos)"""
# This would need to be implemented using face matching
# For now, return empty list
return []
def get_face_quality_distribution(self) -> Dict:
"""Get distribution of face quality scores"""
# This would need to be implemented in the database module
# For now, return empty dict
return {}
def get_processing_timeline(self) -> List[Tuple[str, int]]:
"""Get timeline of photo processing (photos processed per day)"""
# This would need to be implemented in the database module
# For now, return empty list
return []
def export_statistics(self, filename: str = "punimtag_stats.json"):
"""Export statistics to a JSON file"""
import json
stats = self.get_statistics()
try:
with open(filename, 'w') as f:
json.dump(stats, f, indent=2)
print(f"✅ Statistics exported to {filename}")
except Exception as e:
print(f"❌ Error exporting statistics: {e}")
def generate_report(self) -> str:
"""Generate a text report of statistics"""
stats = self.get_statistics()
report = f"""
PunimTag Database Report
Generated: {__import__('datetime').datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
PHOTO STATISTICS:
- Total photos: {stats['total_photos']}
- Processed: {stats['processed_photos']} ({stats['processing_percentage']:.1f}%)
- Unprocessed: {stats['total_photos'] - stats['processed_photos']}
FACE STATISTICS:
- Total faces: {stats['total_faces']}
- Identified: {stats['identified_faces']} ({stats['identification_percentage']:.1f}%)
- Unidentified: {stats['unidentified_faces']}
- Average faces per photo: {stats['faces_per_photo']:.1f}
PEOPLE STATISTICS:
- Total people: {stats['total_people']}
- Average faces per person: {stats['faces_per_person']:.1f}
TAG STATISTICS:
- Total tags: {stats['total_tags']}
- Total photo-tag links: {stats['total_photo_tags']}
- Average tags per photo: {stats['tags_per_photo']:.1f}
"""
return report
src/core/tag_management.py
-266
View File
@@ -1,266 +0,0 @@
#!/usr/bin/env python3
"""
Tag management functionality for PunimTag
"""
from typing import List, Dict, Tuple, Optional
from src.core.config import DEFAULT_BATCH_SIZE
from src.core.database import DatabaseManager
class TagManager:
"""Handles photo tagging and tag management operations"""
def __init__(self, db_manager: DatabaseManager, verbose: int = 0):
"""Initialize tag manager"""
self.db = db_manager
self.verbose = verbose
def deduplicate_tags(self, tag_list: List[str]) -> List[str]:
"""Remove duplicate tags from a list while preserving order (case insensitive)"""
seen = set()
unique_tags = []
for tag in tag_list:
if tag.lower() not in seen:
seen.add(tag.lower())
unique_tags.append(tag)
return unique_tags
def parse_tags_string(self, tags_string: str) -> List[str]:
"""Parse a comma-separated tags string into a list, handling empty strings and whitespace"""
if not tags_string or tags_string.strip() == "":
return []
# Split by comma and strip whitespace from each tag
tags = [tag.strip() for tag in tags_string.split(",")]
# Remove empty strings that might result from splitting
return [tag for tag in tags if tag]
def add_tags_to_photos(self, photo_pattern: str = None, batch_size: int = DEFAULT_BATCH_SIZE) -> int:
"""Add custom tags to photos via command line interface"""
if photo_pattern:
photos = self.db.get_photos_by_pattern(photo_pattern, batch_size)
else:
photos = self.db.get_photos_by_pattern(limit=batch_size)
if not photos:
print("No photos found")
return 0
print(f"🏷️ Tagging {len(photos)} photos (enter comma-separated tags)")
tagged_count = 0
for photo_id, photo_path, filename, date_taken, processed in photos:
print(f"\n📸 {filename}")
tags_input = input("🏷️ Tags: ").strip()
if tags_input.lower() == 'q':
break
if tags_input:
tags = self.parse_tags_string(tags_input)
tags = self.deduplicate_tags(tags)
for tag_name in tags:
# Add tag to database and get its ID
tag_id = self.db.add_tag(tag_name)
if tag_id:
# Link photo to tag
self.db.link_photo_tag(photo_id, tag_id)
print(f" ✅ Added {len(tags)} tags")
tagged_count += 1
print(f"✅ Tagged {tagged_count} photos")
return tagged_count
def add_tags_to_photo(self, photo_id: int, tags: List[str]) -> int:
"""Add tags to a specific photo"""
if not tags:
return 0
tags = self.deduplicate_tags(tags)
added_count = 0
for tag_name in tags:
# Add tag to database and get its ID
tag_id = self.db.add_tag(tag_name)
if tag_id:
# Link photo to tag
self.db.link_photo_tag(photo_id, tag_id)
added_count += 1
return added_count
def remove_tags_from_photo(self, photo_id: int, tags: List[str]) -> int:
"""Remove tags from a specific photo"""
if not tags:
return 0
removed_count = 0
tag_id_to_name, tag_name_to_id = self.db.load_tag_mappings()
for tag_name in tags:
if tag_name in tag_name_to_id:
tag_id = tag_name_to_id[tag_name]
self.db.unlink_photo_tag(photo_id, tag_id)
removed_count += 1
return removed_count
def get_photo_tags(self, photo_id: int) -> List[str]:
"""Get all tags for a specific photo"""
tag_ids = self.db.get_existing_tag_ids_for_photo(photo_id)
tag_id_to_name, _ = self.db.load_tag_mappings()
tags = []
for tag_id in tag_ids:
tag_name = self.db.get_tag_name_by_id(tag_id, tag_id_to_name)
tags.append(tag_name)
return tags
def get_all_tags(self) -> List[Tuple[int, str]]:
"""Get all tags in the database"""
tag_id_to_name, _ = self.db.load_tag_mappings()
return [(tag_id, tag_name) for tag_id, tag_name in tag_id_to_name.items()]
def get_photos_with_tag(self, tag_name: str) -> List[Tuple]:
"""Get all photos that have a specific tag"""
tag_id_to_name, tag_name_to_id = self.db.load_tag_mappings()
if tag_name not in tag_name_to_id:
return []
tag_id = tag_name_to_id[tag_name]
# This would need to be implemented in the database module
# For now, return empty list
return []
def get_tag_statistics(self) -> Dict:
"""Get tag usage statistics"""
tag_id_to_name, _ = self.db.load_tag_mappings()
stats = {
'total_tags': len(tag_id_to_name),
'tag_usage': {}
}
# Count usage for each tag
for tag_id, tag_name in tag_id_to_name.items():
# This would need to be implemented in the database module
# For now, set usage to 0
stats['tag_usage'][tag_name] = 0
return stats
def delete_tag(self, tag_name: str) -> bool:
"""Delete a tag from the database (and all its linkages)"""
tag_id_to_name, tag_name_to_id = self.db.load_tag_mappings()
if tag_name not in tag_name_to_id:
return False
tag_id = tag_name_to_id[tag_name]
# This would need to be implemented in the database module
# For now, return False
return False
def rename_tag(self, old_name: str, new_name: str) -> bool:
"""Rename a tag"""
tag_id_to_name, tag_name_to_id = self.db.load_tag_mappings()
if old_name not in tag_name_to_id:
return False
if new_name in tag_name_to_id:
return False # New name already exists
tag_id = tag_name_to_id[old_name]
# This would need to be implemented in the database module
# For now, return False
return False
def merge_tags(self, source_tag: str, target_tag: str) -> bool:
"""Merge one tag into another (move all linkages from source to target)"""
tag_id_to_name, tag_name_to_id = self.db.load_tag_mappings()
if source_tag not in tag_name_to_id or target_tag not in tag_name_to_id:
return False
source_tag_id = tag_name_to_id[source_tag]
target_tag_id = tag_name_to_id[target_tag]
# This would need to be implemented in the database module
# For now, return False
return False
def get_photos_by_tags(self, tags: List[str], match_all: bool = False) -> List[Tuple]:
"""Get photos that have any (or all) of the specified tags"""
if not tags:
return []
tag_id_to_name, tag_name_to_id = self.db.load_tag_mappings()
tag_ids = []
for tag_name in tags:
# Convert to lowercase for case-insensitive lookup
normalized_tag_name = tag_name.lower().strip()
if normalized_tag_name in tag_name_to_id:
tag_ids.append(tag_name_to_id[normalized_tag_name])
if not tag_ids:
return []
# This would need to be implemented in the database module
# For now, return empty list
return []
def get_common_tags(self, photo_ids: List[int]) -> List[str]:
"""Get tags that are common to all specified photos"""
if not photo_ids:
return []
# Get tags for each photo
all_photo_tags = []
for photo_id in photo_ids:
tags = self.get_photo_tags(photo_id)
all_photo_tags.append(set(tags))
if not all_photo_tags:
return []
# Find intersection of all tag sets
common_tags = set.intersection(*all_photo_tags)
return list(common_tags)
def get_suggested_tags(self, photo_id: int, limit: int = 5) -> List[str]:
"""Get suggested tags based on similar photos"""
# This is a placeholder for tag suggestion logic
# Could be implemented based on:
# - Tags from photos in the same folder
# - Tags from photos taken on the same date
# - Most commonly used tags
# - Machine learning based suggestions
return []
def validate_tag_name(self, tag_name: str) -> Tuple[bool, str]:
"""Validate a tag name and return (is_valid, error_message)"""
if not tag_name or not tag_name.strip():
return False, "Tag name cannot be empty"
tag_name = tag_name.strip()
if len(tag_name) > 50:
return False, "Tag name is too long (max 50 characters)"
if ',' in tag_name:
return False, "Tag name cannot contain commas"
if tag_name.lower() in ['all', 'none', 'untagged']:
return False, "Tag name is reserved"
return True, ""
src/setup.py
-161
View File
@@ -1,161 +0,0 @@
#!/usr/bin/env python3
"""
PunimTag CLI Setup Script
Simple setup for the minimal photo tagger
"""
import os
import sys
import subprocess
from pathlib import Path
def check_python_version():
"""Check if Python version is compatible"""
if sys.version_info < (3, 7):
print("❌ Python 3.7+ is required")
return False
print(f"✅ Python {sys.version_info.major}.{sys.version_info.minor} detected")
return True
def install_system_dependencies():
"""Install system-level packages required for compilation and runtime"""
print("🔧 Installing system dependencies...")
print(" (Build tools, libraries, and image viewer)")
# Check if we're on a Debian/Ubuntu system
if Path("/usr/bin/apt").exists():
try:
# Install required system packages for building Python packages and running tools
packages = [
"cmake", "build-essential", "libopenblas-dev", "liblapack-dev",
"libx11-dev", "libgtk-3-dev", "libboost-python-dev", "feh"
]
print(f"📦 Installing packages: {', '.join(packages)}")
subprocess.run([
"sudo", "apt", "install", "-y"
] + packages, check=True)
print("✅ System dependencies installed successfully")
return True
except subprocess.CalledProcessError as e:
print(f"❌ Failed to install system dependencies: {e}")
print(" You may need to run: sudo apt update")
return False
else:
print("⚠️ System dependency installation not supported on this platform")
print(" Please install manually:")
print(" - cmake, build-essential")
print(" - libopenblas-dev, liblapack-dev")
print(" - libx11-dev, libgtk-3-dev, libboost-python-dev")
print(" - feh (image viewer)")
return True
def install_requirements():
"""Install Python requirements"""
requirements_file = Path("requirements.txt")
if not requirements_file.exists():
print("❌ requirements.txt not found!")
return False
print("📦 Installing Python dependencies...")
try:
subprocess.run([
sys.executable, '-m', 'pip', 'install', '-r', 'requirements.txt'
], check=True)
print("✅ Dependencies installed successfully")
return True
except subprocess.CalledProcessError as e:
print(f"❌ Failed to install dependencies: {e}")
return False
def create_directories():
"""Create necessary directories"""
directories = ['data', 'logs']
for directory in directories:
Path(directory).mkdir(exist_ok=True)
print(f"✅ Created directory: {directory}")
def test_installation():
"""Test if DeepFace face recognition works"""
print("🧪 Testing DeepFace face recognition installation...")
try:
from deepface import DeepFace
import numpy as np
from PIL import Image
import tensorflow as tf
print("✅ All required modules imported successfully")
return True
except ImportError as e:
print(f"❌ Import error: {e}")
return False
def main():
"""Main setup function"""
print("🚀 PunimTag CLI Setup")
print("=" * 40)
# Check Python version
if not check_python_version():
return 1
# Check if we're in a virtual environment (recommended)
if sys.prefix == sys.base_prefix:
print("⚠️ Not in a virtual environment!")
print(" Recommended: python -m venv venv && source venv/bin/activate")
response = input(" Continue anyway? (y/N): ").strip().lower()
if response != 'y':
print("Setup cancelled. Create a virtual environment first.")
return 1
else:
print("✅ Virtual environment detected")
print()
# Install system dependencies
if not install_system_dependencies():
return 1
print()
# Create directories
print("📁 Creating directories...")
create_directories()
print()
# Install requirements
if not install_requirements():
return 1
print()
# Test installation
if not test_installation():
print("⚠️ Installation test failed. You may need to install additional dependencies.")
print(" For Ubuntu/Debian: sudo apt-get install build-essential cmake")
print(" For macOS: brew install cmake")
return 1
print()
print("✅ Setup complete!")
print()
print("🎯 Quick Start:")
print(" 1. Start the web server: python run_api_with_worker.sh")
print(" 2. Access the web interface at http://localhost:8000")
print(" 3. Use the web UI to scan, process, and identify faces")
print()
print("📖 For more information, see README.md")
print()
print("⚠️ IMPORTANT: Always activate virtual environment first!")
print(" source venv/bin/activate")
return 0
if __name__ == '__main__':
sys.exit(main())
src/web/api/auth.py
+51 -1
View File
@@ -189,7 +189,57 @@ def get_current_user_info(
# Check if user exists in main database to get admin status
user = db.query(User).filter(User.username == username).first()
is_admin = user.is_admin if user else False
# If user doesn't exist in main database, check if we should bootstrap them
if not user:
# Check if any admin users exist
admin_count = db.query(User).filter(User.is_admin == True).count()
# If no admins exist, bootstrap current user as admin
if admin_count == 0:
from src.web.utils.password import hash_password
# Generate unique email to avoid conflicts
base_email = f"{username}@example.com"
email = base_email
counter = 1
# Ensure email is unique
while db.query(User).filter(User.email == email).first():
email = f"{username}+{counter}@example.com"
counter += 1
# Create user as admin for bootstrap (they should change password)
default_password_hash = hash_password("changeme")
try:
user = User(
username=username,
password_hash=default_password_hash,
email=email,
full_name=username,
is_active=True,
is_admin=True,
)
db.add(user)
db.commit()
db.refresh(user)
is_admin = True
except Exception:
# If creation fails (e.g., race condition), try to get existing user
db.rollback()
user = db.query(User).filter(User.username == username).first()
if user:
# Update existing user to be admin if no admins exist
if not user.is_admin:
user.is_admin = True
db.commit()
db.refresh(user)
is_admin = user.is_admin
else:
is_admin = False
else:
is_admin = False
else:
is_admin = user.is_admin if user else False
return UserResponse(username=username, is_admin=is_admin)
src/web/api/pending_photos.py
+53 -8
View File
@@ -16,7 +16,7 @@ from sqlalchemy.orm import Session
from src.web.db.session import get_auth_db, get_db
from src.web.api.users import get_current_admin_user
from src.web.api.auth import get_current_user
from src.web.services.photo_service import import_photo_from_path
from src.web.services.photo_service import import_photo_from_path, calculate_file_hash
from src.web.settings import PHOTO_STORAGE_DIR
router = APIRouter(prefix="/pending-photos", tags=["pending-photos"])
@@ -78,6 +78,7 @@ class ReviewResponse(BaseModel):
approved: int
rejected: int
errors: list[str]
warnings: list[str] = [] # Informational messages (e.g., duplicates)
@router.get("", response_model=PendingPhotosListResponse)
@@ -264,6 +265,7 @@ def review_pending_photos(
approved_count = 0
rejected_count = 0
duplicate_count = 0
errors = []
admin_user_id = current_admin.get("user_id")
now = datetime.utcnow()
@@ -314,32 +316,66 @@ def review_pending_photos(
errors.append(f"Photo file not found for pending photo {decision.id}: {source_path}")
continue
# Calculate file hash and check for duplicates BEFORE moving file
try:
file_hash = calculate_file_hash(str(source_path))
except Exception as e:
errors.append(f"Failed to calculate hash for pending photo {decision.id}: {str(e)}")
continue
# Check if photo with same hash already exists in main database
existing_photo = main_db.execute(text("""
SELECT id, path FROM photos WHERE file_hash = :file_hash
"""), {"file_hash": file_hash}).fetchone()
if existing_photo:
# Photo already exists - mark as duplicate and skip import
# Don't add to errors - we'll show a summary message instead
# Update status to rejected with duplicate reason
auth_db.execute(text("""
UPDATE pending_photos
SET status = 'rejected',
reviewed_at = :reviewed_at,
reviewed_by = :reviewed_by,
rejection_reason = 'Duplicate photo already exists in database'
WHERE id = :id
"""), {
"id": decision.id,
"reviewed_at": now,
"reviewed_by": admin_user_id,
})
auth_db.commit()
rejected_count += 1
duplicate_count += 1
continue
# Generate unique filename for main storage to avoid conflicts
file_ext = source_path.suffix
unique_filename = f"{uuid.uuid4()}{file_ext}"
dest_path = main_storage_dir / unique_filename
# Move file to main storage
# Copy file to main storage (keep original in shared location)
try:
shutil.move(str(source_path), str(dest_path))
shutil.copy2(str(source_path), str(dest_path))
except Exception as e:
errors.append(f"Failed to move photo file for {decision.id}: {str(e)}")
errors.append(f"Failed to copy photo file for {decision.id}: {str(e)}")
continue
# Import photo into main database (Scan process)
# This will also check for duplicates by hash, but we've already checked above
try:
photo, is_new = import_photo_from_path(main_db, str(dest_path))
if not is_new:
# Photo already exists - delete the moved file
# Photo already exists (shouldn't happen due to hash check above, but handle gracefully)
if dest_path.exists():
dest_path.unlink()
errors.append(f"Photo already exists in main database: {photo.path}")
continue
except Exception as e:
# If import fails, try to move file back
# If import fails, delete the copied file (original remains in shared location)
if dest_path.exists():
try:
shutil.move(str(dest_path), str(source_path))
dest_path.unlink()
except:
pass
errors.append(f"Failed to import photo {decision.id} into main database: {str(e)}")
@@ -388,9 +424,18 @@ def review_pending_photos(
auth_db.rollback()
main_db.rollback()
# Add friendly message about duplicates if any were found
warnings = []
if duplicate_count > 0:
if duplicate_count == 1:
warnings.append(f"{duplicate_count} photo was not added as it already exists in the database")
else:
warnings.append(f"{duplicate_count} photos were not added as they already exist in the database")
return ReviewResponse(
approved=approved_count,
rejected=rejected_count,
errors=errors
errors=errors,
warnings=warnings
)
src/web/config.py
+26
View File
@@ -0,0 +1,26 @@
"""Configuration values used by the PunimTag web services.
This module replaces the legacy desktop configuration to keep the web
application self-contained.
"""
from __future__ import annotations
# Supported image formats for uploads/imports
SUPPORTED_IMAGE_FORMATS = {".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif"}
# DeepFace behavior
DEEPFACE_ENFORCE_DETECTION = False
DEEPFACE_ALIGN_FACES = True
# Face filtering thresholds
MIN_FACE_CONFIDENCE = 0.4
MIN_FACE_SIZE = 40
MAX_FACE_SIZE = 1500
# Matching tolerance and calibration options
DEFAULT_FACE_TOLERANCE = 0.6
USE_CALIBRATED_CONFIDENCE = True
CONFIDENCE_CALIBRATION_METHOD = "empirical" # "empirical", "linear", or "sigmoid"
src/web/db/models.py
+2
View File
@@ -38,6 +38,7 @@ class Photo(Base):
date_added = Column(DateTime, default=datetime.utcnow, nullable=False)
date_taken = Column(Date, nullable=True, index=True)
processed = Column(Boolean, default=False, nullable=False, index=True)
file_hash = Column(Text, nullable=False, index=True)
faces = relationship("Face", back_populates="photo", cascade="all, delete-orphan")
photo_tags = relationship(
@@ -49,6 +50,7 @@ class Photo(Base):
Index("idx_photos_processed", "processed"),
Index("idx_photos_date_taken", "date_taken"),
Index("idx_photos_date_added", "date_added"),
Index("idx_photos_file_hash", "file_hash"),
)
src/web/services/face_service.py
+6 -15
View File
@@ -20,12 +20,15 @@ try:
except ImportError:
DEEPFACE_AVAILABLE = False
from src.core.config import (
DEEPFACE_ENFORCE_DETECTION,
from src.web.config import (
CONFIDENCE_CALIBRATION_METHOD,
DEFAULT_FACE_TOLERANCE,
DEEPFACE_ALIGN_FACES,
DEEPFACE_ENFORCE_DETECTION,
MAX_FACE_SIZE,
MIN_FACE_CONFIDENCE,
MIN_FACE_SIZE,
MAX_FACE_SIZE,
USE_CALIBRATED_CONFIDENCE,
)
from src.utils.exif_utils import EXIFOrientationHandler
from src.utils.pose_detection import PoseDetector, RETINAFACE_AVAILABLE
@@ -1400,8 +1403,6 @@ def calibrate_confidence(distance: float, tolerance: float = None) -> float:
Returns:
Calibrated confidence percentage (0-100) representing actual match probability
"""
from src.core.config import DEFAULT_FACE_TOLERANCE, USE_CALIBRATED_CONFIDENCE, CONFIDENCE_CALIBRATION_METHOD
if tolerance is None:
tolerance = DEFAULT_FACE_TOLERANCE
@@ -1510,7 +1511,6 @@ def find_similar_faces(
Args:
filter_frontal_only: Only return frontal or tilted faces (not profile)
"""
from src.core.config import DEFAULT_FACE_TOLERANCE
from src.web.db.models import Photo
if tolerance is None:
@@ -1608,9 +1608,6 @@ def calculate_batch_similarities(
where face_id_1 is from the request list, face_id_2 is from all faces in DB
similarity is in [0,1] range and confidence_pct is in [0,100] range
"""
from src.core.config import DEFAULT_FACE_TOLERANCE
from src.web.db.models import Photo
if not face_ids:
return []
@@ -1754,8 +1751,6 @@ def find_auto_match_matches(
List of (person_id, reference_face_id, reference_face, matches) tuples
where matches is list of (face, distance, confidence_pct) tuples
"""
from src.core.config import DEFAULT_FACE_TOLERANCE
if tolerance is None:
tolerance = DEFAULT_FACE_TOLERANCE
@@ -1866,10 +1861,6 @@ def get_auto_match_people_list(
Returns:
List of (person_id, reference_face, person_name, face_count) tuples
"""
from src.web.db.models import Person, Photo
from src.core.config import DEFAULT_FACE_TOLERANCE
from sqlalchemy import func, case
if tolerance is None:
tolerance = DEFAULT_FACE_TOLERANCE
src/web/services/photo_service.py
+57 -5
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import hashlib
import os
from pathlib import Path
from datetime import datetime, date
@@ -10,7 +11,7 @@ from typing import Callable, Optional, Tuple
from PIL import Image
from sqlalchemy.orm import Session
from src.core.config import SUPPORTED_IMAGE_FORMATS
from src.web.config import SUPPORTED_IMAGE_FORMATS
from src.web.db.models import Photo
@@ -102,6 +103,30 @@ def extract_exif_date(image_path: str) -> Optional[date]:
return None
def calculate_file_hash(file_path: str) -> str:
"""Calculate SHA256 hash of file content.
Args:
file_path: Path to the file to hash
Returns:
Hexadecimal string representation of SHA256 hash
"""
sha256_hash = hashlib.sha256()
try:
with open(file_path, "rb") as f:
# Read file in chunks to handle large files efficiently
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
return sha256_hash.hexdigest()
except Exception as e:
# Log error for debugging
import logging
logger = logging.getLogger(__name__)
logger.error(f"Failed to calculate hash for {file_path}: {e}")
raise
def extract_photo_date(image_path: str) -> Optional[date]:
"""Extract date taken from photo with fallback to file modification time.
@@ -169,8 +194,18 @@ def import_photo_from_path(
photo_path = os.path.abspath(photo_path)
filename = os.path.basename(photo_path)
# Check if photo already exists by path
existing = db.query(Photo).filter(Photo.path == photo_path).first()
# Calculate file hash for duplicate detection
try:
file_hash = calculate_file_hash(photo_path)
except Exception as e:
# If hash calculation fails, we can't proceed
import logging
logger = logging.getLogger(__name__)
logger.error(f"Failed to calculate hash for {photo_path}: {e}")
raise
# Check if photo already exists by hash (primary duplicate check)
existing = db.query(Photo).filter(Photo.file_hash == file_hash).first()
if existing:
# If existing photo doesn't have date_taken, try to update it
if existing.date_taken is None:
@@ -181,16 +216,33 @@ def import_photo_from_path(
db.refresh(existing)
return existing, False
# Also check by path as fallback (in case hash wasn't set for some reason)
existing_by_path = db.query(Photo).filter(Photo.path == photo_path).first()
if existing_by_path:
# Update hash if missing
if not existing_by_path.file_hash:
existing_by_path.file_hash = file_hash
db.commit()
db.refresh(existing_by_path)
# If existing photo doesn't have date_taken, try to update it
if existing_by_path.date_taken is None:
date_taken = extract_photo_date(photo_path)
if date_taken:
existing_by_path.date_taken = date_taken
db.commit()
db.refresh(existing_by_path)
return existing_by_path, False
# Extract date taken with fallback to file modification time
date_taken = extract_photo_date(photo_path)
# Create new photo record - match desktop schema exactly
# Desktop schema: id, path, filename, date_added, date_taken (DATE), processed
# Create new photo record with file_hash
photo = Photo(
path=photo_path,
filename=filename,
date_taken=date_taken,
processed=False,
file_hash=file_hash,
)
db.add(photo)