ilia/punimtag
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance API startup script and add file hash management for photos

Browse Source 
This commit improves the `run_api_with_worker.sh` script by ensuring the virtual environment is created if it doesn't exist and dependencies are installed. It also adds a check to ensure the database schema is up to date. Additionally, new functionality has been introduced to calculate and store file hashes for uploaded photos, preventing duplicates. The database schema has been updated to include a `file_hash` column in the `photos` table, along with an index for efficient querying. The frontend has been updated to handle warnings for duplicate photos during the review process. Documentation has been updated to reflect these changes.
This commit is contained in:
tanyar09
2025-11-24 13:16:41 -05:00
parent 93cb4eda5b
commit 661e812193
35 changed files with 593 additions and 5628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/check_identified_poses.py
-102
View File
@@ -1,102 +0,0 @@
#!/usr/bin/env python3
"""Check all identified faces for pose information"""
import sqlite3
import sys
import os
# Add project root to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.core.config import DEFAULT_DB_PATH
def check_identified_faces(db_path: str):
"""Check all identified faces for pose information"""
if not os.path.exists(db_path):
print(f"Database not found: {db_path}")
return
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
cursor = conn.cursor()
# Get all identified faces with pose information
cursor.execute('''
SELECT
f.id,
f.person_id,
p.name || ' ' || p.last_name as person_name,
ph.filename,
f.pose_mode,
f.yaw_angle,
f.pitch_angle,
f.roll_angle,
f.face_confidence,
f.quality_score,
f.location
FROM faces f
JOIN people p ON f.person_id = p.id
JOIN photos ph ON f.photo_id = ph.id
WHERE f.person_id IS NOT NULL
ORDER BY p.id, f.id
''')
faces = cursor.fetchall()
if not faces:
print("No identified faces found.")
return
print(f"\n{'='*80}")
print(f"Found {len(faces)} identified faces")
print(f"{'='*80}\n")
# Group by person
by_person = {}
for face in faces:
person_id = face['person_id']
if person_id not in by_person:
by_person[person_id] = []
by_person[person_id].append(face)
# Print summary
print("SUMMARY BY PERSON:")
print("-" * 80)
for person_id, person_faces in by_person.items():
person_name = person_faces[0]['person_name']
pose_modes = [f['pose_mode'] for f in person_faces]
frontal_count = sum(1 for p in pose_modes if p == 'frontal')
profile_count = sum(1 for p in pose_modes if 'profile' in p)
other_count = len(pose_modes) - frontal_count - profile_count
print(f"\nPerson {person_id}: {person_name}")
print(f" Total faces: {len(person_faces)}")
print(f" Frontal: {frontal_count}")
print(f" Profile: {profile_count}")
print(f" Other: {other_count}")
print(f" Pose modes: {set(pose_modes)}")
# Print detailed information
print(f"\n{'='*80}")
print("DETAILED FACE INFORMATION:")
print(f"{'='*80}\n")
for face in faces:
print(f"Face ID: {face['id']}")
print(f" Person: {face['person_name']} (ID: {face['person_id']})")
print(f" Photo: {face['filename']}")
print(f" Pose Mode: {face['pose_mode']}")
print(f" Yaw: {face['yaw_angle']:.2f}°" if face['yaw_angle'] is not None else " Yaw: None")
print(f" Pitch: {face['pitch_angle']:.2f}°" if face['pitch_angle'] is not None else " Pitch: None")
print(f" Roll: {face['roll_angle']:.2f}°" if face['roll_angle'] is not None else " Roll: None")
print(f" Confidence: {face['face_confidence']:.3f}")
print(f" Quality: {face['quality_score']:.3f}")
print(f" Location: {face['location']}")
print()
conn.close()
if __name__ == "__main__":
db_path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_DB_PATH
check_identified_faces(db_path)
scripts/check_two_faces_pose.py
Executable
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""Check two identified faces and analyze why their pose modes are wrong"""
import sys
import os
import json
# Add project root to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from src.web.db.models import Face, Person, Photo
from src.web.db.session import get_database_url
from src.utils.pose_detection import PoseDetector
def check_two_faces(face_id1: int = None, face_id2: int = None):
"""Check two identified faces and analyze their pose modes"""
db_url = get_database_url()
print(f"Connecting to database: {db_url}")
engine = create_engine(db_url)
Session = sessionmaker(bind=engine)
session = Session()
try:
# Get all identified faces
query = (
session.query(Face, Person, Photo)
.join(Person, Face.person_id == Person.id)
.join(Photo, Face.photo_id == Photo.id)
.filter(Face.person_id.isnot(None))
.order_by(Face.id)
)
if face_id1:
query = query.filter(Face.id == face_id1)
elif face_id2:
query = query.filter(Face.id == face_id2)
faces = query.limit(2).all()
if len(faces) < 2:
print(f"Found {len(faces)} identified face(s). Need 2 faces to compare.")
if len(faces) == 0:
print("No identified faces found.")
return
print("\nShowing available identified faces:")
all_faces = (
session.query(Face, Person, Photo)
.join(Person, Face.person_id == Person.id)
.join(Photo, Face.photo_id == Photo.id)
.filter(Face.person_id.isnot(None))
.order_by(Face.id)
.limit(10)
.all()
)
for face, person, photo in all_faces:
print(f" Face ID: {face.id}, Person: {person.first_name} {person.last_name}, Photo: {photo.filename}, Pose: {face.pose_mode}")
return
print(f"\n{'='*80}")
print("ANALYZING TWO IDENTIFIED FACES")
print(f"{'='*80}\n")
for idx, (face, person, photo) in enumerate(faces, 1):
person_name = f"{person.first_name} {person.last_name}"
print(f"{'='*80}")
print(f"FACE {idx}: ID {face.id}")
print(f"{'='*80}")
print(f"Person: {person_name} (ID: {face.person_id})")
print(f"Photo: {photo.filename}")
print(f"Current Pose Mode: {face.pose_mode}")
print(f"Yaw: {face.yaw_angle:.2f}°" if face.yaw_angle is not None else "Yaw: None")
print(f"Pitch: {face.pitch_angle:.2f}°" if face.pitch_angle is not None else "Pitch: None")
print(f"Roll: {face.roll_angle:.2f}°" if face.roll_angle is not None else "Roll: None")
print(f"Face Width: {face.face_width if hasattr(face, 'face_width') else 'N/A'}")
print(f"Confidence: {face.face_confidence:.3f}")
print(f"Quality: {face.quality_score:.3f}")
print(f"Location: {face.location}")
# Parse landmarks if available
landmarks = None
if face.landmarks:
try:
landmarks = json.loads(face.landmarks)
print(f"\nLandmarks:")
for key, value in landmarks.items():
print(f" {key}: {value}")
except json.JSONDecodeError:
print(f"\nLandmarks: (invalid JSON)")
# Recalculate pose mode using current logic
print(f"\n{'─'*80}")
print("RECALCULATING POSE MODE:")
print(f"{'─'*80}")
# Calculate face width from landmarks if available
face_width = None
if landmarks:
face_width = PoseDetector.calculate_face_width_from_landmarks(landmarks)
print(f"Calculated face_width from landmarks: {face_width}")
# Recalculate pose mode
recalculated_pose = PoseDetector.classify_pose_mode(
face.yaw_angle,
face.pitch_angle,
face.roll_angle,
face_width,
landmarks
)
print(f"Recalculated Pose Mode: {recalculated_pose}")
if recalculated_pose != face.pose_mode:
print(f"⚠️ MISMATCH! Current: '{face.pose_mode}' vs Recalculated: '{recalculated_pose}'")
# Analyze why
print(f"\nAnalysis:")
if face.yaw_angle is None:
print(f" - Yaw is None")
if landmarks:
left_eye = landmarks.get('left_eye')
right_eye = landmarks.get('right_eye')
nose = landmarks.get('nose')
missing = []
if not left_eye:
missing.append('left_eye')
if not right_eye:
missing.append('right_eye')
if not nose:
missing.append('nose')
if missing:
print(f" - Missing landmarks: {', '.join(missing)}")
print(f" - Should be classified as profile (missing landmarks)")
else:
print(f" - All landmarks present")
if face_width:
print(f" - Face width: {face_width}px")
if face_width < 25.0:
print(f" - Face width < 25px, should be profile")
else:
print(f" - Face width >= 25px, should be frontal")
else:
print(f" - No landmarks available")
else:
abs_yaw = abs(face.yaw_angle)
print(f" - Yaw angle: {face.yaw_angle:.2f}° (abs: {abs_yaw:.2f}°)")
if abs_yaw >= 30.0:
expected = "profile_left" if face.yaw_angle < 0 else "profile_right"
print(f" - |yaw| >= 30°, should be '{expected}'")
else:
print(f" - |yaw| < 30°, should be 'frontal'")
else:
print(f"✓ Pose mode matches recalculated value")
print()
finally:
session.close()
if __name__ == "__main__":
face_id1 = None
face_id2 = None
if len(sys.argv) > 1:
try:
face_id1 = int(sys.argv[1])
except ValueError:
print(f"Invalid face ID: {sys.argv[1]}")
sys.exit(1)
if len(sys.argv) > 2:
try:
face_id2 = int(sys.argv[2])
except ValueError:
print(f"Invalid face ID: {sys.argv[2]}")
sys.exit(1)
try:
check_two_faces(face_id1, face_id2)
except Exception as e:
print(f"❌ Error: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
scripts/cleanup_false_positives.py
-39
View File
@@ -1,39 +0,0 @@
#!/usr/bin/env python3
"""
Script to clean up false positive face detections from the database
"""
import sys
import os
# Add the project root to the Python path
project_root = os.path.join(os.path.dirname(__file__), '..')
sys.path.insert(0, project_root)
from src.core.database import DatabaseManager
from src.core.face_processing import FaceProcessor
def main():
"""Clean up false positive faces from the database"""
print("🧹 PunimTag False Positive Face Cleanup")
print("=" * 50)
# Initialize database and face processor
db_manager = DatabaseManager()
face_processor = FaceProcessor(db_manager, verbose=1)
# Clean up false positives
removed_count = face_processor.cleanup_false_positive_faces(verbose=True)
if removed_count > 0:
print(f"\n✅ Cleanup complete! Removed {removed_count} false positive faces.")
print("You can now re-run face processing with improved filtering.")
else:
print("\n✅ No false positive faces found to remove.")
print("\nTo reprocess photos with improved face detection:")
print("1. Use the web interface to process photos")
print("2. Or use the API endpoints to trigger face processing")
if __name__ == "__main__":
main()
scripts/grant_auth_db_delete_permission.sh
+1
View File
@@ -62,3 +62,4 @@ else
exit 1
fi
scripts/grant_auth_db_permissions.py
+1
View File
@@ -97,3 +97,4 @@ def grant_delete_permission() -> None:
if __name__ == "__main__":
grant_delete_permission()
scripts/migrate_to_deepface.py
-117
View File
@@ -1,117 +0,0 @@
#!/usr/bin/env python3
"""
Migration script to prepare database for DeepFace
Drops all existing tables and recreates with new schema
⚠️ WARNING: This will delete ALL existing data!
Run this script before migrating to DeepFace.
"""
import sqlite3
import sys
from pathlib import Path
# Add parent directory to path
sys.path.insert(0, str(Path(__file__).parent.parent))
from src.core.database import DatabaseManager
from src.core.config import DEFAULT_DB_PATH
def migrate_database():
"""Drop all tables and reinitialize with DeepFace schema"""
print("=" * 70)
print("DeepFace Migration Script - Database Reset")
print("=" * 70)
print()
print("⚠️ WARNING: This will delete ALL existing data!")
print()
print("This includes:")
print(" • All photos")
print(" • All faces and face encodings")
print(" • All people and person data")
print(" • All tags and photo-tag linkages")
print()
print("The database will be recreated with the new DeepFace schema.")
print()
response = input("Type 'DELETE ALL DATA' to confirm (or anything else to cancel): ")
if response != "DELETE ALL DATA":
print()
print("❌ Migration cancelled.")
print()
return False
print()
print("🗑️ Dropping all existing tables...")
print()
try:
# Connect directly to database
conn = sqlite3.connect(DEFAULT_DB_PATH)
cursor = conn.cursor()
# Get list of all tables (excluding SQLite system tables)
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
tables = [row[0] for row in cursor.fetchall()]
if not tables:
print(" No tables found in database.")
else:
# Drop all tables in correct order (respecting foreign keys)
drop_order = ['phototaglinkage', 'person_encodings', 'faces', 'tags', 'people', 'photos']
for table in drop_order:
if table in tables:
cursor.execute(f'DROP TABLE IF EXISTS {table}')
print(f" ✓ Dropped table: {table}")
# Drop any remaining tables not in our list (excluding SQLite system tables)
for table in tables:
if table not in drop_order and not table.startswith('sqlite_'):
cursor.execute(f'DROP TABLE IF EXISTS {table}')
print(f" ✓ Dropped table: {table}")
conn.commit()
conn.close()
print()
print("✅ All tables dropped successfully")
print()
print("🔄 Reinitializing database with DeepFace schema...")
print()
# Reinitialize with new schema
db = DatabaseManager(DEFAULT_DB_PATH, verbose=1)
print()
print("=" * 70)
print("✅ Database migration complete!")
print("=" * 70)
print()
print("Next steps:")
print(" 1. Add photos using the dashboard (File → Add Photos)")
print(" 2. Process faces with DeepFace (Tools → Process Faces)")
print(" 3. Identify people in the Identify panel")
print()
print("New DeepFace features:")
print(" • 512-dimensional face encodings (vs 128)")
print(" • Multiple detector backends (RetinaFace, MTCNN, etc.)")
print(" • ArcFace model for improved accuracy")
print(" • Face confidence scores from detector")
print()
return True
except Exception as e:
print()
print(f"❌ Error during migration: {e}")
print()
return False
if __name__ == "__main__":
success = migrate_database()
sys.exit(0 if success else 1)
scripts/setup_postgresql.sh
+1
View File
@@ -60,3 +60,4 @@ echo "3. Run your application - it will connect to PostgreSQL automatically"
scripts/update_reported_photo_status.py
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Update status of a reported photo in the auth database."""
import sys
from pathlib import Path
# Add project root to path
project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))
from sqlalchemy import text
from src.web.db.session import get_auth_database_url, AuthSessionLocal
def update_reported_photo_status(report_id: int, new_status: str):
"""Update the status of a reported photo."""
if AuthSessionLocal is None:
raise ValueError("Auth database not configured. Set DATABASE_URL_AUTH environment variable.")
db = AuthSessionLocal()
try:
# First check if the report exists and get its current status
check_result = db.execute(text("""
SELECT id, status, review_notes
FROM inappropriate_photo_reports
WHERE id = :report_id
"""), {"report_id": report_id})
row = check_result.fetchone()
if not row:
print(f"❌ Reported photo {report_id} not found in database.")
return
current_status = row.status
review_notes = row.review_notes
print(f"📋 Current status: '{current_status}'")
if review_notes:
print(f"📝 Review notes: '{review_notes}'")
if current_status == new_status:
print(f"ℹ️ Status is already '{new_status}'. No update needed.")
return
# Update the status
result = db.execute(text("""
UPDATE inappropriate_photo_reports
SET status = :new_status
WHERE id = :report_id
"""), {
"new_status": new_status,
"report_id": report_id
})
db.commit()
if result.rowcount > 0:
print(f"✅ Successfully updated reported photo {report_id} status from '{current_status}' to '{new_status}'")
else:
print(f"⚠️ No rows updated.")
except Exception as e:
db.rollback()
print(f"❌ Error updating reported photo status: {str(e)}")
raise
finally:
db.close()
def find_reported_photo_by_note(search_note: str):
"""Find reported photos by review notes."""
if AuthSessionLocal is None:
raise ValueError("Auth database not configured. Set DATABASE_URL_AUTH environment variable.")
db = AuthSessionLocal()
try:
result = db.execute(text("""
SELECT id, photo_id, status, review_notes, reported_at
FROM inappropriate_photo_reports
WHERE review_notes LIKE :search_pattern
ORDER BY id DESC
"""), {"search_pattern": f"%{search_note}%"})
rows = result.fetchall()
if not rows:
print(f"❌ No reported photos found with note containing '{search_note}'")
return []
print(f"📋 Found {len(rows)} reported photo(s) with note containing '{search_note}':\n")
for row in rows:
print(f" ID: {row.id}, Photo ID: {row.photo_id}, Status: {row.status}")
print(f" Notes: {row.review_notes}")
print(f" Reported at: {row.reported_at}\n")
return rows
except Exception as e:
print(f"❌ Error searching for reported photos: {str(e)}")
raise
finally:
db.close()
if __name__ == "__main__":
if len(sys.argv) < 3:
print("Usage: python scripts/update_reported_photo_status.py <report_id> <new_status>")
print(" OR: python scripts/update_reported_photo_status.py search <search_text>")
print("Example: python scripts/update_reported_photo_status.py 57 dismissed")
print("Example: python scripts/update_reported_photo_status.py search 'agree. removed'")
sys.exit(1)
if sys.argv[1] == "search":
search_text = sys.argv[2]
find_reported_photo_by_note(search_text)
else:
report_id = int(sys.argv[1])
new_status = sys.argv[2]
update_reported_photo_status(report_id, new_status)