feat: Add new analysis documents and update installation scripts for backend integration

This commit introduces several new analysis documents, including Auto-Match Load Performance Analysis, Folder Picker Analysis, Monorepo Migration Summary, and various performance analysis documents. Additionally, the installation scripts are updated to reflect changes in backend service paths, ensuring proper integration with the new backend structure. These enhancements provide better documentation and streamline the setup process for users.
This commit is contained in:
Tanya
2025-12-30 15:04:32 -05:00
parent 12c62f1deb
commit 68d280e8f5
140 changed files with 5101 additions and 933 deletions
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""
Check what tables exist in the punimtag main database and their record counts.
"""
from __future__ import annotations
import sys
from pathlib import Path
# Add project root to path
project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))
from sqlalchemy import create_engine, inspect, text
from backend.db.session import get_database_url
def check_database_tables() -> None:
"""Check all tables in the database and their record counts."""
database_url = get_database_url()
print("=" * 80)
print("PUNIMTAG MAIN DATABASE - TABLE INFORMATION")
print("=" * 80)
print(f"\nDatabase URL: {database_url.replace('://', '://****') if '://' in database_url else database_url}\n")
# Create engine
connect_args = {}
if database_url.startswith("sqlite"):
connect_args = {"check_same_thread": False}
engine = create_engine(database_url, connect_args=connect_args)
try:
# Get inspector to list tables
inspector = inspect(engine)
all_tables = inspector.get_table_names()
if not all_tables:
print("❌ No tables found in database.")
return
print(f"Found {len(all_tables)} tables:\n")
# Expected tables from models
expected_tables = {
"photos",
"people",
"faces",
"person_encodings",
"tags",
"phototaglinkage",
"photo_favorites",
"users",
"photo_person_linkage",
"role_permissions",
}
# Connect and query each table
with engine.connect() as conn:
print(f"{'Table Name':<30} {'Record Count':<15} {'Status'}")
print("-" * 80)
for table_name in sorted(all_tables):
# Skip SQLite system tables
if table_name.startswith("sqlite_"):
continue
try:
# Get record count
if database_url.startswith("sqlite"):
result = conn.execute(text(f"SELECT COUNT(*) FROM {table_name}"))
else:
result = conn.execute(text(f'SELECT COUNT(*) FROM "{table_name}"'))
count = result.scalar()
# Check if it's an expected table
status = "✅ Expected" if table_name in expected_tables else "⚠️ Unexpected"
print(f"{table_name:<30} {count:<15} {status}")
except Exception as e:
print(f"{table_name:<30} {'ERROR':<15} ❌ {str(e)[:50]}")
print("-" * 80)
# Summary
print("\n📊 Summary:")
with engine.connect() as conn:
total_records = 0
tables_with_data = 0
for table_name in sorted(all_tables):
if table_name.startswith("sqlite_"):
continue
try:
if database_url.startswith("sqlite"):
result = conn.execute(text(f"SELECT COUNT(*) FROM {table_name}"))
else:
result = conn.execute(text(f'SELECT COUNT(*) FROM "{table_name}"'))
count = result.scalar()
total_records += count
if count > 0:
tables_with_data += 1
except:
pass
print(f" Total tables: {len([t for t in all_tables if not t.startswith('sqlite_')])}")
print(f" Tables with records: {tables_with_data}")
print(f" Total records across all tables: {total_records:,}")
# Check for missing expected tables
missing_tables = expected_tables - set(all_tables)
if missing_tables:
print(f"\n⚠️ Missing expected tables: {', '.join(sorted(missing_tables))}")
# Check for unexpected tables
unexpected_tables = set(all_tables) - expected_tables - {"alembic_version"}
unexpected_tables = {t for t in unexpected_tables if not t.startswith("sqlite_")}
if unexpected_tables:
print(f"\nℹ️ Additional tables found: {', '.join(sorted(unexpected_tables))}")
except Exception as e:
print(f"❌ Error connecting to database: {e}")
import traceback
traceback.print_exc()
return
print("\n" + "=" * 80)
if __name__ == "__main__":
check_database_tables()
+2 -2
View File
@@ -9,8 +9,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from src.web.db.models import Face, Person, Photo
from src.web.db.session import get_database_url
from backend.db.models import Face, Person, Photo
from backend.db.session import get_database_url
def check_identified_faces():
"""Check all identified faces for pose information"""
+2 -2
View File
@@ -10,8 +10,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from src.web.db.models import Face, Person, Photo
from src.web.db.session import get_database_url
from backend.db.models import Face, Person, Photo
from backend.db.session import get_database_url
from src.utils.pose_detection import PoseDetector
def check_two_faces(face_id1: int = None, face_id2: int = None):
+2 -2
View File
@@ -15,8 +15,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from src.web.db.models import Face, Person, Photo
from src.web.db.session import get_database_url
from backend.db.models import Face, Person, Photo
from backend.db.session import get_database_url
from src.utils.pose_detection import PoseDetector
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""
Diagnose frontend issues:
1. Check if backend API is running and accessible
2. Check database connection
3. Test search endpoint
"""
import os
import sys
import requests
from pathlib import Path
# Add project root to path
project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))
from backend.db.session import get_database_url, engine
from sqlalchemy import text
def check_backend_api():
"""Check if backend API is running."""
print("=" * 80)
print("BACKEND API CHECK")
print("=" * 80)
try:
# Check if docs endpoint is accessible
response = requests.get("http://127.0.0.1:8000/docs", timeout=5)
if response.status_code == 200:
print("✅ Backend API is running (docs accessible)")
else:
print(f"⚠️ Backend API returned status {response.status_code}")
except requests.exceptions.ConnectionError:
print("❌ Backend API is NOT running or not accessible")
print(" Start it with: cd backend && uvicorn app:app --reload")
return False
except Exception as e:
print(f"❌ Error checking backend API: {e}")
return False
# Check search endpoint (requires auth)
try:
response = requests.get(
"http://127.0.0.1:8000/api/v1/photos",
params={"search_type": "processed", "page": 1, "page_size": 1},
timeout=5
)
if response.status_code == 200:
print("✅ Search endpoint is accessible (no auth required for this query)")
elif response.status_code == 401:
print("⚠️ Search endpoint requires authentication")
print(" User needs to log in through admin frontend")
else:
print(f"⚠️ Search endpoint returned status {response.status_code}")
except Exception as e:
print(f"⚠️ Error checking search endpoint: {e}")
return True
def check_database():
"""Check database connection and photo count."""
print("\n" + "=" * 80)
print("DATABASE CHECK")
print("=" * 80)
db_url = get_database_url()
print(f"Database URL: {db_url.replace('://', '://****') if '://' in db_url else db_url}")
try:
with engine.connect() as conn:
# Check photo count
result = conn.execute(text("SELECT COUNT(*) FROM photos WHERE processed = 1"))
count = result.scalar()
print(f"✅ Database connection successful")
print(f" Processed photos: {count}")
if count == 0:
print("⚠️ No processed photos found in database")
print(" This explains why viewer frontend shows 0 photos")
else:
print(f" Database has {count} processed photos")
except Exception as e:
print(f"❌ Database connection error: {e}")
return False
return True
def check_viewer_frontend_config():
"""Check viewer frontend configuration."""
print("\n" + "=" * 80)
print("VIEWER FRONTEND CONFIGURATION")
print("=" * 80)
viewer_env = project_root / "viewer-frontend" / ".env"
if not viewer_env.exists():
print("❌ viewer-frontend/.env file not found")
return False
with open(viewer_env) as f:
content = f.read()
if "DATABASE_URL" in content:
# Extract DATABASE_URL
for line in content.split("\n"):
if line.startswith("DATABASE_URL="):
db_url = line.split("=", 1)[1].strip().strip('"')
print(f"Viewer frontend DATABASE_URL: {db_url.replace('://', '://****') if '://' in db_url else db_url}")
# Check if it matches actual database
actual_db = get_database_url()
if "postgresql" in db_url and "sqlite" in actual_db:
print("❌ MISMATCH: Viewer frontend configured for PostgreSQL")
print(" but actual database is SQLite")
print("\n SOLUTION OPTIONS:")
print(" 1. Change viewer-frontend/.env DATABASE_URL to SQLite:")
print(f' DATABASE_URL="file:../data/punimtag.db"')
print(" 2. Update Prisma schema to use SQLite provider")
print(" 3. Migrate database to PostgreSQL")
return False
elif "sqlite" in db_url and "sqlite" in actual_db:
print("✅ Viewer frontend configured for SQLite (matches actual database)")
else:
print("⚠️ Database type mismatch or unclear")
else:
print("⚠️ DATABASE_URL not found in viewer-frontend/.env")
return True
def main():
print("\n🔍 DIAGNOSING FRONTEND ISSUES\n")
backend_ok = check_backend_api()
db_ok = check_database()
viewer_config_ok = check_viewer_frontend_config()
print("\n" + "=" * 80)
print("SUMMARY")
print("=" * 80)
if not backend_ok:
print("❌ Backend API is not running - admin frontend search will fail")
else:
print("✅ Backend API is running")
if not db_ok:
print("❌ Database connection issue")
else:
print("✅ Database connection OK")
if not viewer_config_ok:
print("❌ Viewer frontend configuration issue - needs to be fixed")
else:
print("✅ Viewer frontend configuration OK")
print("\n" + "=" * 80)
if __name__ == "__main__":
main()
+2 -2
View File
@@ -8,8 +8,8 @@ import os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sqlalchemy import inspect
from src.web.db.session import engine, get_database_url
from src.web.db.models import Base
from backend.db.session import engine, get_database_url
from backend.db.models import Base
# Ordered list ensures foreign-key dependents drop first
TARGET_TABLES = [
+15
View File
@@ -0,0 +1,15 @@
#!/bin/bash
# Drop the auth database if it exists
set -e
echo "🗑️ Dropping auth database if it exists..."
sudo -u postgres psql -c "DROP DATABASE IF EXISTS punimtag_auth;"
if [ $? -eq 0 ]; then
echo "✅ Auth database dropped successfully"
else
echo "⚠️ Failed to drop auth database (it may not exist)"
fi
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Fix admin user password in database."""
import sys
import os
# Add project root to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from backend.db.session import get_db
from backend.db.models import User
from backend.utils.password import hash_password, verify_password
def fix_admin_password():
"""Set admin user password to 'admin'."""
db = next(get_db())
try:
admin_user = db.query(User).filter(User.username == 'admin').first()
if not admin_user:
print("❌ Admin user not found in database")
return False
# Set password to 'admin'
new_hash = hash_password('admin')
admin_user.password_hash = new_hash
admin_user.is_active = True
admin_user.is_admin = True
db.commit()
# Verify it works
if verify_password('admin', new_hash):
print("✅ Admin password updated successfully")
print(" Username: admin")
print(" Password: admin")
return True
else:
print("❌ Password verification failed after update")
return False
except Exception as e:
print(f"❌ Error: {e}")
import traceback
traceback.print_exc()
db.rollback()
return False
finally:
db.close()
if __name__ == "__main__":
success = fix_admin_password()
sys.exit(0 if success else 1)
+264
View File
@@ -0,0 +1,264 @@
#!/usr/bin/env python3
"""
Migrate data from SQLite to PostgreSQL database.
This script:
1. Creates PostgreSQL databases if they don't exist
2. Creates all tables in PostgreSQL
3. Migrates all data from SQLite to PostgreSQL
"""
from __future__ import annotations
import sys
import os
from pathlib import Path
# Add project root to path
project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))
from sqlalchemy import create_engine, inspect, text
from sqlalchemy.orm import sessionmaker
from backend.db.models import Base
from backend.db.session import get_database_url
import sqlite3
def create_postgresql_databases():
"""Create PostgreSQL databases if they don't exist."""
from urllib.parse import urlparse
# Get database URLs from environment
db_url = os.getenv("DATABASE_URL", "postgresql+psycopg2://punimtag:punimtag_password@localhost:5432/punimtag")
auth_db_url = os.getenv("DATABASE_URL_AUTH", "postgresql://punimtag:punimtag_password@localhost:5432/punimtag_auth")
# Parse URLs
main_parsed = urlparse(db_url.replace("postgresql+psycopg2://", "postgresql://"))
auth_parsed = urlparse(auth_db_url.replace("postgresql+psycopg2://", "postgresql://"))
main_db_name = main_parsed.path.lstrip("/")
auth_db_name = auth_parsed.path.lstrip("/")
# Connect to postgres database to create other databases
postgres_url = f"postgresql://{main_parsed.username}:{main_parsed.password}@{main_parsed.hostname}:{main_parsed.port or 5432}/postgres"
try:
engine = create_engine(postgres_url)
with engine.connect() as conn:
# Check if databases exist
result = conn.execute(text("SELECT 1 FROM pg_database WHERE datname = :name"), {"name": main_db_name})
if not result.fetchone():
conn.execute(text("COMMIT")) # End any transaction
conn.execute(text(f'CREATE DATABASE "{main_db_name}"'))
print(f"✅ Created database: {main_db_name}")
else:
print(f"✅ Database already exists: {main_db_name}")
result = conn.execute(text("SELECT 1 FROM pg_database WHERE datname = :name"), {"name": auth_db_name})
if not result.fetchone():
conn.execute(text("COMMIT"))
conn.execute(text(f'CREATE DATABASE "{auth_db_name}"'))
print(f"✅ Created database: {auth_db_name}")
else:
print(f"✅ Database already exists: {auth_db_name}")
except Exception as e:
print(f"⚠️ Error creating databases: {e}")
print(" Make sure PostgreSQL is running and credentials are correct")
def migrate_data():
"""Migrate data from SQLite to PostgreSQL."""
print("=" * 80)
print("MIGRATING DATA FROM SQLITE TO POSTGRESQL")
print("=" * 80)
# Get database URLs
sqlite_url = "sqlite:///data/punimtag.db"
postgres_url = os.getenv("DATABASE_URL", "postgresql+psycopg2://punimtag:punimtag_password@localhost:5432/punimtag")
if not postgres_url.startswith("postgresql"):
print("❌ DATABASE_URL is not set to PostgreSQL")
print(" Set DATABASE_URL in .env file to PostgreSQL connection string")
return False
# Connect to both databases
sqlite_engine = create_engine(sqlite_url)
postgres_engine = create_engine(postgres_url)
# Create tables in PostgreSQL
print("\n📋 Creating tables in PostgreSQL...")
Base.metadata.create_all(bind=postgres_engine)
print("✅ Tables created")
# Get table names
inspector = inspect(sqlite_engine)
all_tables = inspector.get_table_names()
# Exclude system tables
all_tables = [t for t in all_tables if not t.startswith("sqlite_")]
# Define migration order (respecting foreign key constraints)
# Tables with no dependencies first, then dependent tables
migration_order = [
"alembic_version", # Migration tracking (optional)
"photos", # Base table
"people", # Base table
"tags", # Base table
"users", # Base table
"faces", # Depends on photos, people, users
"person_encodings", # Depends on people, faces
"phototaglinkage", # Depends on photos, tags
"photo_favorites", # Depends on photos
"photo_person_linkage", # Depends on photos, people, users
"role_permissions", # Base table
]
# Filter to only tables that exist
tables = [t for t in migration_order if t in all_tables]
# Add any remaining tables not in the order list
for t in all_tables:
if t not in tables:
tables.append(t)
print(f"\n📊 Found {len(tables)} tables to migrate: {', '.join(tables)}")
# Boolean columns mapping (SQLite stores as integer, PostgreSQL needs boolean)
boolean_columns = {
"photos": ["processed"],
"faces": ["is_primary_encoding", "excluded"],
"users": ["is_active", "is_admin", "password_change_required"],
"role_permissions": ["allowed"],
}
# Columns that might be missing in SQLite but required in PostgreSQL
# Map: table_name -> {column: default_value}
default_values = {
"photos": {"file_hash": "migrated"}, # file_hash might be missing in old SQLite
}
# Migrate each table
with sqlite_engine.connect() as sqlite_conn, postgres_engine.connect() as postgres_conn:
for table in tables:
print(f"\n🔄 Migrating table: {table}")
# Get row count
count_result = sqlite_conn.execute(text(f"SELECT COUNT(*) FROM {table}"))
row_count = count_result.scalar()
if row_count == 0:
print(f" ⏭️ Table is empty, skipping")
continue
print(f" 📦 {row_count} rows to migrate")
# Check if table already has data in PostgreSQL
try:
pg_count_result = postgres_conn.execute(text(f'SELECT COUNT(*) FROM "{table}"'))
pg_count = pg_count_result.scalar()
if pg_count > 0:
print(f" ⚠️ Table already has {pg_count} rows in PostgreSQL")
# Auto-truncate for non-interactive mode, or ask in interactive
print(f" 🗑️ Truncating existing data...")
postgres_conn.execute(text(f'TRUNCATE TABLE "{table}" CASCADE'))
postgres_conn.commit()
except Exception as e:
# Table might not exist yet, that's OK
pass
# Get column names and types from SQLite
columns_result = sqlite_conn.execute(text(f"PRAGMA table_info({table})"))
column_info = columns_result.fetchall()
sqlite_columns = [row[1] for row in column_info]
# Get PostgreSQL column names
pg_inspector = inspect(postgres_engine)
pg_columns_info = pg_inspector.get_columns(table)
pg_columns = [col['name'] for col in pg_columns_info]
# Use PostgreSQL columns (they're the source of truth)
columns = pg_columns
# Get boolean columns for this table
table_bool_cols = boolean_columns.get(table, [])
# Get default values for missing columns
table_defaults = default_values.get(table, {})
# Build SELECT statement for SQLite (only select columns that exist)
select_cols = [col for col in columns if col in sqlite_columns]
select_sql = f"SELECT {', '.join(select_cols)} FROM {table}"
# Fetch all data
data_result = sqlite_conn.execute(text(select_sql))
rows = data_result.fetchall()
# Insert into PostgreSQL
inserted = 0
for row in rows:
try:
# Build insert statement with boolean conversion
values = {}
for i, col in enumerate(select_cols):
val = row[i]
# Convert integer booleans to Python booleans for PostgreSQL
if col in table_bool_cols:
val = bool(val) if val is not None else None
values[col] = val
# Add default values for missing columns
for col, default_val in table_defaults.items():
if col not in values and col in columns:
values[col] = default_val
# Only insert columns we have values for (that exist in PostgreSQL)
insert_cols = [col for col in columns if col in values]
cols_str = ', '.join([f'"{c}"' for c in insert_cols])
placeholders = ', '.join([f':{c}' for c in insert_cols])
insert_sql = f'INSERT INTO "{table}" ({cols_str}) VALUES ({placeholders})'
postgres_conn.execute(text(insert_sql), values)
inserted += 1
if inserted % 100 == 0:
postgres_conn.commit()
print(f" ✅ Inserted {inserted}/{row_count} rows...", end='\r')
except Exception as e:
print(f"\n ❌ Error inserting row: {e}")
print(f" Row data: {dict(zip(columns, row))}")
postgres_conn.rollback()
break
postgres_conn.commit()
print(f" ✅ Migrated {inserted}/{row_count} rows from {table}")
print("\n" + "=" * 80)
print("✅ MIGRATION COMPLETE")
print("=" * 80)
print("\nNext steps:")
print("1. Update .env file to use PostgreSQL:")
print(" DATABASE_URL=postgresql+psycopg2://punimtag:punimtag_password@localhost:5432/punimtag")
print("2. Restart the backend API")
print("3. Restart the viewer frontend")
print("4. Verify data in viewer frontend")
return True
if __name__ == "__main__":
print("🔧 SQLite to PostgreSQL Migration Tool\n")
# Check if SQLite database exists
sqlite_path = project_root / "data" / "punimtag.db"
if not sqlite_path.exists():
print(f"❌ SQLite database not found: {sqlite_path}")
sys.exit(1)
print(f"✅ Found SQLite database: {sqlite_path}")
# Create PostgreSQL databases
print("\n📦 Creating PostgreSQL databases...")
create_postgresql_databases()
# Migrate data
print("\n")
migrate_data()
+2 -2
View File
@@ -7,8 +7,8 @@ import os
# Add project root to path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.web.db.models import Base
from src.web.db.session import engine, get_database_url
from backend.db.models import Base
from backend.db.session import engine, get_database_url
def recreate_tables():
+8 -2
View File
@@ -32,12 +32,17 @@ BEGIN
END
\$\$;
-- Create database if it doesn't exist
-- Create main database if it doesn't exist
SELECT 'CREATE DATABASE punimtag OWNER punimtag'
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'punimtag')\gexec
-- Create auth database if it doesn't exist
SELECT 'CREATE DATABASE punimtag_auth OWNER punimtag'
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'punimtag_auth')\gexec
-- Grant privileges
GRANT ALL PRIVILEGES ON DATABASE punimtag TO punimtag;
GRANT ALL PRIVILEGES ON DATABASE punimtag_auth TO punimtag;
\q
EOF
@@ -46,7 +51,8 @@ echo ""
echo "📋 Database connection details:"
echo " Host: localhost"
echo " Port: 5432"
echo " Database: punimtag"
echo " Main Database: punimtag"
echo " Auth Database: punimtag_auth"
echo " User: punimtag"
echo " Password: punimtag_password"
echo ""
+3 -3
View File
@@ -8,8 +8,8 @@ import os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sqlalchemy import inspect, text
from src.web.db.session import engine, get_database_url
from src.web.db.models import Base
from backend.db.session import engine, get_database_url
from backend.db.models import Base
def show_table_structure(table_name: str, inspector):
@@ -69,7 +69,7 @@ def show_all_tables():
print("\nHere are the table structures from models:")
# Show from models instead
from src.web.db.models import Photo, Person, Face, PersonEmbedding, Tag, PhotoTag
from backend.db.models import Photo, Person, Face, PersonEmbedding, Tag, PhotoTag
models = [
("photos", Photo),
+1 -1
View File
@@ -9,7 +9,7 @@ project_root = Path(__file__).parent.parent
sys.path.insert(0, str(project_root))
from sqlalchemy import text
from src.web.db.session import get_auth_database_url, AuthSessionLocal
from backend.db.session import get_auth_database_url, AuthSessionLocal
def update_reported_photo_status(report_id: int, new_status: str):
"""Update the status of a reported photo."""