feat: Add new analysis documents and update installation scripts for backend integration
This commit introduces several new analysis documents, including Auto-Match Load Performance Analysis, Folder Picker Analysis, Monorepo Migration Summary, and various performance analysis documents. Additionally, the installation scripts are updated to reflect changes in backend service paths, ensuring proper integration with the new backend structure. These enhancements provide better documentation and streamline the setup process for users.
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Check what tables exist in the punimtag main database and their record counts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
from backend.db.session import get_database_url
|
||||
|
||||
|
||||
def check_database_tables() -> None:
|
||||
"""Check all tables in the database and their record counts."""
|
||||
database_url = get_database_url()
|
||||
|
||||
print("=" * 80)
|
||||
print("PUNIMTAG MAIN DATABASE - TABLE INFORMATION")
|
||||
print("=" * 80)
|
||||
print(f"\nDatabase URL: {database_url.replace('://', '://****') if '://' in database_url else database_url}\n")
|
||||
|
||||
# Create engine
|
||||
connect_args = {}
|
||||
if database_url.startswith("sqlite"):
|
||||
connect_args = {"check_same_thread": False}
|
||||
|
||||
engine = create_engine(database_url, connect_args=connect_args)
|
||||
|
||||
try:
|
||||
# Get inspector to list tables
|
||||
inspector = inspect(engine)
|
||||
all_tables = inspector.get_table_names()
|
||||
|
||||
if not all_tables:
|
||||
print("❌ No tables found in database.")
|
||||
return
|
||||
|
||||
print(f"Found {len(all_tables)} tables:\n")
|
||||
|
||||
# Expected tables from models
|
||||
expected_tables = {
|
||||
"photos",
|
||||
"people",
|
||||
"faces",
|
||||
"person_encodings",
|
||||
"tags",
|
||||
"phototaglinkage",
|
||||
"photo_favorites",
|
||||
"users",
|
||||
"photo_person_linkage",
|
||||
"role_permissions",
|
||||
}
|
||||
|
||||
# Connect and query each table
|
||||
with engine.connect() as conn:
|
||||
print(f"{'Table Name':<30} {'Record Count':<15} {'Status'}")
|
||||
print("-" * 80)
|
||||
|
||||
for table_name in sorted(all_tables):
|
||||
# Skip SQLite system tables
|
||||
if table_name.startswith("sqlite_"):
|
||||
continue
|
||||
|
||||
try:
|
||||
# Get record count
|
||||
if database_url.startswith("sqlite"):
|
||||
result = conn.execute(text(f"SELECT COUNT(*) FROM {table_name}"))
|
||||
else:
|
||||
result = conn.execute(text(f'SELECT COUNT(*) FROM "{table_name}"'))
|
||||
|
||||
count = result.scalar()
|
||||
|
||||
# Check if it's an expected table
|
||||
status = "✅ Expected" if table_name in expected_tables else "⚠️ Unexpected"
|
||||
|
||||
print(f"{table_name:<30} {count:<15} {status}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"{table_name:<30} {'ERROR':<15} ❌ {str(e)[:50]}")
|
||||
|
||||
print("-" * 80)
|
||||
|
||||
# Summary
|
||||
print("\n📊 Summary:")
|
||||
with engine.connect() as conn:
|
||||
total_records = 0
|
||||
tables_with_data = 0
|
||||
for table_name in sorted(all_tables):
|
||||
if table_name.startswith("sqlite_"):
|
||||
continue
|
||||
try:
|
||||
if database_url.startswith("sqlite"):
|
||||
result = conn.execute(text(f"SELECT COUNT(*) FROM {table_name}"))
|
||||
else:
|
||||
result = conn.execute(text(f'SELECT COUNT(*) FROM "{table_name}"'))
|
||||
count = result.scalar()
|
||||
total_records += count
|
||||
if count > 0:
|
||||
tables_with_data += 1
|
||||
except:
|
||||
pass
|
||||
|
||||
print(f" Total tables: {len([t for t in all_tables if not t.startswith('sqlite_')])}")
|
||||
print(f" Tables with records: {tables_with_data}")
|
||||
print(f" Total records across all tables: {total_records:,}")
|
||||
|
||||
# Check for missing expected tables
|
||||
missing_tables = expected_tables - set(all_tables)
|
||||
if missing_tables:
|
||||
print(f"\n⚠️ Missing expected tables: {', '.join(sorted(missing_tables))}")
|
||||
|
||||
# Check for unexpected tables
|
||||
unexpected_tables = set(all_tables) - expected_tables - {"alembic_version"}
|
||||
unexpected_tables = {t for t in unexpected_tables if not t.startswith("sqlite_")}
|
||||
if unexpected_tables:
|
||||
print(f"\nℹ️ Additional tables found: {', '.join(sorted(unexpected_tables))}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error connecting to database: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_database_tables()
|
||||
|
||||
@@ -9,8 +9,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from src.web.db.models import Face, Person, Photo
|
||||
from src.web.db.session import get_database_url
|
||||
from backend.db.models import Face, Person, Photo
|
||||
from backend.db.session import get_database_url
|
||||
|
||||
def check_identified_faces():
|
||||
"""Check all identified faces for pose information"""
|
||||
|
||||
@@ -10,8 +10,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from src.web.db.models import Face, Person, Photo
|
||||
from src.web.db.session import get_database_url
|
||||
from backend.db.models import Face, Person, Photo
|
||||
from backend.db.session import get_database_url
|
||||
from src.utils.pose_detection import PoseDetector
|
||||
|
||||
def check_two_faces(face_id1: int = None, face_id2: int = None):
|
||||
|
||||
@@ -15,8 +15,8 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from src.web.db.models import Face, Person, Photo
|
||||
from src.web.db.session import get_database_url
|
||||
from backend.db.models import Face, Person, Photo
|
||||
from backend.db.session import get_database_url
|
||||
from src.utils.pose_detection import PoseDetector
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Diagnose frontend issues:
|
||||
1. Check if backend API is running and accessible
|
||||
2. Check database connection
|
||||
3. Test search endpoint
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from backend.db.session import get_database_url, engine
|
||||
from sqlalchemy import text
|
||||
|
||||
def check_backend_api():
|
||||
"""Check if backend API is running."""
|
||||
print("=" * 80)
|
||||
print("BACKEND API CHECK")
|
||||
print("=" * 80)
|
||||
|
||||
try:
|
||||
# Check if docs endpoint is accessible
|
||||
response = requests.get("http://127.0.0.1:8000/docs", timeout=5)
|
||||
if response.status_code == 200:
|
||||
print("✅ Backend API is running (docs accessible)")
|
||||
else:
|
||||
print(f"⚠️ Backend API returned status {response.status_code}")
|
||||
except requests.exceptions.ConnectionError:
|
||||
print("❌ Backend API is NOT running or not accessible")
|
||||
print(" Start it with: cd backend && uvicorn app:app --reload")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"❌ Error checking backend API: {e}")
|
||||
return False
|
||||
|
||||
# Check search endpoint (requires auth)
|
||||
try:
|
||||
response = requests.get(
|
||||
"http://127.0.0.1:8000/api/v1/photos",
|
||||
params={"search_type": "processed", "page": 1, "page_size": 1},
|
||||
timeout=5
|
||||
)
|
||||
if response.status_code == 200:
|
||||
print("✅ Search endpoint is accessible (no auth required for this query)")
|
||||
elif response.status_code == 401:
|
||||
print("⚠️ Search endpoint requires authentication")
|
||||
print(" User needs to log in through admin frontend")
|
||||
else:
|
||||
print(f"⚠️ Search endpoint returned status {response.status_code}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error checking search endpoint: {e}")
|
||||
|
||||
return True
|
||||
|
||||
def check_database():
|
||||
"""Check database connection and photo count."""
|
||||
print("\n" + "=" * 80)
|
||||
print("DATABASE CHECK")
|
||||
print("=" * 80)
|
||||
|
||||
db_url = get_database_url()
|
||||
print(f"Database URL: {db_url.replace('://', '://****') if '://' in db_url else db_url}")
|
||||
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
# Check photo count
|
||||
result = conn.execute(text("SELECT COUNT(*) FROM photos WHERE processed = 1"))
|
||||
count = result.scalar()
|
||||
print(f"✅ Database connection successful")
|
||||
print(f" Processed photos: {count}")
|
||||
|
||||
if count == 0:
|
||||
print("⚠️ No processed photos found in database")
|
||||
print(" This explains why viewer frontend shows 0 photos")
|
||||
else:
|
||||
print(f" Database has {count} processed photos")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Database connection error: {e}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def check_viewer_frontend_config():
|
||||
"""Check viewer frontend configuration."""
|
||||
print("\n" + "=" * 80)
|
||||
print("VIEWER FRONTEND CONFIGURATION")
|
||||
print("=" * 80)
|
||||
|
||||
viewer_env = project_root / "viewer-frontend" / ".env"
|
||||
if not viewer_env.exists():
|
||||
print("❌ viewer-frontend/.env file not found")
|
||||
return False
|
||||
|
||||
with open(viewer_env) as f:
|
||||
content = f.read()
|
||||
if "DATABASE_URL" in content:
|
||||
# Extract DATABASE_URL
|
||||
for line in content.split("\n"):
|
||||
if line.startswith("DATABASE_URL="):
|
||||
db_url = line.split("=", 1)[1].strip().strip('"')
|
||||
print(f"Viewer frontend DATABASE_URL: {db_url.replace('://', '://****') if '://' in db_url else db_url}")
|
||||
|
||||
# Check if it matches actual database
|
||||
actual_db = get_database_url()
|
||||
if "postgresql" in db_url and "sqlite" in actual_db:
|
||||
print("❌ MISMATCH: Viewer frontend configured for PostgreSQL")
|
||||
print(" but actual database is SQLite")
|
||||
print("\n SOLUTION OPTIONS:")
|
||||
print(" 1. Change viewer-frontend/.env DATABASE_URL to SQLite:")
|
||||
print(f' DATABASE_URL="file:../data/punimtag.db"')
|
||||
print(" 2. Update Prisma schema to use SQLite provider")
|
||||
print(" 3. Migrate database to PostgreSQL")
|
||||
return False
|
||||
elif "sqlite" in db_url and "sqlite" in actual_db:
|
||||
print("✅ Viewer frontend configured for SQLite (matches actual database)")
|
||||
else:
|
||||
print("⚠️ Database type mismatch or unclear")
|
||||
else:
|
||||
print("⚠️ DATABASE_URL not found in viewer-frontend/.env")
|
||||
|
||||
return True
|
||||
|
||||
def main():
|
||||
print("\n🔍 DIAGNOSING FRONTEND ISSUES\n")
|
||||
|
||||
backend_ok = check_backend_api()
|
||||
db_ok = check_database()
|
||||
viewer_config_ok = check_viewer_frontend_config()
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("SUMMARY")
|
||||
print("=" * 80)
|
||||
|
||||
if not backend_ok:
|
||||
print("❌ Backend API is not running - admin frontend search will fail")
|
||||
else:
|
||||
print("✅ Backend API is running")
|
||||
|
||||
if not db_ok:
|
||||
print("❌ Database connection issue")
|
||||
else:
|
||||
print("✅ Database connection OK")
|
||||
|
||||
if not viewer_config_ok:
|
||||
print("❌ Viewer frontend configuration issue - needs to be fixed")
|
||||
else:
|
||||
print("✅ Viewer frontend configuration OK")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -8,8 +8,8 @@ import os
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from sqlalchemy import inspect
|
||||
from src.web.db.session import engine, get_database_url
|
||||
from src.web.db.models import Base
|
||||
from backend.db.session import engine, get_database_url
|
||||
from backend.db.models import Base
|
||||
|
||||
# Ordered list ensures foreign-key dependents drop first
|
||||
TARGET_TABLES = [
|
||||
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
# Drop the auth database if it exists
|
||||
|
||||
set -e
|
||||
|
||||
echo "🗑️ Dropping auth database if it exists..."
|
||||
|
||||
sudo -u postgres psql -c "DROP DATABASE IF EXISTS punimtag_auth;"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Auth database dropped successfully"
|
||||
else
|
||||
echo "⚠️ Failed to drop auth database (it may not exist)"
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix admin user password in database."""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from backend.db.session import get_db
|
||||
from backend.db.models import User
|
||||
from backend.utils.password import hash_password, verify_password
|
||||
|
||||
def fix_admin_password():
|
||||
"""Set admin user password to 'admin'."""
|
||||
db = next(get_db())
|
||||
try:
|
||||
admin_user = db.query(User).filter(User.username == 'admin').first()
|
||||
|
||||
if not admin_user:
|
||||
print("❌ Admin user not found in database")
|
||||
return False
|
||||
|
||||
# Set password to 'admin'
|
||||
new_hash = hash_password('admin')
|
||||
admin_user.password_hash = new_hash
|
||||
admin_user.is_active = True
|
||||
admin_user.is_admin = True
|
||||
db.commit()
|
||||
|
||||
# Verify it works
|
||||
if verify_password('admin', new_hash):
|
||||
print("✅ Admin password updated successfully")
|
||||
print(" Username: admin")
|
||||
print(" Password: admin")
|
||||
return True
|
||||
else:
|
||||
print("❌ Password verification failed after update")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
db.rollback()
|
||||
return False
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = fix_admin_password()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migrate data from SQLite to PostgreSQL database.
|
||||
|
||||
This script:
|
||||
1. Creates PostgreSQL databases if they don't exist
|
||||
2. Creates all tables in PostgreSQL
|
||||
3. Migrates all data from SQLite to PostgreSQL
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from backend.db.models import Base
|
||||
from backend.db.session import get_database_url
|
||||
import sqlite3
|
||||
|
||||
def create_postgresql_databases():
|
||||
"""Create PostgreSQL databases if they don't exist."""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Get database URLs from environment
|
||||
db_url = os.getenv("DATABASE_URL", "postgresql+psycopg2://punimtag:punimtag_password@localhost:5432/punimtag")
|
||||
auth_db_url = os.getenv("DATABASE_URL_AUTH", "postgresql://punimtag:punimtag_password@localhost:5432/punimtag_auth")
|
||||
|
||||
# Parse URLs
|
||||
main_parsed = urlparse(db_url.replace("postgresql+psycopg2://", "postgresql://"))
|
||||
auth_parsed = urlparse(auth_db_url.replace("postgresql+psycopg2://", "postgresql://"))
|
||||
|
||||
main_db_name = main_parsed.path.lstrip("/")
|
||||
auth_db_name = auth_parsed.path.lstrip("/")
|
||||
|
||||
# Connect to postgres database to create other databases
|
||||
postgres_url = f"postgresql://{main_parsed.username}:{main_parsed.password}@{main_parsed.hostname}:{main_parsed.port or 5432}/postgres"
|
||||
|
||||
try:
|
||||
engine = create_engine(postgres_url)
|
||||
with engine.connect() as conn:
|
||||
# Check if databases exist
|
||||
result = conn.execute(text("SELECT 1 FROM pg_database WHERE datname = :name"), {"name": main_db_name})
|
||||
if not result.fetchone():
|
||||
conn.execute(text("COMMIT")) # End any transaction
|
||||
conn.execute(text(f'CREATE DATABASE "{main_db_name}"'))
|
||||
print(f"✅ Created database: {main_db_name}")
|
||||
else:
|
||||
print(f"✅ Database already exists: {main_db_name}")
|
||||
|
||||
result = conn.execute(text("SELECT 1 FROM pg_database WHERE datname = :name"), {"name": auth_db_name})
|
||||
if not result.fetchone():
|
||||
conn.execute(text("COMMIT"))
|
||||
conn.execute(text(f'CREATE DATABASE "{auth_db_name}"'))
|
||||
print(f"✅ Created database: {auth_db_name}")
|
||||
else:
|
||||
print(f"✅ Database already exists: {auth_db_name}")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error creating databases: {e}")
|
||||
print(" Make sure PostgreSQL is running and credentials are correct")
|
||||
|
||||
def migrate_data():
|
||||
"""Migrate data from SQLite to PostgreSQL."""
|
||||
print("=" * 80)
|
||||
print("MIGRATING DATA FROM SQLITE TO POSTGRESQL")
|
||||
print("=" * 80)
|
||||
|
||||
# Get database URLs
|
||||
sqlite_url = "sqlite:///data/punimtag.db"
|
||||
postgres_url = os.getenv("DATABASE_URL", "postgresql+psycopg2://punimtag:punimtag_password@localhost:5432/punimtag")
|
||||
|
||||
if not postgres_url.startswith("postgresql"):
|
||||
print("❌ DATABASE_URL is not set to PostgreSQL")
|
||||
print(" Set DATABASE_URL in .env file to PostgreSQL connection string")
|
||||
return False
|
||||
|
||||
# Connect to both databases
|
||||
sqlite_engine = create_engine(sqlite_url)
|
||||
postgres_engine = create_engine(postgres_url)
|
||||
|
||||
# Create tables in PostgreSQL
|
||||
print("\n📋 Creating tables in PostgreSQL...")
|
||||
Base.metadata.create_all(bind=postgres_engine)
|
||||
print("✅ Tables created")
|
||||
|
||||
# Get table names
|
||||
inspector = inspect(sqlite_engine)
|
||||
all_tables = inspector.get_table_names()
|
||||
|
||||
# Exclude system tables
|
||||
all_tables = [t for t in all_tables if not t.startswith("sqlite_")]
|
||||
|
||||
# Define migration order (respecting foreign key constraints)
|
||||
# Tables with no dependencies first, then dependent tables
|
||||
migration_order = [
|
||||
"alembic_version", # Migration tracking (optional)
|
||||
"photos", # Base table
|
||||
"people", # Base table
|
||||
"tags", # Base table
|
||||
"users", # Base table
|
||||
"faces", # Depends on photos, people, users
|
||||
"person_encodings", # Depends on people, faces
|
||||
"phototaglinkage", # Depends on photos, tags
|
||||
"photo_favorites", # Depends on photos
|
||||
"photo_person_linkage", # Depends on photos, people, users
|
||||
"role_permissions", # Base table
|
||||
]
|
||||
|
||||
# Filter to only tables that exist
|
||||
tables = [t for t in migration_order if t in all_tables]
|
||||
# Add any remaining tables not in the order list
|
||||
for t in all_tables:
|
||||
if t not in tables:
|
||||
tables.append(t)
|
||||
|
||||
print(f"\n📊 Found {len(tables)} tables to migrate: {', '.join(tables)}")
|
||||
|
||||
# Boolean columns mapping (SQLite stores as integer, PostgreSQL needs boolean)
|
||||
boolean_columns = {
|
||||
"photos": ["processed"],
|
||||
"faces": ["is_primary_encoding", "excluded"],
|
||||
"users": ["is_active", "is_admin", "password_change_required"],
|
||||
"role_permissions": ["allowed"],
|
||||
}
|
||||
|
||||
# Columns that might be missing in SQLite but required in PostgreSQL
|
||||
# Map: table_name -> {column: default_value}
|
||||
default_values = {
|
||||
"photos": {"file_hash": "migrated"}, # file_hash might be missing in old SQLite
|
||||
}
|
||||
|
||||
# Migrate each table
|
||||
with sqlite_engine.connect() as sqlite_conn, postgres_engine.connect() as postgres_conn:
|
||||
for table in tables:
|
||||
print(f"\n🔄 Migrating table: {table}")
|
||||
|
||||
# Get row count
|
||||
count_result = sqlite_conn.execute(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
row_count = count_result.scalar()
|
||||
|
||||
if row_count == 0:
|
||||
print(f" ⏭️ Table is empty, skipping")
|
||||
continue
|
||||
|
||||
print(f" 📦 {row_count} rows to migrate")
|
||||
|
||||
# Check if table already has data in PostgreSQL
|
||||
try:
|
||||
pg_count_result = postgres_conn.execute(text(f'SELECT COUNT(*) FROM "{table}"'))
|
||||
pg_count = pg_count_result.scalar()
|
||||
if pg_count > 0:
|
||||
print(f" ⚠️ Table already has {pg_count} rows in PostgreSQL")
|
||||
# Auto-truncate for non-interactive mode, or ask in interactive
|
||||
print(f" 🗑️ Truncating existing data...")
|
||||
postgres_conn.execute(text(f'TRUNCATE TABLE "{table}" CASCADE'))
|
||||
postgres_conn.commit()
|
||||
except Exception as e:
|
||||
# Table might not exist yet, that's OK
|
||||
pass
|
||||
|
||||
# Get column names and types from SQLite
|
||||
columns_result = sqlite_conn.execute(text(f"PRAGMA table_info({table})"))
|
||||
column_info = columns_result.fetchall()
|
||||
sqlite_columns = [row[1] for row in column_info]
|
||||
|
||||
# Get PostgreSQL column names
|
||||
pg_inspector = inspect(postgres_engine)
|
||||
pg_columns_info = pg_inspector.get_columns(table)
|
||||
pg_columns = [col['name'] for col in pg_columns_info]
|
||||
|
||||
# Use PostgreSQL columns (they're the source of truth)
|
||||
columns = pg_columns
|
||||
|
||||
# Get boolean columns for this table
|
||||
table_bool_cols = boolean_columns.get(table, [])
|
||||
|
||||
# Get default values for missing columns
|
||||
table_defaults = default_values.get(table, {})
|
||||
|
||||
# Build SELECT statement for SQLite (only select columns that exist)
|
||||
select_cols = [col for col in columns if col in sqlite_columns]
|
||||
select_sql = f"SELECT {', '.join(select_cols)} FROM {table}"
|
||||
|
||||
# Fetch all data
|
||||
data_result = sqlite_conn.execute(text(select_sql))
|
||||
rows = data_result.fetchall()
|
||||
|
||||
# Insert into PostgreSQL
|
||||
inserted = 0
|
||||
for row in rows:
|
||||
try:
|
||||
# Build insert statement with boolean conversion
|
||||
values = {}
|
||||
for i, col in enumerate(select_cols):
|
||||
val = row[i]
|
||||
# Convert integer booleans to Python booleans for PostgreSQL
|
||||
if col in table_bool_cols:
|
||||
val = bool(val) if val is not None else None
|
||||
values[col] = val
|
||||
|
||||
# Add default values for missing columns
|
||||
for col, default_val in table_defaults.items():
|
||||
if col not in values and col in columns:
|
||||
values[col] = default_val
|
||||
|
||||
# Only insert columns we have values for (that exist in PostgreSQL)
|
||||
insert_cols = [col for col in columns if col in values]
|
||||
cols_str = ', '.join([f'"{c}"' for c in insert_cols])
|
||||
placeholders = ', '.join([f':{c}' for c in insert_cols])
|
||||
insert_sql = f'INSERT INTO "{table}" ({cols_str}) VALUES ({placeholders})'
|
||||
|
||||
postgres_conn.execute(text(insert_sql), values)
|
||||
inserted += 1
|
||||
|
||||
if inserted % 100 == 0:
|
||||
postgres_conn.commit()
|
||||
print(f" ✅ Inserted {inserted}/{row_count} rows...", end='\r')
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n ❌ Error inserting row: {e}")
|
||||
print(f" Row data: {dict(zip(columns, row))}")
|
||||
postgres_conn.rollback()
|
||||
break
|
||||
|
||||
postgres_conn.commit()
|
||||
print(f" ✅ Migrated {inserted}/{row_count} rows from {table}")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("✅ MIGRATION COMPLETE")
|
||||
print("=" * 80)
|
||||
print("\nNext steps:")
|
||||
print("1. Update .env file to use PostgreSQL:")
|
||||
print(" DATABASE_URL=postgresql+psycopg2://punimtag:punimtag_password@localhost:5432/punimtag")
|
||||
print("2. Restart the backend API")
|
||||
print("3. Restart the viewer frontend")
|
||||
print("4. Verify data in viewer frontend")
|
||||
|
||||
return True
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("🔧 SQLite to PostgreSQL Migration Tool\n")
|
||||
|
||||
# Check if SQLite database exists
|
||||
sqlite_path = project_root / "data" / "punimtag.db"
|
||||
if not sqlite_path.exists():
|
||||
print(f"❌ SQLite database not found: {sqlite_path}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"✅ Found SQLite database: {sqlite_path}")
|
||||
|
||||
# Create PostgreSQL databases
|
||||
print("\n📦 Creating PostgreSQL databases...")
|
||||
create_postgresql_databases()
|
||||
|
||||
# Migrate data
|
||||
print("\n")
|
||||
migrate_data()
|
||||
|
||||
@@ -7,8 +7,8 @@ import os
|
||||
# Add project root to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from src.web.db.models import Base
|
||||
from src.web.db.session import engine, get_database_url
|
||||
from backend.db.models import Base
|
||||
from backend.db.session import engine, get_database_url
|
||||
|
||||
|
||||
def recreate_tables():
|
||||
|
||||
@@ -32,12 +32,17 @@ BEGIN
|
||||
END
|
||||
\$\$;
|
||||
|
||||
-- Create database if it doesn't exist
|
||||
-- Create main database if it doesn't exist
|
||||
SELECT 'CREATE DATABASE punimtag OWNER punimtag'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'punimtag')\gexec
|
||||
|
||||
-- Create auth database if it doesn't exist
|
||||
SELECT 'CREATE DATABASE punimtag_auth OWNER punimtag'
|
||||
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'punimtag_auth')\gexec
|
||||
|
||||
-- Grant privileges
|
||||
GRANT ALL PRIVILEGES ON DATABASE punimtag TO punimtag;
|
||||
GRANT ALL PRIVILEGES ON DATABASE punimtag_auth TO punimtag;
|
||||
\q
|
||||
EOF
|
||||
|
||||
@@ -46,7 +51,8 @@ echo ""
|
||||
echo "📋 Database connection details:"
|
||||
echo " Host: localhost"
|
||||
echo " Port: 5432"
|
||||
echo " Database: punimtag"
|
||||
echo " Main Database: punimtag"
|
||||
echo " Auth Database: punimtag_auth"
|
||||
echo " User: punimtag"
|
||||
echo " Password: punimtag_password"
|
||||
echo ""
|
||||
|
||||
@@ -8,8 +8,8 @@ import os
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from sqlalchemy import inspect, text
|
||||
from src.web.db.session import engine, get_database_url
|
||||
from src.web.db.models import Base
|
||||
from backend.db.session import engine, get_database_url
|
||||
from backend.db.models import Base
|
||||
|
||||
|
||||
def show_table_structure(table_name: str, inspector):
|
||||
@@ -69,7 +69,7 @@ def show_all_tables():
|
||||
print("\nHere are the table structures from models:")
|
||||
|
||||
# Show from models instead
|
||||
from src.web.db.models import Photo, Person, Face, PersonEmbedding, Tag, PhotoTag
|
||||
from backend.db.models import Photo, Person, Face, PersonEmbedding, Tag, PhotoTag
|
||||
|
||||
models = [
|
||||
("photos", Photo),
|
||||
|
||||
@@ -9,7 +9,7 @@ project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from sqlalchemy import text
|
||||
from src.web.db.session import get_auth_database_url, AuthSessionLocal
|
||||
from backend.db.session import get_auth_database_url, AuthSessionLocal
|
||||
|
||||
def update_reported_photo_status(report_id: int, new_status: str):
|
||||
"""Update the status of a reported photo."""
|
||||
|
||||
Reference in New Issue
Block a user